Setup
Packages and options
library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
Datasets to test
NNdataSummary(NNdatasets)
## Dataset n_rows n_inputs n_neurons n_parameters
## 1 mDette 500 3 5 26
## 2 mFriedman 500 5 5 36
## 3 mIshigami 500 3 10 51
## 4 mRef153 153 5 3 22
## 5 uDmod1 51 1 6 19
## 6 uDmod2 51 1 5 16
## 7 uDreyfus1 51 1 3 10
## 8 uDreyfus2 51 1 3 10
## 9 uGauss1 250 1 5 16
## 10 uGauss2 250 1 4 13
## 11 uGauss3 250 1 4 13
## 12 uNeuroOne 51 1 2 7
automl trainPredict arguments - inputs x, y
if(dir.exists("D:/GSoC2020/Results/2020run03/"))
{
odir <- "D:/GSoC2020/Results/2020run03/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{
odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
odir <- "~"
nrep <- 10
maxit2ndorder <- 200
maxit1storderA <- 1000
maxit1storderB <- 10000
maxit1storderC <- 100000
#library(automl)
automl.method <- c("trainwgrad_RMSprop","trainwgrad_adam", "trainwpso") #"trainwgrad_Momentum"
hyperParams.automl <- function(optim_method, ...) {
# if (optim_method == "trainwgrad_Momentum") {beta2 <- 0.9; beta2 <- 0 ; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
if (optim_method == "trainwgrad_RMSprop") {beta1 <- 0 ; beta2 <- 0.99; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
if (optim_method == "trainwgrad_adam") {beta1 <- 0.9; beta2 <- 0.99; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
if (optim_method == "trainwpso") {beta1 <- NULL; beta2 <- NULL; iter <- maxit1storderA; lr = NULL}
return(list(beta1 = beta1, beta2 = beta2, iter = iter, lr = lr, hidden_activation = 'tanh', optim_method = optim_method))
}
NNtrain.automl <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
hyper_params <- do.call(hyperParams, list(optim_method, ...))
beta1 <- hyper_params$beta1 ; beta2 <- hyper_params$beta2
iter <- hyper_params$iter ; lr <- hyper_params$lr
hidden_activation <- hyper_params$hidden_activation
# NNreg <- automl::automl_train(Xref = x, Yref = y)
NNreg <- automl::automl_train_manual(Xref = x, Yref = y,
hpar = list(modexec = hyper_params$optim_method,
beta1 = beta1, beta2 = beta2,
numiterations = iter, learningrate = lr,
layersshape = c(hidden_neur, 0),
layersacttype = c(hidden_activation, ""),
verbose = FALSE,
seed = as.integer(runif(1)*10000000)))
return (NNreg)
}
NNpredict.automl <- function(object, x, ...)
automl::automl_predict(model=object, X=x)
NNclose.automl <- function()
if("package:automl" %in% search())
detach("package:automl", unload=TRUE)
automl.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, automl.method, "NNtrain.automl", "hyperParams.automl", "NNpredict.automl",
NNsummary, "NNclose.automl", NA, automl.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
pkgname="automl", pkgfun="automl_train_manual", rdafile=TRUE, odir=odir)
Launch package’s trainPredict
res <- trainPredict_1pkg(1:12, pkgname = "automl", pkgfun = "automl_train_manual", automl.method,
prepareZZ.arg = automl.prepareZZ, nrep = nrep, doplot = FALSE,
csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)
Results
#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "trainwgrad_RMSprop")%>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwgrad_RMSprop
|
|
RMSE
|
MSE
|
MAE
|
WAE
|
time
|
|
mDette
|
0.4734
|
0.2241
|
0.3598
|
2.6230
|
8.37
|
|
mFriedman
|
0.0418
|
0.0017
|
0.0318
|
0.1627
|
8.11
|
|
mIshigami
|
0.7727
|
0.5971
|
0.5606
|
3.2619
|
8.70
|
|
mRef153
|
3.6206
|
13.1088
|
2.5636
|
13.7640
|
2.75
|
|
uDmod1
|
0.1279
|
0.0163
|
0.0990
|
0.2929
|
1.08
|
|
uDmod2
|
0.0861
|
0.0074
|
0.0695
|
0.2004
|
1.08
|
|
uDreyfus1
|
0.0756
|
0.0057
|
0.0552
|
0.1827
|
1.05
|
|
uDreyfus2
|
0.1160
|
0.0135
|
0.0920
|
0.2642
|
1.06
|
|
uGauss1
|
3.5603
|
12.6759
|
2.6931
|
10.4343
|
4.36
|
|
uGauss2
|
4.0986
|
16.7989
|
3.0681
|
13.7961
|
4.16
|
|
uGauss3
|
3.4154
|
11.6647
|
2.7322
|
9.0438
|
1.82
|
|
uNeuroOne
|
0.2846
|
0.0810
|
0.2332
|
0.5660
|
1.04
|
kable(apply(res[,,2,], c(3,1), min), caption = "trainwgrad_adam")%>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwgrad_adam
|
|
RMSE
|
MSE
|
MAE
|
WAE
|
time
|
|
mDette
|
0.4883
|
0.2385
|
0.3833
|
1.7848
|
9.36
|
|
mFriedman
|
0.0332
|
0.0011
|
0.0269
|
0.0984
|
9.42
|
|
mIshigami
|
0.7195
|
0.5177
|
0.5359
|
2.5098
|
9.33
|
|
mRef153
|
3.3858
|
11.4637
|
2.3050
|
13.2932
|
3.14
|
|
uDmod1
|
0.0603
|
0.0036
|
0.0470
|
0.1460
|
1.22
|
|
uDmod2
|
0.0611
|
0.0037
|
0.0491
|
0.1312
|
1.19
|
|
uDreyfus1
|
0.0362
|
0.0013
|
0.0202
|
0.1252
|
0.82
|
|
uDreyfus2
|
0.0944
|
0.0089
|
0.0752
|
0.2221
|
0.68
|
|
uGauss1
|
3.7974
|
14.4201
|
2.7945
|
13.6836
|
4.89
|
|
uGauss2
|
3.6521
|
13.3379
|
2.7436
|
13.1870
|
4.89
|
|
uGauss3
|
2.5172
|
6.3364
|
2.0068
|
8.2477
|
4.82
|
|
uNeuroOne
|
0.2848
|
0.0811
|
0.2307
|
0.5874
|
0.92
|
kable(apply(res[,,3,], c(3,1), min), caption = "trainwpso")%>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwpso
|
|
RMSE
|
MSE
|
MAE
|
WAE
|
time
|
|
mDette
|
2.6100
|
6.8124
|
1.8890
|
14.5915
|
7.42
|
|
mFriedman
|
0.1078
|
0.0116
|
0.0853
|
0.2915
|
9.41
|
|
mIshigami
|
2.4888
|
6.1942
|
2.0244
|
7.1210
|
10.15
|
|
mRef153
|
3.7387
|
13.9779
|
2.7569
|
13.0120
|
4.08
|
|
uDmod1
|
0.1480
|
0.0219
|
0.1159
|
0.4691
|
6.33
|
|
uDmod2
|
0.1563
|
0.0244
|
0.1352
|
0.3168
|
2.70
|
|
uDreyfus1
|
0.0712
|
0.0051
|
0.0477
|
0.1549
|
0.89
|
|
uDreyfus2
|
0.1041
|
0.0108
|
0.0834
|
0.2364
|
2.55
|
|
uGauss1
|
10.5504
|
111.3111
|
8.1994
|
26.1492
|
6.61
|
|
uGauss2
|
4.2267
|
17.8652
|
3.3492
|
13.5698
|
4.02
|
|
uGauss3
|
4.2532
|
18.0900
|
3.4678
|
11.7201
|
6.69
|
|
uNeuroOne
|
0.2833
|
0.0802
|
0.2302
|
0.5432
|
2.55
|