1 Setup

1.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

1.2 Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

1.3 automl trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run03/"))
{  
  odir <- "D:/GSoC2020/Results/2020run03/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(automl)
automl.method <- c("trainwgrad_RMSprop","trainwgrad_adam", "trainwpso") #"trainwgrad_Momentum"
hyperParams.automl <- function(optim_method, ...) {
#  if (optim_method == "trainwgrad_Momentum")     {beta2 <- 0.9; beta2 <- 0   ; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
  if (optim_method == "trainwgrad_RMSprop")      {beta1 <- 0  ; beta2 <- 0.99; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
  if (optim_method == "trainwgrad_adam")         {beta1 <- 0.9; beta2 <- 0.99; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
  if (optim_method == "trainwpso")               {beta1 <- NULL; beta2 <- NULL; iter <- maxit1storderA; lr = NULL}
  
  return(list(beta1 = beta1, beta2 = beta2, iter = iter, lr = lr, hidden_activation = 'tanh', optim_method = optim_method))
}
NNtrain.automl <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
  
  hyper_params <- do.call(hyperParams, list(optim_method, ...))
  
  beta1 <- hyper_params$beta1 ; beta2 <- hyper_params$beta2
  iter <- hyper_params$iter ; lr <- hyper_params$lr
  hidden_activation <- hyper_params$hidden_activation
  
  # NNreg <- automl::automl_train(Xref = x, Yref = y)
  
  NNreg <- automl::automl_train_manual(Xref = x, Yref = y,
                                       hpar = list(modexec = hyper_params$optim_method,
                                                   beta1 = beta1, beta2 = beta2,
                                                   numiterations = iter, learningrate = lr,
                                                   layersshape = c(hidden_neur, 0),
                                                   layersacttype = c(hidden_activation, ""),
                                                   verbose = FALSE,
                                                   seed = as.integer(runif(1)*10000000)))

    return (NNreg)
}
NNpredict.automl <- function(object, x, ...)
    automl::automl_predict(model=object, X=x)
NNclose.automl <- function()
  if("package:automl" %in% search())
    detach("package:automl", unload=TRUE)
automl.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, automl.method, "NNtrain.automl", "hyperParams.automl", "NNpredict.automl", 
                               NNsummary, "NNclose.automl", NA, automl.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="automl", pkgfun="automl_train_manual", rdafile=TRUE, odir=odir)

2 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "automl", pkgfun = "automl_train_manual", automl.method,
  prepareZZ.arg = automl.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

3 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "trainwgrad_RMSprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwgrad_RMSprop
RMSE MSE MAE WAE time
mDette 0.4734 0.2241 0.3598 2.6230 8.37
mFriedman 0.0418 0.0017 0.0318 0.1627 8.11
mIshigami 0.7727 0.5971 0.5606 3.2619 8.70
mRef153 3.6206 13.1088 2.5636 13.7640 2.75
uDmod1 0.1279 0.0163 0.0990 0.2929 1.08
uDmod2 0.0861 0.0074 0.0695 0.2004 1.08
uDreyfus1 0.0756 0.0057 0.0552 0.1827 1.05
uDreyfus2 0.1160 0.0135 0.0920 0.2642 1.06
uGauss1 3.5603 12.6759 2.6931 10.4343 4.36
uGauss2 4.0986 16.7989 3.0681 13.7961 4.16
uGauss3 3.4154 11.6647 2.7322 9.0438 1.82
uNeuroOne 0.2846 0.0810 0.2332 0.5660 1.04
kable(apply(res[,,2,], c(3,1), min), caption = "trainwgrad_adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwgrad_adam
RMSE MSE MAE WAE time
mDette 0.4883 0.2385 0.3833 1.7848 9.36
mFriedman 0.0332 0.0011 0.0269 0.0984 9.42
mIshigami 0.7195 0.5177 0.5359 2.5098 9.33
mRef153 3.3858 11.4637 2.3050 13.2932 3.14
uDmod1 0.0603 0.0036 0.0470 0.1460 1.22
uDmod2 0.0611 0.0037 0.0491 0.1312 1.19
uDreyfus1 0.0362 0.0013 0.0202 0.1252 0.82
uDreyfus2 0.0944 0.0089 0.0752 0.2221 0.68
uGauss1 3.7974 14.4201 2.7945 13.6836 4.89
uGauss2 3.6521 13.3379 2.7436 13.1870 4.89
uGauss3 2.5172 6.3364 2.0068 8.2477 4.82
uNeuroOne 0.2848 0.0811 0.2307 0.5874 0.92
kable(apply(res[,,3,], c(3,1), min), caption = "trainwpso")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwpso
RMSE MSE MAE WAE time
mDette 2.6100 6.8124 1.8890 14.5915 7.42
mFriedman 0.1078 0.0116 0.0853 0.2915 9.41
mIshigami 2.4888 6.1942 2.0244 7.1210 10.15
mRef153 3.7387 13.9779 2.7569 13.0120 4.08
uDmod1 0.1480 0.0219 0.1159 0.4691 6.33
uDmod2 0.1563 0.0244 0.1352 0.3168 2.70
uDreyfus1 0.0712 0.0051 0.0477 0.1549 0.89
uDreyfus2 0.1041 0.0108 0.0834 0.2364 2.55
uGauss1 10.5504 111.3111 8.1994 26.1492 6.61
uGauss2 4.2267 17.8652 3.3492 13.5698 4.02
uGauss3 4.2532 18.0900 3.4678 11.7201 6.69
uNeuroOne 0.2833 0.0802 0.2302 0.5432 2.55