NNbenchmark Generic Function with All Packages

1 Setup

1.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

1.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

1.3 AMORE trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(AMORE)
hyperParams.AMORE <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("ADAPTgd", "ADAPTgdwm", "BATCHgd", "BATCHgdwm"))) stop("Invalid Parameters.")
    if (optim_method == "ADAPTgd") {iter <- maxit1storderA; lr <- 0.01; momentum <- 0; hidden_activation <- "tansig"} 
    if (optim_method == "ADAPTgdwm") {iter <- maxit1storderA; lr <- 0.01; momentum <- 0.8; hidden_activation <- "sigmoid"} 
    if (optim_method == "BATCHgd") {iter <- maxit1storderB; lr <- 0.1; momentum <- 0; hidden_activation <- "tansig"}
    if (optim_method == "BATCHgdwm") {iter <- maxit1storderB; lr <- 0.1; momentum <- 0.8; hidden_activation <- "tansig"}

    out <- list(iter = iter, lr = lr, momentum = momentum, hidden_activation = hidden_activation)
    
    return (out)
}

NNtrain.AMORE <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    momentum <- hyper_params$momentum
    hidden_activation <- hyper_params$hidden_activation
    
    net_structure <- AMORE::newff(n.neurons = c(ncol(x), hidden_neur, 1), 
                                  learning.rate.global = lr, 
                                  error.criterium = "LMS", 
                                  hidden.layer = hidden_activation, 
                                  method = optim_method, 
                                  momentum.global = momentum)
    
    NNreg <- AMORE::train(net = net_structure, 
                          P = x, 
                          T = y, 
                          error.criterium = "LMS", 
                          report = FALSE, 
                          n.shows = iter, 
                          show.step = 1)
    return (NNreg)
}
NNpredict.AMORE <- function(object, x, ...)
    AMORE::sim.MLPnet(object$net, x)    
NNclose.AMORE <- function()
  if("package:AMORE" %in% search())
    detach("package:AMORE", unload=TRUE)
AMORE.method <- c("ADAPTgd", "ADAPTgdwm", "BATCHgd", "BATCHgdwm")
AMORE.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, AMORE.method, "NNtrain.AMORE", "hyperParams.AMORE", "NNpredict.AMORE", 
                               NNsummary, "NNclose.AMORE", NA, AMORE.prepareZZ, nrep=2, echo=TRUE, 
                               doplot=FALSE, echoreport=0,
                               pkgname="AMORE", pkgfun="train", rdafile=TRUE, odir=odir)

2 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "AMORE", pkgfun = "train", AMORE.method,
  prepareZZ.arg = AMORE.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

3 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "ADAPTgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

ADAPTgd

	RMSE	MSE	MAE	WAE	time
mDette	0.4391	0.1928	0.3111	1.9884	0.11
mFriedman	0.0264	0.0007	0.0206	0.0885	0.10
mIshigami	0.7690	0.5914	0.5722	2.7954	0.22
mRef153	3.3184	11.0118	2.3437	13.4872	0.01
uDmod1	0.3082	0.0950	0.2657	0.6759	0.03
uDmod2	0.2579	0.0665	0.2296	0.4945	0.01
uDreyfus1	0.3308	0.1094	0.2567	0.7653	0.00
uDreyfus2	0.3555	0.1264	0.2786	0.8885	0.00
uGauss1	12.5180	156.6999	10.5573	26.2055	0.03
uGauss2	7.4794	55.9420	5.7680	21.6317	0.04
uGauss3	4.7958	22.9994	3.8748	12.4516	0.03
uNeuroOne	0.2958	0.0875	0.2444	0.6422	0.01

kable(apply(res[,,2,], c(3,1), min), caption = "ADAPTgdwm")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

ADAPTgdwm

	RMSE	MSE	MAE	WAE	time
mDette	0.3972	0.1578	0.3053	1.6710	0.17
mFriedman	0.0439	0.0019	0.0315	0.1668	0.17
mIshigami	0.8636	0.7459	0.6286	3.4443	0.31
mRef153	3.3209	11.0286	2.3546	13.5405	0.04
uDmod1	0.2197	0.0483	0.1594	0.6487	0.05
uDmod2	0.1145	0.0131	0.0954	0.2709	0.02
uDreyfus1	0.1804	0.0326	0.1218	0.4287	0.03
uDreyfus2	0.1675	0.0281	0.1324	0.4960	0.03
uGauss1	14.8661	221.0012	9.4252	44.2658	0.07
uGauss2	4.3864	19.2406	3.3319	15.1157	0.06
uGauss3	4.4658	19.9431	3.4080	15.0898	0.08
uNeuroOne	0.2854	0.0814	0.2282	0.6240	0.01

kable(apply(res[,,3,], c(3,1), min), caption = "BATCHgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BATCHgd

	RMSE	MSE	MAE	WAE	time
mDette	1.8688	3.4924	1.4854	8.0169	1.84
mFriedman	0.0177	0.0003	0.0138	0.0578	1.86
mIshigami	2.5215	6.3580	2.1633	6.1806	2.58
mRef153	3.3610	11.2963	2.3941	13.0569	1.42
uDmod1	0.2023	0.0409	0.1592	0.4634	1.76
uDmod2	0.2228	0.0496	0.1859	0.4824	1.64
uDreyfus1	0.3160	0.0998	0.2648	0.7009	1.36
uDreyfus2	0.1778	0.0316	0.1289	0.5439	1.36
uGauss1	12.0830	145.9998	10.2079	26.1784	1.67
uGauss2	9.1582	83.8717	6.6562	26.4822	1.54
uGauss3	5.0863	25.8700	3.9667	14.4411	1.53
uNeuroOne	0.2931	0.0859	0.2417	0.6299	1.22

kable(apply(res[,,4,], c(3,1), min), caption = "BATCHgdwm")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BATCHgdwm

	RMSE	MSE	MAE	WAE	time
mDette	1.8586	3.4543	1.4838	7.9898	1.82
mFriedman	0.0173	0.0003	0.0134	0.0532	1.87
mIshigami	2.4805	6.1529	2.0656	6.1225	2.67
mRef153	3.3094	10.9523	2.3517	13.0089	1.42
uDmod1	0.3265	0.1066	0.2844	0.7165	1.79
uDmod2	0.1585	0.0251	0.1263	0.3859	1.64
uDreyfus1	0.3346	0.1119	0.2777	0.7038	1.36
uDreyfus2	0.2097	0.0440	0.1526	0.6268	1.37
uGauss1	12.0720	145.7339	10.2102	26.2759	1.72
uGauss2	9.2190	84.9900	6.7259	26.6399	1.56
uGauss3	5.0868	25.8757	3.9642	14.3059	1.56
uNeuroOne	0.2924	0.0855	0.2410	0.6281	1.22

4 Setup

4.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

4.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

4.3 ANN2 trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(ANN2)
ANN2.method <- c("sgd", "adam", "rmsprop")
hyperParams.ANN2 <- function(optim_method, ...) {
    
    if (optim_method == "sgd")     { iter <- maxit1storderA; lr <- 0.01} 
    if (optim_method == "adam")    { iter <- maxit1storderA; lr <- 0.01} 
    if (optim_method == "rmsprop") { iter <- maxit1storderA; lr <- 0.01} 
    
    out <- list(iter = iter, lr = lr)
    return(out)
}

NNtrain.ANN2 <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...){
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    
    NNreg <- ANN2::neuralnetwork(X = x, y = y, 
                           val.prop = 0, 
                           standardize = FALSE, 
                           hidden.layers = hidden_neur, 
                           regression = TRUE,
                           loss.type = "squared",
                           n.epochs = iter,
                           optim.type = optim_method,
                           learn.rates = lr,
                           verbose = FALSE,
                           random.seed = as.integer(runif(1)*10000000))
    
   return (NNreg)
}
NNpredict.ANN2 <- function(object, x, ...)
    as.numeric(predict(object, x)$predictions)    
NNclose.ANN2 <- function()
  if("package:ANN2" %in% search())
    detach("package:ANN2", unload=TRUE)
ANN2.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, ANN2.method, "NNtrain.ANN2", "hyperParams.ANN2", "NNpredict.ANN2", 
                               NNsummary, "NNclose.ANN2", NA, ANN2.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="ANN2", pkgfun="neuralnetwork", rdafile=TRUE, odir=odir)

5 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "ANN2", pkgfun = "neuralnetwork", ANN2.method,
  prepareZZ.arg = ANN2.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

6 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "sgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

sgd

	RMSE	MSE	MAE	WAE	time
mDette	1.2208	1.4902	0.9125	6.8603	0.20
mFriedman	0.0178	0.0003	0.0146	0.0502	0.21
mIshigami	0.7787	0.6063	0.5844	3.3678	1.03
mRef153	3.5190	12.3836	2.5641	13.0075	0.03
uDmod1	0.2581	0.0666	0.2098	0.5573	0.01
uDmod2	0.2518	0.0634	0.2187	0.4871	0.00
uDreyfus1	0.3493	0.1220	0.2543	0.8069	0.00
uDreyfus2	0.3581	0.1282	0.2720	0.9086	0.00
uGauss1	11.0994	123.1959	9.2770	25.0191	0.08
uGauss2	7.4578	55.6182	5.7629	22.0830	0.06
uGauss3	4.8318	23.3462	3.7862	13.6047	0.06
uNeuroOne	0.3069	0.0942	0.2520	0.5979	0.00

kable(apply(res[,,2,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adam

	RMSE	MSE	MAE	WAE	time
mDette	1.7980	3.2327	1.4555	7.1598	0.21
mFriedman	0.0183	0.0003	0.0145	0.0512	0.22
mIshigami	0.7560	0.5716	0.5686	2.5635	1.04
mRef153	3.4586	11.9619	2.4652	12.2180	0.03
uDmod1	0.2198	0.0483	0.1683	0.4967	0.00
uDmod2	0.1702	0.0290	0.1343	0.3928	0.00
uDreyfus1	0.2762	0.0763	0.2230	0.6472	0.00
uDreyfus2	0.3222	0.1038	0.2400	0.8638	0.00
uGauss1	2.7029	7.3057	2.1724	7.7204	0.08
uGauss2	3.7310	13.9202	2.9927	11.0996	0.06
uGauss3	3.1354	9.8308	2.4941	9.1757	0.08
uNeuroOne	0.3082	0.0950	0.2513	0.6411	0.00

kable(apply(res[,,3,], c(3,1), min), caption = "rmsprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

rmsprop

	RMSE	MSE	MAE	WAE	time
mDette	1.9463	3.7882	1.4715	6.8555	0.19
mFriedman	0.0250	0.0006	0.0199	0.0931	0.22
mIshigami	0.7045	0.4963	0.5338	2.7789	1.03
mRef153	3.2991	10.8838	2.3512	11.9437	0.04
uDmod1	0.2345	0.0550	0.1789	0.5468	0.01
uDmod2	0.1831	0.0335	0.1425	0.4134	0.00
uDreyfus1	0.2467	0.0609	0.1728	0.6428	0.00
uDreyfus2	0.2338	0.0547	0.1687	0.7745	0.00
uGauss1	5.5210	30.4820	4.0533	16.3089	0.06
uGauss2	3.5637	12.6997	2.7937	11.3547	0.07
uGauss3	2.9727	8.8369	2.3524	8.0896	0.07
uNeuroOne	0.2904	0.0844	0.2327	0.5825	0.00

7 Setup

7.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

7.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

7.3 automl trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(automl)
automl.method <- c("trainwgrad_RMSprop","trainwgrad_adam", "trainwpso") #"trainwgrad_Momentum"
hyperParams.automl <- function(optim_method, ...) {
#  if (optim_method == "trainwgrad_Momentum")     {beta2 <- 0.9; beta2 <- 0   ; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
  if (optim_method == "trainwgrad_RMSprop")      {beta1 <- 0  ; beta2 <- 0.99; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
  if (optim_method == "trainwgrad_adam")         {beta1 <- 0.9; beta2 <- 0.99; iter <- maxit1storderA; lr = 0.01; optim_method = "trainwgrad"}
  if (optim_method == "trainwpso")               {beta1 <- NULL; beta2 <- NULL; iter <- maxit1storderA; lr = NULL}
  
  return(list(beta1 = beta1, beta2 = beta2, iter = iter, lr = lr, hidden_activation = 'tanh', optim_method = optim_method))
}
NNtrain.automl <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
  
  hyper_params <- do.call(hyperParams, list(optim_method, ...))
  
  beta1 <- hyper_params$beta1 ; beta2 <- hyper_params$beta2
  iter <- hyper_params$iter ; lr <- hyper_params$lr
  hidden_activation <- hyper_params$hidden_activation
  
  # NNreg <- automl::automl_train(Xref = x, Yref = y)
  
  NNreg <- automl::automl_train_manual(Xref = x, Yref = y,
                                       hpar = list(modexec = hyper_params$optim_method,
                                                   beta1 = beta1, beta2 = beta2,
                                                   numiterations = iter, learningrate = lr,
                                                   layersshape = c(hidden_neur, 0),
                                                   layersacttype = c(hidden_activation, ""),
                                                   verbose = FALSE,
                                                   seed = as.integer(runif(1)*10000000)))

    return (NNreg)
}
NNpredict.automl <- function(object, x, ...)
    automl::automl_predict(model=object, X=x)
NNclose.automl <- function()
  if("package:automl" %in% search())
    detach("package:automl", unload=TRUE)
automl.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, automl.method, "NNtrain.automl", "hyperParams.automl", "NNpredict.automl", 
                               NNsummary, "NNclose.automl", NA, automl.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="automl", pkgfun="automl_train_manual", rdafile=TRUE, odir=odir)

8 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "automl", pkgfun = "automl_train_manual", automl.method,
  prepareZZ.arg = automl.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

9 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "trainwgrad_RMSprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

trainwgrad_RMSprop

	RMSE	MSE	MAE	WAE	time
mDette	0.4821	0.2324	0.3668	2.2126	8.55
mFriedman	0.0397	0.0016	0.0316	0.1521	8.49
mIshigami	1.8225	3.3214	1.4189	5.7080	8.81
mRef153	3.5937	12.9145	2.4501	13.3238	2.75
uDmod1	0.1052	0.0111	0.0835	0.2446	1.11
uDmod2	0.1245	0.0155	0.1017	0.3522	1.09
uDreyfus1	0.0479	0.0023	0.0327	0.1331	1.10
uDreyfus2	0.1179	0.0139	0.0963	0.2674	1.05
uGauss1	4.9906	24.9062	3.5830	15.9339	4.44
uGauss2	3.9723	15.7795	3.0889	12.6865	4.42
uGauss3	3.5555	12.6418	2.9084	9.4194	4.37
uNeuroOne	0.2842	0.0808	0.2343	0.5601	1.07

kable(apply(res[,,2,], c(3,1), min), caption = "trainwgrad_adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

trainwgrad_adam

	RMSE	MSE	MAE	WAE	time
mDette	0.4255	0.1811	0.3281	1.5416	9.50
mFriedman	0.0277	0.0008	0.0221	0.0927	9.51
mIshigami	0.7511	0.5642	0.5718	2.8043	9.88
mRef153	3.4616	11.9828	2.4074	13.9990	1.25
uDmod1	0.0596	0.0036	0.0465	0.1593	1.25
uDmod2	0.0511	0.0026	0.0416	0.1358	1.24
uDreyfus1	0.0087	0.0001	0.0065	0.0199	0.75
uDreyfus2	0.0933	0.0087	0.0753	0.2454	1.23
uGauss1	4.4523	19.8230	3.2657	14.4227	4.96
uGauss2	8.1454	66.3474	5.2565	29.2542	4.93
uGauss3	3.1214	9.7433	2.4527	8.6465	4.96
uNeuroOne	0.2844	0.0809	0.2309	0.6216	1.13

kable(apply(res[,,3,], c(3,1), min), caption = "trainwpso")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

trainwpso

	RMSE	MSE	MAE	WAE	time
mDette	2.7275	7.4393	2.0527	15.3558	9.96
mFriedman	0.1029	0.0106	0.0871	0.2465	8.02
mIshigami	1.8381	3.3787	1.4239	5.9148	23.02
mRef153	3.7011	13.6979	2.6113	13.3234	4.69
uDmod1	0.2424	0.0587	0.1839	0.6094	2.71
uDmod2	0.2032	0.0413	0.1633	0.4454	8.17
uDreyfus1	0.1052	0.0111	0.0838	0.2283	2.75
uDreyfus2	0.1180	0.0139	0.0959	0.3098	3.11
uGauss1	9.8111	96.2573	8.1760	21.4884	4.27
uGauss2	6.1384	37.6803	4.8975	16.8950	7.21
uGauss3	4.8318	23.3466	3.8433	14.7445	3.09
uNeuroOne	0.2847	0.0811	0.2311	0.5317	4.03

10 Setup

10.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

10.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

10.3 brnn trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(brnn)
brnn.method <- "Gauss-Newton"
hyperParams.brnn <- function(optim_method, ...) {
    return(list(iter = maxit2ndorder))
}
NNtrain.brnn <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.brnn, list(brnn.method))
    iter <- hyper_params$iter
    
    NNreg <- brnn::brnn(x, y, neur, normalize = FALSE, epochs = iter, verbose = FALSE)
    return (NNreg)
}
NNpredict.brnn <- function(object, x, ...)
    predict(object, x)
NNclose.brnn <- function()
  if("package:brnn" %in% search())
    detach("package:brnn", unload=TRUE)
brnn.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, brnn.method, "NNtrain.brnn", "hyperParams.brnn", "NNpredict.brnn", 
                               NNsummary, "NNclose.brnn", NA, brnn.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="brnn", pkgfun="brnn", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, brnn.method[1], "NNtrain.brnn", "hyperParams.brnn", "NNpredict.brnn", 
                               NNsummary, brnn.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="brnn", pkgfun="brnn", csvfile=TRUE, rdafile=TRUE, odir=odir)

11 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "brnn", pkgfun = "brnn", brnn.method,
  prepareZZ.arg = brnn.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

12 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.4578	0.2096	0.3462	2.0525	0.20
mFriedman	0.0046	0.0000	0.0037	0.0140	0.08
mIshigami	0.6588	0.4340	0.5026	2.9308	0.14
mRef153	3.4791	12.1045	2.5059	13.7347	0.01
uDmod1	0.0451	0.0020	0.0364	0.1166	0.00
uDmod2	0.0435	0.0019	0.0346	0.1073	0.01
uDreyfus1	0.0026	0.0000	0.0020	0.0062	0.00
uDreyfus2	0.0913	0.0083	0.0730	0.2241	0.00
uGauss1	2.2434	5.0327	1.7295	7.6157	0.01
uGauss2	2.3781	5.6555	1.8657	7.3796	0.01
uGauss3	2.8273	7.9934	2.2154	7.6585	0.01
uNeuroOne	0.3523	0.1241	0.2848	0.8269	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	1.9537	3.8168	1.4572	11.8945	0.22
mFriedman	0.0052	0.0000	0.0043	0.0154	0.27
mIshigami	0.6635	0.4402	0.5100	2.9395	0.20
mRef153	3.4796	12.1076	2.5068	13.7382	0.02
uDmod1	0.5884	0.3462	0.5069	1.0104	0.00
uDmod2	0.0673	0.0045	0.0522	0.1838	0.02
uDreyfus1	0.0034	0.0000	0.0029	0.0115	0.00
uDreyfus2	0.0913	0.0083	0.0730	0.2241	0.00
uGauss1	2.4366	5.9370	1.8814	8.7824	0.05
uGauss2	3.5508	12.6079	2.9377	10.2283	0.05
uGauss3	3.1966	10.2185	2.5109	10.0153	0.03
uNeuroOne	0.3523	0.1241	0.2848	0.8271	0.00

13 Setup

13.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

13.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

13.3 CaDENCE trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(CaDENCE)
CaDENCE.method <- c("optim", "psoptim", "Rprop")
hyperParams.CaDENCE <- function(optim_method, ...) {
  
    if (optim_method == "optim")    {iter <- maxit2ndorder;  epsilon <- 0.01} 
    if (optim_method == "psoptim")  {iter <- maxit1storderA; epsilon <- 0.01}
    if (optim_method == "Rprop")    {iter <- maxit1storderA; epsilon <- 0.01}
    
    out <- list(iter = iter, epsilon = epsilon)
    return (out)
}

NNtrain.CaDENCE <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    iter <- hyper_params$iter ; epsilon <- hyper_params$epsilon

    NNreg <- CaDENCE::cadence.fit(x = x, y = y, 
                                iter.max = iter, 
                                n.hidden = neur, 
                                hidden.fcn = tanh, 
                                method = optim_method, 
                                n.trials = 1, 
                                trace = 0, 
                                maxit.Nelder = 1, 
                                f.cost = cadence.cost,
                                distribution = list(density.fcn = dnorm,
                                                    parameters = c("mean", "sd"),
                                                    parameters.fixed = NULL,
                                                    output.fcns = c(identity, exp)),
                                epsilon = epsilon)
    return(NNreg)
}
NNpredict.CaDENCE <- function(object, x, ...)
    CaDENCE::cadence.predict(x = x, fit = object)[,1]
NNclose.CaDENCE <- function()
  if("package:CaDENCE" %in% search())
    detach("package:CaDENCE", unload=TRUE)
CaDENCE.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- trainPredict_1data(1, CaDENCE.method, "NNtrain.CaDENCE", "hyperParams.CaDENCE", "NNpredict.CaDENCE", 
                               NNsummary, "NNclose.CaDENCE", NA, CaDENCE.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="CaDENCE", pkgfun="cadence.fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

14 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "CaDENCE", pkgfun = "cadence.fit", CaDENCE.method,
  prepareZZ.arg = CaDENCE.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

15 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "optim")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

optim

	RMSE	MSE	MAE	WAE	time
mDette	0.3277	0.1074	0.2301	2.2656	6.88
mFriedman	0.0160	0.0003	0.0117	0.0597	9.15
mIshigami	1.0465	1.0952	0.7492	4.1310	14.89
mRef153	3.2677	10.6777	2.3374	11.9005	2.76
uDmod1	0.0564	0.0032	0.0429	0.2015	1.64
uDmod2	0.0688	0.0047	0.0501	0.2265	2.25
uDreyfus1	0.0032	0.0000	0.0019	0.0168	0.52
uDreyfus2	0.0924	0.0085	0.0722	0.2132	0.27
uGauss1	2.3392	5.4720	1.8421	7.3698	2.86
uGauss2	2.4041	5.7797	1.8904	7.5217	2.08
uGauss3	2.4116	5.8157	1.9152	7.2932	2.22
uNeuroOne	0.2831	0.0801	0.2310	0.5816	0.25

kable(apply(res[,,2,], c(3,1), min), caption = "psoptim")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

psoptim

	RMSE	MSE	MAE	WAE	time
mDette	3.1663	10.0254	1.7240	20.5350	10.84
mFriedman	0.0950	0.0090	0.0634	0.3137	12.00
mIshigami	2.6775	7.1691	2.2758	7.8350	14.83
mRef153	3.7348	13.9485	2.7123	13.4615	7.61
uDmod1	0.3096	0.0958	0.2636	0.7048	5.36
uDmod2	0.2114	0.0447	0.1837	0.4148	4.83
uDreyfus1	0.4218	0.1779	0.2335	1.2535	4.49
uDreyfus2	0.3210	0.1031	0.2360	0.8750	4.47
uGauss1	25.3213	641.1664	20.3628	51.1232	6.07
uGauss2	11.1566	124.4697	7.4290	31.5858	5.72
uGauss3	10.8502	117.7273	7.2720	33.8175	5.69
uNeuroOne	0.5523	0.3051	0.4192	1.4478	4.19

kable(apply(res[,,3,], c(3,1), min), caption = "Rprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

Rprop

	RMSE	MSE	MAE	WAE	time
mDette	4.6664	21.7750	2.8590	26.1659	16.73
mFriedman	0.0850	0.0072	0.0649	0.1875	22.31
mIshigami	1.3422	1.8016	0.9124	6.3997	36.74
mRef153	3.7332	13.9365	2.7117	12.0118	9.89
uDmod1	0.2005	0.0402	0.1462	0.5493	5.87
uDmod2	0.1820	0.0331	0.1427	0.4619	5.62
uDreyfus1	0.3995	0.1596	0.2758	0.9190	3.53
uDreyfus2	0.1684	0.0284	0.1187	0.5419	3.79
uGauss1	17.7666	315.6525	12.8365	49.8613	7.13
uGauss2	11.7899	139.0007	8.1481	32.3143	5.72
uGauss3	9.1862	84.3868	6.1965	33.9628	5.72
uNeuroOne	0.3054	0.0932	0.2457	0.6182	2.76

16 Setup

16.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

16.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

16.3 caret trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(caret)
caret.method <- "none"
hyperParams.caret <- function(...) {
    return (list(iter=maxit2ndorder, trace=FALSE))
}
NNtrain.caret <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    NNreg <- caret::avNNet(x, y, repeats = 3, size = neur, linout = TRUE, trace=hyper_params$trace, bag = TRUE)
    return(NNreg)
}
NNpredict.caret <- function(object, x, ...)
    predict(object, newdata=x)
NNclose.caret <- function()
  if("package:caret" %in% search())
    detach("package:caret", unload=TRUE)
caret.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, caret.method, "NNtrain.caret", "hyperParams.caret", "NNpredict.caret", 
                               NNsummary, "NNclose.caret", NA, caret.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="caret", pkgfun="caret", csvfile=TRUE, rdafile=TRUE, odir=odir)

17 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "caret", pkgfun = "avNNet", caret.method,
  prepareZZ.arg = caret.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

18 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.6690	0.4475	0.5238	2.8064	0.11
mFriedman	0.0392	0.0015	0.0316	0.1133	0.14
mIshigami	1.1064	1.2242	0.8536	4.1914	0.23
mRef153	3.3032	10.9110	2.3720	11.7195	0.02
uDmod1	0.0567	0.0032	0.0404	0.2172	0.01
uDmod2	0.0683	0.0047	0.0536	0.1505	0.00
uDreyfus1	0.0064	0.0000	0.0047	0.0166	0.00
uDreyfus2	0.0988	0.0098	0.0783	0.2439	0.00
uGauss1	2.4460	5.9832	1.9368	7.3514	0.04
uGauss2	3.3273	11.0706	2.5360	9.9572	0.05
uGauss3	2.5243	6.3720	2.0152	8.0255	0.03
uNeuroOne	0.2870	0.0823	0.2307	0.5361	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	1.2255	1.5019	0.9320	6.3623	0.13
mFriedman	0.0477	0.0023	0.0381	0.1381	0.15
mIshigami	1.8293	3.3462	1.4661	6.9393	0.25
mRef153	3.3647	11.3212	2.3885	12.8780	0.03
uDmod1	0.0855	0.0073	0.0619	0.3245	0.02
uDmod2	0.0944	0.0089	0.0733	0.2027	0.02
uDreyfus1	0.0335	0.0011	0.0256	0.0889	0.01
uDreyfus2	0.1094	0.0120	0.0862	0.3012	0.02
uGauss1	2.9638	8.7841	2.3333	10.2034	0.07
uGauss2	3.7095	13.7602	2.8466	11.3257	0.06
uGauss3	3.3541	11.2497	2.5115	9.7162	0.05
uNeuroOne	0.2874	0.0826	0.2353	0.6322	0.00

19 Setup

19.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

19.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

19.3 deepdive trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(deepdive)
deepdive.method <- c("gradientDescent","momentum","rmsProp","adam")
hyperParams.deepdive <- function(optim_method, ...) {
    
    if (optim_method == "gradientDescent")  {maxiter <- maxit1storderB; eta = 0.4} 
    if (optim_method == "momentum")         {maxiter <- maxit1storderB; eta = 0.8}
    if (optim_method == "rmsProp")          {maxiter <- maxit1storderA; eta = 0.8}
    if (optim_method == "adam")             {maxiter <- maxit1storderA; eta = 0.8}
  
    out <- list(iter = maxiter, method = optim_method, modelType = "regress", eta = eta, print = 1000)
    return (out)
}
NNtrain.deepdive <- function(x, y, dataxy, formula, neur, optim_method, hyperParams, NNfullformula, NNparam,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    NNreg <- deepdive::deepnet(x = x, y = y, c(neur), modelType = hyper_params$modelType,
                     iterations = hyper_params$iter, eta=hyper_params$eta, 
                     optimiser=hyper_params$method, printItrSize=hyper_params$print,
                     normalise = FALSE)
    
    return (NNreg)
}
NNpredict.deepdive <- function(object, x, ...)
  predict.deepnet(object,newData=x)$ypred
NNclose.deepdive <- function()
  if("package:deepdive" %in% search())
    detach("package:deepdive", unload=TRUE)
deepdive.prepareZZ <- list(xdmv = "d", ydmv = "d", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, deepdive.method, "NNtrain.deepdive", "hyperParams.deepdive", "NNpredict.deepdive", 
                               NNsummary, "NNclose.deepdive", NA, deepdive.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="deepdive", pkgfun="ann", csvfile=TRUE, rdafile=TRUE, odir=odir)

20 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "deepdive", pkgfun = "deepnet", deepdive.method,
  prepareZZ.arg = deepdive.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

21 Results

kable(apply(res[,,1,], c(3,1), min), caption = "gradientDescent")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

gradientDescent

	RMSE	MSE	MAE	WAE	time
mDette	4.4310	19.6338	3.2628	20.7622	7.19
mFriedman	0.1474	0.0217	0.1154	0.4581	7.58
mIshigami	3.0218	9.1314	2.4940	10.2360	8.95
mRef153	4.1805	17.4764	3.1843	14.4946	5.90
uDmod1	0.3353	0.1124	0.2912	0.7067	5.27
uDmod2	0.2699	0.0729	0.2369	0.5509	5.27
uDreyfus1	0.3429	0.1176	0.2801	0.7346	5.11
uDreyfus2	0.3570	0.1275	0.2905	0.8478	5.19
uGauss1	23.6597	559.7792	19.1988	57.7174	6.01
uGauss2	28.8118	830.1176	25.1770	67.7823	5.86
uGauss3	32.2441	1039.6844	27.7063	70.1972	5.86
uNeuroOne	0.3666	0.1344	0.3105	0.6748	5.11

kable(apply(res[,,2,], c(3,1), min), caption = "momentum")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

momentum

	RMSE	MSE	MAE	WAE	time
mDette	4.1990	17.6313	3.1011	18.5512	7.39
mFriedman	0.1363	0.0186	0.1061	0.4860	7.77
mIshigami	2.5791	6.6519	2.0107	8.7569	9.11
mRef153	3.8424	14.7643	2.7879	15.0474	6.03
uDmod1	0.3320	0.1103	0.2891	0.7441	5.47
uDmod2	0.2656	0.0705	0.2355	0.5269	5.42
uDreyfus1	0.3429	0.1176	0.2801	0.7341	5.42
uDreyfus2	0.3570	0.1275	0.2907	0.8468	5.32
uGauss1	16.2557	264.2462	13.0878	49.2409	6.21
uGauss2	28.8104	830.0390	25.1756	67.8020	6.11
uGauss3	32.2413	1039.4994	27.7055	70.2077	6.10
uNeuroOne	0.3544	0.1256	0.3001	0.6152	5.32

kable(apply(res[,,3,], c(3,1), min), caption = "rmsProp")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

rmsProp

	RMSE	MSE	MAE	WAE	time
mDette	2.7205	7.4011	1.8705	16.1780	0.73
mFriedman	0.1287	0.0166	0.0990	0.4133	0.76
mIshigami	2.6728	7.1439	2.3060	7.1452	0.87
mRef153	4.0384	16.3091	3.0290	15.7099	0.62
uDmod1	0.1728	0.0299	0.1257	0.4478	0.56
uDmod2	0.2252	0.0507	0.1580	0.5513	0.55
uDreyfus1	0.1184	0.0140	0.0878	0.3446	0.54
uDreyfus2	0.1625	0.0264	0.1232	0.4839	0.53
uGauss1	23.5833	556.1721	19.1113	57.9958	0.62
uGauss2	18.3426	336.4509	12.6957	49.5614	0.61
uGauss3	20.7977	432.5437	15.4531	49.0396	0.61
uNeuroOne	0.3161	0.0999	0.2695	0.5981	0.54

kable(apply(res[,,4,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adam

	RMSE	MSE	MAE	WAE	time
mDette	3.0971	9.5922	2.0640	18.6373	0.73
mFriedman	0.0875	0.0077	0.0764	0.2699	0.76
mIshigami	2.5913	6.7150	2.0819	10.0604	0.89
mRef153	3.7605	14.1414	2.6651	15.3279	0.63
uDmod1	0.1178	0.0139	0.0797	0.4868	0.55
uDmod2	0.3189	0.1017	0.2113	0.7209	0.56
uDreyfus1	0.0304	0.0009	0.0265	0.0644	0.56
uDreyfus2	0.1149	0.0132	0.0907	0.2749	0.56
uGauss1	20.5179	420.9855	16.5474	47.6354	0.62
uGauss2	16.8843	285.0805	11.6361	46.6218	0.61
uGauss3	10.0466	100.9345	7.3484	31.6838	0.61
uNeuroOne	0.2946	0.0868	0.2459	0.5582	0.55

22 Setup

22.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

22.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

22.3 deepnet trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(deepnet)
deepnet.method <- "BP"
hyperParams.deepnet <- function(optim_method, ...) {

    iter <- maxit1storderA
    lr <- 0.8
    dropout <- 0
    momentum <- 0.95
    hidden_activation <- "sigm"

    out <- list(iter = iter, lr = lr, momentum = momentum, hidden_activation = hidden_activation, dropout = dropout)
    
    return (out)
}

NNtrain.deepnet <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    dropout <- hyper_params$dropout
    momentum <- hyper_params$momentum
    hidden_activation <- hyper_params$hidden_activation
    dropout <- hyper_params$dropout

    NNreg <- deepnet::nn.train(x = x, y = y, 
                               hidden = c(hidden_neur), 
                               activationfun = hidden_activation, 
                               learningrate = lr, 
                               output = 'linear', 
                               numepochs = iter, 
                               hidden_dropout = dropout, 
                               momentum = momentum)
    return (NNreg)
}
NNpredict.deepnet <- function(object, x, ...)
    deepnet::nn.predict(nn = object, x = x)
NNclose.deepnet <- function()
  if("package:deepnet" %in% search())
    detach("package:deepnet", unload=TRUE)
deepnet.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, CaDENCE.method, "NNtrain.deepnet", "hyperParams.deepnet", "NNpredict.deepnet", 
                               NNsummary, "NNclose.deepnet", NA, deepnet.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="deepnet", pkgfun="nn.train", rdafile=TRUE, odir=odir)

23 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "deepnet", pkgfun = "nn.train", deepnet.method,
  prepareZZ.arg = deepnet.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

24 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.5308	0.2818	0.3905	2.3515	0.61
mFriedman	0.0396	0.0016	0.0312	0.1549	0.64
mIshigami	1.0536	1.1102	0.7952	3.3040	0.75
mRef153	3.4561	11.9444	2.4791	12.4950	0.20
uDmod1	0.0582	0.0034	0.0478	0.1399	0.08
uDmod2	0.0563	0.0032	0.0468	0.1204	0.08
uDreyfus1	0.0139	0.0002	0.0121	0.0286	0.06
uDreyfus2	0.0928	0.0086	0.0740	0.2347	0.06
uGauss1	3.4191	11.6901	2.5947	10.8474	0.30
uGauss2	3.4032	11.5820	2.7491	11.0331	0.29
uGauss3	3.5001	12.2508	2.8388	8.7228	0.29
uNeuroOne	0.2830	0.0801	0.2314	0.5632	0.08

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.6403	0.4100	0.5135	2.7237	0.63
mFriedman	0.0967	0.0094	0.0838	0.2139	0.64
mIshigami	1.4687	2.1571	1.0190	6.8677	0.77
mRef153	3.8740	15.0082	2.9819	15.8992	0.20
uDmod1	0.1173	0.0138	0.0845	0.3896	0.09
uDmod2	0.0608	0.0037	0.0490	0.1446	0.09
uDreyfus1	0.0704	0.0050	0.0451	0.2219	0.09
uDreyfus2	0.1049	0.0110	0.0824	0.2577	0.08
uGauss1	4.2839	18.3517	3.5260	11.8002	0.30
uGauss2	6.4830	42.0293	4.8425	17.3581	0.30
uGauss3	3.7035	13.7160	2.9180	12.1143	0.30
uNeuroOne	0.2830	0.0801	0.2314	0.5653	0.08

25 Setup

25.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

25.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

25.3 elmNNRcpp Train Function - arguments x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(elmNNRcpp)
elmNNRcpp.method <- "extremeML"
hyperParams.elmNNRcpp <- function(optim_method, ...) {
    
    moorep_pseudoinv_tol <- 0.01
    wt_init <- "normal_gaussian"
    hidden_activation <- "tansig"
    
    out <- list(wt_init = wt_init, hidden_activation = hidden_activation, 
                moorep_pseudoinv_tol=moorep_pseudoinv_tol) 
    return (out)
}
NNtrain.elmNNRcpp <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    wt_init <- hyper_params$wt_init
    hidden_activation <- hyper_params$hidden_activation
    moorep_pseudoinv_tol <- hyper_params$moorep_pseudoinv_tol

    NNreg <- elmNNRcpp::elm_train(x, y, 
                                  nhid = hidden_neur, 
                                  actfun=hidden_activation, 
                                  init_weights = wt_init, 
                                  bias = TRUE, 
                                  moorep_pseudoinv_tol = moorep_pseudoinv_tol, 
                                  verbose = FALSE,
                                  seed = as.integer(runif(1)*10000000))
    return (NNreg)
}
NNpredict.elmNNRcpp <- function(object, x, ...)
    elmNNRcpp::elm_predict(elm_train_object = object, newdata = x, normalize = FALSE)
NNclose.elmNNRcpp <- function()
  if("package:elmNNRcpp" %in% search())
    detach("package:elmNNRcpp", unload=TRUE)
elmNNRcpp.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, elmNNRcpp.method, "NNtrain.elmNNRcpp", "hyperParams.elmNNRcpp", "NNpredict.elmNNRcpp", 
                               NNsummary, "NNclose.elmNNRcpp", NA, elmNNRcpp.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="elmNNRcpp", pkgfun="elm_train", rdafile=TRUE, odir=odir)

26 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "elmNNRcpp", pkgfun = "elm_train", elmNNRcpp.method,
  prepareZZ.arg = elmNNRcpp.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

27 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	7.3193	53.5727	5.6493	31.5849	0
mFriedman	0.1516	0.0230	0.1187	0.4670	0
mIshigami	3.0949	9.5785	2.5515	10.9134	0
mRef153	16.4917	271.9754	13.0574	42.1803	0
uDmod1	0.3320	0.1102	0.2875	0.7089	0
uDmod2	0.2589	0.0670	0.2141	0.5352	0
uDreyfus1	0.3407	0.1160	0.2487	0.7978	0
uDreyfus2	0.4534	0.2055	0.3659	1.0364	0
uGauss1	17.0430	290.4654	12.8901	42.5221	0
uGauss2	20.3911	415.7968	16.1511	42.9417	0
uGauss3	8.4445	71.3093	6.9616	19.9511	0
uNeuroOne	0.8650	0.7483	0.7043	2.1877	0

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	7.6899	59.1340	5.9574	32.3344	0
mFriedman	0.1734	0.0301	0.1379	0.5055	0
mIshigami	3.2590	10.6211	2.6511	11.3823	0
mRef153	18.7626	352.0352	15.3993	53.2953	0
uDmod1	0.3623	0.1312	0.3038	0.8727	0
uDmod2	0.2648	0.0701	0.2308	0.5419	0
uDreyfus1	0.4066	0.1653	0.2973	1.0342	0
uDreyfus2	0.6226	0.3876	0.5077	1.4031	0
uGauss1	18.7830	352.8001	15.4373	50.7429	0
uGauss2	23.5586	555.0082	20.1337	51.5475	0
uGauss3	19.1869	368.1354	16.8753	35.7678	0
uNeuroOne	0.9526	0.9074	0.7905	2.2943	0

28 Setup

28.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

28.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

28.3 ELMR Train Function - arguments fmla, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

ELMR.method <- "extremeML"
hyperParams.ELMR <- function(optim_method, ...) {

    hidden_activation <- "sig"
    size_first_block <- 50
    size_each_chunk <- 50
    
    out <- list(hidden_activation = hidden_activation,
                size_first_block=size_first_block, size_each_chunk=size_each_chunk)
    return (out)
}
NNtrain.ELMR <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    hidden_activation <- hyper_params$hidden_activation
    size_each_chunk <- hyper_params$size_each_chunk
    size_first_block <- hyper_params$size_first_block
    
    #OSelm_train.formula() call OSelm_training()
    NNreg <- ELMR::OSelm_train.formula(formula = formula,
                                       data = dataxy,
                                       Elm_type = "regression",
                                       nHiddenNeurons = hidden_neur,
                                       ActivationFunction = hidden_activation,
                                       N0 = size_first_block, Block = size_each_chunk)

    return (NNreg)
}
NNpredict.ELMR <- function(object, x, xy)
    ELMR::predict_elm(model = object, test = xy)$predicted
NNclose.ELMR <- function()
  if("package:ELMR" %in% search())
    detach("package:ELMR", unload=TRUE)
ELMR.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, ELMR.method, "NNtrain.ELMR", "hyperParams.ELMR", "NNpredict.ELMR", 
                               NNsummary, "NNclose.ELMR", NA, ELMR.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="ELMR", pkgfun="OSelm_train", rdafile=TRUE, odir=odir)

29 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "ELMR", pkgfun = "OSelm_train.formula", ELMR.method,
  prepareZZ.arg = ELMR.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

30 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	6.3469	40.2826	4.7764	30.3175	0
mFriedman	0.1677	0.0281	0.1318	0.5248	0
mIshigami	3.2348	10.4637	2.6236	11.1547	0
mRef153	9.4764	89.8022	7.6555	27.2034	0
uDmod1	0.3003	0.0902	0.2447	0.7099	0
uDmod2	0.2613	0.0683	0.2326	0.4779	0
uDreyfus1	0.3987	0.1590	0.2606	0.9912	0
uDreyfus2	0.4554	0.2074	0.3005	1.3169	0
uGauss1	37.3986	1398.6525	30.3227	103.2220	0
uGauss2	27.8077	773.2681	20.9835	64.9821	0
uGauss3	31.0706	965.3848	24.6930	81.3878	0
uNeuroOne	0.9735	0.9477	0.8164	2.2524	0

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	7.2310	52.2868	5.5344	32.0052	0.02
mFriedman	0.1924	0.0370	0.1538	0.5716	0.00
mIshigami	3.2840	10.7849	2.6674	12.0160	0.01
mRef153	15.5973	243.2754	12.5855	41.9893	0.00
uDmod1	0.3082	0.0950	0.2529	0.7867	0.00
uDmod2	0.2735	0.0748	0.2358	0.5366	0.00
uDreyfus1	0.4505	0.2029	0.3027	1.1845	0.00
uDreyfus2	0.5844	0.3416	0.4293	1.3727	0.00
uGauss1	490.0535	240152.4584	426.0995	1516.4503	0.01
uGauss2	31.2976	979.5385	25.2946	82.5683	0.02
uGauss3	41.2586	1702.2698	33.0051	112.2972	0.01
uNeuroOne	1.0466	1.0953	0.8640	2.4817	0.00

31 Setup

31.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

31.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

31.3 EnsembleBase trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(EnsembleBase)
EnsembleBase.method <- "none"
hyperParams.EnsembleBase <- function(optim_method, ...) {
    out <- list(iter = maxit2ndorder, decay = 0)
    return (out)
}
NNtrain.EnsembleBase <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.EnsembleBase, list(EnsembleBase.method))
    
    iter <- hyper_params$iter ; decay <- hyper_params$decay
    
    NNreg <- EnsembleBase::Regression.Batch.Fit(make.configs("nnet", config.df = expand.grid(decay=decay,size=c(neur),maxit=iter)), formula, dataxy, ncores = 1)
    
    return (NNreg)
}
NNpredict.EnsembleBase <- function(object, x, ...)
    predict(object, x)
NNclose.EnsembleBase <- function()
  if("package:EnsembleBase" %in% search())
    detach("package:EnsembleBase", unload=TRUE)
EnsembleBase.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, EnsembleBase.method, "NNtrain.EnsembleBase", "hyperParams.EnsembleBase", "NNpredict.EnsembleBase", 
                               NNsummary, "NNclose.EnsembleBase", NA, EnsembleBase.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="EnsembleBase", pkgfun="EnsembleBase", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, EnsembleBase.method[1], "NNtrain.EnsembleBase", "hyperParams.EnsembleBase", "NNpredict.EnsembleBase", 
                               NNsummary, EnsembleBase.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="EnsembleBase", pkgfun="EnsembleBase", csvfile=TRUE, rdafile=TRUE, odir=odir)

32 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "EnsembleBase", pkgfun = "Regression.Batch.Fit", EnsembleBase.method,
  prepareZZ.arg = EnsembleBase.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

33 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.8770	0.7692	0.6768	4.5316	0.00
mFriedman	0.0245	0.0006	0.0162	0.1211	0.08
mIshigami	0.6342	0.4022	0.4296	3.7822	0.13
mRef153	3.0740	9.4494	2.1085	12.9550	0.01
uDmod1	0.0733	0.0054	0.0542	0.2560	0.00
uDmod2	0.0618	0.0038	0.0471	0.1521	0.00
uDreyfus1	0.0922	0.0085	0.0521	0.3303	0.00
uDreyfus2	0.1196	0.0143	0.0953	0.2633	0.00
uGauss1	2.5261	6.3812	1.9614	7.3709	0.03
uGauss2	2.7901	7.7846	2.1897	9.7349	0.03
uGauss3	2.3829	5.6782	1.8660	7.4469	0.03
uNeuroOne	0.2826	0.0799	0.2312	0.5472	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	13.9426	194.3962	11.3013	47.5398	0.01
mFriedman	0.0262	0.0007	0.0181	0.1321	0.09
mIshigami	0.8141	0.6628	0.5735	3.9523	0.13
mRef153	3.4380	11.8196	2.4557	13.5954	0.01
uDmod1	0.1033	0.0107	0.0759	0.4193	0.00
uDmod2	0.0638	0.0041	0.0504	0.1716	0.00
uDreyfus1	0.1151	0.0133	0.0834	0.3335	0.00
uDreyfus2	0.1272	0.0162	0.1022	0.3102	0.00
uGauss1	2.6857	7.2130	2.0641	8.3968	0.03
uGauss2	3.8821	15.0706	3.0341	11.9878	0.03
uGauss3	3.1571	9.9671	2.4537	9.9903	0.03
uNeuroOne	0.2831	0.0802	0.2326	0.5543	0.02

34 Setup

34.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

34.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

34.3 h2o trainPredict arguments - inputs “y”, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(h2o)
h2o.method <- "first-order"
hyperParams.h2o <- function(optim_method, ...) {
    
    hidden_activation = "Tanh"
    iter <- maxit1storderB
    rate <- 0.01
    stopping_rounds <- 500
    stopping_tolerance <- 1e-5
    distribution <- "gaussian"

    out <- list(hidden_activation = hidden_activation, iter = iter, rate=rate, 
                stopping_rounds=stopping_rounds, stopping_tolerance=stopping_tolerance,
                distribution=distribution)
    
    return (out)
}
NNtrain.h2o <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    hidden_activation <- hyper_params$hidden_activation
    iter <- hyper_params$iter
    rate <- hyper_params$rate
    stopping_rounds <- hyper_params$stopping_rounds
    stopping_tolerance <- hyper_params$stopping_tolerance
    distribution <- hyper_params$distribution
    if(class(dataxy) != "H2OFrame")
      dataxy <- h2o::as.h2o(dataxy)

    NNreg <-   h2o::h2o.deeplearning(y = "y",
                                     training_frame = dataxy,
                                     overwrite_with_best_model = TRUE, 
                                     standardize = FALSE,
                                     activation = hidden_activation,
                                     adaptive_rate = FALSE,
                                     rate = rate,
                                     hidden = hidden_neur,
                                     epochs = iter,
                                     train_samples_per_iteration = -1,
                                     initial_weight_distribution = "Normal",
                                     initial_weight_scale = 0.1,
                                     loss = "Quadratic",
                                     distribution = distribution,
                                     stopping_rounds = stopping_rounds,
                                     stopping_metric = "RMSE",
                                     stopping_tolerance = stopping_tolerance,
                                     seed = as.integer(runif(1)*10000000),
                                     verbose = FALSE
                                     )
    return (NNreg)
}
NNpredict.h2o <- function(object, x, ...)
{
  predictions <- h2o::h2o.predict(object, newdata=h2o::as.h2o(x))
  as.data.frame(predictions)$predict
}
NNclose.h2o <- function()
{
  h2o::h2o.shutdown(FALSE)
  if("package:h2o" %in% search())
    detach("package:h2o", unload=TRUE)
}
NNstart.h2o <- function()
{
  require("h2o", character.only = TRUE)
  h2o::h2o.init()
  h2o::h2o.no_progress()
}
h2o.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, h2o.method, "NNtrain.h2o", "hyperParams.h2o", "NNpredict.h2o", 
                               NNsummary, "NNclose.h2o", "NNstart.h2o", h2o.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="h2o", pkgfun="deeplearning", rdafile=TRUE, odir=odir)

35 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "h2o", pkgfun = "h2o.deeplearning", h2o.method,
  prepareZZ.arg = h2o.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

36 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.3696	0.1366	0.2853	1.1748	5.39
mFriedman	0.0225	0.0005	0.0171	0.0843	5.37
mIshigami	0.8347	0.6968	0.6217	3.5282	6.44
mRef153	3.2514	10.5717	2.2604	12.8888	4.36
uDmod1	0.0480	0.0023	0.0383	0.1169	3.28
uDmod2	0.0474	0.0022	0.0374	0.1051	3.29
uDreyfus1	0.0131	0.0002	0.0100	0.0386	3.29
uDreyfus2	0.0926	0.0086	0.0737	0.2188	3.34
uGauss1	2.2985	5.2832	1.7959	7.3896	4.27
uGauss2	2.8574	8.1646	2.2415	9.4798	4.35
uGauss3	2.9525	8.7170	2.2677	8.8168	4.38
uNeuroOne	0.2831	0.0801	0.2309	0.5497	3.31

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.3789	0.1436	0.2948	1.3228	5.47
mFriedman	0.0261	0.0007	0.0204	0.0902	6.40
mIshigami	0.8467	0.7169	0.6295	3.6234	6.46
mRef153	3.5349	12.4958	2.5500	14.8677	4.41
uDmod1	0.0494	0.0024	0.0402	0.1185	3.34
uDmod2	0.0482	0.0023	0.0394	0.1126	3.34
uDreyfus1	0.0146	0.0002	0.0112	0.0432	3.34
uDreyfus2	0.0933	0.0087	0.0740	0.2242	3.36
uGauss1	2.3587	5.5635	1.8349	7.6990	4.39
uGauss2	3.4454	11.8707	2.7928	11.6466	4.41
uGauss3	3.2931	10.8445	2.5635	9.9032	4.38
uNeuroOne	0.2832	0.0802	0.2331	0.5539	3.34

37 Setup

37.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

37.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

37.3 keras Train Function - arguments x,y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(keras)
keras.method <- c("adadelta", "adagrad", "adam", "adamax","nadam","rmsprop", "sgd")
hyperParams.keras <- function(optim_method, ...) {

    hidden_activation = "tanh"
    iter <- maxit1storderB
    lr <- 0.1
    
    out <- list(hidden_activation = hidden_activation, iter = iter, lr = lr)
    return (out)
}

NNtrain.keras <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
        
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    iter <- hyper_params$iter
    
    early_stop <- callback_early_stopping(monitor = "loss", patience = 20, restore_best_weights = TRUE, mode = "auto", min_delta = 0.0001)
    #should we have a higher min_delta
    
    hidden_activation <- hyper_params$hidden_activation
    lr <- hyper_params$lr
    
    if (optim_method == "adadelta")  { op <- optimizer_adadelta(lr = lr)}
    if (optim_method == "adagrad")   { op <- optimizer_adagrad(lr = lr)}
    if (optim_method == "adam")      { op <- optimizer_adam(lr = lr)}
    if (optim_method == "adamax")    { op <- optimizer_adamax(lr = lr)}
    if (optim_method == "nadam")     { op <- optimizer_nadam(lr = lr)}
    if (optim_method == "rmsprop")   { op <- optimizer_rmsprop(lr = lr)}
    if (optim_method == "sgd")       { op <- optimizer_sgd(lr = lr)}

    model <- keras_model_sequential() %>%
        layer_dense(units = hidden_neur, activation = hidden_activation, input_shape = ncol(x)) %>%
        layer_dense(units = 1)

    model %>% compile(
        loss = "mse",
        optimizer = op,
        metrics = list("mean_absolute_error")
    )

    historylog <- model %>% fit(x, y, epochs = iter, verbose = 0, callbacks = list(early_stop))

    return (model)
}  

NNpredict.keras <- function(object, x, ...)
{
  object %>% predict(x)
}
NNclose.keras <- function()
{
  if("package:keras" %in% search())
    detach("package:keras", unload=TRUE)
}

keras.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, keras.method, "NNtrain.keras", "hyperParams.keras", "NNpredict.keras", 
                               NNsummary, "NNclose.keras", NA, keras.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="keras", pkgfun="fit", rdafile=TRUE, odir=odir, echoreport=2)

38 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "keras", pkgfun = "fit", keras.method,
  prepareZZ.arg = keras.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

39 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "adadelta")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adadelta

	RMSE	MSE	MAE	WAE	time
mDette	0.9613	0.9242	0.7272	4.7287	167.25
mFriedman	0.0317	0.0010	0.0249	0.1059	120.85
mIshigami	2.6386	6.9622	2.2554	6.9079	103.13
mRef153	4.0384	16.3085	2.9531	11.1678	23.79
uDmod1	0.3587	0.1287	0.3072	0.8421	76.08
uDmod2	0.2927	0.0857	0.2378	0.6959	75.37
uDreyfus1	0.3617	0.1309	0.2663	0.8027	29.63
uDreyfus2	0.3652	0.1333	0.2757	0.9863	33.43
uGauss1	14.8378	220.1611	11.9527	42.6112	1.12
uGauss2	7.9524	63.2413	5.8779	24.8152	58.08
uGauss3	5.4903	30.1438	4.2834	15.1099	79.95
uNeuroOne	0.3096	0.0958	0.2622	0.5655	67.06

kable(apply(res[,,2,], c(3,1), min), caption = "adagrad")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adagrad

	RMSE	MSE	MAE	WAE	time
mDette	1.5412	2.3752	1.0466	9.6760	10.23
mFriedman	0.0296	0.0009	0.0236	0.0929	8.97
mIshigami	0.8522	0.7263	0.6255	3.3311	6.37
mRef153	3.8952	15.1728	2.9143	13.2657	2.18
uDmod1	0.2252	0.0507	0.1765	0.5236	3.45
uDmod2	0.1597	0.0255	0.1260	0.3900	10.51
uDreyfus1	0.1630	0.0266	0.1288	0.4104	1.17
uDreyfus2	0.1847	0.0341	0.1340	0.5863	2.05
uGauss1	6.5047	42.3113	5.1168	19.5841	27.92
uGauss2	5.0447	25.4490	3.6068	17.8629	8.42
uGauss3	3.4760	12.0827	2.7864	10.8061	10.20
uNeuroOne	0.2893	0.0837	0.2393	0.5161	5.37

kable(apply(res[,,3,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adam

	RMSE	MSE	MAE	WAE	time
mDette	0.7615	0.5798	0.5937	2.8001	1.42
mFriedman	0.0636	0.0040	0.0508	0.2348	1.48
mIshigami	0.9777	0.9559	0.7217	3.9155	2.04
mRef153	3.9276	15.4261	2.8619	14.1381	0.98
uDmod1	0.1376	0.0189	0.1120	0.2974	1.34
uDmod2	0.0963	0.0093	0.0782	0.2230	1.39
uDreyfus1	0.0706	0.0050	0.0581	0.1792	1.71
uDreyfus2	0.1105	0.0122	0.0919	0.2513	2.02
uGauss1	2.8614	8.1876	2.2907	8.1367	3.12
uGauss2	3.9732	15.7860	3.0663	15.0530	1.61
uGauss3	3.2791	10.7523	2.5796	10.0008	1.53
uNeuroOne	0.2869	0.0823	0.2314	0.5678	1.02

kable(apply(res[,,4,], c(3,1), min), caption = "adamax")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adamax

	RMSE	MSE	MAE	WAE	time
mDette	0.6492	0.4215	0.4713	2.5227	3.45
mFriedman	0.0326	0.0011	0.0262	0.1051	3.00
mIshigami	0.8307	0.6901	0.6065	2.9560	3.51
mRef153	3.7083	13.7517	2.6777	14.4399	1.35
uDmod1	0.0883	0.0078	0.0708	0.2643	2.31
uDmod2	0.1082	0.0117	0.0891	0.2339	2.44
uDreyfus1	0.0365	0.0013	0.0285	0.0979	1.24
uDreyfus2	0.1118	0.0125	0.0901	0.2467	3.86
uGauss1	2.5783	6.6475	2.0082	7.0409	5.54
uGauss2	3.8559	14.8682	2.8623	12.5173	4.89
uGauss3	2.5583	6.5450	2.0418	6.9307	3.84
uNeuroOne	0.2841	0.0807	0.2309	0.5351	1.32

kable(apply(res[,,5,], c(3,1), min), caption = "nadam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

nadam

	RMSE	MSE	MAE	WAE	time
mDette	1.0271	1.0548	0.8112	3.4910	2.79
mFriedman	0.0732	0.0054	0.0591	0.2260	1.79
mIshigami	1.0800	1.1664	0.8086	3.9812	1.84
mRef153	4.1342	17.0915	3.1288	14.6948	1.23
uDmod1	0.1786	0.0319	0.1361	0.5892	1.24
uDmod2	0.1201	0.0144	0.1019	0.2913	1.29
uDreyfus1	0.0648	0.0042	0.0498	0.1882	0.97
uDreyfus2	0.1341	0.0180	0.1091	0.3395	1.25
uGauss1	5.2915	28.0003	3.9137	14.5660	2.64
uGauss2	4.2819	18.3348	3.1232	14.0616	2.10
uGauss3	3.2682	10.6813	2.6333	8.3149	2.03
uNeuroOne	0.2855	0.0815	0.2303	0.5351	1.17

kable(apply(res[,,6,], c(3,1), min), caption = "rmsprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

rmsprop

	RMSE	MSE	MAE	WAE	time
mDette	2.6780	7.1715	1.7824	11.9935	1.61
mFriedman	0.1010	0.0102	0.0815	0.3391	1.74
mIshigami	2.8335	8.0289	2.3789	7.9823	1.42
mRef153	4.5931	21.0966	3.2339	16.3043	0.77
uDmod1	0.2375	0.0564	0.1781	0.6557	0.95
uDmod2	0.1629	0.0265	0.1269	0.4713	0.79
uDreyfus1	0.3101	0.0961	0.2545	0.6950	0.83
uDreyfus2	0.2221	0.0493	0.1632	0.7247	0.78
uGauss1	6.6698	44.4858	5.3250	16.5607	1.64
uGauss2	8.1016	65.6362	6.4959	22.1370	1.61
uGauss3	5.4568	29.7772	4.3038	15.1532	1.59
uNeuroOne	0.3042	0.0925	0.2548	0.6270	0.98

kable(apply(res[,,7,], c(3,1), min), caption = "sgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

sgd

	RMSE	MSE	MAE	WAE	time
mDette	0.5726	0.3278	0.4611	2.0003	3.12
mFriedman	0.0365	0.0013	0.0293	0.1017	2.86
mIshigami	2.7076	7.3312	2.3025	7.1576	1.98
mRef153	3.7419	14.0019	2.7635	14.0891	1.45
uDmod1	0.2044	0.0418	0.1608	0.4821	1.64
uDmod2	0.2431	0.0591	0.2005	0.5898	1.25
uDreyfus1	0.3373	0.1138	0.2662	0.7565	1.45
uDreyfus2	0.3523	0.1241	0.2740	0.9029	1.30
uGauss1	3.4196	11.6940	2.6687	10.0790	5.91
uGauss2	5.1907	26.9434	3.6742	19.6587	4.97
uGauss3	4.8610	23.6289	3.8061	13.0815	4.14
uNeuroOne	0.2901	0.0842	0.2385	0.5656	2.89

40 Setup

40.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

40.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

40.3 MachineShop trainPredict arguments - inputs fmla, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(MachineShop)
MachineShop.method <- "none"
hyperParams.MachineShop <- function(...) {
    return (list(iter=maxit2ndorder, trace=FALSE, linout=TRUE))
}
NNtrain.MachineShop <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    trace <- hyper_params$trace
    maxit <- hyper_params$iter
    linout <- hyper_params$linout #linearoutpputunit
    myNN <- MachineShop::NNetModel(size = neur, linout = linout, maxit = maxit,
                                   trace=trace)
    MachineShop::fit(formula, data = dataxy, model = myNN)
    
}
NNpredict.MachineShop <- function(object, x, ...)
    as.numeric(predict(object, newdata=x, type="response"))
NNclose.MachineShop <- function()
  if("package:MachineShop" %in% search())
    detach("package:MachineShop", unload=TRUE)
MachineShop.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, MachineShop.method, "NNtrain.MachineShop", "hyperParams.MachineShop", "NNpredict.MachineShop", 
                               NNsummary, "NNclose.MachineShop", NA, MachineShop.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="MachineShop", pkgfun="fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

41 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "MachineShop", pkgfun = "fit", MachineShop.method,
  prepareZZ.arg = MachineShop.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

42 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.2570	0.0660	0.2065	0.9549	0.06
mFriedman	0.0085	0.0001	0.0068	0.0265	0.09
mIshigami	0.6685	0.4468	0.5103	2.7169	0.14
mRef153	3.1772	10.0949	2.1868	12.3914	0.00
uDmod1	0.0442	0.0020	0.0353	0.1026	0.00
uDmod2	0.0406	0.0016	0.0325	0.1002	0.00
uDreyfus1	0.0023	0.0000	0.0015	0.0075	0.00
uDreyfus2	0.0906	0.0082	0.0724	0.2203	0.00
uGauss1	2.2521	5.0721	1.7482	6.9300	0.02
uGauss2	2.5986	6.7528	2.0803	7.8638	0.01
uGauss3	2.3086	5.3294	1.8457	6.3333	0.02
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	1.2314	1.5165	0.9854	8.0327	0.08
mFriedman	0.0116	0.0001	0.0092	0.0360	0.11
mIshigami	2.1956	4.8206	1.7857	5.3089	0.15
mRef153	3.4737	12.0666	2.4888	13.8862	0.01
uDmod1	0.0456	0.0021	0.0365	0.1181	0.01
uDmod2	0.0494	0.0024	0.0374	0.1288	0.02
uDreyfus1	0.0034	0.0000	0.0028	0.0102	0.01
uDreyfus2	0.1415	0.0200	0.1045	0.4507	0.01
uGauss1	2.2681	5.1443	1.7734	7.5718	0.03
uGauss2	3.1210	9.7405	2.4847	9.3644	0.02
uGauss3	3.1576	9.9705	2.4940	9.7409	0.03
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.01

43 Setup

43.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

43.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

43.3 minpack.lm trainPredict arguments - inputs full fmla, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(minpack.lm)
minpack.lm.method <- "none"
hyperParams.minpack.lm <- function(...) {
    return (list(iter=maxit2ndorder))
}
NNtrain.minpack.lm <- function(x, y, dataxy, formula, neur, method, hyperParams, NNfullformula, NNparam, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))

    minpack.lm::nlsLM(NNfullformula, data = dataxy,
                      control = list(maxiter = hyper_params$iter))
}
NNpredict.minpack.lm <- function(object, x, ...)
  predict(object, newdata=as.data.frame(x))
NNclose.minpack.lm <- function()
  if("package:minpack.lm" %in% search())
    detach("package:minpack.lm", unload=TRUE)
minpack.lm.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, minpack.lm.method, "NNtrain.minpack.lm", "hyperParams.minpack.lm", "NNpredict.minpack.lm", 
                               NNsummary, "NNclose.minpack.lm", NA, minpack.lm.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="minpack.lm", pkgfun="nlsLM", csvfile=TRUE, rdafile=TRUE, odir=odir)

44 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "minpack.lm", pkgfun = "nlsLM", minpack.lm.method,
  prepareZZ.arg = minpack.lm.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

45 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.6081	0.3698	0.4989	1.9776	0.23
mFriedman	0.1269	0.0161	0.1009	0.3714	0.36
mIshigami	2.5379	6.4408	2.0524	7.6035	0.89
mRef153	3.4318	11.7771	2.4646	13.2825	0.06
uDmod1	0.0445	0.0020	0.0362	0.1153	0.03
uDmod2	0.0427	0.0018	0.0333	0.1058	0.01
uDreyfus1	0.0000	0.0000	0.0000	0.0001	0.00
uDreyfus2	0.0906	0.0082	0.0723	0.2197	0.01
uGauss1	2.2329	4.9857	1.7383	6.9429	0.06
uGauss2	2.9795	8.8775	2.3890	9.0540	0.04
uGauss3	3.1472	9.9050	2.4837	9.7293	0.03
uNeuroOne	1.2720	1.6181	1.1104	2.5150	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.6081	0.3698	0.4989	1.9776	0.24
mFriedman	0.1269	0.0161	0.1009	0.3714	0.38
mIshigami	2.5379	6.4408	2.0524	7.6035	0.90
mRef153	3.4318	11.7771	2.4646	13.2825	0.07
uDmod1	0.0445	0.0020	0.0362	0.1153	0.03
uDmod2	0.0427	0.0018	0.0333	0.1058	0.02
uDreyfus1	0.0000	0.0000	0.0000	0.0001	0.00
uDreyfus2	0.0906	0.0082	0.0723	0.2197	0.02
uGauss1	2.2329	4.9857	1.7383	6.9429	0.07
uGauss2	2.9795	8.8775	2.3890	9.0540	0.05
uGauss3	3.1472	9.9050	2.4837	9.7293	0.04
uNeuroOne	1.2720	1.6181	1.1104	2.5150	0.00

46 Setup

46.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

46.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

46.3 monmlp trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(monmlp)
monmlp.method <- c("BFGS", "Nelder-Mead")
hyperParams.monmlp <- function(optim_method, ...) {
    if (optim_method == "BFGS")         {iter <- maxit2ndorder}
    if (optim_method == "Nelder-Mead")  {iter <- maxit1storderB} 
    return (list(iter=iter, silent=TRUE, scale=TRUE))
}
NNtrain.monmlp <- function(x, y, dataxy, formula, neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    iter <- hyper_params$iter
    scale <- hyper_params$scale
    silent <- hyper_params$silent
    
    monmlp::monmlp.fit(x, y, hidden1 = neur, scale.y = scale, silent=silent,
                         method = optim_method, iter.max = iter)
}
NNpredict.monmlp <- function(object, x, ...)
  as.numeric(monmlp::monmlp.predict(x, weights=object))
NNclose.monmlp <- function()
  if("package:monmlp" %in% search())
    detach("package:monmlp", unload=TRUE)
monmlp.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, monmlp.method, "NNtrain.monmlp", "hyperParams.monmlp", "NNpredict.monmlp", 
                               NNsummary, "NNclose.monmlp", NA, monmlp.prepareZZ, nrep=2,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="monmlp", pkgfun="monmlp.fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

47 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "monmlp", pkgfun = "monmlp.fit", monmlp.method,
  prepareZZ.arg = monmlp.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = TRUE)

48 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "BFGS")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BFGS

	RMSE	MSE	MAE	WAE	time
mDette	0.3732	0.1393	0.2834	1.6050	0.29
mFriedman	0.0132	0.0002	0.0105	0.0434	0.29
mIshigami	0.8185	0.6699	0.6145	3.2824	0.45
mRef153	3.2360	10.4717	2.1899	13.2846	0.22
uDmod1	0.0919	0.0084	0.0716	0.3325	0.19
uDmod2	0.0522	0.0027	0.0421	0.1148	0.20
uDreyfus1	0.0323	0.0010	0.0257	0.0789	0.18
uDreyfus2	0.0917	0.0084	0.0736	0.2281	0.20
uGauss1	2.7246	7.4233	2.1603	8.9636	0.22
uGauss2	3.0438	9.2645	2.4413	9.0617	0.22
uGauss3	2.9135	8.4887	2.2996	7.4899	0.22
uNeuroOne	0.2831	0.0801	0.2308	0.5562	0.19

kable(apply(res[,,2,], c(3,1), min), caption = "Nelder-Mead")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

Nelder-Mead

	RMSE	MSE	MAE	WAE	time
mDette	3.0247	9.1490	2.1300	16.9350	1.03
mFriedman	0.1155	0.0133	0.0848	0.3409	1.00
mIshigami	2.7368	7.4900	2.3038	7.8591	1.50
mRef153	3.7710	14.2203	2.7940	14.0924	0.49
uDmod1	0.1381	0.0191	0.1119	0.3158	0.41
uDmod2	0.1342	0.0180	0.1059	0.3334	0.33
uDreyfus1	0.1425	0.0203	0.1168	0.3015	0.22
uDreyfus2	0.1762	0.0310	0.1421	0.4840	0.23
uGauss1	12.1868	148.5171	9.5869	32.7148	0.48
uGauss2	8.2728	68.4388	6.5270	21.4965	0.39
uGauss3	5.8001	33.6414	4.4551	16.4890	0.36
uNeuroOne	0.3020	0.0912	0.2525	0.5852	0.21

49 Setup

49.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

49.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

49.3 neuralnet trainPredict arguments - inputs fmla, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(neuralnet)
neuralnet.method <- c("slr", "sag", "rprop-", "rprop+", "backprop")
hyperParams.neuralnet <- function(method, ...) {
    return(list(iter=maxit1storderC, threshold=0.2, linear.output=TRUE))
}
NNtrain.neuralnet <- function(x, y, dataxy, formula, hidden_neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(method, ...))
    
    if(method == "backprop") {learningrate <- 0.001} else {learningrate <- NULL}
    
    neuralnet::neuralnet(formula = formula, data = dataxy, hidden = hidden_neur, algorithm=method,
                        threshold=hyper_params$threshold, linear.output=hyper_params$linear.output,
                                    stepmax = hyper_params$iter, startweights = NULL, act.fct = "tanh",
                                    learningrate = learningrate)
}
NNpredict.neuralnet <- function(object, x, ...)
  as.numeric(predict(object, newdata = x))

NNclose.neuralnet <- function()
  if("package:neuralnet" %in% search())
    detach("package:neuralnet", unload=TRUE)
neuralnet.prepareZZ <- list(xdmv = "d", ydmv = "d", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, neuralnet.method, "NNtrain.neuralnet", "hyperParams.neuralnet", "NNpredict.neuralnet", 
                               NNsummary, "NNclose.neuralnet", NA, neuralnet.prepareZZ, nrep=2,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="neuralnet", pkgfun="neuralnet", rdafile=TRUE, odir=odir)

50 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "neuralnet", pkgfun = "neuralnet", neuralnet.method,
  prepareZZ.arg = neuralnet.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

51 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "slr")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

slr

	RMSE	MSE	MAE	WAE	time
mDette	0.5494	0.3018	0.4227	2.2406	1.07
mFriedman	0.0690	0.0048	0.0498	0.1835	1.19
mIshigami	0.6816	0.4646	0.5019	2.6920	21.84
mRef153	3.8201	14.5934	2.8021	14.8220	0.00
uDmod1	0.0839	0.0070	0.0670	0.2125	0.06
uDmod2	0.0840	0.0071	0.0692	0.2061	0.04
uDreyfus1	0.2981	0.0888	0.2316	0.6732	0.00
uDreyfus2	0.3374	0.1138	0.2600	0.8539	0.00
uGauss1	2.8881	8.3414	2.2891	9.1323	0.15
uGauss2	3.5678	12.7289	2.6347	14.0133	0.08
uGauss3	2.8318	8.0188	2.2679	8.3311	0.03
uNeuroOne	0.2923	0.0854	0.2404	0.6649	0.00

kable(apply(res[,,2,], c(3,1), min), caption = "sag")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

sag

	RMSE	MSE	MAE	WAE	time
mDette	2.1196	4.4929	1.4792	13.2483	2.53
mFriedman	0.0806	0.0065	0.0716	0.1713	2.78
mIshigami	3.6898	13.6148	2.9776	13.1137	25.03
mRef153	3.3829	11.4438	2.3192	14.1818	0.03
uDmod1	0.0583	0.0034	0.0481	0.1680	0.16
uDmod2	0.0811	0.0066	0.0691	0.1949	0.25
uDreyfus1	0.1963	0.0386	0.1545	0.4932	0.00
uDreyfus2	0.1663	0.0277	0.1215	0.5369	0.04
uGauss1	2.2972	5.2769	1.7692	7.6070	3.39
uGauss2	3.3723	11.3726	2.7158	9.5832	0.77
uGauss3	2.7247	7.4241	2.1604	8.3531	0.02
uNeuroOne	0.2893	0.0837	0.2393	0.5963	0.01

kable(apply(res[,,3,], c(3,1), min), caption = "rprop-")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

rprop-

	RMSE	MSE	MAE	WAE	time
mDette	0.5338	0.2849	0.4053	2.3496	0.31
mFriedman	0.0095	0.0001	0.0075	0.0364	0.92
mIshigami	0.6728	0.4526	0.4998	2.6300	1.08
mRef153	3.4657	12.0109	2.4968	13.8651	0.00
uDmod1	0.1634	0.0267	0.1247	0.4303	0.02
uDmod2	0.0955	0.0091	0.0731	0.2699	0.03
uDreyfus1	0.1014	0.0103	0.0815	0.2279	0.00
uDreyfus2	0.1632	0.0266	0.1195	0.5401	0.01
uGauss1	2.6198	6.8633	2.0879	8.9886	0.12
uGauss2	3.5904	12.8911	2.6835	12.9142	0.06
uGauss3	2.6706	7.1323	2.1530	7.4196	0.01
uNeuroOne	0.2864	0.0820	0.2347	0.5735	0.00

kable(apply(res[,,4,], c(3,1), min), caption = "rprop+")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

rprop+

	RMSE	MSE	MAE	WAE	time
mDette	0.4859	0.2361	0.3681	1.9684	0.29
mFriedman	0.0102	0.0001	0.0080	0.0340	3.19
mIshigami	0.5788	0.3350	0.4256	2.6219	0.71
mRef153	3.4475	11.8851	2.3500	14.9019	0.01
uDmod1	0.1086	0.0118	0.0837	0.2374	0.03
uDmod2	0.1077	0.0116	0.0886	0.2424	0.02
uDreyfus1	0.2119	0.0449	0.1689	0.5040	0.00
uDreyfus2	0.2846	0.0810	0.2146	0.7950	0.00
uGauss1	2.9603	8.7633	2.3236	9.1523	0.07
uGauss2	3.6133	13.0558	2.6981	13.8602	0.01
uGauss3	2.5491	6.4979	2.0661	7.4271	0.03
uNeuroOne	0.2848	0.0811	0.2341	0.5706	0.00

kable(apply(res[,,5,], c(3,1), min), caption = "backprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

backprop

	RMSE	MSE	MAE	WAE	time
mDette	8.1656	66.6770	6.5262	36.2385	14.04
mFriedman	0.2348	0.0551	0.1880	0.6346	14.31
mIshigami	3.6898	13.6148	2.9776	13.1137	23.84
mRef153	3.8593	14.8938	2.8804	13.3655	0.05
uDmod1	0.1521	0.0231	0.1156	0.4429	0.41
uDmod2	0.1091	0.0119	0.0884	0.2494	0.22
uDreyfus1	0.3201	0.1024	0.2620	0.6745	0.01
uDreyfus2	0.3205	0.1027	0.2469	0.8518	0.02
uGauss1	2.9109	8.4732	2.3100	9.5344	0.50
uGauss2	4.2479	18.0445	3.0708	12.4784	0.72
uGauss3	3.8802	15.0562	3.0447	10.7745	0.23
uNeuroOne	0.2898	0.0840	0.2377	0.5770	0.12

52 Setup

52.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

52.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

52.3 nlsr trainPredict arguments - inputs full fmla, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(nlsr)
nlsr.method <- "none"
hyperParams.nlsr <- function(...) {
    return (list(iter = maxit2ndorder, sdnormstart = 0.1))
}
NNtrain.nlsr <- function(x, y, dataxy, formula, neur, method, hyperParams, NNfullformula, NNparam, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    start <- round(rnorm(NNparam, sd = hyper_params$sdnormstart), 4)
    names(start)  <- paste0("b", 1:NNparam)
    
    NNreg <- nlsr::nlxb(NNfullformula, data = dataxy, start = start, control = list(femax = hyper_params$iter))
    return(NNreg)
}
NNpredict.nlsr <- function(object, x, ...)
  as.numeric(predict(object, x))
NNclose.nlsr <- function()
  if("package:nlsr" %in% search())
    detach("package:nlsr", unload=TRUE)
nlsr.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, nlsr.method, "NNtrain.nlsr", "hyperParams.nlsr", "NNpredict.nlsr", 
                               NNsummary, "NNclose.nlsr", NA, nlsr.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="nlsr", pkgfun="nlxb", csvfile=TRUE, rdafile=TRUE, odir=odir)

53 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "nlsr", pkgfun = "nlxb", nlsr.method,
  prepareZZ.arg = nlsr.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

54 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.1400	0.0196	0.1124	0.6392	0.41
mFriedman	0.0045	0.0000	0.0036	0.0129	0.65
mIshigami	0.6602	0.4359	0.5081	2.5692	1.39
mRef153	3.1128	9.6892	2.1804	12.6227	0.03
uDmod1	0.0433	0.0019	0.0341	0.1038	0.08
uDmod2	0.0427	0.0018	0.0333	0.1058	0.02
uDreyfus1	0.0000	0.0000	0.0000	0.0001	0.00
uDreyfus2	0.0906	0.0082	0.0723	0.2197	0.05
uGauss1	2.2321	4.9824	1.7377	6.9331	0.15
uGauss2	2.3327	5.4417	1.8303	7.3268	0.05
uGauss3	2.2991	5.2858	1.8376	7.0773	0.02
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.4500	0.2025	0.3497	2.7841	0.42
mFriedman	0.0061	0.0000	0.0048	0.0196	0.68
mIshigami	2.2311	4.9778	1.8053	5.7864	1.43
mRef153	3.4349	11.7984	2.4566	14.5501	0.15
uDmod1	0.0433	0.0019	0.0349	0.1063	0.09
uDmod2	0.0427	0.0018	0.0333	0.1058	0.04
uDreyfus1	0.0000	0.0000	0.0000	0.0001	0.02
uDreyfus2	0.0906	0.0082	0.0723	0.2197	0.06
uGauss1	2.6129	6.8271	2.0484	8.2482	0.16
uGauss2	2.9754	8.8533	2.3838	9.0619	0.13
uGauss3	2.8185	7.9439	2.2078	7.5077	0.06
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

55 Setup

55.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

55.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

55.3 nnet trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(nnet)
nnet.method <- "none"
hyperParams.nnet <- function(...) {
    return (list(iter=maxit2ndorder, trace=FALSE))
}
NNtrain.nnet <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    NNreg <- nnet::nnet(x, y, size = neur, linout = TRUE, maxit = hyper_params$iter, trace=hyper_params$trace)
    return(NNreg)
}
NNpredict.nnet <- function(object, x, ...)
    predict(object, newdata=x)
NNclose.nnet <- function()
  if("package:nnet" %in% search())
    detach("package:nnet", unload=TRUE)
nnet.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, nnet.method, "NNtrain.nnet", "hyperParams.nnet", "NNpredict.nnet", 
                               NNsummary, "NNclose.nnet", NA, nnet.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="nnet", pkgfun="nnet", csvfile=TRUE, rdafile=TRUE, odir=odir)

56 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "nnet", pkgfun = "nnet", nnet.method,
  prepareZZ.arg = nnet.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

57 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.2650	0.0702	0.2120	0.9640	0.06
mFriedman	0.0091	0.0001	0.0072	0.0287	0.09
mIshigami	0.5462	0.2984	0.4009	2.3158	0.14
mRef153	3.0865	9.5264	2.1869	11.6499	0.00
uDmod1	0.0437	0.0019	0.0360	0.1198	0.00
uDmod2	0.0602	0.0036	0.0486	0.1212	0.00
uDreyfus1	0.0026	0.0000	0.0021	0.0081	0.00
uDreyfus2	0.0906	0.0082	0.0724	0.2199	0.00
uGauss1	2.2380	5.0087	1.7370	7.0010	0.02
uGauss2	2.3625	5.5814	1.8552	7.4016	0.01
uGauss3	2.3554	5.5480	1.8465	7.1275	0.00
uNeuroOne	0.2830	0.0801	0.2313	0.5673	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.4735	0.2242	0.3557	2.0121	0.08
mFriedman	0.0120	0.0001	0.0094	0.0404	0.10
mIshigami	0.6959	0.4843	0.5147	3.0034	0.15
mRef153	3.2253	10.4023	2.2336	13.9974	0.01
uDmod1	0.0865	0.0075	0.0636	0.3435	0.01
uDmod2	0.0615	0.0038	0.0489	0.1408	0.00
uDreyfus1	0.0716	0.0051	0.0449	0.2254	0.00
uDreyfus2	0.0906	0.0082	0.0725	0.2202	0.00
uGauss1	2.2557	5.0883	1.7541	7.3381	0.03
uGauss2	3.0894	9.5444	2.4567	9.4881	0.03
uGauss3	3.1706	10.0529	2.5057	9.8058	0.02
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

58 Setup

58.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

58.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

58.3 qrnn trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(qrnn)
qrnn.method <- "none"
hyperParams.qrnn <- function(optim_method, ...) {
    
    iter <- maxit2ndorder
    init.range = c(-0.1, 0.1, -0.1, 0.1)
  
    out <- list(iter = iter, init.range=init.range)
    return (out)
}
NNtrain.qrnn <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    NNreg <- qrnn::qrnn.fit(x, y, n.hidden = neur, 
                     iter.max = hyper_params$iter, n.trials = 1,
                     init.range = hyper_params$init.range, trace=FALSE)
    
    return (NNreg)
}
NNpredict.qrnn <- function(object, x, ...)
  qrnn::qrnn.predict(x, object)
NNclose.qrnn <- function()
  if("package:qrnn" %in% search())
    detach("package:qrnn", unload=TRUE)
qrnn.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, qrnn.method, "NNtrain.qrnn", "hyperParams.qrnn", "NNpredict.qrnn", 
                               NNsummary, "NNclose.qrnn", NA, qrnn.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="qrnn", pkgfun="qrnn.fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

59 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "qrnn", pkgfun = "qrnn.fit", qrnn.method,
  prepareZZ.arg = qrnn.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

60 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.3632	0.1319	0.2412	1.8839	0.37
mFriedman	0.0105	0.0001	0.0078	0.0346	0.29
mIshigami	0.7656	0.5862	0.4803	3.7497	0.84
mRef153	3.2636	10.6514	2.0063	14.7512	0.13
uDmod1	0.1162	0.0135	0.0705	0.3143	0.08
uDmod2	0.0511	0.0026	0.0373	0.1373	0.16
uDreyfus1	0.2781	0.0773	0.1721	0.8460	0.08
uDreyfus2	0.1601	0.0256	0.1212	0.3555	0.06
uGauss1	2.7155	7.3739	2.0868	7.9015	0.11
uGauss2	2.6571	7.0601	2.0248	8.7844	0.14
uGauss3	2.7773	7.7135	2.1210	7.7161	0.11
uNeuroOne	0.2939	0.0864	0.2258	0.7231	0.04

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.7514	0.5646	0.4482	6.6249	0.39
mFriedman	0.0296	0.0009	0.0190	0.1330	0.53
mIshigami	0.7907	0.6251	0.4951	4.0838	1.20
mRef153	3.4540	11.9300	2.1362	15.2840	0.17
uDmod1	0.1349	0.0182	0.0830	0.6014	0.22
uDmod2	0.0821	0.0067	0.0598	0.2411	0.21
uDreyfus1	0.2841	0.0807	0.1815	0.9095	0.12
uDreyfus2	0.2693	0.0725	0.1983	0.6891	0.10
uGauss1	2.7208	7.4029	2.0888	8.1916	0.17
uGauss2	3.7782	14.2747	2.6654	16.0998	0.19
uGauss3	3.9015	15.2216	2.8959	13.2058	0.19
uNeuroOne	0.2939	0.0864	0.2258	0.7231	0.11

61 Setup

61.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

61.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

61.3 radiant.model trainPredict arguments - inputs “y”, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(radiant.model)
radiant.model.method <- "none"
hyperParams.radiant.model <- function(...) {
    return (list(iter=maxit2ndorder, type="regression", decay=0))
}
NNtrain.radiant.model <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    radiant.model::nn(dataxy, rvar = "y", evar = attr(terms(formula), "term.labels"),
                      type = hyper_params$type, size = neur, maxit = hyper_params$iter,
                      decay = hyper_params$decay)
    
}
NNpredict.radiant.model <- function(object, x, ...)
   predict(object, pred_data=as.data.frame(x))$Prediction
NNclose.radiant.model <- function()
  if("package:radiant.model" %in% search())
    detach("package:radiant.model", unload=TRUE)
radiant.model.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, radiant.model.method, "NNtrain.radiant.model", "hyperParams.radiant.model", "NNpredict.radiant.model", 
                               NNsummary, "NNclose.radiant.model", NA, radiant.model.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="radiant.model", pkgfun="nn", csvfile=TRUE, rdafile=TRUE, odir=odir)

62 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "radiant.model", pkgfun = "nn", radiant.model.method,
  prepareZZ.arg = radiant.model.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

63 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.2621	0.0687	0.2119	1.0202	0.09
mFriedman	0.0084	0.0001	0.0068	0.0250	0.12
mIshigami	0.4934	0.2434	0.3570	2.1803	0.17
mRef153	3.1803	10.1145	2.1894	13.3586	0.04
uDmod1	0.0800	0.0064	0.0636	0.3080	0.01
uDmod2	0.0647	0.0042	0.0529	0.1478	0.00
uDreyfus1	0.0121	0.0001	0.0082	0.0385	0.01
uDreyfus2	0.0907	0.0082	0.0723	0.2189	0.01
uGauss1	2.3275	5.4173	1.8307	7.2534	0.04
uGauss2	2.5784	6.6482	2.0434	7.8916	0.05
uGauss3	2.6848	7.2080	2.1288	7.8135	0.04
uNeuroOne	0.2830	0.0801	0.2313	0.5556	0.01

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.5412	0.2929	0.4096	2.1475	0.11
mFriedman	0.0150	0.0002	0.0106	0.0664	0.12
mIshigami	0.7868	0.6190	0.5896	3.1250	0.17
mRef153	3.3593	11.2851	2.4590	13.9894	0.05
uDmod1	0.1088	0.0118	0.0817	0.3346	0.02
uDmod2	0.0771	0.0059	0.0602	0.2202	0.02
uDreyfus1	0.0682	0.0046	0.0555	0.1546	0.03
uDreyfus2	0.0917	0.0084	0.0732	0.2275	0.02
uGauss1	6.3010	39.7029	5.1479	17.4222	0.06
uGauss2	4.5795	20.9722	3.7498	11.3454	0.05
uGauss3	3.4127	11.6466	2.7026	10.5356	0.05
uNeuroOne	0.2830	0.0801	0.2313	0.5677	0.01

64 Setup

64.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

64.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

64.3 rminer trainPredict arguments - inputs formula, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(rminer)
rminer.method <- "none"
hyperParams.rminer <- function(...) {
    return (list(task="reg", iter=maxit2ndorder))
}
NNtrain.rminer <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    rminer::fit(formula, data = dataxy, model = "mlp", task = hyper_params$task, 
                                        size = neur, maxit = hyper_params$iter)
}
NNpredict.rminer <- function(object, x, ...)
   as.numeric(rminer::predict(object, newdata=as.data.frame(x)))
NNclose.rminer <- function()
  if("package:rminer" %in% search())
    detach("package:rminer", unload=TRUE)
rminer.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, rminer.method, "NNtrain.rminer", "hyperParams.rminer", "NNpredict.rminer", 
                               NNsummary, "NNclose.rminer", NA, rminer.prepareZZ, nrep=2,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="rminer", pkgfun="fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

65 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "rminer", pkgfun = "fit", rminer.method,
  prepareZZ.arg = rminer.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

66 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.2335	0.0545	0.1864	0.8434	0.23
mFriedman	0.0095	0.0001	0.0074	0.0298	0.26
mIshigami	0.6490	0.4212	0.4940	2.8053	0.42
mRef153	3.1128	9.6892	2.1804	12.6402	0.03
uDmod1	0.0449	0.0020	0.0363	0.1199	0.02
uDmod2	0.0405	0.0016	0.0327	0.0983	0.01
uDreyfus1	0.0020	0.0000	0.0016	0.0046	0.00
uDreyfus2	0.0906	0.0082	0.0724	0.2196	0.01
uGauss1	2.2452	5.0411	1.7452	6.8539	0.07
uGauss2	2.3604	5.5713	1.8537	7.4500	0.08
uGauss3	2.3033	5.3053	1.8359	6.3721	0.06
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.3147	0.0990	0.2456	1.2905	0.25
mFriedman	0.0112	0.0001	0.0088	0.0373	0.29
mIshigami	0.6668	0.4446	0.5016	3.0019	0.45
mRef153	3.1819	10.1244	2.1870	13.5191	0.05
uDmod1	0.0495	0.0025	0.0418	0.1258	0.03
uDmod2	0.0579	0.0034	0.0479	0.1065	0.01
uDreyfus1	0.0023	0.0000	0.0018	0.0057	0.01
uDreyfus2	0.0906	0.0082	0.0724	0.2202	0.02
uGauss1	2.2628	5.1201	1.7485	7.3946	0.08
uGauss2	2.3690	5.6122	1.8630	7.5477	0.08
uGauss3	2.3232	5.3972	1.8528	7.0323	0.06
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

67 Setup

67.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

67.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

67.3 RSNNS trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(RSNNS)
RSNNS.method <- c("Rprop","BackpropBatch","BackpropChunk","BackpropMomentum",
                  "BackpropWeightDecay","Quickprop","SCG","Std_Backpropagation")
hyperParams.RSNNS <- function(optim_method, ...) {
    
    if (optim_method == "Std_Backpropagation")   {iter <- maxit1storderA} 
    if (optim_method == "BackpropBatch")         {iter <- maxit1storderB}
    if (optim_method == "BackpropChunk")         {iter <- maxit1storderA}
    if (optim_method == "BackpropMomentum")      {iter <- maxit1storderA}
    if (optim_method == "BackpropWeightDecay")   {iter <- maxit1storderA}
    if (optim_method == "Rprop")                 {iter <- maxit1storderA}
    if (optim_method == "Quickprop")             {iter <- maxit1storderB}
    if (optim_method == "SCG")                   {iter <- maxit1storderA}

    out <- list(iter = iter)
    return(out)
}
NNtrain.RSNNS <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, NNfullformula, NNparam,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    if (optim_method == "Std_Backpropagation" || optim_method == "BackpropBatch"){
          NNreg <- RSNNS::mlp(x, y, size = hidden_neur, learnFunc = optim_method, learnFuncParams = c(0.1), #lr 
                        maxit = hyper_params$iter, linOut = TRUE)
    } else {
          NNreg <- RSNNS::mlp(x, y, size = hidden_neur, learnFunc = optim_method,
                        maxit = hyper_params$iter, linOut = TRUE)
    }
    
    return (NNreg)
}
NNpredict.RSNNS <- function(object, x, ...)
  predict(object, x)
NNclose.RSNNS <- function()
  if("package:RSNNS" %in% search())
    detach("package:RSNNS", unload=TRUE)
RSNNS.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)


if(FALSE)
res <- train_and_predict_1data(1, RSNNS.method, "NNtrain.RSNNS", "hyperParams.RSNNS", "NNpredict.RSNNS", 
                               NNsummary, "NNclose.RSNNS", NA, RSNNS.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="RSNNS", pkgfun="mlp", rdafile=TRUE, odir=odir)

68 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "RSNNS", pkgfun = "mlp", RSNNS.method,
  prepareZZ.arg = RSNNS.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

69 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "Rprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

Rprop

	RMSE	MSE	MAE	WAE	time
mDette	0.7757	0.6017	0.5534	5.1570	0.65
mFriedman	0.0307	0.0009	0.0245	0.1283	0.66
mIshigami	1.3146	1.7282	1.0090	4.9058	0.78
mRef153	3.1838	10.1368	2.2135	13.3449	0.19
uDmod1	0.1232	0.0152	0.0790	0.4218	0.09
uDmod2	0.0447	0.0020	0.0349	0.1114	0.07
uDreyfus1	0.0617	0.0038	0.0424	0.1973	0.08
uDreyfus2	0.1145	0.0131	0.0905	0.2628	0.07
uGauss1	2.9859	8.9153	2.2453	11.3172	0.31
uGauss2	3.5732	12.7680	2.6538	13.6403	0.31
uGauss3	2.9609	8.7671	2.3621	8.0132	0.31
uNeuroOne	0.2830	0.0801	0.2312	0.5671	0.06

kable(apply(res[,,2,], c(3,1), min), caption = "BackpropBatch")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BackpropBatch

	RMSE	MSE	MAE	WAE	time
mDette	1.9746	3.8992	1.5353	9.4548	6.64
mFriedman	0.0434	0.0019	0.0313	0.1958	6.75
mIshigami	2.6668	7.1121	2.2937	6.9794	8.39
mRef153	3.5172	12.3706	2.5655	14.1675	1.95
uDmod1	0.2568	0.0659	0.2102	0.5705	0.86
uDmod2	0.2601	0.0676	0.2278	0.5397	0.83
uDreyfus1	0.3120	0.0973	0.2434	0.7269	0.76
uDreyfus2	0.3063	0.0938	0.2369	0.8153	0.77
uGauss1	14.5624	212.0648	11.6416	39.2616	3.31
uGauss2	12.1638	147.9568	9.8948	26.7486	3.11
uGauss3	6.7104	45.0297	5.4781	16.7390	3.12
uNeuroOne	0.6867	0.4715	0.5614	1.6444	0.75

kable(apply(res[,,3,], c(3,1), min), caption = "BackpropChunk")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BackpropChunk

	RMSE	MSE	MAE	WAE	time
mDette	0.5892	0.3472	0.4376	2.7526	0.66
mFriedman	0.0541	0.0029	0.0423	0.1966	0.68
mIshigami	1.3784	1.9000	1.0023	6.3303	0.80
mRef153	3.5933	12.9118	2.7408	13.4810	0.20
uDmod1	0.1298	0.0169	0.0903	0.5015	0.09
uDmod2	0.0829	0.0069	0.0672	0.1946	0.08
uDreyfus1	0.0838	0.0070	0.0634	0.1874	0.08
uDreyfus2	0.1199	0.0144	0.0952	0.2980	0.07
uGauss1	2.9425	8.6585	2.3081	9.0275	0.32
uGauss2	3.2955	10.8604	2.5618	11.5690	0.33
uGauss3	2.9280	8.5732	2.2447	7.7402	0.31
uNeuroOne	0.2912	0.0848	0.2329	0.6286	0.06

kable(apply(res[,,4,], c(3,1), min), caption = "BackpropMomentum")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BackpropMomentum

	RMSE	MSE	MAE	WAE	time
mDette	0.6547	0.4286	0.5035	2.9202	0.64
mFriedman	0.0558	0.0031	0.0434	0.2015	0.66
mIshigami	2.6138	6.8317	2.0342	9.3769	0.76
mRef153	3.4853	12.1476	2.5131	13.9347	0.20
uDmod1	0.1445	0.0209	0.1071	0.5454	0.08
uDmod2	0.0752	0.0056	0.0632	0.1975	0.09
uDreyfus1	0.0719	0.0052	0.0587	0.1749	0.07
uDreyfus2	0.1209	0.0146	0.0956	0.2851	0.08
uGauss1	2.8791	8.2895	2.2161	8.4937	0.32
uGauss2	3.3532	11.2438	2.5715	12.7735	0.31
uGauss3	2.9271	8.5676	2.3397	7.4477	0.31
uNeuroOne	0.2968	0.0881	0.2443	0.5759	0.06

kable(apply(res[,,5,], c(3,1), min), caption = "BackpropWeightDecay")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BackpropWeightDecay

	RMSE	MSE	MAE	WAE	time
mDette	0.6328	0.4005	0.4999	2.5642	0.64
mFriedman	0.0429	0.0018	0.0333	0.1583	0.68
mIshigami	1.2711	1.6157	0.9539	6.1831	0.79
mRef153	3.4319	11.7782	2.3273	13.6367	0.19
uDmod1	0.1314	0.0173	0.0934	0.5080	0.09
uDmod2	0.0799	0.0064	0.0641	0.1725	0.07
uDreyfus1	0.0797	0.0063	0.0605	0.1837	0.08
uDreyfus2	0.1186	0.0141	0.0941	0.2681	0.07
uGauss1	2.8682	8.2266	2.2421	8.7730	0.33
uGauss2	4.5703	20.8880	3.7014	12.3743	0.31
uGauss3	2.8723	8.2503	2.2442	7.5050	0.30
uNeuroOne	0.3096	0.0959	0.2550	0.7078	0.08

kable(apply(res[,,6,], c(3,1), min), caption = "Quickprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

Quickprop

	RMSE	MSE	MAE	WAE	time
mDette	7.1667	51.3615	5.8690	29.6111	7.35
mFriedman	0.1664	0.0277	0.1312	0.5334	7.28
mIshigami	3.4245	11.7272	2.7706	12.3377	9.59
mRef153	6.3871	40.7949	4.9348	16.9337	1.99
uDmod1	0.5775	0.3336	0.5047	0.9896	0.87
uDmod2	0.2570	0.0660	0.2039	0.8023	0.83
uDreyfus1	0.2177	0.0474	0.1733	0.5071	0.78
uDreyfus2	0.2122	0.0450	0.1714	0.5594	0.76
uGauss1	23.6323	558.4840	19.3689	57.4787	3.53
uGauss2	24.5455	602.4818	19.4492	51.6199	3.38
uGauss3	27.8595	776.1537	22.2511	52.3658	3.36
uNeuroOne	0.5304	0.2813	0.4235	1.2829	0.72

kable(apply(res[,,7,], c(3,1), min), caption = "SCG")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

SCG

	RMSE	MSE	MAE	WAE	time
mDette	0.4652	0.2164	0.3537	2.1410	1.03
mFriedman	0.0202	0.0004	0.0160	0.0703	0.97
mIshigami	0.6980	0.4872	0.5118	2.7409	1.28
mRef153	3.2498	10.5614	2.2186	14.3780	0.30
uDmod1	0.0970	0.0094	0.0685	0.3475	0.13
uDmod2	0.0555	0.0031	0.0460	0.1292	0.12
uDreyfus1	0.0851	0.0072	0.0688	0.2077	0.10
uDreyfus2	0.1238	0.0153	0.0998	0.2811	0.10
uGauss1	2.7001	7.2908	2.1575	9.3216	0.53
uGauss2	6.2398	38.9349	4.7463	17.2565	0.47
uGauss3	3.3416	11.1662	2.6060	9.4291	0.49
uNeuroOne	0.2855	0.0815	0.2326	0.6131	0.09

kable(apply(res[,,8,], c(3,1), min), caption = "Std_Backpropagation")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

Std_Backpropagation

	RMSE	MSE	MAE	WAE	time
mDette	0.4789	0.2294	0.3487	1.8761	0.62
mFriedman	0.0420	0.0018	0.0342	0.1451	0.64
mIshigami	2.7659	7.6502	2.1817	10.1281	0.76
mRef153	3.5707	12.7501	2.5817	12.9387	0.18
uDmod1	0.1215	0.0148	0.0932	0.3645	0.08
uDmod2	0.0788	0.0062	0.0664	0.1973	0.07
uDreyfus1	0.1127	0.0127	0.0864	0.2478	0.06
uDreyfus2	0.1298	0.0169	0.1009	0.2947	0.07
uGauss1	3.0594	9.3601	2.4477	8.5800	0.32
uGauss2	3.4215	11.7065	2.5777	13.4441	0.31
uGauss3	3.0294	9.1771	2.3720	8.3779	0.31
uNeuroOne	0.2834	0.0803	0.2306	0.5587	0.07

70 Setup

70.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

70.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

70.3 snnR trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

snnR.method <- "none"
hyperParams.snnR <- function(optim_method, ...) {
    out <- list(iter = maxit2ndorder)
    return (out)
}
NNtrain.snnR <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    NNreg <- snnR::snnR(x, y, nHidden = as.matrix(hidden_neur), 
                 iteramax = hyper_params$iter, verbose=FALSE)
    return (NNreg)
}
NNpredict.snnR <- function(object, x, ...)
  predict(object, x)
NNclose.snnR <- function()
  if("package:snnR" %in% search())
    detach("package:snnR", unload=TRUE)
snnR.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)


if(FALSE)
res <- train_and_predict_1data(1, simpleNeural.method, "NNtrain.snnR", "hyperParams.snnR", "NNpredict.snnR", 
                               NNsummary, "NNclose.snnR", NA, simpleNeural.prepareZZ, nrep=5, echo=TRUE, doplot=TRUE,
                               pkgname="snnR", pkgfun="snnR", rdafile=TRUE, odir=odir, echoreport=1)

71 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "snnR", pkgfun = "snnR", snnR.method,
  prepareZZ.arg = snnR.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

72 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	1.9864	3.9459	1.5889	8.8501	0.05
mFriedman	0.0457	0.0021	0.0353	0.1994	0.08
mIshigami	0.7757	0.6017	0.5560	3.2825	0.25
mRef153	4.3529	18.9480	3.2423	15.2576	0.01
uDmod1	0.2927	0.0857	0.2512	0.6561	0.02
uDmod2	0.2585	0.0668	0.2264	0.5023	0.00
uDreyfus1	0.3691	0.1362	0.2756	0.8531	0.00
uDreyfus2	0.3837	0.1473	0.2773	1.0352	0.00
uGauss1	11.6175	134.9657	9.5749	25.9146	0.02
uGauss2	8.8419	78.1789	6.4413	25.4487	0.02
uGauss3	5.2818	27.8977	4.0957	15.6474	0.01
uNeuroOne	0.6793	0.4615	0.5564	1.6288	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	1.9864	3.9459	1.5889	8.8501	0.15
mFriedman	0.0839	0.0070	0.0747	0.2113	0.10
mIshigami	0.8621	0.7433	0.6030	3.4730	0.36
mRef153	4.3529	18.9480	3.2423	15.2576	0.02
uDmod1	0.2927	0.0857	0.2512	0.6561	0.04
uDmod2	0.2984	0.0890	0.2556	0.6651	0.02
uDreyfus1	0.3691	0.1362	0.2756	0.8531	0.00
uDreyfus2	0.3837	0.1473	0.2773	1.0352	0.01
uGauss1	11.6175	134.9658	9.5749	25.9147	0.03
uGauss2	9.4678	89.6394	6.9147	30.1105	0.05
uGauss3	5.2818	27.8978	4.0957	15.6475	0.03
uNeuroOne	0.6793	0.4615	0.5564	1.6288	0.00

73 Setup

73.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

73.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

73.3 traineR trainPredict arguments - inputs fmla, data

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(traineR)
traineR.method <- "none"
hyperParams.traineR <- function(...) {
    return (list(iter=maxit2ndorder))
}
NNtrain.traineR <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
  hyper_params <- do.call(hyperParams, list(...))
  NNreg <- traineR::train.nnet(formula = formula, data = dataxy, size = neur, maxit = hyper_params$iter, linout = TRUE)
  return(NNreg)
}
NNpredict.traineR <- function(object, x, dataxy, ...){
  object$fitted.values 
}
NNclose.traineR <- function()
  if("package:traineR" %in% search())
    detach("package:traineR", unload=TRUE)
traineR.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, traineR.method, "NNtrain.traineR", "hyperParams.traineR", "NNpredict.traineR", 
                               NNsummary, "NNclose.traineR", NA, traineR.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="traineR", pkgfun="traineR", csvfile=TRUE, rdafile=TRUE, odir=odir)

74 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "traineR", pkgfun = "train.nnet", traineR.method,
  prepareZZ.arg = traineR.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

75 Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.4539	0.2060	0.3431	1.8447	0.07
mFriedman	0.0112	0.0001	0.0087	0.0446	0.08
mIshigami	0.6846	0.4686	0.5051	3.0097	0.14
mRef153	3.1128	9.6893	2.1800	12.6557	0.00
uDmod1	0.0410	0.0017	0.0311	0.1208	0.00
uDmod2	0.0505	0.0025	0.0420	0.1195	0.01
uDreyfus1	0.0019	0.0000	0.0015	0.0040	0.00
uDreyfus2	0.0906	0.0082	0.0725	0.2199	0.00
uGauss1	2.2431	5.0316	1.7377	7.2239	0.03
uGauss2	2.3713	5.6228	1.8674	7.5691	0.00
uGauss3	2.2976	5.2791	1.8303	6.4135	0.01
uNeuroOne	0.2830	0.0801	0.2313	0.5673	0.00

kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

	RMSE	MSE	MAE	WAE	time
mDette	0.5799	0.3363	0.4649	2.6448	0.08
mFriedman	0.0263	0.0007	0.0209	0.0954	0.10
mIshigami	0.7400	0.5476	0.5453	3.2851	0.15
mRef153	3.3913	11.5012	2.4098	13.3731	0.02
uDmod1	0.0470	0.0022	0.0393	0.1293	0.00
uDmod2	0.0649	0.0042	0.0529	0.1392	0.02
uDreyfus1	0.0022	0.0000	0.0019	0.0076	0.00
uDreyfus2	0.1123	0.0126	0.0901	0.2736	0.00
uGauss1	2.3022	5.3003	1.7999	7.5122	0.04
uGauss2	2.6020	6.7705	2.0822	8.0308	0.02
uGauss3	2.8669	8.2190	2.2422	7.9607	0.01
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.00

76 Setup

76.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

76.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

76.3 validann trainPredict arguments - inputs x, y

if(dir.exists("D:/GSoC2020/Results/2020run04/"))
{  
  odir <- "D:/GSoC2020/Results/2020run04/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(validann)
validann.method <- c("Nelder-Mead", "BFGS", "CG", "L-BFGS-B", "SANN")
hyperParams.validann <- function(optim_method, ...) {
    
    if (optim_method == "Nelder-Mead")  {iter <- maxit1storderB} 
    if (optim_method == "BFGS")         {iter <- maxit2ndorder}
    if (optim_method == "CG")           {iter <- maxit1storderA}
    if (optim_method == "L-BFGS-B")     {iter <- maxit2ndorder}
    if (optim_method == "SANN")         {iter <- maxit1storderA}
  
    out <- list(iter = iter)
    return (out)
}
NNtrain.validann <- function(x, y, dataxy, formula, neur, optim_method, hyperParams, NNfullformula, NNparam,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    method <- hyper_params$method
    
    NNreg <- validann::ann(x, y, size = neur, 
                           method = optim_method, maxit = iter)
    
    return (NNreg)
}
NNpredict.validann <- function(object, x, ...)
  predict(object, x)
NNclose.validann <- function()
  if("package:validann" %in% search())
    detach("package:validann", unload=TRUE)
validann.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, validann.method, "NNtrain.validann", "hyperParams.validann", "NNpredict.validann", 
                               NNsummary, "NNclose.validann", NA, validann.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="validann", pkgfun="ann", csvfile=TRUE, rdafile=TRUE, odir=odir)

77 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "validann", pkgfun = "ann", validann.method,
  prepareZZ.arg = validann.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

78 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "Nelder-Mead")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

Nelder-Mead

	RMSE	MSE	MAE	WAE	time
mDette	3.1073	9.6551	2.1835	15.8479	1.86
mFriedman	0.0991	0.0098	0.0725	0.3234	4.99
mIshigami	2.6029	6.7749	2.1288	7.0617	14.94
mRef153	3.9234	15.3927	2.8301	13.7759	20.48
uDmod1	0.1034	0.0107	0.0793	0.2793	21.46
uDmod2	0.0633	0.0040	0.0497	0.1682	23.24
uDreyfus1	0.0833	0.0069	0.0624	0.2237	8.53
uDreyfus2	0.1300	0.0169	0.1019	0.2899	11.77
uGauss1	9.2009	84.6565	7.5305	22.1062	32.30
uGauss2	6.7221	45.1871	4.8292	20.4808	26.07
uGauss3	4.9577	24.5788	4.1348	13.5018	19.13
uNeuroOne	0.3256	0.1060	0.2629	0.6496	3.58

kable(apply(res[,,2,], c(3,1), min), caption = "BFGS")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

BFGS

	RMSE	MSE	MAE	WAE	time
mDette	0.2730	0.0745	0.2113	1.0832	1.69
mFriedman	0.0096	0.0001	0.0076	0.0281	2.37
mIshigami	0.6342	0.4022	0.4793	2.5390	4.89
mRef153	3.2262	10.4086	2.1814	13.0232	0.80
uDmod1	0.0435	0.0019	0.0353	0.1104	0.77
uDmod2	0.0405	0.0016	0.0327	0.0981	0.67
uDreyfus1	0.0022	0.0000	0.0016	0.0058	0.22
uDreyfus2	0.0906	0.0082	0.0724	0.2199	0.33
uGauss1	2.2606	5.1103	1.7484	7.2027	0.81
uGauss2	2.3571	5.5560	1.8540	7.4543	0.78
uGauss3	2.3046	5.3110	1.8455	7.0735	0.51
uNeuroOne	0.2830	0.0801	0.2313	0.5675	0.07

kable(apply(res[,,3,], c(3,1), min), caption = "CG")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

CG

	RMSE	MSE	MAE	WAE	time
mDette	0.3813	0.1454	0.2920	1.5129	10.30
mFriedman	0.0163	0.0003	0.0127	0.0519	22.16
mIshigami	0.6427	0.4131	0.4730	2.6976	53.94
mRef153	3.1175	9.7189	2.1558	13.1011	29.02
uDmod1	0.0506	0.0026	0.0400	0.1204	28.32
uDmod2	0.0536	0.0029	0.0444	0.1235	30.50
uDreyfus1	0.0035	0.0000	0.0028	0.0100	24.73
uDreyfus2	0.0910	0.0083	0.0729	0.2224	26.27
uGauss1	2.3620	5.5792	1.8686	7.1781	41.00
uGauss2	3.5928	12.9084	2.8969	10.7264	33.27
uGauss3	2.4990	6.2450	1.9989	7.0980	36.84
uNeuroOne	0.2830	0.0801	0.2313	0.5675	20.41

kable(apply(res[,,4,], c(3,1), min), caption = "L-BFGS-B")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

L-BFGS-B

	RMSE	MSE	MAE	WAE	time
mDette	0.4455	0.1985	0.3406	1.8421	1.80
mFriedman	0.0211	0.0004	0.0169	0.0668	2.53
mIshigami	0.8502	0.7228	0.6144	3.4423	5.25
mRef153	3.2380	10.4849	2.1936	13.9338	0.98
uDmod1	0.0489	0.0024	0.0389	0.1096	0.85
uDmod2	0.0691	0.0048	0.0554	0.1828	0.76
uDreyfus1	0.0038	0.0000	0.0030	0.0116	0.48
uDreyfus2	0.0907	0.0082	0.0722	0.2149	0.48
uGauss1	2.7065	7.3250	2.1354	7.5003	1.03
uGauss2	3.0632	9.3835	2.4348	9.8076	0.81
uGauss3	2.5172	6.3361	2.0251	6.7721	0.81
uNeuroOne	0.2830	0.0801	0.2313	0.5674	0.13

kable(apply(res[,,5,], c(3,1), min), caption = "SANN")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

SANN

	RMSE	MSE	MAE	WAE	time
mDette	3.3417	11.1668	2.2857	16.2472	0.15
mFriedman	0.1414	0.0200	0.1093	0.5223	0.19
mIshigami	2.9199	8.5256	2.4341	9.4201	0.26
mRef153	5.7592	33.1688	4.2642	15.5036	0.13
uDmod1	0.2296	0.0527	0.1864	0.5463	0.12
uDmod2	0.2274	0.0517	0.1895	0.5007	0.14
uDreyfus1	0.2692	0.0725	0.2108	0.6316	0.12
uDreyfus2	0.2712	0.0736	0.2115	0.6264	0.14
uGauss1	12.6829	160.8548	10.5485	34.6583	0.20
uGauss2	9.9162	98.3301	8.5558	24.4015	0.20
uGauss3	6.9649	48.5102	5.3655	19.1877	0.19
uNeuroOne	0.3084	0.0951	0.2457	0.6330	0.14