NNbenchmark Generic Function with keras

1 Setup

1.1 Packages and options

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)

1.2 Datasets to test

NNdataSummary(NNdatasets)

##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

1.3 keras Train Function - arguments x,y

if(dir.exists("D:/GSoC2020/Results/2020run03/"))
{  
  odir <- "D:/GSoC2020/Results/2020run03/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-  10000
maxit1storderC <- 100000

#library(keras)
keras.method <- c("adadelta", "adagrad", "adam", "adamax","nadam","rmsprop", "sgd")
hyperParams.keras <- function(optim_method, ...) {

    hidden_activation = "tanh"
    iter <- maxit1storderA
    lr <- 0.1
    
    out <- list(hidden_activation = hidden_activation, iter = iter, lr = lr)
    return (out)
}

NNtrain.keras <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
        
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    iter <- hyper_params$iter
    
    early_stop <- callback_early_stopping(monitor = "loss", patience = 20, restore_best_weights = TRUE, mode = "auto", min_delta = 0.0001)
    #should we have a higher min_delta
    
    hidden_activation <- hyper_params$hidden_activation
    lr <- hyper_params$lr
    
    if (optim_method == "adadelta")  { op <- optimizer_adadelta(lr = lr)}
    if (optim_method == "adagrad")   { op <- optimizer_adagrad(lr = lr)}
    if (optim_method == "adam")      { op <- optimizer_adam(lr = lr)}
    if (optim_method == "adamax")    { op <- optimizer_adamax(lr = lr)}
    if (optim_method == "nadam")     { op <- optimizer_nadam(lr = lr)}
    if (optim_method == "rmsprop")   { op <- optimizer_rmsprop(lr = lr)}
    if (optim_method == "sgd")       { op <- optimizer_sgd(lr = lr)}

    model <- keras_model_sequential() %>%
        layer_dense(units = hidden_neur, activation = hidden_activation, input_shape = ncol(x)) %>%
        layer_dense(units = 1)

    model %>% compile(
        loss = "mse",
        optimizer = op,
        metrics = list("mean_absolute_error")
    )

    historylog <- model %>% fit(x, y, epochs = iter, verbose = 0, callbacks = list(early_stop))

    return (model)
}  

NNpredict.keras <- function(object, x, ...)
{
  object %>% predict(x)
}
NNclose.keras <- function()
{
  if("package:keras" %in% search())
    detach("package:keras", unload=TRUE)
}

keras.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, keras.method, "NNtrain.keras", "hyperParams.keras", "NNpredict.keras", 
                               NNsummary, "NNclose.keras", NA, keras.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="keras", pkgfun="fit", rdafile=TRUE, odir=odir, echoreport=2)

2 Launch package’s trainPredict

res <- trainPredict_1pkg(1:12, pkgname = "keras", pkgfun = "fit", keras.method,
  prepareZZ.arg = keras.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

3 Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "adadelta")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adadelta

	RMSE	MSE	MAE	WAE	time
mDette	4.0883	16.7143	2.9191	20.3976	33.49
mFriedman	0.0953	0.0091	0.0720	0.3557	32.90
mIshigami	2.7829	7.7445	2.3563	8.7551	33.02
mRef153	5.1377	26.3958	4.0651	13.7149	13.38
uDmod1	0.5258	0.2764	0.4592	0.8824	7.60
uDmod2	0.4134	0.1709	0.3580	0.7153	7.90
uDreyfus1	0.5705	0.3255	0.4627	1.1187	7.64
uDreyfus2	0.5800	0.3365	0.4587	1.2584	7.97
uGauss1	27.3533	748.2023	23.2934	59.7053	2.90
uGauss2	16.9919	288.7261	13.6212	38.6946	17.89
uGauss3	16.9396	286.9488	14.3125	35.0210	17.76
uNeuroOne	0.8810	0.7762	0.7120	2.1293	7.45

kable(apply(res[,,2,], c(3,1), min), caption = "adagrad")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adagrad

	RMSE	MSE	MAE	WAE	time
mDette	0.6392	0.4085	0.4578	3.3321	8.39
mFriedman	0.0285	0.0008	0.0223	0.0854	8.27
mIshigami	2.4522	6.0132	2.0422	6.6262	6.70
mRef153	3.5486	12.5926	2.6155	13.4431	2.25
uDmod1	0.2390	0.0571	0.1868	0.5690	3.02
uDmod2	0.1926	0.0371	0.1512	0.4451	2.55
uDreyfus1	0.2684	0.0720	0.2086	0.6405	1.38
uDreyfus2	0.3003	0.0902	0.2280	0.8005	1.21
uGauss1	6.3733	40.6193	5.1701	15.8331	17.61
uGauss2	7.0160	49.2248	5.1667	22.0921	9.96
uGauss3	4.0025	16.0201	3.1409	11.6127	8.06
uNeuroOne	0.2887	0.0833	0.2372	0.5450	5.27

kable(apply(res[,,3,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adam

	RMSE	MSE	MAE	WAE	time
mDette	0.8331	0.6940	0.6483	3.4312	1.70
mFriedman	0.0499	0.0025	0.0390	0.1402	1.73
mIshigami	1.0199	1.0402	0.7508	3.6539	1.46
mRef153	3.8065	14.4893	2.8286	14.2538	0.92
uDmod1	0.1482	0.0220	0.1119	0.4010	1.39
uDmod2	0.0850	0.0072	0.0682	0.2042	1.26
uDreyfus1	0.0494	0.0024	0.0347	0.1311	0.96
uDreyfus2	0.1105	0.0122	0.0873	0.2910	0.73
uGauss1	2.5111	6.3054	1.9631	7.8498	2.77
uGauss2	3.7745	14.2466	2.8786	13.6929	2.14
uGauss3	2.7447	7.5332	2.1863	7.6121	1.47
uNeuroOne	0.2855	0.0815	0.2307	0.5330	0.96

kable(apply(res[,,4,], c(3,1), min), caption = "adamax")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

adamax

	RMSE	MSE	MAE	WAE	time
mDette	0.5285	0.2793	0.4036	1.5864	2.59
mFriedman	0.0357	0.0013	0.0278	0.1267	2.38
mIshigami	0.8875	0.7876	0.6515	3.2132	3.17
mRef153	3.4778	12.0949	2.4227	13.1939	1.36
uDmod1	0.1524	0.0232	0.1224	0.3930	2.33
uDmod2	0.0920	0.0085	0.0754	0.2154	1.66
uDreyfus1	0.0375	0.0014	0.0295	0.0850	1.14
uDreyfus2	0.1042	0.0109	0.0846	0.2642	1.95
uGauss1	2.5606	6.5568	2.0345	7.0122	5.40
uGauss2	3.6299	13.1761	2.7489	12.3500	4.28
uGauss3	2.6809	7.1871	2.1236	8.1841	3.43
uNeuroOne	0.2845	0.0809	0.2317	0.5458	1.59

kable(apply(res[,,5,], c(3,1), min), caption = "nadam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

nadam

	RMSE	MSE	MAE	WAE	time
mDette	0.9627	0.9268	0.7373	4.1360	2.17
mFriedman	0.0787	0.0062	0.0624	0.2179	1.82
mIshigami	1.0523	1.1073	0.7605	3.8427	1.66
mRef153	4.1931	17.5817	3.0333	14.8135	1.13
uDmod1	0.1663	0.0277	0.1366	0.3997	1.45
uDmod2	0.1361	0.0185	0.1113	0.3208	1.28
uDreyfus1	0.1025	0.0105	0.0782	0.2520	0.85
uDreyfus2	0.1446	0.0209	0.1157	0.3421	1.08
uGauss1	3.8080	14.5010	2.8566	12.9275	2.42
uGauss2	3.9635	15.7090	3.0202	12.9083	2.14
uGauss3	2.7128	7.3591	2.1659	8.5573	1.70
uNeuroOne	0.2846	0.0810	0.2290	0.5112	1.12

kable(apply(res[,,6,], c(3,1), min), caption = "rmsprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

rmsprop

	RMSE	MSE	MAE	WAE	time
mDette	2.7613	7.6250	2.0790	11.1316	1.51
mFriedman	0.0910	0.0083	0.0724	0.3078	1.35
mIshigami	2.3632	5.5849	1.9418	6.0876	1.48
mRef153	4.9479	24.4817	3.5765	14.6881	0.92
uDmod1	0.2075	0.0431	0.1598	0.5485	0.85
uDmod2	0.1573	0.0247	0.1215	0.4326	0.70
uDreyfus1	0.2153	0.0464	0.1585	0.5660	0.69
uDreyfus2	0.1978	0.0391	0.1476	0.5868	0.88
uGauss1	5.4917	30.1585	4.2049	15.1014	1.21
uGauss2	8.0779	65.2517	6.6360	18.9556	1.33
uGauss3	4.5770	20.9492	3.5824	14.0601	1.34
uNeuroOne	0.3219	0.1036	0.2675	0.6755	0.89

kable(apply(res[,,7,], c(3,1), min), caption = "sgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))

sgd

	RMSE	MSE	MAE	WAE	time
mDette	0.6211	0.3858	0.4626	2.6528	3.58
mFriedman	0.0384	0.0015	0.0298	0.0991	4.19
mIshigami	2.5351	6.4269	2.0951	6.8377	2.44
mRef153	3.7398	13.9864	2.6984	13.0575	1.73
uDmod1	0.2111	0.0446	0.1671	0.5061	1.25
uDmod2	0.2732	0.0746	0.2348	0.6078	1.44
uDreyfus1	0.2287	0.0523	0.1674	0.6017	1.16
uDreyfus2	0.2571	0.0661	0.1948	0.7867	1.24
uGauss1	3.8072	14.4944	2.9389	11.1864	2.80
uGauss2	6.7221	45.1870	5.2147	18.4245	2.73
uGauss3	4.5966	21.1285	3.6147	12.8795	4.35
uNeuroOne	0.2900	0.0841	0.2373	0.5508	3.00