NNbenchmark | RSNNS

0.1 Set Environment

options(scipen = 9999)
options("digits.secs" = 2)
timer  <- createTimer(verbose = FALSE)

0.2 Datasets to Test

NNdataSummary(NNdatasets)

0.3 NN Train Function

hyperParams <- function(optim_method) {
    
    if (!is.element(optim_method, c("Std_Backpropagation", "BackpropBatch", "BackpropChunk", "BackpropMomentum", "BackpropWeightDecay", "Rprop", "Quickprop", "SCG"))) stop("Invalid Parameters.")
    
    iter <- 5000
    
    params <- paste0("method=", optim_method, "_iter=", iter)
    
    out <- list(iter = iter, params = params)
    
    return (out)
}



NNtrain <- function(x, y, optim_method) {
    
    hyper_params <- hyperParams(optim_method)
    
    iter <- hyper_params$iter

    NNreg <- RSNNS::mlp(x, y, size = neur, 
                        learnFunc = optim_method, 
                        maxit = iter, 
                        linOut = T)
    
    return(NNreg)
}

0.4 Main Loop

for (dset in names(NNdatasets)) {

    ## =============================================
    ## EXTRACT INFORMATION FROM THE SELECTED DATASET
    ## =============================================
    ds     <- NNdatasets[[dset]]$ds
    Z      <- NNdatasets[[dset]]$Z
    neur   <- NNdatasets[[dset]]$neur
    nparNN <- NNdatasets[[dset]]$nparNN
    fmlaNN <- NNdatasets[[dset]]$fmlaNN
    donotremove  <- c("dset", "dsets", "ds", "Z", "neur", "TF", "nrep", "timer",
                      "donotremove", "donotremove2")
    donotremove2 <- c("dset", "dsets") 

    ## ===================================================
    ## SELECT THE FORMAT REQUIRED BY THE PACKAGE/ALGORITHM
    ## d = data.frame, m = matrix, v = vector/numeric
    ## ATTACH THE OBJECTS CREATED (x, y, Zxy, ... )
    ## ===================================================
    ZZ     <- prepareZZ(Z, xdmv = "m", ydmv = "m", scale = TRUE)
    attach(ZZ)

    ## =============================================
    ## SELECT THE PACKAGE USED FOR TRAINING
    ## nrep => SELECT THE NUMBER OF INDEPENDANT RUNS
    ## iter => SELECT THE MAX NUMBER OF ITERATIONS
    ## TF   => PLOT THE RESULTS
    ## =============================================

    
    nrep   <- 10
    TF     <- TRUE 

    method <- c("Std_Backpropagation", "BackpropBatch", "BackpropChunk", "BackpropMomentum", "BackpropWeightDecay", "Rprop", "Quickprop", "SCG")
        
    for (m in method) {
        descr  <- paste(dset, "RSNNS::mlp", m, sep = "_")

        ##AUTO
        Ypred  <- list()
        Rmse   <- numeric(length = nrep)
        Mae    <- numeric(length = nrep)
    
        for(i in 1:nrep){
            event      <- paste0(descr, sprintf("_%.2d", i))
            timer$start(event)
            #### ADJUST THE FOLLOWING LINES TO THE PACKAGE::ALGORITHM
            
            hyper_params <- hyperParams(optim_method = m)

            NNreg      <- tryCatch(  
                            NNtrain(x = x, y = y, optim_method = m),
                            error = function(y) {lm(y ~ 0, data = Zxy)}
                          )    
            y_pred     <- tryCatch(
                            ym0 + ysd0*fitted(NNreg),
                            error = function(NNreg) rep(ym0, nrow(Zxy))
                          )    
            ####
            Ypred[[i]] <- y_pred
            Rmse[i]    <- funRMSE(y_pred, y0)
            Mae[i]     <- funMAE(y_pred, y0)
            timer$stop(event, RMSE = Rmse[i], MAE = Mae[i], params = hyper_params$params, printmsg = FALSE)
            
        }
        best <- which(Rmse == min(Rmse, na.rm = TRUE))[1]
        best ; Rmse[[best]]
        
        ## ================================================
        ## PLOT ALL MODELS AND THE MODEL WITH THE BEST RMSE
        ## par OPTIONS CAN BE IMPROVED FOR A BETTER DISPLAY
        ## ================================================
        op <- par(mfcol = c(1,2))
        plotNN(xory, y0, uni, TF, main = descr)
        for (i in 1:nrep){lipoNN(xory, Ypred[[i]], uni, TF, col = i, lwd = 1)}
        
        plotNN(xory, y0, uni, TF, main = descr)
        lipoNN(xory, Ypred[[best]], uni, TF, col = 4, lwd = 4)
        par(op)
    }


## ===========================
## DETACH ZZ - END OF THE LOOP
## ===========================
    detach(ZZ)
}

0.5 Results

dfr0 <- getTimer(timer) 

dfr  <- data.frame(
    ds_pkg.fun_algo = stringr::str_sub(dfr0[ ,1], 1, -4),
    run     = stringr::str_sub(dfr0[ ,1], -2, -1),
    dfr0[, c("RMSE","MAE")],
    dataset = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "^\\w*_"), fixed("_"), ""),
    method = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "_\\w*_"), fixed("_"), ""),
    Elapsed = round(dfr0[ ,4], 5),
    params = dfr0$params
)


dfr %>%
    select(-c(dataset, method))

0.6 Best Results

dfr %>%
    group_by(dataset, method) %>%
    summarise(minRMSE = min(RMSE), meanRMSE = mean(RMSE), meanTime = mean(Elapsed)) %>%
    kable() %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive")) %>%
    collapse_rows(columns = 1:2, valign = "top")

dataset	method	minRMSE	meanRMSE	meanTime
mDette	BackpropBatch	1.5126	2.09243	3.120
	BackpropChunk	0.6166	0.69300	3.173
	BackpropMomentum	0.6073	0.75832	3.104
	BackpropWeightDecay	0.5860	0.73905	3.161
	Quickprop	6.8886	7.37864	3.473
	Rprop	0.4873	0.74563	3.061
	SCG	0.3391	0.97305	5.108
	StdBackpropagation	0.7011	0.91878	3.094
mFriedman	BackpropBatch	0.0337	0.07166	3.187
	BackpropChunk	0.0573	0.07453	3.288
	BackpropMomentum	0.0412	0.07752	3.193
	BackpropWeightDecay	0.0412	0.07646	3.293
	Quickprop	0.0000	0.18626	3.484
	Rprop	0.0177	0.04735	3.194
	SCG	0.0136	0.02925	5.220
	StdBackpropagation	0.0444	0.07383	3.196
mIshigami	BackpropBatch	2.6228	2.65887	4.023
	BackpropChunk	2.6162	3.05936	3.904
	BackpropMomentum	1.7447	2.91546	3.788
	BackpropWeightDecay	2.6404	3.00806	3.911
	Quickprop	3.5499	3.61001	4.599
	Rprop	0.7143	2.14441	3.998
	SCG	0.6344	0.80955	6.466
	StdBackpropagation	2.6211	2.93411	3.730
mRef153	BackpropBatch	3.5928	3.79431	0.949
	BackpropChunk	3.4016	3.69395	0.964
	BackpropMomentum	3.4042	3.77054	0.945
	BackpropWeightDecay	3.4148	3.72293	0.966
	Quickprop	6.9868	10.13821	0.970
	Rprop	3.2768	3.58972	0.942
	SCG	3.2296	3.41122	0.827
	StdBackpropagation	3.3410	3.71050	0.941
uDmod1	BackpropBatch	0.3299	0.33860	0.422
	BackpropChunk	0.0510	0.10688	0.422
	BackpropMomentum	0.0556	0.09163	0.409
	BackpropWeightDecay	0.0763	0.11899	0.414
	Quickprop	0.5882	0.58839	0.436
	Rprop	0.0479	0.14804	0.411
	SCG	0.0447	0.05294	0.600
	StdBackpropagation	0.0516	0.10775	0.413
uDmod2	BackpropBatch	0.1857	0.25680	0.401
	BackpropChunk	0.0486	0.07066	0.401
	BackpropMomentum	0.0670	0.07114	0.395
	BackpropWeightDecay	0.0502	0.06994	0.402
	Quickprop	0.1172	0.43004	0.423
	Rprop	0.0461	0.09443	0.399
	SCG	0.0433	0.10375	0.479
	StdBackpropagation	0.0542	0.07053	0.398
uDreyfus1	BackpropBatch	0.3017	0.33485	0.371
	BackpropChunk	0.0662	0.07959	0.374
	BackpropMomentum	0.0736	0.09164	0.366
	BackpropWeightDecay	0.0670	0.08700	0.376
	Quickprop	0.1861	0.23825	0.373
	Rprop	0.0113	0.06862	0.367
	SCG	0.0756	0.13553	0.502
	StdBackpropagation	0.0717	0.08664	0.372
uDreyfus2	BackpropBatch	0.3350	0.34533	0.373
	BackpropChunk	0.1142	0.12360	0.379
	BackpropMomentum	0.1138	0.13053	0.368
	BackpropWeightDecay	0.1130	0.13162	0.372
	Quickprop	0.1920	0.37154	0.381
	Rprop	0.1119	0.14096	0.369
	SCG	0.1141	0.13927	0.484
	StdBackpropagation	0.1132	0.12932	0.372
uGauss1	BackpropBatch	14.5956	18.22645	1.623
	BackpropChunk	2.3584	5.61152	1.607
	BackpropMomentum	2.4135	2.59344	1.579
	BackpropWeightDecay	2.3888	2.71807	1.592
	Quickprop	25.4882	31.06230	1.795
	Rprop	2.3136	6.58644	1.643
	SCG	2.3157	2.90334	2.680
	StdBackpropagation	2.3165	5.05888	1.570
uGauss2	BackpropBatch	10.9662	14.25282	1.531
	BackpropChunk	4.1966	5.73886	1.546
	BackpropMomentum	3.9252	5.39521	1.519
	BackpropWeightDecay	3.3878	6.04545	1.531
	Quickprop	24.1241	26.63210	1.661
	Rprop	4.1139	9.16546	1.512
	SCG	3.7046	5.43156	2.146
	StdBackpropagation	4.2255	6.41723	1.511
uGauss3	BackpropBatch	5.7185	9.68809	1.531
	BackpropChunk	2.8344	4.54416	1.546
	BackpropMomentum	2.6186	4.46481	1.519
	BackpropWeightDecay	2.8215	3.93162	1.540
	Quickprop	24.6372	28.40334	1.655
	Rprop	2.3210	8.89114	1.513
	SCG	2.9323	3.56621	2.443
	StdBackpropagation	2.6051	4.32769	1.513
uNeuroOne	BackpropBatch	0.2874	0.49200	0.365
	BackpropChunk	0.2861	0.57206	0.360
	BackpropMomentum	0.2842	0.50781	0.361
	BackpropWeightDecay	0.2866	0.54377	0.358
	Quickprop	0.5011	0.63597	0.360
	Rprop	0.2830	0.48436	0.364
	SCG	0.2837	0.42104	0.491
	StdBackpropagation	0.2878	0.53696	0.361

clearNN(donotremove)

NNbenchmark | RSNNS

Salsabila Mahdi

0.1 Set Environment

0.2 Datasets to Test

0.3 NN Train Function

0.4 Main Loop

0.5 Results

0.6 Best Results