NNbenchmark | AutoML

library(NNbenchmark)
library(automl)

library(stringr)
library(dplyr)
library(kableExtra)

0.1 Set Environment

options(scipen = 9999)
options("digits.secs" = 2)
timer  <- createTimer(verbose = FALSE)

0.2 Datasets to Test

NNdataSummary(NNdatasets)

0.3 NN Train Function

hyperParams <- function(optim_method) {
    
    if (!is.element(optim_method, c("trainwgrad", "trainwpso"))) stop("Invalid Parameters.")
    
    hidden_activation = "tanh"
    params <- paste0("method=", optim_method, "_", "hidden_activation=", hidden_activation)
    
    out <- list(hidden_activation = hidden_activation, params = params)
    
    return (out)
}



NNtrain <- function(x, y, hidden_neur, optim_method) {
    
    hyper_params <- hyperParams(optim_method)
    
    hidden_activation <- hyper_params$hidden_activation
    # NNreg <- automl::automl_train(Xref = x, Yref = y)
    
    NNreg <- automl::automl_train_manual(Xref = x, Yref = y,
                                         hpar = list(modexec = optim_method,
                                                     layersshape = c(hidden_neur, 0),
                                                     layersacttype = c(hidden_activation, ""),
                                                     verbose = FALSE,
                                                     seed = as.integer(runif(1)*10000000)))

    return (NNreg)
}

0.4 Main Loop

for (dset in names(NNdatasets)) {

    ## =============================================
    ## EXTRACT INFORMATION FROM THE SELECTED DATASET
    ## =============================================
    ds     <- NNdatasets[[dset]]$ds
    Z      <- NNdatasets[[dset]]$Z
    neur   <- NNdatasets[[dset]]$neur
    nparNN <- NNdatasets[[dset]]$nparNN
    fmlaNN <- NNdatasets[[dset]]$fmlaNN
    donotremove  <- c("dset", "dsets", "ds", "Z", "neur", "TF", "nrep", "timer",
                      "donotremove", "donotremove2")
    donotremove2 <- c("dset", "dsets") 

    ## ===================================================
    ## SELECT THE FORMAT REQUIRED BY THE PACKAGE/ALGORITHM
    ## d = data.frame, m = matrix, v = vector/numeric
    ## ATTACH THE OBJECTS CREATED (x, y, Zxy, ... )
    ## ===================================================
    ZZ     <- prepareZZ(Z, xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
    attach(ZZ)

    ## =============================================
    ## SELECT THE PACKAGE USED FOR TRAINING
    ## nrep => SELECT THE NUMBER OF INDEPENDANT RUNS
    ## iter => SELECT THE MAX NUMBER OF ITERATIONS
    ## TF   => PLOT THE RESULTS
    ## =============================================

    
    nrep   <- 10
    TF     <- TRUE 

    method <- c("trainwgrad", "trainwpso")
        
    for (m in method) {
        
        descr  <- paste(dset, "automl::automl_train_manual", m, sep = "_")

        ## AUTO
        Ypred  <- list()
        Rmse   <- numeric(length = nrep)
        Mae    <- numeric(length = nrep)
    
        for(i in 1:nrep){
            event      <- paste0(descr, sprintf("_%.2d", i))
            timer$start(event)
            #### ADJUST THE FOLLOWING LINES TO THE PACKAGE::ALGORITHM
            
            hyper_params <- hyperParams(optim_method = m)

            NNreg      <- tryCatch(
                            NNtrain(x = x, y = y, hidden_neur = neur, optim_method = m),
                            error = function(y) {lm(y ~ 0, data = Zxy)}
                          )     
            y_pred     <- tryCatch(
                            ym0 + ysd0 * automl::automl_predict(model = NNreg, X = x),
                            error = ym0
                          )     
            ####
            Ypred[[i]] <- y_pred
            Rmse[i]    <- funRMSE(y_pred, y0)
            Mae[i]     <- funMAE(y_pred, y0)
            timer$stop(event, RMSE = Rmse[i], MAE = Mae[i], params = hyper_params$params, printmsg = FALSE)
        }
        best <- which(Rmse == min(Rmse, na.rm = TRUE))[1]
        best ; Rmse[[best]]
        
        ## ================================================
        ## PLOT ALL MODELS AND THE MODEL WITH THE BEST RMSE
        ## par OPTIONS CAN BE IMPROVED FOR A BETTER DISPLAY
        ## ================================================
        op <- par(mfcol = c(1,2))
        plotNN(xory, y0, uni, TF, main = descr)
        for (i in 1:nrep) lipoNN(xory, Ypred[[i]], uni, TF, col = i, lwd = 1)
        
        plotNN(xory, y0, uni, TF, main = descr)
        lipoNN(xory, Ypred[[best]], uni, TF, col = 4, lwd = 4)
        par(op)
    }


## ===========================
## DETACH ZZ - END OF THE LOOP
## ===========================
    detach(ZZ)
}

0.5 Results

dfr0 <- getTimer(timer) 

dfr  <- data.frame(
    ds_pkg.fun_algo = stringr::str_sub(dfr0[ ,1], 1, -4),
    run     = stringr::str_sub(dfr0[ ,1], -2, -1),
    dfr0[, c("RMSE","MAE")],
    dataset = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "^\\w*_"), fixed("_"), ""),
    method = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "_\\w*_"), fixed("_"), ""),
    Elapsed = round(dfr0[ ,4], 5),
    params = dfr0$params
)


dfr %>%
    select(-c(dataset, method))

0.6 Best Results

dfr %>%
    group_by(dataset, method) %>%
    summarise(minRMSE = min(RMSE), meanRMSE = mean(RMSE), meanTime = mean(Elapsed)) %>%
    kable() %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive")) %>%
    collapse_rows(columns = 1:2, valign = "top")

dataset	method	minRMSE	meanRMSE	meanTime
mDette	trainmanualtrainwgrad	2.7462	4.15200	1.493
	trainmanualtrainwpso	4.5662	6.95967	3.120
mFriedman	trainmanualtrainwgrad	0.1409	0.14688	1.504
	trainmanualtrainwpso	0.1210	0.12617	3.252
mIshigami	trainmanualtrainwgrad	2.4558	2.72810	2.206
	trainmanualtrainwpso	2.7465	2.80357	6.759
mRef153	trainmanualtrainwgrad	5.8325	7.90956	0.545
	trainmanualtrainwpso	3.8110	4.95765	2.780
uDmod1	trainmanualtrainwgrad	0.6504	0.79179	0.148
	trainmanualtrainwpso	0.3324	0.34150	2.195
uDmod2	trainmanualtrainwgrad	0.5217	0.76631	0.214
	trainmanualtrainwpso	0.2624	0.26643	2.737
uDreyfus1	trainmanualtrainwgrad	0.6207	0.94795	0.195
	trainmanualtrainwpso	0.3150	0.36901	3.087
uDreyfus2	trainmanualtrainwgrad	0.6997	1.04069	0.260
	trainmanualtrainwpso	0.3320	0.39396	2.606
uGauss1	trainmanualtrainwgrad	41.7126	67.98823	1.161
	trainmanualtrainwpso	17.8156	22.33244	3.931
uGauss2	trainmanualtrainwgrad	29.2349	48.62522	0.911
	trainmanualtrainwpso	13.3829	16.81382	3.023
uGauss3	trainmanualtrainwgrad	32.5774	36.26530	1.095
	trainmanualtrainwpso	14.0474	19.41113	3.894
uNeuroOne	trainmanualtrainwgrad	0.7576	1.65706	0.224
	trainmanualtrainwpso	0.3020	0.52754	3.665

clearNN(donotremove)