NNbenchmark | CaDENCE

library(NNbenchmark)
library(CaDENCE)

library(stringr)
library(dplyr)
library(kableExtra)

0.1 Set Environment

options(scipen = 9999)
options("digits.secs" = 2)
timer  <- createTimer(verbose = FALSE)

0.2 Datasets to Test

NNdataSummary(NNdatasets)

0.3 NN Train Function

hyperParams <- function(optim_method) {
    
    if (!is.element(optim_method, c("nelderMead"))) stop("Invalid Parameters.")

    iter = 150
    params <- paste0("method=", optim_method, "_iter=", iter)
    
    out <- list(iter = iter, params = params)
    
    return (out)
}



NNtrain <- function(x, y, hidden_neur, optim_method) {
    
    hyper_params <- hyperParams(optim_method)
    
    iter <- hyper_params$iter

    NNreg <- CaDENCE::cadence.fit(x = x, y = y, 
                                  iter.max = iter, 
                                  n.hidden = hidden_neur, 
                                  hidden.fcn = tanh, 
                                  method = "optim", 
                                  n.trials = 1, 
                                  trace = 0, 
                                  maxit.Nelder = 1, 
                                  f.cost = cadence.cost)

    
    return (NNreg)
}

0.4 Main Loop

for (dset in names(NNdatasets)) {

    ## =============================================
    ## EXTRACT INFORMATION FROM THE SELECTED DATASET
    ## =============================================
    ds     <- NNdatasets[[dset]]$ds
    Z      <- NNdatasets[[dset]]$Z
    neur   <- NNdatasets[[dset]]$neur
    nparNN <- NNdatasets[[dset]]$nparNN
    fmlaNN <- NNdatasets[[dset]]$fmlaNN
    donotremove  <- c("dset", "dsets", "ds", "Z", "neur", "TF", "nrep", "timer",
                      "donotremove", "donotremove2")
    donotremove2 <- c("dset", "dsets") 

    ## ===================================================
    ## SELECT THE FORMAT REQUIRED BY THE PACKAGE/ALGORITHM
    ## d = data.frame, m = matrix, v = vector/numeric
    ## ATTACH THE OBJECTS CREATED (x, y, Zxy, ... )
    ## ===================================================
    ZZ     <- prepareZZ(Z, xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
    attach(ZZ)

    ## =============================================
    ## SELECT THE PACKAGE USED FOR TRAINING
    ## nrep => SELECT THE NUMBER OF INDEPENDANT RUNS
    ## iter => SELECT THE MAX NUMBER OF ITERATIONS
    ## TF   => PLOT THE RESULTS
    ## =============================================

    
    nrep   <- 10
    TF     <- TRUE 

    method <- c("nelderMead")
        
    for (m in method) {
        
        descr  <- paste(dset, "CaDENCE::cadence.fit", m, sep = "_")

        ## AUTO
        Ypred  <- list()
        Rmse   <- numeric(length = nrep)
        Mae    <- numeric(length = nrep)
    
        for(i in 1:nrep){
            event      <- paste0(descr, sprintf("_%.2d", i))
            timer$start(event)
            #### ADJUST THE FOLLOWING LINES TO THE PACKAGE::ALGORITHM
            
            hyper_params <- hyperParams(optim_method = m)

            NNreg      <- tryCatch(
                            NNtrain(x = x, y = y, hidden_neur = neur, optim_method = m),
                            error = function(y) {lm(y ~ 0, data = Zxy)}
                          )     
            y_pred     <- tryCatch(
                            ym0 + ysd0 * CaDENCE::cadence.predict(x = x, fit = NNreg)[,1],
                            error = ym0
                          )     
            ####
            Ypred[[i]] <- y_pred
            Rmse[i]    <- funRMSE(y_pred, y0)
            Mae[i]     <- funMAE(y_pred, y0)
            timer$stop(event, RMSE = Rmse[i], MAE = Mae[i], params = hyper_params$params, printmsg = FALSE)
        }
        best <- which(Rmse == min(Rmse, na.rm = TRUE))[1]
        best ; Rmse[[best]]
        
        ## ================================================
        ## PLOT ALL MODELS AND THE MODEL WITH THE BEST RMSE
        ## par OPTIONS CAN BE IMPROVED FOR A BETTER DISPLAY
        ## ================================================
        op <- par(mfcol = c(1,2))
        plotNN(xory, y0, uni, TF, main = descr)
        for (i in 1:nrep) lipoNN(xory, Ypred[[i]], uni, TF, col = i, lwd = 1)
        
        plotNN(xory, y0, uni, TF, main = descr)
        lipoNN(xory, Ypred[[best]], uni, TF, col = 4, lwd = 4)
        par(op)
    }


## ===========================
## DETACH ZZ - END OF THE LOOP
## ===========================
    detach(ZZ)
}

## n.hidden = 5 --> 1 * NLL = -856.3929 ; penalty = 0; BIC = -1513.918 ; AICc = -1644.263 ; AIC = -1648.786
## n.hidden = 5 --> 1 * NLL = -324.9208 ; penalty = 0; BIC = -450.974 ; AICc = -581.319 ; AIC = -585.8415
## n.hidden = 5 --> 1 * NLL = -609.7323 ; penalty = 0; BIC = -1020.597 ; AICc = -1150.942 ; AIC = -1155.465
## n.hidden = 5 --> 1 * NLL = -827.291 ; penalty = 0; BIC = -1455.715 ; AICc = -1586.06 ; AIC = -1590.582
## n.hidden = 5 --> 1 * NLL = -259.3357 ; penalty = 0; BIC = -319.804 ; AICc = -450.149 ; AIC = -454.6715
## n.hidden = 5 --> 1 * NLL = -857.2912 ; penalty = 0; BIC = -1515.715 ; AICc = -1646.06 ; AIC = -1650.582
## n.hidden = 5 --> 1 * NLL = -509.7742 ; penalty = 0; BIC = -820.6809 ; AICc = -951.0259 ; AIC = -955.5484
## n.hidden = 5 --> 1 * NLL = -749.5447 ; penalty = 0; BIC = -1300.222 ; AICc = -1430.567 ; AIC = -1435.089
## n.hidden = 5 --> 1 * NLL = -692.6938 ; penalty = 0; BIC = -1186.52 ; AICc = -1316.865 ; AIC = -1321.388
## n.hidden = 5 --> 1 * NLL = -806.7905 ; penalty = 0; BIC = -1414.713 ; AICc = -1545.058 ; AIC = -1549.581

## n.hidden = 5 --> 1 * NLL = 182.6092 ; penalty = 0; BIC = 626.2319 ; AICc = 457.122 ; AIC = 449.2183
## n.hidden = 5 --> 1 * NLL = -497.9753 ; penalty = 0; BIC = -734.9371 ; AICc = -904.0469 ; AIC = -911.9506
## n.hidden = 5 --> 1 * NLL = -328.4908 ; penalty = 0; BIC = -395.968 ; AICc = -565.0778 ; AIC = -572.9816
## n.hidden = 5 --> 1 * NLL = -282.4019 ; penalty = 0; BIC = -303.7903 ; AICc = -472.9001 ; AIC = -480.8038
## n.hidden = 5 --> 1 * NLL = -457.5973 ; penalty = 0; BIC = -654.181 ; AICc = -823.2908 ; AIC = -831.1946
## n.hidden = 5 --> 1 * NLL = -423.7375 ; penalty = 0; BIC = -586.4614 ; AICc = -755.5713 ; AIC = -763.475
## n.hidden = 5 --> 1 * NLL = 185.6175 ; penalty = 0; BIC = 632.2485 ; AICc = 463.1387 ; AIC = 455.235
## n.hidden = 5 --> 1 * NLL = -273.6014 ; penalty = 0; BIC = -286.1893 ; AICc = -455.2991 ; AIC = -463.2028
## n.hidden = 5 --> 1 * NLL = -468.1898 ; penalty = 0; BIC = -675.366 ; AICc = -844.4759 ; AIC = -852.3796
## n.hidden = 5 --> 1 * NLL = -476.6259 ; penalty = 0; BIC = -692.2382 ; AICc = -861.348 ; AIC = -869.2517

## n.hidden = 10 --> 1 * NLL = 434.2217 ; penalty = 0; BIC = 1253.749 ; AICc = 1010.32 ; AIC = 992.4433
## n.hidden = 10 --> 1 * NLL = 219.0179 ; penalty = 0; BIC = 823.3415 ; AICc = 579.9123 ; AIC = 562.0358
## n.hidden = 10 --> 1 * NLL = 51.88832 ; penalty = 0; BIC = 489.0823 ; AICc = 245.6531 ; AIC = 227.7766
## n.hidden = 10 --> 1 * NLL = 376.8114 ; penalty = 0; BIC = 1138.928 ; AICc = 895.4991 ; AIC = 877.6227
## n.hidden = 10 --> 1 * NLL = -95.01161 ; penalty = 0; BIC = 195.2825 ; AICc = -48.1468 ; AIC = -66.02323
## n.hidden = 10 --> 1 * NLL = -292.2636 ; penalty = 0; BIC = -199.2215 ; AICc = -442.6508 ; AIC = -460.5272
## n.hidden = 10 --> 1 * NLL = -79.42298 ; penalty = 0; BIC = 226.4597 ; AICc = -16.96954 ; AIC = -34.84597
## n.hidden = 10 --> 1 * NLL = 511.566 ; penalty = 0; BIC = 1408.438 ; AICc = 1165.008 ; AIC = 1147.132
## n.hidden = 10 --> 1 * NLL = 44.38543 ; penalty = 0; BIC = 474.0766 ; AICc = 230.6473 ; AIC = 212.7709
## n.hidden = 10 --> 1 * NLL = 404.833 ; penalty = 0; BIC = 1194.972 ; AICc = 951.5425 ; AIC = 933.666

## n.hidden = 3 --> 1 * NLL = -120.3741 ; penalty = 0; BIC = -109.9569 ; AICc = -177.6054 ; AIC = -188.7483
## n.hidden = 3 --> 1 * NLL = -105.0919 ; penalty = 0; BIC = -79.39233 ; AICc = -147.0409 ; AIC = -158.1837
## n.hidden = 3 --> 1 * NLL = -110.8481 ; penalty = 0; BIC = -90.90488 ; AICc = -158.5534 ; AIC = -169.6963
## n.hidden = 3 --> 1 * NLL = -129.0288 ; penalty = 0; BIC = -127.2662 ; AICc = -194.9147 ; AIC = -206.0576
## n.hidden = 3 --> 1 * NLL = -126.1454 ; penalty = 0; BIC = -121.4995 ; AICc = -189.148 ; AIC = -200.2908
## n.hidden = 3 --> 1 * NLL = -108.3885 ; penalty = 0; BIC = -85.98568 ; AICc = -153.6342 ; AIC = -164.7771
## n.hidden = 3 --> 1 * NLL = -134.4385 ; penalty = 0; BIC = -138.0857 ; AICc = -205.7342 ; AIC = -216.8771
## n.hidden = 3 --> 1 * NLL = -139.786 ; penalty = 0; BIC = -148.7806 ; AICc = -216.4292 ; AIC = -227.572
## n.hidden = 3 --> 1 * NLL = -115.3122 ; penalty = 0; BIC = -99.8331 ; AICc = -167.4816 ; AIC = -178.6245
## n.hidden = 3 --> 1 * NLL = -130.6806 ; penalty = 0; BIC = -130.5698 ; AICc = -198.2183 ; AIC = -209.3612

## n.hidden = 6 --> 1 * NLL = -41.18925 ; penalty = 0; BIC = 19.84896 ; AICc = 28.12149 ; AIC = -30.37851
## n.hidden = 6 --> 1 * NLL = -51.45951 ; penalty = 0; BIC = -0.6915539 ; AICc = 7.58098 ; AIC = -50.91902
## n.hidden = 6 --> 1 * NLL = -39.61801 ; penalty = 0; BIC = 22.99144 ; AICc = 31.26398 ; AIC = -27.23602
## n.hidden = 6 --> 1 * NLL = -40.15742 ; penalty = 0; BIC = 21.91263 ; AICc = 30.18517 ; AIC = -28.31483
## n.hidden = 6 --> 1 * NLL = -3.334989 ; penalty = 0; BIC = 95.55749 ; AICc = 103.83 ; AIC = 45.33002
## n.hidden = 6 --> 1 * NLL = -23.0294 ; penalty = 0; BIC = 56.16866 ; AICc = 64.4412 ; AIC = 5.941195
## n.hidden = 6 --> 1 * NLL = -4.756855 ; penalty = 0; BIC = 92.71376 ; AICc = 100.9863 ; AIC = 42.48629
## n.hidden = 6 --> 1 * NLL = -5.224493 ; penalty = 0; BIC = 91.77848 ; AICc = 100.051 ; AIC = 41.55101
## n.hidden = 6 --> 1 * NLL = -62.90091 ; penalty = 0; BIC = -23.57435 ; AICc = -15.30181 ; AIC = -73.80181
## n.hidden = 6 --> 1 * NLL = -23.08643 ; penalty = 0; BIC = 56.0546 ; AICc = 64.32713 ; AIC = 5.827131

## n.hidden = 5 --> 1 * NLL = -35.42295 ; penalty = 0; BIC = 15.65426 ; AICc = 9.296956 ; AIC = -26.8459
## n.hidden = 5 --> 1 * NLL = -11.349 ; penalty = 0; BIC = 63.80217 ; AICc = 57.44486 ; AIC = 21.30201
## n.hidden = 5 --> 1 * NLL = -53.46307 ; penalty = 0; BIC = -20.42598 ; AICc = -26.78328 ; AIC = -62.92614
## n.hidden = 5 --> 1 * NLL = -6.08945 ; penalty = 0; BIC = 74.32126 ; AICc = 67.96396 ; AIC = 31.8211
## n.hidden = 5 --> 1 * NLL = -30.25995 ; penalty = 0; BIC = 25.98027 ; AICc = 19.62296 ; AIC = -16.5199
## n.hidden = 5 --> 1 * NLL = 58.95955 ; penalty = 0; BIC = 204.4193 ; AICc = 198.0619 ; AIC = 161.9191
## n.hidden = 5 --> 1 * NLL = -58.79738 ; penalty = 0; BIC = -31.09461 ; AICc = -37.45191 ; AIC = -73.59477
## n.hidden = 5 --> 1 * NLL = -29.65224 ; penalty = 0; BIC = 27.19569 ; AICc = 20.83838 ; AIC = -15.30447
## n.hidden = 5 --> 1 * NLL = -42.82342 ; penalty = 0; BIC = 0.8533156 ; AICc = -5.503991 ; AIC = -41.64685
## n.hidden = 5 --> 1 * NLL = -46.73604 ; penalty = 0; BIC = -6.971921 ; AICc = -13.32923 ; AIC = -49.47209

## n.hidden = 3 --> 1 * NLL = -131.3813 ; penalty = 0; BIC = -207.717 ; AICc = -223.0959 ; AIC = -234.7626
## n.hidden = 3 --> 1 * NLL = -137.9877 ; penalty = 0; BIC = -220.9298 ; AICc = -236.3087 ; AIC = -247.9754
## n.hidden = 3 --> 1 * NLL = -234.9127 ; penalty = 0; BIC = -414.7798 ; AICc = -430.1587 ; AIC = -441.8254
## n.hidden = 3 --> 1 * NLL = -176.4882 ; penalty = 0; BIC = -297.9308 ; AICc = -313.3097 ; AIC = -324.9764
## n.hidden = 3 --> 1 * NLL = -162.4164 ; penalty = 0; BIC = -269.7872 ; AICc = -285.166 ; AIC = -296.8327
## n.hidden = 3 --> 1 * NLL = -157.9817 ; penalty = 0; BIC = -260.9179 ; AICc = -276.2968 ; AIC = -287.9635
## n.hidden = 3 --> 1 * NLL = -165.6873 ; penalty = 0; BIC = -276.3291 ; AICc = -291.708 ; AIC = -303.3747
## n.hidden = 3 --> 1 * NLL = -130.7769 ; penalty = 0; BIC = -206.5083 ; AICc = -221.8872 ; AIC = -233.5538
## n.hidden = 3 --> 1 * NLL = -152.6299 ; penalty = 0; BIC = -250.2142 ; AICc = -265.5931 ; AIC = -277.2597
## n.hidden = 3 --> 1 * NLL = -135.0211 ; penalty = 0; BIC = -214.9966 ; AICc = -230.3755 ; AIC = -242.0422

## n.hidden = 3 --> 1 * NLL = -66.7875 ; penalty = 0; BIC = -78.52945 ; AICc = -93.90834 ; AIC = -105.575
## n.hidden = 3 --> 1 * NLL = -28.31963 ; penalty = 0; BIC = -1.593701 ; AICc = -16.97259 ; AIC = -28.63926
## n.hidden = 3 --> 1 * NLL = -65.86351 ; penalty = 0; BIC = -76.68147 ; AICc = -92.06036 ; AIC = -103.727
## n.hidden = 3 --> 1 * NLL = -63.66865 ; penalty = 0; BIC = -72.29175 ; AICc = -87.67064 ; AIC = -99.3373
## n.hidden = 3 --> 1 * NLL = -56.2599 ; penalty = 0; BIC = -57.47424 ; AICc = -72.85313 ; AIC = -84.5198
## n.hidden = 3 --> 1 * NLL = -44.49044 ; penalty = 0; BIC = -33.93532 ; AICc = -49.31421 ; AIC = -60.98088
## n.hidden = 3 --> 1 * NLL = -32.75037 ; penalty = 0; BIC = -10.45517 ; AICc = -25.83406 ; AIC = -37.50073
## n.hidden = 3 --> 1 * NLL = -58.60883 ; penalty = 0; BIC = -62.17211 ; AICc = -77.551 ; AIC = -89.21767
## n.hidden = 3 --> 1 * NLL = -28.34421 ; penalty = 0; BIC = -1.642865 ; AICc = -17.02176 ; AIC = -28.68842
## n.hidden = 3 --> 1 * NLL = -67.27262 ; penalty = 0; BIC = -79.49968 ; AICc = -94.87857 ; AIC = -106.5452

## n.hidden = 5 --> 1 * NLL = -164.9336 ; penalty = 0; BIC = -208.395 ; AICc = -281.409 ; AIC = -285.8671
## n.hidden = 5 --> 1 * NLL = -251.6222 ; penalty = 0; BIC = -381.7722 ; AICc = -454.7862 ; AIC = -459.2444
## n.hidden = 5 --> 1 * NLL = -341.292 ; penalty = 0; BIC = -561.1119 ; AICc = -634.1259 ; AIC = -638.584
## n.hidden = 5 --> 1 * NLL = -141.1328 ; penalty = 0; BIC = -160.7934 ; AICc = -233.8074 ; AIC = -238.2656
## n.hidden = 5 --> 1 * NLL = -177.2194 ; penalty = 0; BIC = -232.9666 ; AICc = -305.9806 ; AIC = -310.4388
## n.hidden = 5 --> 1 * NLL = -217.744 ; penalty = 0; BIC = -314.0158 ; AICc = -387.0298 ; AIC = -391.488
## n.hidden = 5 --> 1 * NLL = -89.20618 ; penalty = 0; BIC = -56.94021 ; AICc = -129.9542 ; AIC = -134.4124
## n.hidden = 5 --> 1 * NLL = -156.4062 ; penalty = 0; BIC = -191.3402 ; AICc = -264.3542 ; AIC = -268.8124
## n.hidden = 5 --> 1 * NLL = 14.83266 ; penalty = 0; BIC = 151.1375 ; AICc = 78.12348 ; AIC = 73.66533
## n.hidden = 5 --> 1 * NLL = -372.04 ; penalty = 0; BIC = -622.6078 ; AICc = -695.6218 ; AIC = -700.0799

## n.hidden = 4 --> 1 * NLL = -199.2492 ; penalty = 0; BIC = -299.1122 ; AICc = -359.5374 ; AIC = -362.4985
## n.hidden = 4 --> 1 * NLL = -148.805 ; penalty = 0; BIC = -198.2237 ; AICc = -258.6489 ; AIC = -261.61
## n.hidden = 4 --> 1 * NLL = -88.82183 ; penalty = 0; BIC = -78.25736 ; AICc = -138.6826 ; AIC = -141.6437
## n.hidden = 4 --> 1 * NLL = -233.883 ; penalty = 0; BIC = -368.3797 ; AICc = -428.805 ; AIC = -431.766
## n.hidden = 4 --> 1 * NLL = -228.3312 ; penalty = 0; BIC = -357.2761 ; AICc = -417.7013 ; AIC = -420.6624
## n.hidden = 4 --> 1 * NLL = -95.92676 ; penalty = 0; BIC = -92.46723 ; AICc = -152.8925 ; AIC = -155.8535
## n.hidden = 4 --> 1 * NLL = -97.67445 ; penalty = 0; BIC = -95.96261 ; AICc = -156.3879 ; AIC = -159.3489
## n.hidden = 4 --> 1 * NLL = -200.2363 ; penalty = 0; BIC = -301.0862 ; AICc = -361.5115 ; AIC = -364.4725
## n.hidden = 4 --> 1 * NLL = -94.62086 ; penalty = 0; BIC = -89.85542 ; AICc = -150.2807 ; AIC = -153.2417
## n.hidden = 4 --> 1 * NLL = -74.75398 ; penalty = 0; BIC = -50.12167 ; AICc = -110.5469 ; AIC = -113.508

## n.hidden = 4 --> 1 * NLL = -314.924 ; penalty = 0; BIC = -530.4617 ; AICc = -590.8869 ; AIC = -593.848
## n.hidden = 4 --> 1 * NLL = -239.3126 ; penalty = 0; BIC = -379.2388 ; AICc = -439.6641 ; AIC = -442.6251
## n.hidden = 4 --> 1 * NLL = -300.666 ; penalty = 0; BIC = -501.9457 ; AICc = -562.3709 ; AIC = -565.332
## n.hidden = 4 --> 1 * NLL = -203.736 ; penalty = 0; BIC = -308.0857 ; AICc = -368.511 ; AIC = -371.472
## n.hidden = 4 --> 1 * NLL = -298.4393 ; penalty = 0; BIC = -497.4923 ; AICc = -557.9176 ; AIC = -560.8786
## n.hidden = 4 --> 1 * NLL = -77.76468 ; penalty = 0; BIC = -56.14306 ; AICc = -116.5683 ; AIC = -119.5294
## n.hidden = 4 --> 1 * NLL = -358.9597 ; penalty = 0; BIC = -618.5331 ; AICc = -678.9584 ; AIC = -681.9194
## n.hidden = 4 --> 1 * NLL = -303.7173 ; penalty = 0; BIC = -508.0483 ; AICc = -568.4736 ; AIC = -571.4346
## n.hidden = 4 --> 1 * NLL = -284.1073 ; penalty = 0; BIC = -468.8284 ; AICc = -529.2536 ; AIC = -532.2147
## n.hidden = 4 --> 1 * NLL = -352.5419 ; penalty = 0; BIC = -605.6976 ; AICc = -666.1228 ; AIC = -669.0839

## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = 16.31917 ; penalty = 0; BIC = 71.95659 ; AICc = 58.13833 ; AIC = 52.63833
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912
## n.hidden = 2 --> 1 * NLL = -5.321044 ; penalty = 0; BIC = 28.67617 ; AICc = 14.85791 ; AIC = 9.357912

0.5 Results

dfr0 <- getTimer(timer) 

dfr  <- data.frame(
    ds_pkg.fun_algo = stringr::str_sub(dfr0[ ,1], 1, -4),
    run     = stringr::str_sub(dfr0[ ,1], -2, -1),
    dfr0[, c("RMSE","MAE")],
    dataset = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "^\\w*_"), fixed("_"), ""),
    method = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "_\\w*_"), fixed("_"), ""),
    Elapsed = round(dfr0[ ,4], 5),
    params = dfr0$params
)


dfr %>%
    select(-c(dataset, method))

0.6 Best Results

dfr %>%
    group_by(dataset, method) %>%
    summarise(minRMSE = min(RMSE), meanRMSE = mean(RMSE), meanTime = mean(Elapsed)) %>%
    kable() %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive")) %>%
    collapse_rows(columns = 1:2, valign = "middle")

dataset	method	minRMSE	meanRMSE	meanTime
mDette	nelderMead	0.3822	1.35826	7.938
mFriedman		0.0236	0.08978	9.667
mIshigami		1.2023	2.14527	15.955
mRef153		3.3426	4.01390	4.276
uDmod1		0.0474	0.18002	2.926
uDmod2		0.0436	0.17374	2.365
uDreyfus1		0.0627	0.51783	1.314
uDreyfus2		0.1179	0.24284	1.561
uGauss1		2.2864	15.00395	3.004
uGauss2		5.3378	16.44676	2.498
uGauss3		2.3296	5.07050	2.451
uNeuroOne		0.2831	0.31823	0.495

clearNN(donotremove)