1 Environment and datasets

1.1 Setup environment

library(NNbenchmark)
library(kableExtra)
library(dplyr)   
library(stringr) 
options(scipen = 999)
odir <- "D:/GSoC2020/Results/2020run04/"

2 Read csv files and calculate some statistics for the metrics

lf   <- lapply(list.files(odir, pattern = "-results.csv", full.names = TRUE), csv::as.csv)
names(lf) <- names(NNdatasets)
lf <- lf[c(1:4,6,7,10,12)]
gfr <- lapply(lf, function(dfr) cbind(
                      ds   = str_remove(str_extract(dfr$event, "\\w+_"), "_"),
                      pfa  = str_sub(str_remove(dfr$event, str_extract(dfr$event, "\\w+_")),  1, -4),
                      run  = str_sub(dfr$event, -2, -1),
                      dfr[,c("RMSE","MAE","WAE","time")]
                      ))

yfr <- lapply(gfr, function(dfr) {
            as.data.frame(dfr %>%
            group_by(pfa) %>%
            summarise(time.mean = mean(time), 
                      RMSE.min = min(RMSE), 
                      RMSE.med = median(RMSE),
                      RMSE.d51 = median(RMSE) - min(RMSE),
                      MAE.med  = median(MAE),
                      WAE.med  = median(WAE)
                      )
            )})
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
yfr <- lapply(yfr, function(dfr) transform(dfr, npfa = 1:nrow(dfr)))

3 Calculate ranks per datasets and merge results

rankMOFtime <- function(dfr) {
    dfrtime <- dfr[order(dfr$time.mean),]
    dfrRMSE <- dfr[order(dfr$RMSE.min, dfr$time.mean, dfr$RMSE.med),]
    dfrRMSEmed  <- dfr[order(dfr$RMSE.med, dfr$RMSE.min, dfr$time.mean),]
    dfrRMSEd51  <- dfr[order(dfr$RMSE.d51),]
    dfrMAE      <- dfr[order(dfr$MAE.med),]
    dfrWAE      <- dfr[order(dfr$WAE.med),]
    transform(dfr, 
              time.rank = order(dfrtime$npfa),
              RMSE.rank = order(dfrRMSE$npfa),
              RMSEmed.rank  = order(dfrRMSEmed$npfa),
              RMSEd51.rank  = order(dfrRMSEd51$npfa),
              MAE.rank = order(dfrMAE$npfa),
              WAE.rank = order(dfrWAE$npfa)
              )
}
sfr     <- lapply(yfr, rankMOFtime)
sfrwide <- do.call(cbind, sfr)

4 Global scores on combined datasets (final table)

sfr.time   <- sfrwide[, c(grep("time.rank", colnames(sfrwide)))]
time.score <- rank(apply(sfr.time, 1, sum), ties.method = "min")
sfr.RMSE       <- sfrwide[, c(grep("RMSE.rank", colnames(sfrwide)))]
RMSE.score     <- rank(apply(sfr.RMSE, 1, sum), ties.method = "min")
sfr.RMSEmed    <- sfrwide[, c(grep("RMSEmed.rank", colnames(sfrwide)))]
RMSEmed.score  <- rank(apply(sfr.RMSEmed, 1, sum), ties.method = "min")
sfr.RMSEd51    <- sfrwide[, c(grep("RMSEd51.rank", colnames(sfrwide)))]
RMSEd51.score  <- rank(apply(sfr.RMSEd51, 1, sum), ties.method = "min")
sfr.MAE       <- sfrwide[, c(grep("MAE.rank", colnames(sfrwide)))]
MAE.score     <- rank(apply(sfr.MAE, 1, sum), ties.method = "min")
sfr.WAE       <- sfrwide[, c(grep("WAE.rank", colnames(sfrwide)))]
WAE.score     <- rank(apply(sfr.WAE, 1, sum), ties.method = "min")

scoredfr0 <- data.frame(sfr$mDette[,"pfa",drop=FALSE], 
# scoredfr0 <- data.frame(sfr$uNeuroOne[,c("pfa")], 
                        time.score, 
                        RMSE.score, 
                        RMSEmed.score,
                        RMSEd51.score,
              MAE.score,
              WAE.score)

scoredfr <- scoredfr0[order(scoredfr0$RMSE.score),]
rownames(scoredfr) <- NULL

kable(scoredfr)%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
pfa time.score RMSE.score RMSEmed.score RMSEd51.score MAE.score WAE.score
nlsr::nlxb_none 18 1 3 16 3 6
rminer::fit_none 14 2 1 6 1 1
nnet::nnet_none 3 3 2 17 2 3
MachineShop::fit_none 6 4 9 20 10 7
validann::ann_BFGS 35 4 4 11 4 5
traineR::train.nnet_none 4 6 5 15 6 2
radiant.model::nn_none 11 7 8 32 11 9
validann::ann_CG 59 8 6 10 5 4
monmlp::monmlp.fit_BFGS 26 9 10 17 9 10
CaDENCE::cadence.fit_optim 46 10 28 48 23 39
h2o::h2o.deeplearning_first-order 51 11 7 7 8 8
EnsembleBase::Regression.Batch.Fit_none 5 12 15 33 15 15
brnn::brnn_Gauss-Newton 8 13 11 9 12 11
validann::ann_L-BFGS-B 36 14 12 29 13 12
qrnn::qrnn.fit_none 28 15 14 22 7 35
RSNNS::mlp_Rprop 24 16 24 51 25 29
RSNNS::mlp_SCG 30 17 16 22 18 19
automl::automl_train_manual_trainwgrad_adam 50 18 20 35 16 19
deepnet::nn.train_BP 23 18 18 38 24 17
neuralnet::neuralnet_rprop+ 18 20 23 40 21 24
neuralnet::neuralnet_rprop- 21 21 21 41 22 18
caret::avNNet_none 9 22 13 30 14 12
keras::fit_adamax 48 23 16 19 20 16
minpack.lm::nlsLM_none 13 24 18 5 19 14
AMORE::train_ADAPTgdwm 17 25 22 28 17 26
automl::automl_train_manual_trainwgrad_RMSprop 47 26 31 50 29 38
RSNNS::mlp_Std_Backpropagation 22 27 32 30 31 36
ANN2::neuralnetwork_rmsprop 14 28 26 33 27 22
RSNNS::mlp_BackpropChunk 26 29 34 41 32 34
neuralnet::neuralnet_slr 31 30 38 37 38 46
RSNNS::mlp_BackpropMomentum 25 30 35 38 34 30
RSNNS::mlp_BackpropWeightDecay 29 32 30 43 33 32
ANN2::neuralnetwork_adam 16 33 27 26 28 23
AMORE::train_ADAPTgd 10 34 24 8 26 19
keras::fit_adam 42 35 28 44 30 25
keras::fit_adagrad 58 36 42 51 41 33
keras::fit_nadam 44 36 38 58 39 40
neuralnet::neuralnet_sag 41 38 50 59 46 52
AMORE::train_BATCHgdwm 40 39 33 14 36 27
AMORE::train_BATCHgd 39 40 37 24 42 31
ANN2::neuralnetwork_sgd 12 41 36 20 35 28
automl::automl_train_manual_trainwpso 57 42 40 49 40 37
keras::fit_sgd 48 43 44 47 43 43
validann::ann_Nelder-Mead 55 44 43 45 45 41
deepdive::deepnet_adam 32 45 41 1 37 44
monmlp::monmlp.fit_Nelder-Mead 32 46 46 46 43 47
keras::fit_adadelta 60 47 48 27 51 41
neuralnet::neuralnet_backprop 37 48 50 11 48 45
RSNNS::mlp_BackpropBatch 43 49 47 25 50 49
snnR::snnR_none 7 49 48 13 49 47
CaDENCE::cadence.fit_Rprop 56 51 54 60 52 58
keras::fit_rmsprop 37 52 54 57 55 54
deepdive::deepnet_rmsProp 34 53 45 4 47 50
CaDENCE::cadence.fit_psoptim 54 54 56 56 54 56
validann::ann_SANN 20 55 53 53 56 55
deepdive::deepnet_momentum 53 56 52 3 53 51
RSNNS::mlp_Quickprop 45 57 58 36 58 57
deepdive::deepnet_gradientDescent 52 58 57 2 57 53
elmNNRcpp::elm_train_extremeML 1 59 59 55 59 59
ELMR::OSelm_train.formula_extremeML 2 60 60 54 60 60

5 Figures

5.1 Some plots

plot(scoredfr[,c("time.score", "RMSE.score", "RMSEmed.score", "RMSEd51.score")], las = 1)

op <- par(mfrow = c(1,3), las = 1, mar = c(0,0.5,0,0.5), oma = c(2,2,3.5,2), cex = 1.1)
plot(scoredfr[,c("RMSE.score", "RMSEmed.score")]); abline(v=10.5, lty = 2)
mtext("x=RMSE.score,  y=RMSEmed.score", line = 1.5, font = 2)
plot(scoredfr[,c("RMSE.score", "time.score")], yaxt = "n"); abline(v=10.5, lty = 2)
mtext("x=RMSE.score,  y=time.score", line = 1.5, font = 2)
plot(scoredfr[,c("RMSE.score", "RMSEd51.score")], yaxt = "n"); Axis(scoredfr[,5], side = 4)
mtext("x=RMSE.score,  y=RMSEd51.score", line = 1.5, font = 2)

# mtext("(x=RMSE.score, y=RMSEmed.score)    (x=RMSE.score, y=time.score)    (x=RMSE.score, y=RMSEd51.score)", 
      # outer = TRUE, line = 2, font = 2)
par(op)

5.2 Comparison of global scores and scores per dataset

## =====================================
## GLOBAL SCORE APPLIED TO EVERY DATASET
## =====================================
merge_sfr_dfr <- function(x, y) {
    z <- cbind(
            x[,c("npfa","pfa","time.mean","RMSE.min","time.rank","RMSE.rank")], 
            y[,c("time.score","RMSE.score")]
        )
    z[order(z$RMSE.score),]
}
zfr <- lapply(sfr, merge_sfr_dfr, y = scoredfr0)


## =========================
## GRAPHIC RMSEscore_RMSEmin
## =========================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "RMSE.min"]),
     xlab = "RMSE.score", ylab = "RMSE.min", # main = names(zfr)[j], 
     las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "RMSE.min"]),
     labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.score (global)   y=RMSE.min (per dataset)", outer = TRUE, line = 1)

## ==============================
## GRAPHIC RMSEscore_timemean
## ==============================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "time.mean"]),
     xlab = "RMSE.score", ylab = "RMSE.min", # main = names(zfr)[j], 
     las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "time.mean"]),
     labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.score (global)   y=time.mean (per dataset)", outer = TRUE, line = 1)

5.3 By different number of algorithms

## =======================================
## GRAPHIC RMSEmin_timemean - 49 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.min"]), log1p(zfr[[j]][, "time.mean"]),
     xlab = "RMSE.min", ylab = "time.mean", # main = names(zfr)[j], 
     las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.min"]), log1p(zfr[[j]][, "time.mean"]),
     labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset)   y=time.mean (per dataset)    49 algos", outer = TRUE, line = 1)

## =======================================
## GRAPHIC RMSEmin_timemean - 12 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][1:12, "RMSE.min"]), log1p(zfr[[j]][1:12, "time.mean"]),
     xlab = "RMSE.min", ylab = "time.mean", # main = names(zfr)[j], 
     las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][1:12, "RMSE.min"]), log1p(zfr[[j]][1:12, "time.mean"]),
     labels = zfr[[j]][1:12, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset)   y=time.mean (per dataset)    12 algos", outer = TRUE, line = 1)

## =======================================
## GRAPHIC RMSEmin_timemean - 09 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][1:9, "RMSE.min"]), log1p(zfr[[j]][1:9, "time.mean"]),
     xlab = "RMSE.min", ylab = "time.mean", # main = names(zfr)[j], 
     las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][1:9, "RMSE.min"]), log1p(zfr[[j]][1:9, "time.mean"]),
     labels = zfr[[j]][1:9, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset)   y=time.mean (per dataset)    9 algos", outer = TRUE, line = 1)

## THE END
## THE END