NNbenchmark Ranking Function (univariate datasets)
All members of GSoC 2019 & 2020
2020/08/09
1 Setup
1.1 Packages and options
library(NNbenchmark)
library(kableExtra)
library(dplyr)
library(stringr)
options(scipen = 999)
odir <- "D:/GSoC2020/Results/2020run04/"2 Read csv files and calculate some statistics for the metrics
lf <- lapply(list.files(odir, pattern = "-results.csv", full.names = TRUE), csv::as.csv)
names(lf) <- names(NNdatasets)
lf <- lf[c(5:12)] #selecting datasets
ht(lf)## $uDmod1
## event RMSE MSE MAE WAE time
## 1 uDmod1_AMORE::train_ADAPTgd_01 0.3275 0.1072 0.2829 0.7309 0.03
## 2 uDmod1_AMORE::train_ADAPTgd_02 0.3346 0.1120 0.2893 0.7489 0.04
## 3 uDmod1_AMORE::train_ADAPTgd_03 0.3271 0.1070 0.2830 0.7263 0.03
## 313 uDmod1_yager::grnn.fit_none*_03 0.1503 0.0226 0.1139 0.4593 0.01
## 314 uDmod1_yager::grnn.fit_none*_04 0.1503 0.0226 0.1139 0.4593 0.02
## 315 uDmod1_yager::grnn.fit_none*_05 0.1503 0.0226 0.1139 0.4593 0.01
##
## $uDmod2
## event RMSE MSE MAE WAE time
## 1 uDmod2_AMORE::train_ADAPTgd_01 0.2632 0.0693 0.2333 0.5307 0.04
## 2 uDmod2_AMORE::train_ADAPTgd_02 0.2653 0.0704 0.2342 0.5504 0.02
## 3 uDmod2_AMORE::train_ADAPTgd_03 0.2651 0.0703 0.2344 0.5405 0.01
## 313 uDmod2_yager::grnn.fit_none*_03 0.1091 0.0119 0.0874 0.2767 0.02
## 314 uDmod2_yager::grnn.fit_none*_04 0.1091 0.0119 0.0874 0.2767 0.00
## 315 uDmod2_yager::grnn.fit_none*_05 0.1091 0.0119 0.0874 0.2767 0.02
##
## $uGauss3
## event RMSE MSE MAE WAE time
## 1 uGauss3_AMORE::train_ADAPTgd_01 4.7958 22.9994 3.8766 12.5557 0.04
## 2 uGauss3_AMORE::train_ADAPTgd_02 4.9461 24.4637 4.0536 12.8336 0.05
## 3 uGauss3_AMORE::train_ADAPTgd_03 4.8918 23.9301 4.0048 12.5672 0.06
## 313 uGauss3_yager::grnn.fit_none*_03 5.1689 26.7180 4.0100 15.5364 0.21
## 314 uGauss3_yager::grnn.fit_none*_04 5.1689 26.7180 4.0100 15.5364 0.22
## 315 uGauss3_yager::grnn.fit_none*_05 5.1689 26.7180 4.0100 15.5364 0.22
##
## $uNeuroOne
## event RMSE MSE MAE WAE time
## 1 uNeuroOne_AMORE::train_ADAPTgd_01 0.2965 0.0879 0.2451 0.6432 0.01
## 2 uNeuroOne_AMORE::train_ADAPTgd_02 0.2965 0.0879 0.2450 0.6433 0.03
## 3 uNeuroOne_AMORE::train_ADAPTgd_03 0.2965 0.0879 0.2451 0.6434 0.01
## 313 uNeuroOne_yager::grnn.fit_none*_03 0.3110 0.0967 0.2456 0.7642 0.00
## 314 uNeuroOne_yager::grnn.fit_none*_04 0.3110 0.0967 0.2456 0.7642 0.02
## 315 uNeuroOne_yager::grnn.fit_none*_05 0.3110 0.0967 0.2456 0.7642 0.01gfr <- lapply(lf, function(dfr) cbind(
ds = str_remove(str_extract(dfr$event, "\\w+_"), "_"),
pfa = str_sub(str_remove(dfr$event, str_extract(dfr$event, "\\w+_")), 1, -4),
run = str_sub(dfr$event, -2, -1),
dfr[,c("RMSE","MAE","WAE","time")]
))
yfr <- lapply(gfr, function(dfr) {
as.data.frame(dfr %>%
group_by(pfa) %>%
summarise(time.mean = mean(time),
RMSE.min = min(RMSE),
RMSE.med = median(RMSE),
RMSE.d51 = median(RMSE) - min(RMSE),
MAE.med = median(MAE),
WAE.med = median(WAE)
)
)})## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)yfr <- lapply(yfr, function(dfr) transform(dfr, npfa = 1:nrow(dfr)))
ht9(yfr)## $uDmod1
## pfa time.mean RMSE.min RMSE.med RMSE.d51 MAE.med
## 1 AMORE::train_ADAPTgd 0.036 0.3082 0.3271 0.0189 0.2829
## 2 AMORE::train_ADAPTgdwm 0.054 0.2197 0.2765 0.0568 0.2204
## 3 AMORE::train_BATCHgd 1.780 0.2023 0.2922 0.0899 0.2393
## 61 validann::ann_Nelder-Mead 28.208 0.1034 0.1810 0.0776 0.1538
## 62 validann::ann_SANN 0.128 0.2296 0.3046 0.0750 0.2441
## 63 yager::grnn.fit_none* 0.012 0.1503 0.1503 0.0000 0.1139
## WAE.med npfa
## 1 0.7263 1
## 2 0.6575 2
## 3 0.6802 3
## 61 0.4017 61
## 62 0.6614 62
## 63 0.4593 63
##
## $uDmod2
## pfa time.mean RMSE.min RMSE.med RMSE.d51 MAE.med
## 1 AMORE::train_ADAPTgd 0.022 0.2579 0.2632 0.0053 0.2333
## 2 AMORE::train_ADAPTgdwm 0.034 0.1145 0.1924 0.0779 0.1573
## 3 AMORE::train_BATCHgd 1.650 0.2228 0.2644 0.0416 0.2347
## 61 validann::ann_Nelder-Mead 30.686 0.0633 0.1987 0.1354 0.1673
## 62 validann::ann_SANN 0.148 0.2274 0.2546 0.0272 0.1990
## 63 yager::grnn.fit_none* 0.008 0.1091 0.1091 0.0000 0.0874
## WAE.med npfa
## 1 0.5307 1
## 2 0.4195 2
## 3 0.4989 3
## 61 0.4305 61
## 62 0.5099 62
## 63 0.2767 63
##
## $uGauss3
## pfa time.mean RMSE.min RMSE.med RMSE.d51 MAE.med
## 1 AMORE::train_ADAPTgd 0.046 4.7958 4.8043 0.0085 3.9113
## 2 AMORE::train_ADAPTgdwm 0.080 4.4658 5.1079 0.6421 3.6709
## 3 AMORE::train_BATCHgd 1.556 5.0863 5.2682 0.1819 4.1337
## 61 validann::ann_Nelder-Mead 30.822 4.9577 5.3229 0.3652 4.3142
## 62 validann::ann_SANN 0.210 6.9649 10.8474 3.8825 8.3651
## 63 yager::grnn.fit_none* 0.214 5.1689 5.1689 0.0000 4.0100
## WAE.med npfa
## 1 12.5672 1
## 2 20.0320 2
## 3 14.8772 3
## 61 15.0154 61
## 62 26.5278 62
## 63 15.5364 63
##
## $uNeuroOne
## pfa time.mean RMSE.min RMSE.med RMSE.d51 MAE.med
## 1 AMORE::train_ADAPTgd 0.020 0.2958 0.2965 0.0007 0.2451
## 2 AMORE::train_ADAPTgdwm 0.028 0.2854 0.2854 0.0000 0.2285
## 3 AMORE::train_BATCHgd 1.232 0.2931 0.2935 0.0004 0.2421
## 61 validann::ann_Nelder-Mead 9.006 0.3256 0.3341 0.0085 0.2793
## 62 validann::ann_SANN 0.168 0.3084 0.3344 0.0260 0.2773
## 63 yager::grnn.fit_none* 0.012 0.3110 0.3110 0.0000 0.2456
## WAE.med npfa
## 1 0.6433 1
## 2 0.6436 2
## 3 0.6309 3
## 61 0.8397 61
## 62 0.6937 62
## 63 0.7642 633 Calculate ranks per datasets and merge results
rankMOFtime <- function(dfr) {
dfrtime <- dfr[order(dfr$time.mean),]
dfrRMSE <- dfr[order(dfr$RMSE.min, dfr$time.mean, dfr$RMSE.med),]
dfrRMSEmed <- dfr[order(dfr$RMSE.med, dfr$RMSE.min, dfr$time.mean),]
dfrRMSEd51 <- dfr[order(dfr$RMSE.d51),]
dfrMAE <- dfr[order(dfr$MAE.med),]
dfrWAE <- dfr[order(dfr$WAE.med),]
transform(dfr,
time.rank = order(dfrtime$npfa),
RMSE.rank = order(dfrRMSE$npfa),
RMSEmed.rank = order(dfrRMSEmed$npfa),
RMSEd51.rank = order(dfrRMSEd51$npfa),
MAE.rank = order(dfrMAE$npfa),
WAE.rank = order(dfrWAE$npfa)
)
}
sfr <- lapply(yfr, rankMOFtime)
sfrwide <- do.call(cbind, sfr)4 Global scores on combined datasets (final table)
sfr.time <- sfrwide[, c(grep("time.rank", colnames(sfrwide)))]
time.score <- rank(apply(sfr.time, 1, sum), ties.method = "min")
sfr.RMSE <- sfrwide[, c(grep("RMSE.rank", colnames(sfrwide)))]
RMSE.score <- rank(apply(sfr.RMSE, 1, sum), ties.method = "min")
sfr.RMSEmed <- sfrwide[, c(grep("RMSEmed.rank", colnames(sfrwide)))]
RMSEmed.score <- rank(apply(sfr.RMSEmed, 1, sum), ties.method = "min")
sfr.RMSEd51 <- sfrwide[, c(grep("RMSEd51.rank", colnames(sfrwide)))]
RMSEd51.score <- rank(apply(sfr.RMSEd51, 1, sum), ties.method = "min")
sfr.MAE <- sfrwide[, c(grep("MAE.rank", colnames(sfrwide)))]
MAE.score <- rank(apply(sfr.MAE, 1, sum), ties.method = "min")
sfr.WAE <- sfrwide[, c(grep("WAE.rank", colnames(sfrwide)))]
WAE.score <- rank(apply(sfr.WAE, 1, sum), ties.method = "min")
scoredfr0 <- data.frame(sfr$uNeuroOne[,"pfa",drop=FALSE],
# scoredfr0 <- data.frame(sfr$uNeuroOne[,c("pfa")],
time.score,
RMSE.score,
RMSEmed.score,
RMSEd51.score,
MAE.score,
WAE.score)
scoredfr <- scoredfr0[order(scoredfr0$RMSE.score),]
rownames(scoredfr) <- NULL
kable(scoredfr)%>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed"))| pfa | time.score | RMSE.score | RMSEmed.score | RMSEd51.score | MAE.score | WAE.score |
|---|---|---|---|---|---|---|
| nlsr::nlxb_none | 21 | 1 | 2 | 10 | 1 | 2 |
| traineR::train.nnet_none | 4 | 2 | 5 | 13 | 5 | 4 |
| rminer::fit_none | 15 | 3 | 1 | 8 | 2 | 1 |
| validann::ann_BFGS | 35 | 4 | 4 | 12 | 4 | 3 |
| MachineShop::fit_none | 7 | 5 | 7 | 14 | 6 | 7 |
| nnet::nnet_none | 3 | 5 | 6 | 17 | 7 | 8 |
| TrafficBDE::TrainCR_none* | 58 | 7 | 3 | 6 | 3 | 4 |
| CaDENCE::cadence.fit_optim | 44 | 8 | 22 | 39 | 18 | 24 |
| radiant.model::nn_none | 17 | 9 | 14 | 31 | 15 | 14 |
| validann::ann_CG | 62 | 10 | 10 | 18 | 10 | 10 |
| h2o::h2o.deeplearning_first-order | 50 | 11 | 8 | 11 | 8 | 6 |
| validann::ann_L-BFGS-B | 39 | 11 | 13 | 27 | 13 | 12 |
| brnn::brnn_Gauss-Newton | 5 | 13 | 18 | 16 | 20 | 18 |
| minpack.lm::nlsLM_none | 9 | 14 | 9 | 5 | 9 | 9 |
| EnsembleBase::Regression.Batch.Fit_none | 6 | 15 | 11 | 20 | 12 | 11 |
| caret::avNNet_none | 11 | 16 | 12 | 25 | 11 | 12 |
| monmlp::monmlp.fit_BFGS | 31 | 17 | 15 | 30 | 14 | 16 |
| deepnet::nn.train_BP | 23 | 18 | 16 | 26 | 16 | 15 |
| keras::fit_adamax | 52 | 19 | 17 | 34 | 19 | 17 |
| RSNNS::mlp_Rprop | 25 | 19 | 28 | 49 | 26 | 29 |
| automl::automl_train_manual_trainwgrad_adam | 47 | 21 | 20 | 43 | 21 | 23 |
| neuralnet::neuralnet_sag | 38 | 22 | 41 | 62 | 41 | 41 |
| qrnn::qrnn.fit_none | 29 | 23 | 22 | 27 | 17 | 32 |
| RSNNS::mlp_SCG | 33 | 24 | 26 | 41 | 30 | 27 |
| RSNNS::mlp_Std_Backpropagation | 24 | 25 | 24 | 32 | 24 | 22 |
| RSNNS::mlp_BackpropChunk | 27 | 26 | 26 | 38 | 27 | 25 |
| neuralnet::neuralnet_rprop- | 18 | 27 | 30 | 48 | 31 | 26 |
| RSNNS::mlp_BackpropMomentum | 26 | 28 | 19 | 23 | 22 | 19 |
| keras::fit_adam | 48 | 29 | 25 | 44 | 25 | 20 |
| automl::automl_train_manual_trainwgrad_RMSprop | 46 | 30 | 28 | 46 | 29 | 29 |
| neuralnet::neuralnet_rprop+ | 14 | 31 | 36 | 55 | 36 | 35 |
| RSNNS::mlp_BackpropWeightDecay | 29 | 31 | 21 | 22 | 23 | 21 |
| neuralnet::neuralnet_slr | 22 | 33 | 32 | 37 | 31 | 28 |
| keras::fit_nadam | 49 | 34 | 33 | 54 | 35 | 32 |
| validann::ann_Nelder-Mead | 61 | 35 | 38 | 51 | 42 | 40 |
| automl::automl_train_manual_trainwpso | 57 | 36 | 39 | 47 | 40 | 36 |
| ANN2::neuralnetwork_rmsprop | 13 | 37 | 34 | 42 | 33 | 36 |
| neuralnet::neuralnet_backprop | 32 | 38 | 35 | 27 | 34 | 31 |
| keras::fit_adagrad | 60 | 39 | 44 | 52 | 44 | 42 |
| AMORE::train_ADAPTgdwm | 20 | 40 | 43 | 52 | 39 | 49 |
| ANN2::neuralnetwork_adam | 10 | 41 | 41 | 45 | 43 | 39 |
| yager::grnn.fit_none* | 18 | 42 | 31 | 7 | 28 | 32 |
| BNN::BNNsel_none* | 59 | 43 | 37 | 15 | 38 | 43 |
| deepdive::deepnet_adam | 36 | 44 | 39 | 1 | 37 | 38 |
| monmlp::monmlp.fit_Nelder-Mead | 34 | 44 | 45 | 55 | 45 | 45 |
| keras::fit_sgd | 51 | 46 | 49 | 40 | 47 | 48 |
| AMORE::train_BATCHgd | 43 | 47 | 47 | 35 | 48 | 44 |
| keras::fit_rmsprop | 42 | 48 | 53 | 61 | 54 | 53 |
| AMORE::train_BATCHgdwm | 45 | 49 | 48 | 20 | 49 | 46 |
| deepdive::deepnet_rmsProp | 36 | 50 | 46 | 4 | 46 | 46 |
| AMORE::train_ADAPTgd | 16 | 51 | 50 | 19 | 51 | 52 |
| validann::ann_SANN | 28 | 52 | 52 | 59 | 52 | 51 |
| ANN2::neuralnetwork_sgd | 11 | 53 | 50 | 24 | 50 | 50 |
| CaDENCE::cadence.fit_Rprop | 54 | 54 | 57 | 63 | 55 | 60 |
| RSNNS::mlp_BackpropBatch | 40 | 55 | 55 | 50 | 55 | 58 |
| snnR::snnR_none | 8 | 56 | 54 | 9 | 53 | 55 |
| keras::fit_adadelta | 63 | 57 | 58 | 33 | 57 | 57 |
| RSNNS::mlp_Quickprop | 41 | 57 | 59 | 35 | 58 | 59 |
| CaDENCE::cadence.fit_psoptim | 53 | 59 | 62 | 60 | 60 | 61 |
| elmNNRcpp::elm_train_extremeML | 1 | 60 | 61 | 58 | 62 | 62 |
| deepdive::deepnet_momentum | 56 | 61 | 56 | 3 | 59 | 53 |
| deepdive::deepnet_gradientDescent | 55 | 62 | 59 | 2 | 60 | 56 |
| ELMR::OSelm_train.formula_extremeML | 2 | 63 | 63 | 57 | 63 | 63 |
5 Figures
5.1 Some plots
plot(scoredfr[,c("time.score", "RMSE.score", "RMSEmed.score", "RMSEd51.score")], las = 1)
op <- par(mfrow = c(1,3), las = 1, mar = c(0,0.5,0,0.5), oma = c(2,2,3.5,2), cex = 1.1)
plot(scoredfr[,c("RMSE.score", "RMSEmed.score")]); abline(v=10.5, lty = 2)
mtext("x=RMSE.score, y=RMSEmed.score", line = 1.5, font = 2)
plot(scoredfr[,c("RMSE.score", "time.score")], yaxt = "n"); abline(v=10.5, lty = 2)
mtext("x=RMSE.score, y=time.score", line = 1.5, font = 2)
plot(scoredfr[,c("RMSE.score", "RMSEd51.score")], yaxt = "n"); Axis(scoredfr[,5], side = 4)
mtext("x=RMSE.score, y=RMSEd51.score", line = 1.5, font = 2)
# mtext("(x=RMSE.score, y=RMSEmed.score) (x=RMSE.score, y=time.score) (x=RMSE.score, y=RMSEd51.score)",
# outer = TRUE, line = 2, font = 2)
par(op)5.2 Comparison of global scores and scores per dataset
## =====================================
## GLOBAL SCORE APPLIED TO EVERY DATASET
## =====================================
merge_sfr_dfr <- function(x, y) {
z <- cbind(
x[,c("npfa","pfa","time.mean","RMSE.min","time.rank","RMSE.rank")],
y[,c("time.score","RMSE.score")]
)
z[order(z$RMSE.score),]
}
zfr <- lapply(sfr, merge_sfr_dfr, y = scoredfr0)
## =========================
## GRAPHIC RMSEscore_RMSEmin
## =========================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "RMSE.min"]),
xlab = "RMSE.score", ylab = "RMSE.min", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "RMSE.min"]),
labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.score (global) y=RMSE.min (per dataset)", outer = TRUE, line = 1)
## ==============================
## GRAPHIC RMSEscore_timemean
## ==============================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "time.mean"]),
xlab = "RMSE.score", ylab = "RMSE.min", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "time.mean"]),
labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.score (global) y=time.mean (per dataset)", outer = TRUE, line = 1)
5.3 By different number of algorithms
## =======================================
## GRAPHIC RMSEmin_timemean - 49 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.min"]), log1p(zfr[[j]][, "time.mean"]),
xlab = "RMSE.min", ylab = "time.mean", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.min"]), log1p(zfr[[j]][, "time.mean"]),
labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset) y=time.mean (per dataset) 49 algos", outer = TRUE, line = 1)
## =======================================
## GRAPHIC RMSEmin_timemean - 12 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][1:12, "RMSE.min"]), log1p(zfr[[j]][1:12, "time.mean"]),
xlab = "RMSE.min", ylab = "time.mean", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][1:12, "RMSE.min"]), log1p(zfr[[j]][1:12, "time.mean"]),
labels = zfr[[j]][1:12, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset) y=time.mean (per dataset) 12 algos", outer = TRUE, line = 1)
## =======================================
## GRAPHIC RMSEmin_timemean - 09 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][1:9, "RMSE.min"]), log1p(zfr[[j]][1:9, "time.mean"]),
xlab = "RMSE.min", ylab = "time.mean", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][1:9, "RMSE.min"]), log1p(zfr[[j]][1:9, "time.mean"]),
labels = zfr[[j]][1:9, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset) y=time.mean (per dataset) 9 algos", outer = TRUE, line = 1)
## THE END
## THE END