library(NNbenchmark)
library(kableExtra)
library(dplyr)
library(stringr)
options(scipen = 999)
odir <- "D:/GSoC2020/Results/2020run04/"
lf <- lapply(list.files(odir, pattern = "-results.csv", full.names = TRUE), csv::as.csv)
names(lf) <- names(NNdatasets)
lf <- lf[c(1:4,6,7,10,12)]
gfr <- lapply(lf, function(dfr) cbind(
ds = str_remove(str_extract(dfr$event, "\\w+_"), "_"),
pfa = str_sub(str_remove(dfr$event, str_extract(dfr$event, "\\w+_")), 1, -4),
run = str_sub(dfr$event, -2, -1),
dfr[,c("RMSE","MAE","WAE","time")]
))
yfr <- lapply(gfr, function(dfr) {
as.data.frame(dfr %>%
group_by(pfa) %>%
summarise(time.mean = mean(time),
RMSE.min = min(RMSE),
RMSE.med = median(RMSE),
RMSE.d51 = median(RMSE) - min(RMSE),
MAE.med = median(MAE),
WAE.med = median(WAE)
)
)})
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
## `summarise()` ungrouping output (override with `.groups` argument)
yfr <- lapply(yfr, function(dfr) transform(dfr, npfa = 1:nrow(dfr)))
rankMOFtime <- function(dfr) {
dfrtime <- dfr[order(dfr$time.mean),]
dfrRMSE <- dfr[order(dfr$RMSE.min, dfr$time.mean, dfr$RMSE.med),]
dfrRMSEmed <- dfr[order(dfr$RMSE.med, dfr$RMSE.min, dfr$time.mean),]
dfrRMSEd51 <- dfr[order(dfr$RMSE.d51),]
dfrMAE <- dfr[order(dfr$MAE.med),]
dfrWAE <- dfr[order(dfr$WAE.med),]
transform(dfr,
time.rank = order(dfrtime$npfa),
RMSE.rank = order(dfrRMSE$npfa),
RMSEmed.rank = order(dfrRMSEmed$npfa),
RMSEd51.rank = order(dfrRMSEd51$npfa),
MAE.rank = order(dfrMAE$npfa),
WAE.rank = order(dfrWAE$npfa)
)
}
sfr <- lapply(yfr, rankMOFtime)
sfrwide <- do.call(cbind, sfr)
sfr.time <- sfrwide[, c(grep("time.rank", colnames(sfrwide)))]
time.score <- rank(apply(sfr.time, 1, sum), ties.method = "min")
sfr.RMSE <- sfrwide[, c(grep("RMSE.rank", colnames(sfrwide)))]
RMSE.score <- rank(apply(sfr.RMSE, 1, sum), ties.method = "min")
sfr.RMSEmed <- sfrwide[, c(grep("RMSEmed.rank", colnames(sfrwide)))]
RMSEmed.score <- rank(apply(sfr.RMSEmed, 1, sum), ties.method = "min")
sfr.RMSEd51 <- sfrwide[, c(grep("RMSEd51.rank", colnames(sfrwide)))]
RMSEd51.score <- rank(apply(sfr.RMSEd51, 1, sum), ties.method = "min")
sfr.MAE <- sfrwide[, c(grep("MAE.rank", colnames(sfrwide)))]
MAE.score <- rank(apply(sfr.MAE, 1, sum), ties.method = "min")
sfr.WAE <- sfrwide[, c(grep("WAE.rank", colnames(sfrwide)))]
WAE.score <- rank(apply(sfr.WAE, 1, sum), ties.method = "min")
scoredfr0 <- data.frame(sfr$mDette[,"pfa",drop=FALSE],
# scoredfr0 <- data.frame(sfr$uNeuroOne[,c("pfa")],
time.score,
RMSE.score,
RMSEmed.score,
RMSEd51.score,
MAE.score,
WAE.score)
scoredfr <- scoredfr0[order(scoredfr0$RMSE.score),]
rownames(scoredfr) <- NULL
kable(scoredfr)%>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
pfa | time.score | RMSE.score | RMSEmed.score | RMSEd51.score | MAE.score | WAE.score |
---|---|---|---|---|---|---|
nlsr::nlxb_none | 18 | 1 | 3 | 16 | 3 | 6 |
rminer::fit_none | 14 | 2 | 1 | 6 | 1 | 1 |
nnet::nnet_none | 3 | 3 | 2 | 17 | 2 | 3 |
MachineShop::fit_none | 6 | 4 | 9 | 20 | 10 | 7 |
validann::ann_BFGS | 35 | 4 | 4 | 11 | 4 | 5 |
traineR::train.nnet_none | 4 | 6 | 5 | 15 | 6 | 2 |
radiant.model::nn_none | 11 | 7 | 8 | 32 | 11 | 9 |
validann::ann_CG | 59 | 8 | 6 | 10 | 5 | 4 |
monmlp::monmlp.fit_BFGS | 26 | 9 | 10 | 17 | 9 | 10 |
CaDENCE::cadence.fit_optim | 46 | 10 | 28 | 48 | 23 | 39 |
h2o::h2o.deeplearning_first-order | 51 | 11 | 7 | 7 | 8 | 8 |
EnsembleBase::Regression.Batch.Fit_none | 5 | 12 | 15 | 33 | 15 | 15 |
brnn::brnn_Gauss-Newton | 8 | 13 | 11 | 9 | 12 | 11 |
validann::ann_L-BFGS-B | 36 | 14 | 12 | 29 | 13 | 12 |
qrnn::qrnn.fit_none | 28 | 15 | 14 | 22 | 7 | 35 |
RSNNS::mlp_Rprop | 24 | 16 | 24 | 51 | 25 | 29 |
RSNNS::mlp_SCG | 30 | 17 | 16 | 22 | 18 | 19 |
automl::automl_train_manual_trainwgrad_adam | 50 | 18 | 20 | 35 | 16 | 19 |
deepnet::nn.train_BP | 23 | 18 | 18 | 38 | 24 | 17 |
neuralnet::neuralnet_rprop+ | 18 | 20 | 23 | 40 | 21 | 24 |
neuralnet::neuralnet_rprop- | 21 | 21 | 21 | 41 | 22 | 18 |
caret::avNNet_none | 9 | 22 | 13 | 30 | 14 | 12 |
keras::fit_adamax | 48 | 23 | 16 | 19 | 20 | 16 |
minpack.lm::nlsLM_none | 13 | 24 | 18 | 5 | 19 | 14 |
AMORE::train_ADAPTgdwm | 17 | 25 | 22 | 28 | 17 | 26 |
automl::automl_train_manual_trainwgrad_RMSprop | 47 | 26 | 31 | 50 | 29 | 38 |
RSNNS::mlp_Std_Backpropagation | 22 | 27 | 32 | 30 | 31 | 36 |
ANN2::neuralnetwork_rmsprop | 14 | 28 | 26 | 33 | 27 | 22 |
RSNNS::mlp_BackpropChunk | 26 | 29 | 34 | 41 | 32 | 34 |
neuralnet::neuralnet_slr | 31 | 30 | 38 | 37 | 38 | 46 |
RSNNS::mlp_BackpropMomentum | 25 | 30 | 35 | 38 | 34 | 30 |
RSNNS::mlp_BackpropWeightDecay | 29 | 32 | 30 | 43 | 33 | 32 |
ANN2::neuralnetwork_adam | 16 | 33 | 27 | 26 | 28 | 23 |
AMORE::train_ADAPTgd | 10 | 34 | 24 | 8 | 26 | 19 |
keras::fit_adam | 42 | 35 | 28 | 44 | 30 | 25 |
keras::fit_adagrad | 58 | 36 | 42 | 51 | 41 | 33 |
keras::fit_nadam | 44 | 36 | 38 | 58 | 39 | 40 |
neuralnet::neuralnet_sag | 41 | 38 | 50 | 59 | 46 | 52 |
AMORE::train_BATCHgdwm | 40 | 39 | 33 | 14 | 36 | 27 |
AMORE::train_BATCHgd | 39 | 40 | 37 | 24 | 42 | 31 |
ANN2::neuralnetwork_sgd | 12 | 41 | 36 | 20 | 35 | 28 |
automl::automl_train_manual_trainwpso | 57 | 42 | 40 | 49 | 40 | 37 |
keras::fit_sgd | 48 | 43 | 44 | 47 | 43 | 43 |
validann::ann_Nelder-Mead | 55 | 44 | 43 | 45 | 45 | 41 |
deepdive::deepnet_adam | 32 | 45 | 41 | 1 | 37 | 44 |
monmlp::monmlp.fit_Nelder-Mead | 32 | 46 | 46 | 46 | 43 | 47 |
keras::fit_adadelta | 60 | 47 | 48 | 27 | 51 | 41 |
neuralnet::neuralnet_backprop | 37 | 48 | 50 | 11 | 48 | 45 |
RSNNS::mlp_BackpropBatch | 43 | 49 | 47 | 25 | 50 | 49 |
snnR::snnR_none | 7 | 49 | 48 | 13 | 49 | 47 |
CaDENCE::cadence.fit_Rprop | 56 | 51 | 54 | 60 | 52 | 58 |
keras::fit_rmsprop | 37 | 52 | 54 | 57 | 55 | 54 |
deepdive::deepnet_rmsProp | 34 | 53 | 45 | 4 | 47 | 50 |
CaDENCE::cadence.fit_psoptim | 54 | 54 | 56 | 56 | 54 | 56 |
validann::ann_SANN | 20 | 55 | 53 | 53 | 56 | 55 |
deepdive::deepnet_momentum | 53 | 56 | 52 | 3 | 53 | 51 |
RSNNS::mlp_Quickprop | 45 | 57 | 58 | 36 | 58 | 57 |
deepdive::deepnet_gradientDescent | 52 | 58 | 57 | 2 | 57 | 53 |
elmNNRcpp::elm_train_extremeML | 1 | 59 | 59 | 55 | 59 | 59 |
ELMR::OSelm_train.formula_extremeML | 2 | 60 | 60 | 54 | 60 | 60 |
plot(scoredfr[,c("time.score", "RMSE.score", "RMSEmed.score", "RMSEd51.score")], las = 1)
op <- par(mfrow = c(1,3), las = 1, mar = c(0,0.5,0,0.5), oma = c(2,2,3.5,2), cex = 1.1)
plot(scoredfr[,c("RMSE.score", "RMSEmed.score")]); abline(v=10.5, lty = 2)
mtext("x=RMSE.score, y=RMSEmed.score", line = 1.5, font = 2)
plot(scoredfr[,c("RMSE.score", "time.score")], yaxt = "n"); abline(v=10.5, lty = 2)
mtext("x=RMSE.score, y=time.score", line = 1.5, font = 2)
plot(scoredfr[,c("RMSE.score", "RMSEd51.score")], yaxt = "n"); Axis(scoredfr[,5], side = 4)
mtext("x=RMSE.score, y=RMSEd51.score", line = 1.5, font = 2)
# mtext("(x=RMSE.score, y=RMSEmed.score) (x=RMSE.score, y=time.score) (x=RMSE.score, y=RMSEd51.score)",
# outer = TRUE, line = 2, font = 2)
par(op)
## =====================================
## GLOBAL SCORE APPLIED TO EVERY DATASET
## =====================================
merge_sfr_dfr <- function(x, y) {
z <- cbind(
x[,c("npfa","pfa","time.mean","RMSE.min","time.rank","RMSE.rank")],
y[,c("time.score","RMSE.score")]
)
z[order(z$RMSE.score),]
}
zfr <- lapply(sfr, merge_sfr_dfr, y = scoredfr0)
## =========================
## GRAPHIC RMSEscore_RMSEmin
## =========================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "RMSE.min"]),
xlab = "RMSE.score", ylab = "RMSE.min", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "RMSE.min"]),
labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.score (global) y=RMSE.min (per dataset)", outer = TRUE, line = 1)
## ==============================
## GRAPHIC RMSEscore_timemean
## ==============================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "time.mean"]),
xlab = "RMSE.score", ylab = "RMSE.min", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.score"]), log1p(zfr[[j]][, "time.mean"]),
labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.score (global) y=time.mean (per dataset)", outer = TRUE, line = 1)
## =======================================
## GRAPHIC RMSEmin_timemean - 49 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][, "RMSE.min"]), log1p(zfr[[j]][, "time.mean"]),
xlab = "RMSE.min", ylab = "time.mean", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][, "RMSE.min"]), log1p(zfr[[j]][, "time.mean"]),
labels = zfr[[j]][, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset) y=time.mean (per dataset) 49 algos", outer = TRUE, line = 1)
## =======================================
## GRAPHIC RMSEmin_timemean - 12 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][1:12, "RMSE.min"]), log1p(zfr[[j]][1:12, "time.mean"]),
xlab = "RMSE.min", ylab = "time.mean", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][1:12, "RMSE.min"]), log1p(zfr[[j]][1:12, "time.mean"]),
labels = zfr[[j]][1:12, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset) y=time.mean (per dataset) 12 algos", outer = TRUE, line = 1)
## =======================================
## GRAPHIC RMSEmin_timemean - 09 algos
## =======================================
op <- par(mfrow = c(4,2), las = 1, mar = c(0,0,0,0), oma = c(1,1,3,1))
for (j in seq_along(zfr)) {
names(zfr)[j]
plot(log1p(zfr[[j]][1:9, "RMSE.min"]), log1p(zfr[[j]][1:9, "time.mean"]),
xlab = "RMSE.min", ylab = "time.mean", #Â main = names(zfr)[j],
las = 1, col = 0, xaxt = "n", yaxt = "n")
mtext(names(zfr)[j], line = -1.2, cex = 0.8)
text(log1p(zfr[[j]][1:9, "RMSE.min"]), log1p(zfr[[j]][1:9, "time.mean"]),
labels = zfr[[j]][1:9, "RMSE.score"])
}
mtext("x=RMSE.min (per dataset) y=time.mean (per dataset) 9 algos", outer = TRUE, line = 1)
## THE END
## THE END