From 03061a8fac1afdd597d21f45813eab82841cfac7 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 1 Jul 2021 16:20:37 +0200 Subject: [PATCH 01/28] Increase subset support for arbitraty attributes; add attribute-related functionality; improve concatenation of datasets --- DESCRIPTION | 2 +- R/DataSet.R | 74 ++++++++++++++++++++++++++++++++++++++---- R/DataSetList.R | 85 +++++++++++++++++++++++++++++++++---------------- 3 files changed, 126 insertions(+), 35 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9d169a77..ab6902df 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: IOHanalyzer Type: Package Title: Data Analysis Part of 'IOHprofiler' -Version: 0.1.5.2 +Version: 0.1.6.0 Author: Hao Wang [cre, aut], Diederick Vermetten [aut], Carola Doerr [aut], Thomas Bäck [aut] Maintainer: Hao Wang Authors@R: c( diff --git a/R/DataSet.R b/R/DataSet.R index 59d6f147..4a6d829d 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -194,7 +194,7 @@ c.DataSet <- function(...) { if (!any((class(ds)) == 'DataSet')) stop("Operation only possible when all arguments are DataSets") } - + fixed_attrs <- c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') info <- list() for (attr_str in fixed_attrs) { @@ -206,20 +206,26 @@ c.DataSet <- function(...) { info <- c(info,temp) } names(info) <- fixed_attrs - + + #Record number of runs to make masks of static attributes + nr_runs <- unlist(lapply(dsl, function(x) ncol(x$FV))) for (attr_str in names(attributes(dsl[[1]]))) { if (attr_str %in% fixed_attrs || attr_str %in% c("names", "class")) next temp <- unlist(lapply(dsl, function(x) attr(x, attr_str))) if (length(unique(temp)) == 1) temp <- unique(temp) - else - temp <- list(temp_name = temp) + else { + if (length(temp) == length(nr_runs)) + temp <- list(temp_name = rep(temp, nr_runs)) + else + temp <- list(temp_name = temp) + } names(temp) <- attr_str info <- c(info, temp) } - + format <- info[['format']] #attr(dsl[[1]], "format") - + RT_raw <- unlist(lapply(dsl, function(ds) { lapply(seq_len(ncol(ds$RT)), function(cnr) { rt_temp <- as.matrix(ds$RT[, cnr]) @@ -229,7 +235,7 @@ c.DataSet <- function(...) { RT <- align_running_time(RT_raw, format = "TWO_COL", maximization = info$maximization)$RT FV <- align_function_value(RT_raw, format = "TWO_COL")$FV - + # TODO: to deal with cases where aligned parameters are present in original DataSets PAR <- list( 'by_FV' = RT[names(RT) != 'RT'], @@ -241,7 +247,61 @@ c.DataSet <- function(...) { if (!par_name %in% c('by_FV', 'by_RT')) PAR[[par_name]] <- unlist(lapply(dsl, function(x) {x$PAR[[par_name]]}), recursive = F) } + + do.call( + function(...) + structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), + c(info) + ) +} +#' S3 subset function for DataSet +#' +#' @description Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet +#' and turned into a new DataSet object. +#' +#' @param x The DataSet from which to get a subset +#' @param mask The mask to use when subsetting. The length should be equal to the number of runs present in the +#' provided dataset object x. +#' @param ... Arguments passed to underlying subset method (not yet supported) +#' +#' @return A new DataSet +#' @export +#' @examples +#' subset(ds, [0,1,1,1]) +subset.DataSet <- function(x, mask, ...) { + + if (length(mask) != ncol(x$FV)) + stop("Operation only possible when maks is the same length as the number of runs in the DataSet") + + info <- list() + for (attr_str in names(attributes(x))) { + if (attr_str %in% c('names', 'class')) next + temp <- attr(x, attr_str) + if (length(unique(temp)) == 1) + temp <- unique(temp) + else { + if (length(temp) == length(mask)) + temp <- list(temp[mask]) + else{ + warning(paste0("Attribute detected (", attr_str, ") with incorrect length for the mask-based subsetting!")) + next + } + } + names(temp) <- attr_str + info <- c(info, temp) + } + + format <- info[['format']] + + RT <- x$RT[, mask] + FV <- x$FV[, mask] + + PAR <- list( + 'by_FV' = ifelse(ncol(x$PAR$by_FV) == length(mask), x$PAR$by_FV[, mask], NULL), + 'by_RT' = ifelse(ncol(x$PAR$by_RT) == length(mask), x$PAR$by_RT[, mask], NULL) + ) + do.call( function(...) structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), diff --git a/R/DataSetList.R b/R/DataSetList.R index 3e4818b5..f5fc5e85 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -45,6 +45,9 @@ read_dir <- function(path, verbose = T, print_fun = NULL, maximization = TRUE, #' These formats are specified in more detail in our github wiki. #' @param subsampling Logical. Whether *.cdat files are subsampled? #' @param print_fun Function used to print output when in verbose mode +#' @param full_aggregation If True, individual DataSets are aggregated as much as possible: all DataSets +#' with the same algorithmname, function id and dimension are combined together. This leads to information loss +#' related to static variables, so only use if that information is not required. #' #' @return A DataSetList object #' @export @@ -52,7 +55,7 @@ read_dir <- function(path, verbose = T, print_fun = NULL, maximization = TRUE, #' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") #' DataSetList(path) DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization = NULL, - format = IOHprofiler, subsampling = FALSE) { + format = IOHprofiler, subsampling = FALSE, full_aggregation = TRUE) { if (is.null(path)) return(structure(list(), class = c('DataSetList', 'list'))) @@ -164,7 +167,10 @@ DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization attr(object, 'suite') <- suite attr(object, 'maximization') <- maximization - clean_DataSetList(object) + if (full_aggregation) + clean_DataSetList(object) + else + object } @@ -700,30 +706,40 @@ get_runtimes <- function(dsList) { )))) } -# #' Get the best function value reached in a DataSetList -# #' -# #' @param dsList The DataSetLsit -# #' -# #' @return A list matrices of all runtime values which occur in the DataSetList -# #' @export -# get_best_targets <- function(dsList, by = 'funcId', maximize = T) { -# targets <- c() -# funcIds <- get_funcId(dsList) - -# for (i in seq_along(aggr_attr)) { -# data <- subset(dsList, funcId == funcIds[i]) +#' Get all attributes which can be used to subset a DataSetList +#' +#' @param dsl The DataSetList +#' +#' @return The list of available attributes +#' @export +#' @examples +#' get_static_attributes(dsl) +get_static_attributes <- function(dsl) { + full_names <- unique(unlist(lapply(dsl, function(ds) {names(attributes(ds))}))) + + reserved_attributes <- c("names", "class", "suite", "maximization", "algInfo", "comment", + "datafile", "maxRT", "finalFV", "format") + setdiff(full_names, reserved_attributes) +} -# Fall <- get_funvals(data) -# Fval <- ifelse(maximize, max(Fall), min(Fall)) -# targets <- c(targets, Fval) -# } -# targets -# } +#' Get all options for a specific attribute which can be used to subset a DataSetList +#' +#' This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. +#' Note the only attributes returned by `get_static_attributes` are supported in this funcion +#' +#' @param dsl The DataSetList +#' @param attribute the name of the attribute for which to get the available options in dsl +#' @return The list of options for the specified attribute +#' @export +#' @examples +#' get_static_attribute_values(dsl, 'funcId') +get_static_attribute_values <- function(dsl, attribute) { + unique(unlist(lapply(dsl, function(ds) {attr(ds, attribute)}))) +} -# TODO: the attribute list should als be sliced here... #' Filter a DataSetList by some criteria #' -#' @param x The DataSetLsit +#' @param x The DataSetList #' @param ... The condition to filter on. Can be any expression which assigns True or False #' to a DataSet object, such as DIM == 625 or funcId == 2 #' @@ -734,14 +750,29 @@ get_runtimes <- function(dsList) { subset.DataSetList <- function(x, ...) { condition_call <- substitute(list(...)) enclos <- parent.frame() - idx <- sapply(x, - function(ds) - all(unlist( + obj <- lapply(x, + function(ds){ + tryCatch({ + mask <- unlist( eval(condition_call, attributes(ds), enclos) - ))) - x[idx] + )}, error = return(NULL)) + if (length(mask) == 1) return(ds) + return(subset(ds, mask)) + + }) + class(obj) <- c('DataSetList', class(obj)) + + # also slice the attributes accordingly + attr(obj, 'suite') <- attr(x, 'suite') + attr(obj, 'maximization') <- attr(x, 'maximization') + attr(obj, 'DIM') <- sapply(x, function(ds) attr(ds, 'DIM')) + attr(obj, 'funcId') <- sapply(x, function(ds) attr(ds, 'funcId')) + attr(obj, 'algId') <- sapply(x, function(ds) attr(ds, 'algId')) + + obj } + #' Save DataTable in multiple formats #' #' @param df The DataTable to store From 625ba03de8adb277ab07eea18cd19f25d7432744 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 1 Jul 2021 16:21:32 +0200 Subject: [PATCH 02/28] Add documentation files for newly added functions --- NAMESPACE | 3 +++ man/DataSetList.Rd | 7 ++++++- man/get_static_attribute_values.Rd | 23 +++++++++++++++++++++++ man/get_static_attributes.Rd | 20 ++++++++++++++++++++ man/subset.DataSet.Rd | 26 ++++++++++++++++++++++++++ man/subset.DataSetList.Rd | 2 +- 6 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 man/get_static_attribute_values.Rd create mode 100644 man/get_static_attributes.Rd create mode 100644 man/subset.DataSet.Rd diff --git a/NAMESPACE b/NAMESPACE index 70b680ae..ab1ba3dd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -60,6 +60,7 @@ S3method(pairwise.test,DataSetList) S3method(pairwise.test,list) S3method(print,DataSet) S3method(print,DataSetList) +S3method(subset,DataSet) S3method(subset,DataSetList) S3method(summary,DataSet) S3method(summary,DataSetList) @@ -135,6 +136,8 @@ export(get_overview) export(get_parId) export(get_runtimes) export(get_shapley_values) +export(get_static_attribute_values) +export(get_static_attributes) export(get_target_dt) export(glicko2_ranking) export(max_ERTs) diff --git a/man/DataSetList.Rd b/man/DataSetList.Rd index 624504a3..a8bc05d5 100644 --- a/man/DataSetList.Rd +++ b/man/DataSetList.Rd @@ -5,7 +5,8 @@ \title{S3 constructor of the 'DataSetList'} \usage{ DataSetList(path = NULL, verbose = T, print_fun = NULL, - maximization = NULL, format = IOHprofiler, subsampling = FALSE) + maximization = NULL, format = IOHprofiler, subsampling = FALSE, + full_aggregation = TRUE) } \arguments{ \item{path}{Path to the data files. Will look for all .info-files in this directory and use @@ -29,6 +30,10 @@ the corresponding datafiles to create the DataSetList} These formats are specified in more detail in our github wiki.} \item{subsampling}{Logical. Whether *.cdat files are subsampled?} + +\item{full_aggregation}{If True, individual DataSets are aggregated as much as possible: all DataSets +with the same algorithmname, function id and dimension are combined together. This leads to information loss +related to static variables, so only use if that information is not required.} } \value{ A DataSetList object diff --git a/man/get_static_attribute_values.Rd b/man/get_static_attribute_values.Rd new file mode 100644 index 00000000..fae27df0 --- /dev/null +++ b/man/get_static_attribute_values.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_static_attribute_values} +\alias{get_static_attribute_values} +\title{Get all options for a specific attribute which can be used to subset a DataSetList} +\usage{ +get_static_attribute_values(dsl, attribute) +} +\arguments{ +\item{dsl}{The DataSetList} + +\item{attribute}{the name of the attribute for which to get the available options in dsl} +} +\value{ +The list of options for the specified attribute +} +\description{ +This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. +Note the only attributes returned by `get_static_attributes` are supported in this funcion +} +\examples{ +get_static_attribute_values(dsl, 'funcId') +} diff --git a/man/get_static_attributes.Rd b/man/get_static_attributes.Rd new file mode 100644 index 00000000..ea7489d5 --- /dev/null +++ b/man/get_static_attributes.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_static_attributes} +\alias{get_static_attributes} +\title{Get all attributes which can be used to subset a DataSetList} +\usage{ +get_static_attributes(dsl) +} +\arguments{ +\item{dsl}{The DataSetList} +} +\value{ +The list of available attributes +} +\description{ +Get all attributes which can be used to subset a DataSetList +} +\examples{ +get_static_attributes(dsl) +} diff --git a/man/subset.DataSet.Rd b/man/subset.DataSet.Rd new file mode 100644 index 00000000..92bec90b --- /dev/null +++ b/man/subset.DataSet.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{subset.DataSet} +\alias{subset.DataSet} +\title{S3 subset function for DataSet} +\usage{ +\method{subset}{DataSet}(x, mask, ...) +} +\arguments{ +\item{x}{The DataSet from which to get a subset} + +\item{mask}{The mask to use when subsetting. The length should be equal to the number of runs present in the +provided dataset object x.} + +\item{...}{Arguments passed to underlying subset method (not yet supported)} +} +\value{ +A new DataSet +} +\description{ +Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet +and turned into a new DataSet object. +} +\examples{ +subset(ds, [0,1,1,1]) +} diff --git a/man/subset.DataSetList.Rd b/man/subset.DataSetList.Rd index 1928cb51..5618870d 100644 --- a/man/subset.DataSetList.Rd +++ b/man/subset.DataSetList.Rd @@ -7,7 +7,7 @@ \method{subset}{DataSetList}(x, ...) } \arguments{ -\item{x}{The DataSetLsit} +\item{x}{The DataSetList} \item{...}{The condition to filter on. Can be any expression which assigns True or False to a DataSet object, such as DIM == 625 or funcId == 2} From 5ac2310a868271012f00e078c79949fd7270f0b5 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 1 Jul 2021 16:30:29 +0200 Subject: [PATCH 03/28] Fix example of dataset subset --- R/DataSet.R | 2 +- man/subset.DataSet.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/DataSet.R b/R/DataSet.R index 4a6d829d..bdc529ee 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -268,7 +268,7 @@ c.DataSet <- function(...) { #' @return A new DataSet #' @export #' @examples -#' subset(ds, [0,1,1,1]) +#' subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) subset.DataSet <- function(x, mask, ...) { if (length(mask) != ncol(x$FV)) diff --git a/man/subset.DataSet.Rd b/man/subset.DataSet.Rd index 92bec90b..b30225d1 100644 --- a/man/subset.DataSet.Rd +++ b/man/subset.DataSet.Rd @@ -22,5 +22,5 @@ Subset for DataSets. Based on the provided mask, the relevant data is taken from and turned into a new DataSet object. } \examples{ -subset(ds, [0,1,1,1]) +subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) } From 4d0a24b0ec7129670fee01792bee458bb70b1534 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 1 Jul 2021 17:29:09 +0200 Subject: [PATCH 04/28] Fix several issues in subset-handling --- R/DataSetList.R | 29 +++++++++++++++++------------ man/subset.DataSetList.Rd | 5 +++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/R/DataSetList.R b/R/DataSetList.R index f5fc5e85..ee61886f 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -740,8 +740,9 @@ get_static_attribute_values <- function(dsl, attribute) { #' Filter a DataSetList by some criteria #' #' @param x The DataSetList -#' @param ... The condition to filter on. Can be any expression which assigns True or False -#' to a DataSet object, such as DIM == 625 or funcId == 2 +#' @param ... The conditions to filter on. Can be any expression which assigns True or False +#' to a DataSet object, such as DIM == 625 or funcId == 2. Multiple conditions should be seperated by commas and will +#' be treated as AND. Support for custom conditions using AND and OR are not supported as of version 0.1.6.0 #' #' @return The filtered DataSetList #' @export @@ -752,24 +753,28 @@ subset.DataSetList <- function(x, ...) { enclos <- parent.frame() obj <- lapply(x, function(ds){ - tryCatch({ - mask <- unlist( - eval(condition_call, attributes(ds), enclos) - )}, error = return(NULL)) - if (length(mask) == 1) return(ds) - return(subset(ds, mask)) + mask <- tryCatch(expr = { + unlist( + eval(condition_call, attributes(ds), enclos = enclos) + ) + }, error = function(e) {F}) + if (length(mask) == 1 && mask) return(ds) + else if (length(mask) == 1) return(NULL) + return(subset(ds, mask)) }) + class(obj) <- c('DataSetList', class(obj)) + obj <- Filter(Negate(is.null), obj) # also slice the attributes accordingly attr(obj, 'suite') <- attr(x, 'suite') attr(obj, 'maximization') <- attr(x, 'maximization') - attr(obj, 'DIM') <- sapply(x, function(ds) attr(ds, 'DIM')) - attr(obj, 'funcId') <- sapply(x, function(ds) attr(ds, 'funcId')) - attr(obj, 'algId') <- sapply(x, function(ds) attr(ds, 'algId')) + attr(obj, 'DIM') <- sapply(obj, function(ds) attr(ds, 'DIM')) + attr(obj, 'funcId') <- sapply(obj, function(ds) attr(ds, 'funcId')) + attr(obj, 'algId') <- sapply(obj, function(ds) attr(ds, 'algId')) - obj + return(obj) } diff --git a/man/subset.DataSetList.Rd b/man/subset.DataSetList.Rd index 5618870d..2e9c216e 100644 --- a/man/subset.DataSetList.Rd +++ b/man/subset.DataSetList.Rd @@ -9,8 +9,9 @@ \arguments{ \item{x}{The DataSetList} -\item{...}{The condition to filter on. Can be any expression which assigns True or False -to a DataSet object, such as DIM == 625 or funcId == 2} +\item{...}{The conditions to filter on. Can be any expression which assigns True or False +to a DataSet object, such as DIM == 625 or funcId == 2. Multiple conditions should be seperated by commas and will +be treated as AND. Support for custom conditions using AND and OR are not supported as of version 0.1.6.0} } \value{ The filtered DataSetList From c718b5dbf3d5fbb3cee9a9d11dc7254075a90910 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 2 Jul 2021 12:11:37 +0200 Subject: [PATCH 05/28] Fixes to subsetting on custom attributes --- R/DataSetList.R | 34 ++++++++++++++++++++++++++++------ man/subset.DataSetList.Rd | 9 +++++++-- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/R/DataSetList.R b/R/DataSetList.R index ee61886f..8476befd 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -741,26 +741,48 @@ get_static_attribute_values <- function(dsl, attribute) { #' #' @param x The DataSetList #' @param ... The conditions to filter on. Can be any expression which assigns True or False -#' to a DataSet object, such as DIM == 625 or funcId == 2. Multiple conditions should be seperated by commas and will -#' be treated as AND. Support for custom conditions using AND and OR are not supported as of version 0.1.6.0 +#' to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes +#' (funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should +#' be used respectively #' #' @return The filtered DataSetList #' @export #' @examples #' subset(dsl, funcId == 1) +#' subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes +#' subset(dsl, instance == 1) +#' subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes +#' subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND subset.DataSetList <- function(x, ...) { condition_call <- substitute(list(...)) enclos <- parent.frame() obj <- lapply(x, function(ds){ mask <- tryCatch(expr = { - unlist( - eval(condition_call, attributes(ds), enclos = enclos) - ) + mask <- NULL + for (idx in seq(2,length(condition_call))) { + mask_temp <- unlist( + eval(condition_call[[idx]], attributes(ds)) + ) + if (is.null(mask)) mask <- mask_temp + else { + if (length(mask_temp) == 1 && !mask_temp) { + mask <- F + } else if (length(mask_temp) == 1) { + mask <- mask + } else if (length(mask_temp) == length(mask) || length(mask) == 1) { + mask <- mask & mask_temp + } else { + stop("Error creating mask") + } + + } + } + mask }, error = function(e) {F}) if (length(mask) == 1 && mask) return(ds) - else if (length(mask) == 1) return(NULL) + else if (length(mask) == 1 || !any(mask)) return(NULL) return(subset(ds, mask)) }) diff --git a/man/subset.DataSetList.Rd b/man/subset.DataSetList.Rd index 2e9c216e..9f6be59d 100644 --- a/man/subset.DataSetList.Rd +++ b/man/subset.DataSetList.Rd @@ -10,8 +10,9 @@ \item{x}{The DataSetList} \item{...}{The conditions to filter on. Can be any expression which assigns True or False -to a DataSet object, such as DIM == 625 or funcId == 2. Multiple conditions should be seperated by commas and will -be treated as AND. Support for custom conditions using AND and OR are not supported as of version 0.1.6.0} +to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes +(funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should +be used respectively} } \value{ The filtered DataSetList @@ -21,4 +22,8 @@ Filter a DataSetList by some criteria } \examples{ subset(dsl, funcId == 1) +subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes +subset(dsl, instance == 1) +subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes +subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND } From fb3c0eefce827a53632c90ee62a34ce68ead7505 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 2 Jul 2021 12:35:25 +0200 Subject: [PATCH 06/28] Re-add the enclosure environment for evaluation the conditions of subset --- R/DataSetList.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/DataSetList.R b/R/DataSetList.R index 8476befd..03345d80 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -762,7 +762,7 @@ subset.DataSetList <- function(x, ...) { mask <- NULL for (idx in seq(2,length(condition_call))) { mask_temp <- unlist( - eval(condition_call[[idx]], attributes(ds)) + eval(condition_call[[idx]], attributes(ds), enclos = enclos) ) if (is.null(mask)) mask <- mask_temp else { From 7372d554a44eaccd64cfd659506051176fad05b3 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 2 Jul 2021 14:45:03 +0200 Subject: [PATCH 07/28] Add functionality for adding unique identifier to DataSetLists --- NAMESPACE | 2 ++ R/DataSetList.R | 79 ++++++++++++++++++++++++++++++++++++++++++-- man/add_unique_id.Rd | 25 ++++++++++++++ man/get_unique_id.Rd | 21 ++++++++++++ 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 man/add_unique_id.Rd create mode 100644 man/get_unique_id.Rd diff --git a/NAMESPACE b/NAMESPACE index ab1ba3dd..2827e5dd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -93,6 +93,7 @@ export(Plot.Stats.Glicko2_Candlestick) export(Plot.Stats.Significance_Graph) export(Plot.Stats.Significance_Heatmap) export(SP) +export(add_unique_id) export(arrange) export(bootstrap_RT) export(cat.DataSet) @@ -139,6 +140,7 @@ export(get_shapley_values) export(get_static_attribute_values) export(get_static_attributes) export(get_target_dt) +export(get_unique_id) export(glicko2_ranking) export(max_ERTs) export(mean_FVs) diff --git a/R/DataSetList.R b/R/DataSetList.R index 03345d80..cdf83b62 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -756,11 +756,29 @@ get_static_attribute_values <- function(dsl, attribute) { subset.DataSetList <- function(x, ...) { condition_call <- substitute(list(...)) enclos <- parent.frame() + + condition_call <- tryCatch({ + temp <- eval(condition_call, enclos = enclos) + if (class(temp) == "list") { + temp <- lapply(unlist(temp), function(x) { + if (class(x) == 'character') { + parse(text = x) + } + else x + }) + unlist(temp) + } else if (class(temp) == "call") { + list(temp) + } else { + NULL + } + }, error = function(e) {condition_call[2:length(condition_call)]}) + obj <- lapply(x, function(ds){ mask <- tryCatch(expr = { mask <- NULL - for (idx in seq(2,length(condition_call))) { + for (idx in seq(length(condition_call))) { mask_temp <- unlist( eval(condition_call[[idx]], attributes(ds), enclos = enclos) ) @@ -787,7 +805,7 @@ subset.DataSetList <- function(x, ...) { }) class(obj) <- c('DataSetList', class(obj)) - obj <- Filter(Negate(is.null), obj) + obj <- Filter(Negate(is.null), obj) # also slice the attributes accordingly attr(obj, 'suite') <- attr(x, 'suite') @@ -799,6 +817,63 @@ subset.DataSetList <- function(x, ...) { return(obj) } +#' Add unique identifiers to each DataSet in the provided DataSetList based on static attributes +#' +#' Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to +#' ensure each dataset has exactly one unique identifier. +#' Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. +#' +#' @param dsl The DataSetList +#' @param attrs The list of attributes to combine into a unique identifier +#' @return A new DataSetList object where the split has been done based on the provided attributes, and the unique +#' identifier has been added. +#' @export +#' @examples +#' add_unique_id(dsl, c('instance')) +add_unique_id <- function(dsl, attrs) { + if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") + grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(ds1, x)})) + colnames(grid) <- attrs + + dsl_new <- DataSetList() + attr_vals <- c() + for (x in transpose(grid)) { + conditions <- unlist(lapply(seq(length(attrs)), function(idx) { + parse(text = paste0(attrs[[idx]], ' == ', x[[idx]])) + })) + dsl_temp <- subset2(dsl, conditions) + if (length(attrs) == 1) + attr_val <- x + else + attr_val <- paste0(x, collapse = "_") + + attr_vals <- c(attr_vals, rep(attr_val, length(dsl_temp))) + dsl_new <- c(dsl_new, dsl_temp) + } + attr(dsl_new, 'identifying_variables') <- attrs + attr(dsl_new, 'unique_ids') <- attr_vals + for (idx in seq(length(dsl_temp))) { + attr(dsl_new[[idx]], 'unique_id') <- attr_vals[[idx]] + } + dsl_new +} + +#' Get the unique identifiers for each DataSet in the provided DataSetList +#' +#' If no unique identifier is set (using `add_unique_id`), this function falls back on returning the algorith id +#' (from `get_aldId`)to ensure backwards compatibility +#' +#' @param dsl The DataSetList +#' @return The list of unique identiefiers present in dsl +#' @export +#' @examples +#' get_unique_id(dsl) +get_unique_id <- function(dsl) { + temp <- attr(dsl, 'unique_id') + if (is.null(temp)) return(get_algId(dsl)) + return(unique(temp)) +} + #' Save DataTable in multiple formats #' diff --git a/man/add_unique_id.Rd b/man/add_unique_id.Rd new file mode 100644 index 00000000..4719d48f --- /dev/null +++ b/man/add_unique_id.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{add_unique_id} +\alias{add_unique_id} +\title{Add unique identifiers to each DataSet in the provided DataSetList based on static attributes} +\usage{ +add_unique_id(dsl, attrs) +} +\arguments{ +\item{dsl}{The DataSetList} + +\item{attrs}{The list of attributes to combine into a unique identifier} +} +\value{ +A new DataSetList object where the split has been done based on the provided attributes, and the unique +identifier has been added. +} +\description{ +Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to +ensure each dataset has exactly one unique identifier. +Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. +} +\examples{ +add_unique_id(dsl, c('instance')) +} diff --git a/man/get_unique_id.Rd b/man/get_unique_id.Rd new file mode 100644 index 00000000..8a2255f4 --- /dev/null +++ b/man/get_unique_id.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_unique_id} +\alias{get_unique_id} +\title{Get the unique identifiers for each DataSet in the provided DataSetList} +\usage{ +get_unique_id(dsl) +} +\arguments{ +\item{dsl}{The DataSetList} +} +\value{ +The list of unique identiefiers present in dsl +} +\description{ +If no unique identifier is set (using `add_unique_id`), this function falls back on returning the algorith id +(from `get_aldId`)to ensure backwards compatibility +} +\examples{ +get_unique_id(dsl) +} From 45969dee139512473a3313b999c9a3005cb6dbab Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 2 Jul 2021 14:58:20 +0200 Subject: [PATCH 08/28] Fix examples of adding unique ids --- NAMESPACE | 1 + R/DataSetList.R | 4 ++-- R/IOHanalyzer.R | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 2827e5dd..a1e39614 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -170,6 +170,7 @@ importFrom(data.table,melt) importFrom(data.table,rbindlist) importFrom(data.table,setnames) importFrom(data.table,setorderv) +importFrom(data.table,transpose) importFrom(dplyr,"%>%") importFrom(dplyr,mutate) importFrom(ggplot2,aes) diff --git a/R/DataSetList.R b/R/DataSetList.R index cdf83b62..d483c94e 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -832,7 +832,7 @@ subset.DataSetList <- function(x, ...) { #' add_unique_id(dsl, c('instance')) add_unique_id <- function(dsl, attrs) { if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") - grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(ds1, x)})) + grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(dsl, x)})) colnames(grid) <- attrs dsl_new <- DataSetList() @@ -841,7 +841,7 @@ add_unique_id <- function(dsl, attrs) { conditions <- unlist(lapply(seq(length(attrs)), function(idx) { parse(text = paste0(attrs[[idx]], ' == ', x[[idx]])) })) - dsl_temp <- subset2(dsl, conditions) + dsl_temp <- subset(dsl, conditions) if (length(attrs) == 1) attr_val <- x else diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index 8a5fd72d..2e0d3bdf 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -8,7 +8,7 @@ #' @importFrom RColorBrewer brewer.pal #' @importFrom colorRamps primary.colors #' @importFrom data.table as.data.table rbindlist data.table fread := melt is.data.table -#' @importFrom data.table setorderv frank setnames rbindlist copy +#' @importFrom data.table setorderv frank setnames rbindlist copy transpose #' @importFrom plotly add_annotations add_trace orca plot_ly rename_ subplot layout #' @importFrom ggplot2 aes geom_jitter geom_line geom_ribbon geom_violin ggplot element_text #' @importFrom ggplot2 guides scale_color_manual scale_colour_manual scale_fill_manual From 06525d02234140547f17a3e5ac22d65617a6c7bb Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Mon, 5 Jul 2021 14:36:04 +0200 Subject: [PATCH 09/28] Fix subset again to allow string-based conditons for evaluation in code (e.g. add_unique_id) --- R/DataSetList.R | 63 +++++++++++++++++---------------------- man/subset.DataSetList.Rd | 3 +- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/R/DataSetList.R b/R/DataSetList.R index d483c94e..df0df40e 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -420,13 +420,8 @@ get_ERT.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', . } #' @rdname get_RT_summary -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. #' @export -get_RT_summary.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - +get_RT_summary.DataSetList <- function(ds, ftarget, budget = NULL, ...) { rbindlist(lapply(ds, function(ds) { res <- cbind(attr(ds, 'DIM'), @@ -743,7 +738,8 @@ get_static_attribute_values <- function(dsl, attribute) { #' @param ... The conditions to filter on. Can be any expression which assigns True or False #' to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes #' (funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should -#' be used respectively +#' be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a +#' string to be parsed. This allows exectution of subset commands on arbitrary variables in code. #' #' @return The filtered DataSetList #' @export @@ -754,25 +750,14 @@ get_static_attribute_values <- function(dsl, attribute) { #' subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes #' subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND subset.DataSetList <- function(x, ...) { - condition_call <- substitute(list(...)) enclos <- parent.frame() - - condition_call <- tryCatch({ - temp <- eval(condition_call, enclos = enclos) - if (class(temp) == "list") { - temp <- lapply(unlist(temp), function(x) { - if (class(x) == 'character') { - parse(text = x) - } - else x - }) - unlist(temp) - } else if (class(temp) == "call") { - list(temp) - } else { - NULL - } - }, error = function(e) {condition_call[2:length(condition_call)]}) + if (hasArg('text')) { + text <- list(...)$text + condition_call <- parse(text = text) + } else { + condition_call <- substitute(list(...)) + condition_call <- condition_call[2:length(condition_call)] + } obj <- lapply(x, function(ds){ @@ -813,7 +798,10 @@ subset.DataSetList <- function(x, ...) { attr(obj, 'DIM') <- sapply(obj, function(ds) attr(ds, 'DIM')) attr(obj, 'funcId') <- sapply(obj, function(ds) attr(ds, 'funcId')) attr(obj, 'algId') <- sapply(obj, function(ds) attr(ds, 'algId')) - + unique_ids <- unlist(sapply(obj, function(ds) attr(ds, 'unique_id'))) + if (!any(is.null(unique_ids))) { + attr(obj, 'unique_ids') <- unique_ids + } return(obj) } @@ -838,10 +826,13 @@ add_unique_id <- function(dsl, attrs) { dsl_new <- DataSetList() attr_vals <- c() for (x in transpose(grid)) { - conditions <- unlist(lapply(seq(length(attrs)), function(idx) { - parse(text = paste0(attrs[[idx]], ' == ', x[[idx]])) - })) - dsl_temp <- subset(dsl, conditions) + # conditions <- unlist(lapply(seq(length(attrs)), function(idx) { + # parse(text = paste0(attrs[[idx]], ' == ', x[[idx]])) + # })) + conditions <- paste0(unlist(lapply(seq(length(attrs)), function(idx) { + paste0(attrs[[idx]], ' == ', x[[idx]]) + })), collapse = " & ") + dsl_temp <- subset(dsl, text=conditions) if (length(attrs) == 1) attr_val <- x else @@ -852,7 +843,7 @@ add_unique_id <- function(dsl, attrs) { } attr(dsl_new, 'identifying_variables') <- attrs attr(dsl_new, 'unique_ids') <- attr_vals - for (idx in seq(length(dsl_temp))) { + for (idx in seq(length(dsl_new))) { attr(dsl_new[[idx]], 'unique_id') <- attr_vals[[idx]] } dsl_new @@ -984,11 +975,11 @@ generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, Xseq <- seq_FV(all, start, stop, length.out = 60, scale = ifelse(scale_log, 'log', 'linear')) if (include_opts) { - for (algid in get_algId(dsList)) { + for (uid in get_unique_id(dsList)) { if (maximization) - Xseq <- c(Xseq, max(get_funvals(subset(dsList, algId == algid)))) + Xseq <- c(Xseq, max(get_funvals(subset(dsList, unique_id == uid)))) else - Xseq <- c(Xseq, min(get_funvals(subset(dsList, algId == algid)))) + Xseq <- c(Xseq, min(get_funvals(subset(dsList, unique_id == uid)))) } Xseq <- unique(sort(Xseq)) } @@ -998,8 +989,8 @@ generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, Xseq <- seq_RT(all, start, stop, length.out = 60, scale = ifelse(scale_log, 'log', 'linear')) if (include_opts) { - for (algid in get_algId(dsList)) { - Xseq <- c(Xseq, max(get_funvals(subset(dsList, algId == algid)))) + for (uid in get_unique_id(dsList)) { + Xseq <- c(Xseq, max(get_funvals(subset(dsList, algId == uid)))) } Xseq <- unique(sort(Xseq)) } diff --git a/man/subset.DataSetList.Rd b/man/subset.DataSetList.Rd index 9f6be59d..e9d36a79 100644 --- a/man/subset.DataSetList.Rd +++ b/man/subset.DataSetList.Rd @@ -12,7 +12,8 @@ \item{...}{The conditions to filter on. Can be any expression which assigns True or False to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes (funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should -be used respectively} +be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a +string to be parsed. This allows exectution of subset commands on arbitrary variables in code.} } \value{ The filtered DataSetList From 64de55cf59f8b1c55c0c1f3229c0f3b16f354eff Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Mon, 5 Jul 2021 14:36:22 +0200 Subject: [PATCH 10/28] Add unique id to RT summary --- R/DataSet.R | 7 ++++++- man/get_RT_summary.Rd | 5 +---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/R/DataSet.R b/R/DataSet.R index bdc529ee..50c5590d 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -709,6 +709,7 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') else maxRT <- as.numeric(budget) algId <- attr(ds, 'algId') + unique_id <- attr(ds, 'unique_id') maximization <- attr(ds, 'maximization') ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) @@ -728,7 +729,7 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { } data <- data[matched, , drop = FALSE] - apply(data, 1, IOHanalyzer_env$D_quantile) %>% + dt_temp <- apply(data, 1, IOHanalyzer_env$D_quantile) %>% t %>% as.data.table %>% cbind(as.data.table(SP(data, maxRT))) %>% @@ -739,6 +740,10 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { set_colnames(c('algId', 'target', 'mean', 'median', 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), 'ERT', 'runs', 'ps')) + if (!is.null(unique_id)) { + dt_temp[, 'unique_id' := unique_id] + } + dt_temp } #' Get the maximal running time diff --git a/man/get_RT_summary.Rd b/man/get_RT_summary.Rd index 03bc7335..32e5d85b 100644 --- a/man/get_RT_summary.Rd +++ b/man/get_RT_summary.Rd @@ -10,8 +10,7 @@ get_RT_summary(ds, ftarget, budget, ...) \method{get_RT_summary}{DataSet}(ds, ftarget, budget = NULL, ...) -\method{get_RT_summary}{DataSetList}(ds, ftarget, budget = NULL, - algorithm = "all", ...) +\method{get_RT_summary}{DataSetList}(ds, ftarget, budget = NULL, ...) } \arguments{ \item{ds}{A DataSet or DataSetList object} @@ -21,8 +20,6 @@ get_RT_summary(ds, ftarget, budget, ...) \item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} \item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} } \value{ A data.table containing the runtime statistics for each provided target From e796318d891aceab6e42083224eda077875738a3 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Wed, 7 Jul 2021 14:46:15 +0200 Subject: [PATCH 11/28] Change unique id to ID and initialize to algId by default --- NAMESPACE | 5 ++-- R/DataSet.R | 6 ++-- R/DataSetList.R | 40 +++++++++++++++----------- R/IOHanalyzer.R | 3 +- man/{add_unique_id.Rd => change_id.Rd} | 8 +++--- man/{get_unique_id.Rd => get_id.Rd} | 12 ++++---- 6 files changed, 41 insertions(+), 33 deletions(-) rename man/{add_unique_id.Rd => change_id.Rd} (88%) rename man/{get_unique_id.Rd => get_id.Rd} (54%) diff --git a/NAMESPACE b/NAMESPACE index a1e39614..6db360f3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -93,10 +93,10 @@ export(Plot.Stats.Glicko2_Candlestick) export(Plot.Stats.Significance_Graph) export(Plot.Stats.Significance_Heatmap) export(SP) -export(add_unique_id) export(arrange) export(bootstrap_RT) export(cat.DataSet) +export(change_id) export(check_dsc_configured) export(check_format) export(clean_DataSetList) @@ -130,6 +130,7 @@ export(get_dsc_posthoc) export(get_dsc_rank) export(get_funcId) export(get_funvals) +export(get_id) export(get_line_style) export(get_marg_contrib_ecdf) export(get_maxRT) @@ -140,7 +141,6 @@ export(get_shapley_values) export(get_static_attribute_values) export(get_static_attributes) export(get_target_dt) -export(get_unique_id) export(glicko2_ranking) export(max_ERTs) export(mean_FVs) @@ -204,6 +204,7 @@ importFrom(magrittr,mod) importFrom(magrittr,set_colnames) importFrom(magrittr,set_names) importFrom(magrittr,set_rownames) +importFrom(methods,hasArg) importFrom(plotly,add_annotations) importFrom(plotly,add_trace) importFrom(plotly,layout) diff --git a/R/DataSet.R b/R/DataSet.R index 50c5590d..376e7d0a 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -163,7 +163,7 @@ DataSet <- function(info, verbose = F, maximization = NULL, format = IOHprofiler function(...) structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), c(info, list(maxRT = maxRT, finalFV = finalFV, format = format, - maximization = maximization, suite = suite)) + maximization = maximization, suite = suite, ID = info$algId)) ) } else @@ -709,7 +709,7 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') else maxRT <- as.numeric(budget) algId <- attr(ds, 'algId') - unique_id <- attr(ds, 'unique_id') + unique_id <- attr(ds, 'ID') maximization <- attr(ds, 'maximization') ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) @@ -741,7 +741,7 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), 'ERT', 'runs', 'ps')) if (!is.null(unique_id)) { - dt_temp[, 'unique_id' := unique_id] + dt_temp[, 'ID' := unique_id] } dt_temp } diff --git a/R/DataSetList.R b/R/DataSetList.R index df0df40e..d07c7451 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -167,6 +167,8 @@ DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization attr(object, 'suite') <- suite attr(object, 'maximization') <- maximization + attr(object, 'ID') <- attr(object, 'algId') + attr(object, 'ID_attributes') <- c('algId') if (full_aggregation) clean_DataSetList(object) else @@ -817,8 +819,8 @@ subset.DataSetList <- function(x, ...) { #' identifier has been added. #' @export #' @examples -#' add_unique_id(dsl, c('instance')) -add_unique_id <- function(dsl, attrs) { +#' change_id(dsl, c('instance')) +change_id <- function(dsl, attrs) { if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(dsl, x)})) colnames(grid) <- attrs @@ -832,7 +834,7 @@ add_unique_id <- function(dsl, attrs) { conditions <- paste0(unlist(lapply(seq(length(attrs)), function(idx) { paste0(attrs[[idx]], ' == ', x[[idx]]) })), collapse = " & ") - dsl_temp <- subset(dsl, text=conditions) + dsl_temp <- subset(dsl, text = conditions) if (length(attrs) == 1) attr_val <- x else @@ -841,27 +843,31 @@ add_unique_id <- function(dsl, attrs) { attr_vals <- c(attr_vals, rep(attr_val, length(dsl_temp))) dsl_new <- c(dsl_new, dsl_temp) } - attr(dsl_new, 'identifying_variables') <- attrs - attr(dsl_new, 'unique_ids') <- attr_vals + attr(dsl_new, 'ID_attributes') <- attrs + attr(dsl_new, 'ID') <- attr_vals for (idx in seq(length(dsl_new))) { - attr(dsl_new[[idx]], 'unique_id') <- attr_vals[[idx]] + attr(dsl_new[[idx]], 'ID') <- attr_vals[[idx]] } dsl_new } #' Get the unique identifiers for each DataSet in the provided DataSetList #' -#' If no unique identifier is set (using `add_unique_id`), this function falls back on returning the algorith id -#' (from `get_aldId`)to ensure backwards compatibility +#' If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +#' this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility #' #' @param dsl The DataSetList #' @return The list of unique identiefiers present in dsl #' @export #' @examples -#' get_unique_id(dsl) -get_unique_id <- function(dsl) { - temp <- attr(dsl, 'unique_id') - if (is.null(temp)) return(get_algId(dsl)) +#' get_id(dsl) +get_id <- function(dsl) { + temp <- attr(dsl, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'add_ID' function.") + return(get_algId(dsl)) + } return(unique(temp)) } @@ -975,11 +981,11 @@ generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, Xseq <- seq_FV(all, start, stop, length.out = 60, scale = ifelse(scale_log, 'log', 'linear')) if (include_opts) { - for (uid in get_unique_id(dsList)) { + for (uid in get_id(dsList)) { if (maximization) - Xseq <- c(Xseq, max(get_funvals(subset(dsList, unique_id == uid)))) + Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) else - Xseq <- c(Xseq, min(get_funvals(subset(dsList, unique_id == uid)))) + Xseq <- c(Xseq, min(get_funvals(subset(dsList, ID == uid)))) } Xseq <- unique(sort(Xseq)) } @@ -989,8 +995,8 @@ generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, Xseq <- seq_RT(all, start, stop, length.out = 60, scale = ifelse(scale_log, 'log', 'linear')) if (include_opts) { - for (uid in get_unique_id(dsList)) { - Xseq <- c(Xseq, max(get_funvals(subset(dsList, algId == uid)))) + for (uid in get_id(dsList)) { + Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) } Xseq <- unique(sort(Xseq)) } diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index 2e0d3bdf..53ba845e 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -19,6 +19,7 @@ #' @importFrom httr POST add_headers content authenticate #' @importFrom reshape2 acast #' @importFrom knitr kable +#' @importFrom methods hasArg #' @useDynLib IOHanalyzer NULL # Ugly hack, but appears to be required to appease CRAN @@ -26,7 +27,7 @@ utils::globalVariables(c(".", "algId", "run", "ERT", "RT", "group", "DIM", "Fvalue", "lower", "upper", "target", "format", "runtime", "parId", "instance", "input", "funcId", "budget", "dimension", "loss", "name", "optimizer_name", - "rescale", "maxRT", "algnames", ".SD", "function_class")) + "rescale", "maxRT", "algnames", ".SD", "function_class", "ID")) options(shiny.port = 4242) diff --git a/man/add_unique_id.Rd b/man/change_id.Rd similarity index 88% rename from man/add_unique_id.Rd rename to man/change_id.Rd index 4719d48f..96b99795 100644 --- a/man/add_unique_id.Rd +++ b/man/change_id.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/DataSetList.R -\name{add_unique_id} -\alias{add_unique_id} +\name{change_id} +\alias{change_id} \title{Add unique identifiers to each DataSet in the provided DataSetList based on static attributes} \usage{ -add_unique_id(dsl, attrs) +change_id(dsl, attrs) } \arguments{ \item{dsl}{The DataSetList} @@ -21,5 +21,5 @@ ensure each dataset has exactly one unique identifier. Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. } \examples{ -add_unique_id(dsl, c('instance')) +change_id(dsl, c('instance')) } diff --git a/man/get_unique_id.Rd b/man/get_id.Rd similarity index 54% rename from man/get_unique_id.Rd rename to man/get_id.Rd index 8a2255f4..f64f8922 100644 --- a/man/get_unique_id.Rd +++ b/man/get_id.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/DataSetList.R -\name{get_unique_id} -\alias{get_unique_id} +\name{get_id} +\alias{get_id} \title{Get the unique identifiers for each DataSet in the provided DataSetList} \usage{ -get_unique_id(dsl) +get_id(dsl) } \arguments{ \item{dsl}{The DataSetList} @@ -13,9 +13,9 @@ get_unique_id(dsl) The list of unique identiefiers present in dsl } \description{ -If no unique identifier is set (using `add_unique_id`), this function falls back on returning the algorith id -(from `get_aldId`)to ensure backwards compatibility +If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility } \examples{ -get_unique_id(dsl) +get_id(dsl) } From 608042e56674b52c7855c6f27d1210378cb64693 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Tue, 20 Jul 2021 15:52:51 +0200 Subject: [PATCH 12/28] Update description file --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ab6902df..3e03d747 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,7 +39,8 @@ Imports: kableExtra, stringi, httr, - knitr + knitr, + methods LinkingTo: Rcpp SystemRequirements: C++11 RoxygenNote: 7.0.0 From 6303fe0b420cbb64a3204a5ec08ca738246c3b53 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Wed, 21 Jul 2021 12:57:48 +0200 Subject: [PATCH 13/28] Add ID to all relevant functions --- NAMESPACE | 2 + R/DataSet.R | 93 ++++++++++++++++---------- R/DataSetList.R | 79 +++++++++++----------- R/IOHanalyzer.R | 2 +- R/plot.R | 60 ++++++++--------- R/plotDataSetList.R | 50 +++++++------- R/stats.R | 44 ++++++------ data/dsl.rda | Bin 4285 -> 4322 bytes data/dsl_large.rda | Bin 910424 -> 910495 bytes man/clean_DataSetList.Rd | 2 +- man/get_color_scheme.Rd | 4 +- man/get_color_scheme_dt.Rd | 2 +- man/get_id.Rd | 20 ++++-- man/get_line_style.Rd | 4 +- man/get_marg_contrib_ecdf.Rd | 6 +- man/plot_general_data.Rd | 4 +- man/set_color_scheme.Rd | 4 +- tests/testthat/test_data_generation.R | 32 ++++----- 18 files changed, 222 insertions(+), 186 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 6db360f3..b352ec66 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -50,6 +50,8 @@ S3method(get_RT_sample,DataSet) S3method(get_RT_sample,DataSetList) S3method(get_RT_summary,DataSet) S3method(get_RT_summary,DataSetList) +S3method(get_id,DataSet) +S3method(get_id,DataSetList) S3method(get_maxRT,DataSet) S3method(get_maxRT,DataSetList) S3method(get_overview,DataSet) diff --git a/R/DataSet.R b/R/DataSet.R index 376e7d0a..46fa4722 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -570,6 +570,23 @@ get_RT_overview <- function(ds, ...) UseMethod("get_RT_overview", ds) #' @export get_overview <- function(ds, ...) UseMethod("get_overview", ds) +#' Get condensed overview of datasets +#' +#' Get the unique identifiers for each DataSet in the provided DataSetList +#' +#' If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +#' this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility +#' +#' @param ds The DataSetList +#' @param ... Arguments passed to other methods +#' +#' @return The list of unique identiefiers present in dsl +#' @examples +#' get_id(dsl) +#' get_id(dsl[[1]]) +#' @export +get_id <- function(ds, ...) UseMethod("get_id", ds) + #' @rdname get_FV_overview #' @export get_FV_overview.DataSet <- function(ds, ...) { @@ -589,7 +606,7 @@ get_FV_overview.DataSet <- function(ds, ...) { median_fv <- median(last_row, na.rm = T) runs_reached <- sum(last_row == best_fv) - data.table(algId = attr(ds, 'algId'), + data.table(ID = get_id(ds), DIM = attr(ds, 'DIM'), funcId = attr(ds, 'funcId'), `worst recorded` = worst_recorded_fv, @@ -623,7 +640,7 @@ get_RT_overview.DataSet <- function(ds, ...) { max_rt <- max(data, na.rm = T) } - data.table(algId = attr(ds, 'algId'), + data.table(ID = get_id(ds), DIM = attr(ds, 'DIM'), funcId = attr(ds, 'funcId'), `miminal runtime` = min_rt, @@ -659,7 +676,7 @@ get_overview.DataSet <- function(ds, ...) { median_fv <- median(last_row, na.rm = T) runs_reached <- sum(last_row == best_fv) - data.table(`algId` = attr(ds, 'algId'), + data.table(ID = get_id(ds), `DIM` = attr(ds, 'DIM'), `funcId` = attr(ds, 'funcId'), `runs` = runs, @@ -696,8 +713,8 @@ get_ERT.DataSet <- function(ds, ftarget, budget = NULL, ...) { return(data.table()) data <- data[matched, , drop = FALSE] - dt <- as.data.table(cbind(algId, ftarget, SP(data, maxRT)$ERT)) - colnames(dt) <- c('algId', 'target', 'ERT') + dt <- as.data.table(cbind(get_id(ds), ftarget, SP(data, maxRT)$ERT)) + colnames(dt) <- c('ID', 'target', 'ERT') dt } @@ -708,8 +725,7 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { data <- ds$RT if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') else maxRT <- as.numeric(budget) - algId <- attr(ds, 'algId') - unique_id <- attr(ds, 'ID') + ID <- get_id(ds) maximization <- attr(ds, 'maximization') ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) @@ -733,16 +749,13 @@ get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { t %>% as.data.table %>% cbind(as.data.table(SP(data, maxRT))) %>% - cbind(algId, ftarget, + cbind(ID, ftarget, apply(data, 1, .mean), apply(data, 1, .median), apply(data, 1, .sd), .) %>% - set_colnames(c('algId', 'target', 'mean', 'median', + set_colnames(c('ID', 'target', 'mean', 'median', 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), 'ERT', 'runs', 'ps')) - if (!is.null(unique_id)) { - dt_temp[, 'ID' := unique_id] - } dt_temp } @@ -764,14 +777,14 @@ get_maxRT <- function(ds, ...) UseMethod("get_maxRT", ds) #' @export #' get_maxRT.DataSet <- function(ds, output = 'wide', ...) { - algId <- attr(ds, 'algId') + ID <- get_id(ds) N <- ncol(ds$RT) - res <- t(c(algId, attr(ds, 'maxRT'))) %>% + res <- t(c(ID, attr(ds, 'maxRT'))) %>% as.data.table %>% - set_colnames(c('algId', paste0('run.', seq(N)))) + set_colnames(c('ID', paste0('run.', seq(N)))) if (output == 'long') { - res <- melt(res, id = 'algId', variable.name = 'run', value.name = 'maxRT') + res <- melt(res, id = 'ID', variable.name = 'run', value.name = 'maxRT') res[, run := as.numeric(gsub('run.', '', run)) %>% as.integer ][, maxRT := as.integer(maxRT) ][order(run)] @@ -785,7 +798,7 @@ get_maxRT.DataSet <- function(ds, output = 'wide', ...) { get_RT_sample.DataSet <- function(ds, ftarget, output = 'wide', ...) { data <- ds$RT N <- ncol(data) - algId <- attr(ds, 'algId') + ID <- get_id(ds) maximization <- attr(ds, 'maximization') ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) @@ -800,12 +813,12 @@ get_RT_sample.DataSet <- function(ds, ftarget, output = 'wide', ...) { } ) - res <- cbind(algId, ftarget, as.data.table(data[matched, , drop = FALSE])) %>% - set_colnames(c('algId', 'target', paste0('run.', seq(N)))) + res <- cbind(ID, ftarget, as.data.table(data[matched, , drop = FALSE])) %>% + set_colnames(c('ID', 'target', paste0('run.', seq(N)))) if (output == 'long') { # TODO: option to not add run etc to speed up performance of ECDF calculation? - res <- melt(res, id = c('algId', 'target'), variable.name = 'run', value.name = 'RT') + res <- melt(res, id = c('ID', 'target'), variable.name = 'run', value.name = 'RT') res[, run := as.integer(as.numeric(gsub('run.', '', run))) ][, RT := as.integer(RT) ][order(target, run)] @@ -836,7 +849,7 @@ get_FV_summary.DataSet <- function(ds, runtime, ...) { data <- ds$FV NC <- ncol(data) NR <- nrow(data) - algId <- attr(ds, 'algId') + ID <- get_id(ds) maximization <- attr(ds, 'maximization') runtime <- sort(as.numeric(unique(c(runtime)))) @@ -846,12 +859,12 @@ get_FV_summary.DataSet <- function(ds, runtime, ...) { matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) data <- data[matched, , drop = FALSE] - cbind(algId, runtime, NC, + cbind(ID, runtime, NC, apply(data, 1, .mean), apply(data, 1, .median), apply(data, 1, .sd), as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile)))) %>% - set_colnames(c('algId', 'runtime', 'runs', 'mean', 'median', 'sd', + set_colnames(c('ID', 'runtime', 'runs', 'mean', 'median', 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'))) } @@ -864,7 +877,7 @@ get_FV_sample.DataSet <- function(ds, runtime, output = 'wide', ...) { data <- ds$FV N <- ncol(data) n_row <- nrow(data) - algId <- attr(ds, 'algId') + ID <- get_id(ds) maximization <- attr(ds, 'maximization') runtime <- sort(as.numeric(unique(c(runtime)))) @@ -872,11 +885,11 @@ get_FV_sample.DataSet <- function(ds, runtime, output = 'wide', ...) { idx <- seq_along(RT) matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) - res <- cbind(algId, runtime, as.data.table(data[matched, , drop = FALSE])) %>% - set_colnames(c('algId', 'runtime', paste0('run.', seq(N)))) + res <- cbind(ID, runtime, as.data.table(data[matched, , drop = FALSE])) %>% + set_colnames(c('ID', 'runtime', paste0('run.', seq(N)))) if (output == 'long') { - res <- melt(res, id = c('algId', 'runtime'), variable.name = 'run', value.name = 'f(x)') + res <- melt(res, id = c('ID', 'runtime'), variable.name = 'run', value.name = 'f(x)') res[, run := as.integer(as.numeric(gsub('run.', '', run))) ][order(runtime, run)] } @@ -909,7 +922,7 @@ get_PAR_summary.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV' } idx <- seq_along(RefValues) - algId <- attr(ds, 'algId') + ID <- get_id(ds) par_name <- get_PAR_name(ds, which = which) if (parId != 'all') @@ -933,14 +946,14 @@ get_PAR_summary.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV' function(par) { data <- ds_par[[par]][matched, , drop = FALSE] df <- cbind( - algId, par, idxValue, + ID, par, idxValue, apply(data, 1, function(x) length(x[!is.na(x)])), apply(data, 1, .mean), apply(data, 1, .median), apply(data, 1, .sd), as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile))) ) - colnames(df) <- c('algId', 'parId', idx_name, 'runs', 'mean', 'median', 'sd', + colnames(df) <- c('ID', 'parId', idx_name, 'runs', 'mean', 'median', 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%')) df }) %>% @@ -969,7 +982,7 @@ get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', } idx <- seq_along(RefValues) - algId <- attr(ds, 'algId') + ID <- get_id(ds) par_name <- get_PAR_name(ds, which = which) if (parId != 'all') @@ -987,14 +1000,14 @@ get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', function(parId) { data <- ds_par[[parId]] data <- as.data.table(data[matched, , drop = FALSE]) - data <- cbind(algId, parId, idxValue, data) - colnames(data) <- c('algId', 'parId', idx_name, paste0('run.', seq(N))) + data <- cbind(ID, parId, idxValue, data) + colnames(data) <- c('ID', 'parId', idx_name, paste0('run.', seq(N))) data }) res <- rbindlist(res) if (output == 'long') { - res <- melt(res, id = c('algId', 'parId', idx_name), variable.name = 'run', value.name = 'value') + res <- melt(res, id = c('ID', 'parId', idx_name), variable.name = 'run', value.name = 'value') res[, run := as.integer(as.numeric(gsub('run.', '', run)))] if (which == 'by_FV') res[order(target, run)] @@ -1003,3 +1016,15 @@ get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', } res } + +#' @rdname get_id +#' @export +get_id.DataSet <- function(ds, ...) { + temp <- attr(ds, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'change_id' function.") + return(attr(ds, 'algId')) + } + return(unique(temp)) +} \ No newline at end of file diff --git a/R/DataSetList.R b/R/DataSetList.R index d07c7451..56bdb551 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -178,7 +178,7 @@ DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization #' Clean DataSetList object by concatenating DataSets #' -#' Concatenates all DataSets with the same algorith name, function id and dimension +#' Concatenates all DataSets with the same ID, function id and dimension #' #' @param dsList The DataSetList object to clean #' @export @@ -192,6 +192,7 @@ clean_DataSetList <- function(dsList) { attr(dsList, 'funcId'), attr(dsList, 'DIM'), attr(dsList, 'algId'), + attr(dsList, 'ID'), SIMPLIFY = T, USE.NAMES = F ) @@ -211,6 +212,7 @@ clean_DataSetList <- function(dsList) { attr(dsList, 'DIM') <- sapply(dsList, function(ds) attr(ds, 'DIM')) attr(dsList, 'funcId') <- sapply(dsList, function(ds) attr(ds, 'funcId')) attr(dsList, 'algId') <- sapply(dsList, function(ds) attr(ds, 'algId')) + attr(dsList, 'ID') <- sapply(dsList, function(ds) attr(ds, 'ID')) } dsList } @@ -235,7 +237,7 @@ c.DataSetList <- function(...) { if (!any((class(object)) == 'DataSetList')) class(object) <- c('DataSetList', class(object)) - for (attr_str in c('DIM', 'funcId', 'algId')) { + for (attr_str in c('DIM', 'funcId', 'algId', 'ID')) { attr(object, attr_str) <- unlist(lapply(dsl, function(x) attr(x, attr_str))) } @@ -247,7 +249,7 @@ c.DataSetList <- function(...) { ) # These attributes NEED to be the same across the datasetlist - for (attr_str in c('maximization')) { + for (attr_str in c('maximization', 'ID_attributes')) { temp <- unique( unlist(lapply(dsl, function(x) attr(x, attr_str))) ) @@ -278,6 +280,7 @@ c.DataSetList <- function(...) { # also slice the attributes accordingly attr(obj, 'DIM') <- attr(x, 'DIM')[i] + attr(obj, 'ID') <- attr(x, 'ID')[i] attr(obj, 'funcId') <- attr(x, 'funcId')[i] attr(obj, 'algId') <- attr(x, 'algId')[i] attr(obj, 'suite') <- attr(x, 'suite') @@ -337,7 +340,7 @@ summary.DataSetList <- function(object, ...) { suite = attr(d, 'suite'), funcId = attr(d, 'funcId'), DIM = attr(d, 'DIM'), - algId = attr(d, 'algId'), + ID = attr(d, 'ID'), datafile = attr(d, 'datafile'), comment = attr(d, 'comment') ) @@ -379,7 +382,7 @@ arrange.DataSetList <- function(dsl, ...) { } v <- as.character(v) - if (v %in% c('DIM', 'funcId', 'algId')) + if (v %in% c('DIM', 'funcId', 'ID')) cols[[i]] <- v else { cols[[i]] <- NULL @@ -398,7 +401,7 @@ arrange.DataSetList <- function(dsl, ...) { dsl <- dsl[idx] # TODO: perhaps we do not need those attributes at all... - for (v in c('DIM', 'funcId', 'algId')) + for (v in c('DIM', 'funcId', 'ID')) attr(dsl, v) <- sapply(dsl, function(d) attr(d, v)) dsl } @@ -586,6 +589,18 @@ get_PAR_sample.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { rbindlist(lapply(ds, function(ds) get_PAR_sample(ds, idxValue, ...)), fill = T) } +#' @rdname get_id +#' @export +get_id.DataSetList <- function(ds, ...) { + temp <- attr(ds, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'change_id' function.") + return(get_algId(ds)) + } + return(unique(temp)) +} + #' Get all dimensions present in a DataSetList #' #' @param dsList The DataSetLsit @@ -851,25 +866,7 @@ change_id <- function(dsl, attrs) { dsl_new } -#' Get the unique identifiers for each DataSet in the provided DataSetList -#' -#' If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), -#' this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility -#' -#' @param dsl The DataSetList -#' @return The list of unique identiefiers present in dsl -#' @export -#' @examples -#' get_id(dsl) -get_id <- function(dsl) { - temp <- attr(dsl, 'ID') - if (is.null(temp)) { - warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added - to new datasets, see the 'add_ID' function.") - return(get_algId(dsl)) - } - return(unique(temp)) -} + #' Save DataTable in multiple formats @@ -1063,7 +1060,7 @@ generate_data.hist <- function(dsList, target, use.equal.bins = F, which = 'by_R lapply( dsList, function(ds) { - algId <- attr(ds, 'algId') + ID <- get_id(ds) if (which == 'by_RT') data <- get_RT_sample(ds, target, output = 'long')$RT @@ -1091,7 +1088,7 @@ generate_data.hist <- function(dsList, target, use.equal.bins = F, which = 'by_R plot_data <- data.frame( x = res$mids, y = res$counts, - algId = algId, + ID = ID, width = breaks[2] - breaks[1], text = plot_text ) @@ -1154,7 +1151,7 @@ generate_data.ECDF <- function(dsList, targets, scale_log = F, which = 'by_RT', } dt <- as.data.table(rbindlist(lapply(dsList, function(df) { - algId <- attr(df, 'algId') + ID <- get_id(df) if (by_rt) { temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] targets_ <- temp[funcId == attr(df, 'funcId')][['target']] @@ -1177,9 +1174,9 @@ generate_data.ECDF <- function(dsList, targets, scale_log = F, which = 'by_RT', data.frame(x = x, mean = apply(m, 2, . %>% mean(na.rm = T)), sd = apply(m, 2, . %>% sd(na.rm = T))) %>% - mutate(upper = mean + sd, lower = mean - sd, algId = algId) + mutate(upper = mean + sd, lower = mean - sd, ID = ID) }))) - dt[, mean(mean), by = .(x, algId)][, .(mean = V1, algId = algId, x = x)] + dt[, mean(mean), by = .(x, ID)][, .(mean = V1, ID = ID, x = x)] } @@ -1209,17 +1206,17 @@ generate_data.AUC <- function(dsList, targets, scale_log = F, which = 'by_RT', d dt_ecdf <- generate_data.ECDF(dsList, targets, scale_log, which) } max_idx <- nrow(unique(dt_ecdf[,'x'])) - dt_ecdf[, idx := seq(max_idx), by = 'algId'] + dt_ecdf[, idx := seq(max_idx), by = 'ID'] dt3 = copy(dt_ecdf) dt3[, idx := idx - 1] - dt_merged = merge(dt_ecdf, dt3, by = c('algId', 'idx')) - colnames(dt_merged) <- c("algId", "idx", "mean_pre", "x_pre", "mean_post", "x") + dt_merged = merge(dt_ecdf, dt3, by = c('ID', 'idx')) + colnames(dt_merged) <- c("ID", "idx", "mean_pre", "x_pre", "mean_post", "x") dt_merged[, auc_contrib := ((mean_pre + mean_post)/2)*(x - x_pre)] - dt_merged[, auc := cumsum(auc_contrib)/x, by = 'algId'] + dt_merged[, auc := cumsum(auc_contrib)/x, by = 'ID'] #TODO: just for max x if (multiple_x) - return(dt_merged[, c('algId','x','auc') ]) - return(dt_merged[idx == (max_idx - 1), c('algId','x','auc') ]) + return(dt_merged[, c('ID','x','auc') ]) + return(dt_merged[idx == (max_idx - 1), c('ID','x','auc') ]) } @@ -1340,7 +1337,7 @@ generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which } aggr_attr <- if (aggr_on == 'funcId') get_funcId(dsList) else get_dim(dsList) - N <- length(get_algId(dsList)) + N <- length(get_id(dsList)) dt <- rbindlist(lapply(aggr_attr, function(agg_val) { if (by_rt) { @@ -1348,7 +1345,7 @@ generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which dt <- get_RT_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) else dt <- get_RT_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) - dt[, c('algId', value = 'ERT', 'funcId', 'DIM')] + dt[, c('ID', value = 'ERT', 'funcId', 'DIM')] setnames(dt, 'ERT', 'value') } else{ @@ -1356,7 +1353,7 @@ generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which dt <- get_FV_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) else dt <- get_FV_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) - dt[, c('algId', value = 'mean', 'funcId', 'DIM')] + dt[, c('ID', value = 'mean', 'funcId', 'DIM')] setnames(dt, 'mean', 'value') } })) @@ -1403,7 +1400,7 @@ generate_data.ECDF_raw <- function(dsList, targets, scale_log = F) { } dt <- as.data.table(rbindlist(lapply(dsList, function(df) { - algId <- attr(df, 'algId') + ID <- get_id(df) temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] targets_ <- temp[funcId == attr(df, 'funcId')][['target']] @@ -1415,7 +1412,7 @@ generate_data.ECDF_raw <- function(dsList, targets, scale_log = F) { fun <- ecdf(data) hit <- if (is.function(fun)) fun(x) else NA } - data.table(hit = hit, rt = x, target = target, funcId = attr(df, 'funcId'), DIM = attr(df, 'DIM'), algId = attr(df, 'algId')) + data.table(hit = hit, rt = x, target = target, funcId = attr(df, 'funcId'), DIM = attr(df, 'DIM'), ID = ID) }) %>% do.call(rbind, .) m diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index 53ba845e..e09f46a7 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -27,7 +27,7 @@ utils::globalVariables(c(".", "algId", "run", "ERT", "RT", "group", "DIM", "Fvalue", "lower", "upper", "target", "format", "runtime", "parId", "instance", "input", "funcId", "budget", "dimension", "loss", "name", "optimizer_name", - "rescale", "maxRT", "algnames", ".SD", "function_class", "ID")) + "rescale", "maxRT", "algnames", ".SD", "function_class", "ID", "ids")) options(shiny.port = 4242) diff --git a/R/plot.R b/R/plot.R index 90fb1018..c6f96e81 100644 --- a/R/plot.R +++ b/R/plot.R @@ -179,7 +179,7 @@ Set3 <- function(n) colorspace::sequential_hcl(n, c(-88, 59), c. = c(60, 75, 55) fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) IOHanalyzer_env$used_colorscheme <- Set2 -IOHanalyzer_env$alg_colors <- NULL +IOHanalyzer_env$id_colors <- NULL #' Set the colorScheme of the IOHanalyzer plots #' @@ -191,7 +191,7 @@ IOHanalyzer_env$alg_colors <- NULL #' \item Variant 3 #' } #' And it is also possible to select "Custom", which allows uploading of a custom set of colors -#' @param algnames The names of the algorithms for which to set the colors +#' @param ids The names of the algorithms (or custom ids, see `change_id`) for which to set the colors #' @param path The path to the file containing the colors to use. Only used if #' schemename is "Custom" #' @@ -199,14 +199,14 @@ IOHanalyzer_env$alg_colors <- NULL #' #' @examples #' set_color_scheme("Default", get_algId(dsl)) -set_color_scheme <- function(schemename, algnames, path = NULL){ +set_color_scheme <- function(schemename, ids, path = NULL){ if (schemename == "Custom" && !is.null(path)) { dt <- fread(path, header = T) - if (any(colnames(dt) != c("algnames", "colors", "linestyles"))) { + if (any(colnames(dt) != c("ids", "colors", "linestyles"))) { warning("Incorrect file-format has been uploaded.") } else - IOHanalyzer_env$alg_colors <- dt + IOHanalyzer_env$id_colors <- dt return() } else { @@ -217,71 +217,71 @@ set_color_scheme <- function(schemename, algnames, path = NULL){ if (schemename == "Variant 1") IOHanalyzer_env$used_colorscheme <- Set1 else if (schemename == "Variant 2") IOHanalyzer_env$used_colorscheme <- Set2 else if (schemename == "Variant 3") IOHanalyzer_env$used_colorscheme <- Set3 - options(IOHanalyzer.max_colors = length(algnames)) + options(IOHanalyzer.max_colors = length(ids)) } - create_color_scheme(algnames) + create_color_scheme(ids) } } #' Get datatable of current color (and linestyle) scheme to file #' -#' @return data.table object with 3 columns: algnames, colors, linestyles +#' @return data.table object with 3 columns: ids, colors, linestyles #' @export #' @examples #' get_color_scheme_dt() get_color_scheme_dt <- function(){ - return(IOHanalyzer_env$alg_colors) + return(IOHanalyzer_env$id_colors) } #' Helper function to create default color scheme #' #' @noRd -create_color_scheme <- function(algnames) { - if (length(algnames) == 0) { +create_color_scheme <- function(ids) { + if (length(ids) == 0) { return(NULL) } - colors <- color_palettes(length(algnames)) + colors <- color_palettes(length(ids)) linestyles <- rep(c("solid", "dash", "dot"), ceiling(length(colors)/3))[1:length(colors)] - IOHanalyzer_env$alg_colors <- data.table(algnames, colors, linestyles) + IOHanalyzer_env$id_colors <- data.table(ids, colors, linestyles) } #' Get colors according to the current colorScheme of the IOHanalyzer #' -#' @param algnames_in List of algorithms for which to get colors +#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get colors #' #' @export #' #' @examples #' get_color_scheme(get_algId(dsl)) -get_color_scheme <- function(algnames_in){ - if (is.null(IOHanalyzer_env$alg_colors)) - create_color_scheme(algnames_in) - cdt <- IOHanalyzer_env$alg_colors - colors <- subset(cdt, algnames %in% algnames_in)[['colors']] - if (is.null(colors) || length(colors) != length(algnames_in)) { - return(color_palettes(length(algnames_in))) +get_color_scheme <- function(ids_in){ + if (is.null(IOHanalyzer_env$id_colors)) + create_color_scheme(ids_in) + cdt <- IOHanalyzer_env$id_colors + colors <- subset(cdt, ids %in% ids_in)[['colors']] + if (is.null(colors) || length(colors) != length(ids_in)) { + return(color_palettes(length(ids_in))) } - names(colors) <- algnames_in + names(colors) <- ids_in return(colors) } #' Get line styles according to the current styleScheme of the IOHanalyzer #' -#' @param algnames_in List of algorithms for which to get linestyles +#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get linestyles #' #' @export #' #' @examples #' get_line_style(get_algId(dsl)) -get_line_style <- function(algnames_in){ - if (is.null(IOHanalyzer_env$alg_colors)) - create_color_scheme(algnames_in) - cdt <- IOHanalyzer_env$alg_colors - linestyles <- subset(cdt, algnames %in% algnames_in)[['linestyles']] - if (is.null(linestyles) || length(linestyles) != length(algnames_in)) { +get_line_style <- function(ids_in){ + if (is.null(IOHanalyzer_env$id_colors)) + create_color_scheme(ids_in) + cdt <- IOHanalyzer_env$id_colors + linestyles <- subset(cdt, ids %in% ids_in)[['linestyles']] + if (is.null(linestyles) || length(linestyles) != length(ids_in)) { return(rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), - ceiling(length(algnames_in)/3))[1:length(algnames_in)]) + ceiling(length(ids_in)/3))[1:length(ids_in)]) } return(linestyles) } diff --git a/R/plotDataSetList.R b/R/plotDataSetList.R index 76f9ba6c..53b50005 100644 --- a/R/plotDataSetList.R +++ b/R/plotDataSetList.R @@ -3,7 +3,7 @@ symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", get_legends <- function(dsList) { N <- length(dsList) - legends <- sapply(dsList, function(d) attr(d, 'algId')) + legends <- sapply(dsList, function(d) get_id(d)) if (length(unique(legends)) < N) { funcId <- sapply(dsList, function(d) attr(d, 'funcId')) @@ -460,7 +460,7 @@ Plot.RT.Single_Func.DataSetList <- function(dsList, Fstart = NULL, Fstop = NULL, show_legend <- T if (length(y_attrs) > 0) { p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, + type = 'line', legend_attr = 'ID', show.legend = show_legend, scale.ylog = scale.ylog, p = p, scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", y_title = "Function Evaluations", @@ -469,7 +469,7 @@ Plot.RT.Single_Func.DataSetList <- function(dsList, Fstart = NULL, Fstop = NULL, } if (show.CI) { p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', upper_attr = 'upper', p = p, show.legend = show_legend, scale.ylog = scale.ylog, scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", @@ -513,7 +513,7 @@ Plot.FV.Single_Func.DataSetList <- function(dsList, RTstart = NULL, RTstop = NUL show_legend <- T if (length(y_attrs) > 0) { p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, + type = 'line', legend_attr = 'ID', show.legend = show_legend, scale.ylog = scale.ylog, scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", y_title = "Function Evaluations", @@ -524,7 +524,7 @@ Plot.FV.Single_Func.DataSetList <- function(dsList, RTstart = NULL, RTstop = NUL p <- NULL if (show.CI) { p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', upper_attr = 'upper', p = p, show.legend = show_legend, scale.ylog = scale.ylog, scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", @@ -557,7 +557,7 @@ Plot.RT.PMF.DataSetList <- function(dsList, ftarget, show.sample = F, data <- generate_data.PMF(dsList, ftarget, 'by_RT') - plot_general_data(data, 'algId', 'RT', scale.ylog = scale.ylog, + plot_general_data(data, 'ID', 'RT', scale.ylog = scale.ylog, x_title = "Algorithm", y_title = "Function Evaluations") } @@ -571,7 +571,7 @@ Plot.RT.Histogram.DataSetList <- function(dsList, ftarget, plot_mode = 'overlay' } data <- generate_data.hist(dsList, ftarget, use.equal.bins, 'by_RT') - subplot_attr <- if (plot_mode == 'subplot') 'algId' else NULL + subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', subplot_attr = subplot_attr, x_title = "Function Evaluations", y_title = "Runs") @@ -624,7 +624,7 @@ Plot.FV.PDF.DataSetList <- function(dsList, runtime, show.sample = F, scale.ylog data <- generate_data.PMF(dsList, runtime, 'by_FV') - plot_general_data(data, 'algId', 'f(x)', scale.ylog = scale.ylog, + plot_general_data(data, 'ID', 'f(x)', scale.ylog = scale.ylog, x_title = "Algorithm", y_title = "Target Value") } @@ -638,7 +638,7 @@ Plot.FV.Histogram.DataSetList <- function(dsList, runtime, plot_mode='overlay', } data <- generate_data.hist(dsList, runtime, use.equal.bins, 'by_FV') - subplot_attr <- if (plot_mode == 'subplot') 'algId' else NULL + subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', subplot_attr = subplot_attr, x_title = "Target Values", y_title = "Runs") @@ -702,7 +702,7 @@ Plot.RT.Parameters.DataSetList <- function(dsList, f_min = NULL, f_max = NULL, show_legend <- T if (length(y_attrs) > 0) { p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, + type = 'line', legend_attr = 'ID', show.legend = show_legend, scale.ylog = scale.ylog, subplot_attr = 'parId', scale.xlog = scale.xlog) show_legend <- F @@ -711,7 +711,7 @@ Plot.RT.Parameters.DataSetList <- function(dsList, f_min = NULL, f_max = NULL, p <- NULL if (show.CI) { p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', upper_attr = 'upper', p = p, show.legend = show_legend, scale.ylog = scale.ylog, subplot_attr = 'parId', scale.xlog = scale.xlog) @@ -735,7 +735,7 @@ Plot.FV.Parameters.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, show_legend <- T if (length(y_attrs) > 0) { p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, + type = 'line', legend_attr = 'ID', show.legend = show_legend, scale.ylog = scale.ylog, subplot_attr = 'parId', scale.xlog = scale.xlog) show_legend <- F @@ -744,7 +744,7 @@ Plot.FV.Parameters.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, p <- NULL if (show.CI) { p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', upper_attr = 'upper', p = p, show.legend = show_legend, scale.ylog = scale.ylog, subplot_attr = 'parId', scale.xlog = scale.xlog) @@ -843,7 +843,7 @@ Plot.Stats.Significance_Heatmap.DataSetList <- function(dsList, ftarget, alpha = bootstrap.size = 30, which = 'by_FV'){ if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || - length(get_algId(dsList)) < 2) + length(get_id(dsList)) < 2) return(NULL) p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) @@ -880,7 +880,7 @@ Plot.Stats.Significance_Graph.DataSetList <- function(dsList, ftarget, alpha = 0 stop("Package \"pkg\" needed for this function to work. Please install it.", call. = FALSE) } - if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || length(get_algId(dsList)) < 2) { + if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || length(get_id(dsList)) < 2) { return(NULL) } p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) @@ -902,13 +902,13 @@ Plot.Stats.Glicko2_Candlestick.DataSetList <- function(dsList, nr_rounds = 100, if (is.null(df)) { df <- glicko2_ranking(dsList, nr_rounds, which, target_dt = target_dt)$ratings - algIds <- df$Player$algId + Ids <- df$Player$ID } else{ - algIds <- df$algId + Ids <- df$ID } p <- IOH_plot_ly_default(title = "Glicko2-rating", - x.title = "Algorithm", + x.title = "ID", y.title = "Rating") df$Rating %<>% as.numeric df$Deviation %<>% as.numeric @@ -918,17 +918,17 @@ Plot.Stats.Glicko2_Candlestick.DataSetList <- function(dsList, nr_rounds = 100, close <- df$Rating - df$Deviation N <- length(df$Rating) - colors <- get_color_scheme(algIds) + colors <- get_color_scheme(Ids) if (length(colors != N)) { - colors <- get_color_scheme(get_algId(dsList)) + colors <- get_color_scheme(get_id(dsList)) } for (i in seq(N)) { # rgba_str <- paste0('rgba(', paste0(col2rgb(colors[i]), collapse = ','), ',0.52)') color <- list(line = list(color = colors[[i]])) - p %<>% add_trace(type = "candlestick", x = algIds[[i]], open = open[[i]], close = close[[i]], - high = high[[i]], low = low[[i]], legendgroup = algIds[[i]], - name = algIds[[i]], increasing = color, decreasing = color, + p %<>% add_trace(type = "candlestick", x = Ids[[i]], open = open[[i]], close = close[[i]], + high = high[[i]], low = low[[i]], legendgroup = Ids[[i]], + name = Ids[[i]], increasing = color, decreasing = color, hovertext = paste0(format(df$Rating[[i]], digits = 3), '+-', format(df$Deviation[[i]], digits = 3)), hoverinfo = "text") @@ -976,8 +976,8 @@ add_transparancy <- function(colors, percentage){ #' @param ... Additional parameters for the add_trace function #' #' @export -plot_general_data <- function(df, x_attr = 'algId', y_attr = 'vals', type = 'violin', - legend_attr = 'algId', scale.xlog = F, scale.ylog = F, +plot_general_data <- function(df, x_attr = 'ID', y_attr = 'vals', type = 'violin', + legend_attr = 'ID', scale.xlog = F, scale.ylog = F, scale.reverse = F, p = NULL, x_title = NULL, y_title = NULL, plot_title = NULL, upper_attr = NULL, lower_attr = NULL, subplot_attr = NULL, show.legend = F, diff --git a/R/stats.R b/R/stats.R index d27a8ce8..57b7c013 100644 --- a/R/stats.R +++ b/R/stats.R @@ -139,8 +139,8 @@ pairwise.test.DataSetList <- function(x, ftarget, bootstrap.size = 0, which = 'b if (which == 'by_FV') { dt <- get_RT_sample(x, ftarget, output = 'long') maxRT <- get_maxRT(x, output = 'long') - maxRT <- split(maxRT$maxRT, maxRT$algId) - s <- split(dt$RT, dt$algId) + maxRT <- split(maxRT$maxRT, maxRT$ID) + s <- split(dt$RT, dt$ID) } else if (which == 'by_RT') { dt <- get_FV_sample(x, ftarget, output = 'long') @@ -148,7 +148,7 @@ pairwise.test.DataSetList <- function(x, ftarget, bootstrap.size = 0, which = 'b if (bootstrap.size > 0) warning("Bootstrapping is currently not supported for fixed-budget statistics.") bootstrap.size = 0 - s <- split(dt$`f(x)`, dt$algId) + s <- split(dt$`f(x)`, dt$ID) } else stop("Unsupported argument 'which'. Available options are 'by_FV' and 'by_RT'") @@ -514,7 +514,7 @@ glicko2_ranking <- function(dsl, nr_rounds = 100, which = 'by_FV', target_dt = N stop("Package \"pkg\" needed for this function to work. Please install it.", call. = FALSE) } - req(length(get_algId(dsl)) > 1) + req(length(get_id(dsl)) > 1) if (!is.null(target_dt) && !('data.table' %in% class(target_dt))) { warning("Provided `target_dt` argument is not a data.table") @@ -544,7 +544,7 @@ glicko2_ranking <- function(dsl, nr_rounds = 100, which = 'by_FV', target_dt = N } funcs } - n_algs = length(get_algId(dsl)) + n_algs = length(get_id(dsl)) alg_names <- NULL for (k in seq(1,nr_rounds)) { for (dim in get_dims()) { @@ -776,8 +776,8 @@ convert_to_dsc_compliant <- function(dsList, targets = NULL, which = 'by_RT', targets <- get_target_dt(dsList, which) } - final_list <- lapply(get_algId(dsList), function(algname) { - dsl_sub <- subset(dsList, algId == algname) + final_list <- lapply(get_id(dsList), function(id) { + dsl_sub <- subset(dsList, ID == id) problems <- lapply(dsl_sub, function(ds) { target <- targets[funcId == attr(ds, 'funcId') & DIM == attr(ds, 'DIM'), 'target'] if (!is.numeric(target)) { #TODO: more robust solution @@ -820,7 +820,7 @@ convert_to_dsc_compliant <- function(dsList, targets = NULL, which = 'by_RT', return(list(name = paste0("F", attr(ds, 'funcId'), "_", attr(ds, 'DIM'),"D"), data = data)) }) - return(list(algorithm = algname, problems = problems)) + return(list(ID = id, problems = problems)) }) @@ -950,7 +950,7 @@ get_dsc_posthoc <- function(omni_res, nr_algs, nr_problems, base_algorithm = NUL #' #' Based on the contribution to the ECDF-curve of the VBS of the portfolio #' -#' @param alg The result from a call to `get_dsc_omnibus` +#' @param id The id for which to get the contribution #' @param perm The permutation of algorithms to which is being contributed #' @param j At which point in the permutation the contribution should be measured #' @param dt The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`) @@ -958,17 +958,17 @@ get_dsc_posthoc <- function(omni_res, nr_algs, nr_problems, base_algorithm = NUL #' @export #' @examples #' dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) -#' get_marg_contrib_ecdf(get_algId(dsl)[[1]], get_algId(dsl), 1, dt) -get_marg_contrib_ecdf <- function(alg, perm, j, dt) { +#' get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) +get_marg_contrib_ecdf <- function(id, perm, j, dt) { hit <- NULL #Bind to avoid notes - algs <- perm[0:j] - v_pre <- sum(dt[algId %in% algs, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - algs <- perm[0:(j - 1)] + ids <- perm[0:j] + v_pre <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + ids <- perm[0:(j - 1)] if (j == 1) { v_post <- 0 } else - v_post <- sum(dt[algId %in% algs, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + v_post <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) v_pre - v_post } @@ -990,17 +990,17 @@ get_marg_contrib_ecdf <- function(alg, perm, j, dt) { #' get_shapley_values(dsl, get_ECDF_targets(dsl)) get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, max_perm_size = 10, normalize = T){ hit <- NULL #Bind to avoid notes - algs_full <- get_algId(dsList) + ids_full <- get_id(dsList) dt <- generate_data.ECDF_raw(dsList, targets, scale_log = scale.log) - nr_players <- length(algs_full) + nr_players <- length(ids_full) - perms <- lapply(seq_len(nr_players * group_size), function(i) {sample(algs_full)}) + perms <- lapply(seq_len(nr_players * group_size), function(i) {sample(ids_full)}) max_val <- sum(dt[, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) ### For each algorithm, calculate shapley value - shapleys <- lapply(algs_full, function(alg) { + shapleys <- lapply(ids_full, function(id) { ### For each group of permutations temp <- mean(unlist( lapply(seq_len(max_perm_size), function(j) { @@ -1008,8 +1008,8 @@ get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, m ### mean(unlist( lapply(perm_sub, function(perm) { - perm <- replace(perm, c(j, which(perm == alg)), c(alg,perm[[j]])) - get_marg_contrib_ecdf(alg, perm, j, dt) + perm <- replace(perm, c(j, which(perm == id)), c(id,perm[[j]])) + get_marg_contrib_ecdf(id, perm, j, dt) }) )) @@ -1020,6 +1020,6 @@ get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, m temp }) - data.table(algId = algs_full, shapley = shapleys) + data.table(ID = ids_full, shapley = shapleys) } diff --git a/data/dsl.rda b/data/dsl.rda index 70d3473223556ced80e5fef7ab4f05b48e62e7ad..dcfd8838ddc39992b45b37cd7fcd820628bbbb33 100644 GIT binary patch delta 4112 zcmV+r5by83A>tttLRx4!F+o`-Q(3|tYA^t2D1VU>BY*v%01ET~00003oWAsVO{4-c z0{{SG*Z=?k00Trc01yBXQ%svlH1aZ<0000000000WM}{Y00000000000001hny0Ce zAoVo#LnA-~)Wip=h&1w=27np^5t9hW$PEAtnGaCN10Z4m05(w3kjOM-8ZcNpkxo# z1}04gfskk#7=}ThG|}ZBsvA)CJ*5Jqkc@ruCkG>^R~94M5^UIfBiwqJ#u0 zVAL`aK|*){&w&{*Qb4JzGht*VhHCkaY!!?Az8}{lJSasXi!qldR$DEtY@wFoMz+RO z#*?oBuYdbBoNi7NS4K`U>ou(`Db43v$;-wX46mGAyraMwI=sr_-rSr8UF}Ffr8Fh9I#NvE-=T{d+%?mbaJS0 z%74PLL#7G@K_Q5$&aA6x6oqPUeO&pJteX;W#gJOUwIyUnwLS+3Yb~}46+L4llerd1 ztX}}i)RNmHS~iBagto;MEff?jwG>j)sd=Y*IOeOOEHz6MD|)+4G1A#>CAJ9LRyxeY zoPeZI1&4~-f{|Tjg;pqv2NuCW;@qK7MSm33Y?(_H%1~|f87E05Zo;s4Nm3P~p9@1Y z@sOE?6iyk&YUV;H^LcP*F7v0c-I7YkXKBEQ!lx87D=naw(T$~+owC5v60%G}jwL|T z3d-9Sv0#H@!VHe7mjM&81Wx*RIx04J@fjExGYIp%i%Tw>D;Pznp|n zG&*F52^4y}xpR~aaY(qsY&4p2w5kS^Zqshtmsxk(sPycq3XJ}_#}MqSB(cu z<|C$%W8dH|Zo;C(b#XS=POQwW-hW}8`7@ex-LSltK;v{+^wrI0TAlmfb9KSaS(cPe zTdh+Swwr6h9s{3{qnD-&DkvJZyd6`&D2^rWxz9H?;)O=hK|NbL*_xx%bzZsN<|uL7 zoHg3bZh-36R2I^lrE|YCEd^BNlvGp}50^{P&j({%-wqZWwTCV-SBHz+3V)M>cQ>pT z9}e!Cu9_@dXq~L7#;CaCrg_g6WfTj8DMaLPW;%B%tVshy*|cLmgKD$mbz45h)fz8celXrovJZ z7PSZ28^0M55=dQ(KZKn#1~MVxVu2|Y5 zl}3Yt9Y`=J$22b@NWugNs9sBnad8q-LPaE9bF;n?o{GXPDF=7s7X;=Q9kGe(N3lJz zft;bDHO%F{ZkZ&oKKjpG!+`tyJ_(1!y@R401q}G^-;al}XMb`-*yWhXi3O9OfzTiz zc)-ITC? zY_3%@mrSM_O@Aa!-B}DVFv`O$86}oUWs#Od%LK~?St@0Lh72(zESttlkO{F+fRTks z5HL97$PKe});CSDI5U`)NOK7~Cg7^7knKUm2NWHEZjVY94S;F}k~D=WRaHYkGy_0r z84LyG#Tg7mh66cb#f*j~B^enMUNaPCEI7u`1h~M&$bZmYYBdyQH5rTzMq!-9)MQH; z7#Nl^Ga7m_p}3=l+>18M3nswSXtNj87;mJPF({TQOMeM~Qx#fKp$t`6DycRwz8+E^ zD=pe*cAx3PYcY%o9?NXBC78oXMwuq+ zl*qD(cYmc#8dPGUU3y{m%OkeHx4CRY$uHpzs#8j) z31wC!!Zs%JGc3%;ZL3;IYm9+rM;R-szXiVWUyZVb^wT6;qaoV^`qXHJ_hn?QXO%GQ zrK&~T2*j(yEz`-OopI0;QZ|vWG*9G+y%HJNqVodz5)^l!#L{Ov z^9LF61U1h6;5cV1&8s6x&o$B{O|h&Fpm|Z$93y}_lEz_}*(_&0c{B;t8brVu4yee~ zX*U`YYo0{!T##~4s!?E3V!+le79xrT6bdLB#wEuryeueDq{W=xwgj*?M*Oj`Sn2b( z4Syp^iD8LxVo_p^+2N#xRL<3?lQ9}bMHvi@MTLe31h`yjWZs$yGlMw9;G0F}Mu9~c zixx7I5MT{MK+agi&M_`9E*BY$R2YJTA-ybUIPokjVqj3jqay>sB#jmt0YgcyZJNeI zL71?yjA9H%gGr#?7Z{is85a{NkX|zlV}DTB85#_W%Z$cxurx{jh@K2uF(i75illqG zkUuvpSP@{bL?`AXpLW9-MUYY{6p-*BRGVa<5XeIih9Fi!Kp(sZz3~SIKOFjbvV) zPb!mM-$JvCLGoJ2qT^XRAz2Yvi99SS8W?ymprNKL6qN(OqdYHxn1c{g7z#?RC7udm z=2WU13!mrEM&S)u5a69T~baXr~!*Qe#>wDO#R{wUx}e*pt2sFn?^^ z=~WR?e28x)Qln)Rj9yx0Ikz_RhY0np)7N}r5Qm#qevhN|CeJIvo?Vqi1yvt1l?N$N zSe}l;ixe}|xEbggrM33v$5wh4VlE|!GCfjM6!u6Jh=8ygY7t0k&`}jpVnvFJI<;dI zSR)cE5J5(DX$C0;7BSLTs-mKz41W|AeF+)pEV-O-(54%EvJl%2 z86l}{PfnSNGuk5x(>Qsstr=RB_Rsp%4()09#XWLW@N z%r&FT!w0|AA?0@Ku1h;Gl0}dw8L>JY5)j)V#(LgO028)`9@N=fE{Fuq$BE(mloXs8 zZzAys6PdzJ2O%LfH{P2aciKoy+4JF7$q^V-1@$C^;?+u!oJtAGMt?9SFi@PZ3}7*W zBFLnR6pm0Rtca+Bh^%7UL{%b+K|y4Pr-E93F%R26ATal`!k%dqT?BON8npgHxjUQt z(_gMJ&hOC{j_8sy-k2A=D2XUL#B))$E=lbpKR@wfcyA8QCrhD$Sx@##IVg?h{yapN zy14quFC5GE;Z5L$c7LbF!)V`_I>^pgZbGGs=OYdOkI~bsHVDT4=Qe6h~v!=a94&Ge42RU4(Ek)Pi=UD>cp*msA7Uxq_wDmTNOi0Csl zsV9mLMtd!R-w14HkCs~eoC9Ywk33NhaXZ7L9bYP|S>p(4-+%DvbtVoIYA`h@_Xd04 zt~E&wk-q4C(BN%%OdBZHow;!7SRy?0 zwS<4|TPOB};3adCTs0A_^wl4}9|%WvJCa;ljzpE>4m_78O%6>bKl=8+-f7os z%Z|47BFyD&yXjwJ#(iD|v}Jwf~V4BY(UA0Bg_y0000EbopF00x2@01yBXO*Axw{U*{TqX~%9Mol)R0UnW)WSC4w3FMxHFa%^|GB6Ox z9*AOKjRu-v6A7l8VKf+^nwn%l0000q000000000000000000000iXaN(U54+&@|IP zWHbXofqwu7ngO5y!3_)mfuWN?7=fmmX`>TB0|Wp8ph={NNQ6wMlOUKzfB~QY8Z-cC zXfy$#pazWq0MVcT10VnlhJXM8N>UUkdY-4LqbI5AdYUvE00HV6Xbm2xs2X~M)bdY6 zGy$N^Kx8x?pfnl=f$9Kc14oj6s69i}@j$60CSoBIMuwRRHklenqIyjaQK6$v8Utz> z13&{n0iXZ?02&Pq8f{Gt9;Sc*GzYW$qvCBXe{~;6>pZ`(jp~E`PoLphjm#&7*IGlTXF*Y-=Z%r>d>}y-@=Li^(GM`yMp32>q4Z3EwU~kCW@G^AffVH-RZ>m0Cih|Z zj3=cT{|G~Yg3eeRuu#S>Fvrw-e%`gwf6Af5D+lq}S$g)LZ_y$&_mf0H7v^BgXwkWA+prLK3qL!6Q z%{$S@HC+*5s#u|0)!J!}mdj}^utwUk)@CH+1tNefJXX{cit98gu|!Zfwh9jxf8`2- zD5j%i%2=*agKwqDI!QBj6@$7;kgXi}iW#Sjgv=&?qO7Pb=H@m9 zNGS-QWwH@SVB$<-%EP$~W?0bpf83#o6T5=qBX55C!feWIa^st@uWDyunrd_8-EP)x zDxmhgcCu(`WlB+P&y39oqsMi*von4CBjj~>&~(mXI%x(y{sQLgDlA7=6K!z(dmhaI`YU98sV4y|QDZ7IrEJM%KoRZdw&ML}Ts zG`$?~b~VlL;bF^Ia^n?ve|WvHsW>-tdckqGI;Kg=eq3yG^M7q9D+1y;-&7<)2FH9apPcJ?+a-FpE5*lFwq&Q(Q z3|0n`3n{}pgFAMxG>aa2z`>j+2;mJfFh8PGP82|zJtP#J0*HY;_%j{@JQV4sxD#r@ z$lY}Ye>7;oq62~TBt!rV(4}t%&DWw^LD1soGX%5eMitu9?%5wJ{^;;fzAcJ$v-~@=0 zZbU{!qoi2~V~{Go^WLN=$i!74jm5QbzRAQ=Ze~1VH6GWfDW(H!g5htJ` z4(U$uwB2@**+xUJsHGJWilek3J!Kp-Bv@rppy7vD3=A^S3&@f%fdVLAONnuD62%n6 zYSrtt#M|MUCYC|azJ_WGwJ<#yk?9AlJ&A#dLriasOMJajNnm~bpKQZ`{0lx6hsHgF zqHq*C_B%b3e>)&nR!^Tus!T{?P@%FeUr{BJUS3$1ELk>9B@tvtn>>oCQRbD{D%Udu zGap0$z)xfrS{^`IG3rSG4#1K?r=SrLV!;GZQ1FO}kdg$1)@rKIuWzp0YAIH_thgHd z2LpuPadrJ(OI1nClc?@{4$}6)PSYm4PT+*^k{}2qe=y3!EEy%1NoA3i36>F-46>xl z0}N7b&DVO>N~ycS0V4`R1_vN@Lv-Yu2PEAWNpV(UIHh#$@XXAkhz^t;C^#X>9iUh? zfvg&c)D)#vRSh804I!jvFc+5;W-=BS3`-U)W->A;%*>+kprb)y$T($-%uI~S%|@b) z2BSfNf2hzgGBp{J#%2aZjLioJmQ#rgIF?G zL<~+S&_|3<6m;{o$v3q$X_{Ri*RH) z0RFWaA$^%yD_P;pJ2`5RcES{iC-x>!5R~i=NY*sN=Mwd~;6a%m*;~Wflb%EDeI;Vkn?dK%#-LWL$I0!or0bjF_AD z(3S?+;FdOv9cC^$sA@7SGA=AiEK$mxfBHyOOj`AlXd_U}qcNGRu&}`4mkW(d+w(~# zAjD*F&BF6DNTQ7eiy6s~Fb1)tVpzz;WL#ieE;Jb^G6e=>epu3Uu`DcPU{J`TGXuRO zjW!xVLs7mrje{|y&{$Z?G6q9QsM2hg85kLvmlHXdUNjAZSlge+*zxXv{-RLU~NaOkq5Z@3_ui4}Nb`)a9xC3Edc^ zRXpQZd44NAhfMJ@2*5l^Gq>7(Mu*qkc-?s~i+9=6%0tdDQm26!knx#Nw6n}WrL>vaqRA zvZYtdwm`sv@HKh;JoQqh%G0URq^2w>I;K2=%Sg?S4H7 zL)q1zqv$;`v&ZnKlVwpsRY%IDLCRECC!nyR#SHY$26@)0ZGFkH)t+lH7ZSu79;qq{ zdqfJvKv)g62&6S=D2k}DBE?0WTCs{O5s4LuAfr08gA{^`80jokQBhF_e+mk|X>1=X zJ;CyyE%T|Q+jQ9r=1|k1Ra|SPrApRYWp9}In&X6`#2mPo>7x|kUYv5<(pefC(1Y*dd zs>LFTprEQniz1WJp;&m#f2;)}F;NtNDI&#@K@?R%V2H3$MIwRV>UEvX!9u#gvqh>m zO=w{FT*_$zxpTv+WU`~()+Ul&T*EWM&1o#wRAQkK0bvvvibuY|O2P{OQDVX=09h1J zHx){dPzzpe4lC|Bc>6uKt}F>ybGeorjDjRk08%LkEQCGTgsk^ge?p)gL&BAT!vY+m z)l!EPxnZ5T&mX+1dK9O0TKTlFrN|kz@(RY^F-RA~2~M zF72^i37zVXNo3JWAXBofo{ ziTh`_dX&@4&gcpye>)W0*$}Tpd@MLee+P9)vn*?QU=6b;79S7K%nb>zne8 zqF%$eJL3E^9B8QE4vs?xx=G*z&Ko1T5ZTcmAhrCr1mI|L9sq|#ox#)&x0_Y4|H2x! z_>?)u4aW{Je>JG~WzW+ZET-95zbe%tZjGj^XhrtTqDzn?$x9l9O_d^M=+NO#_ID}b)d?*Sg@E~0$E%cbrV5$30YS4>1X=2MHKRd zf;nZ*O3gSn@s!!U Qs{i8dNT&)C6_Ub}0B{ydxBvhE diff --git a/data/dsl_large.rda b/data/dsl_large.rda index 4266add51b18b888c45f4f4be8a636df4dbe071a..3d3339ab37e2b5bd9aa67b13206d247f7ccc52ed 100644 GIT binary patch delta 11629 zcmXwe2UJr{^EbWMh;$H0fQX2I5RoP&l_G>10g)0QG(mdP&|9bp9YIOx3KE(iqS8AO znglHLA}E3=pguky&wKvgoU?a!X7}9L-8=I;vvU`QuRR*M7MsYJ$dt&O$dbsK$d<^S z$dSmI$d$;Q$kWO@-pT=mhi}DZOuR0fZRG{xgWu}0a^*2ZFP|`|+ket$f6RchwPaur zk2rnAu+US%z`y`LYYc)pJH8C^Zyqyn{U`m8&Sw2bjP3tD{%eQ)7h(>UW6!#0#FWZd zGlgZWpOM+m_5(Bg`2MzjaB0jVT{@Yv8muX{tnB-Hox02X44+&dbZIu?WPPGYiBgSKY9gF*q~C7OUY^ zFAa_~V2gV!&c0*7FeT2wz;CJk-y;4;EG0-QPVvlhGnQuq48sfz8Vrm%%x2l>$70Op z_}Vk2IOAs=TTz0+|E4op*wkB<;QrTVHrMzcXJDB6KmLD=%Y?{L%$8A+VB{02ZYJ(i ztY@Jh=Vv`CVQqolxq&7}s+U@OtLp)0dvQ}y32rHMHFXsfpochaW>&ho{INQ}<I(QU)@c8?u zg?|<8cK^9Le5?U*inc<+u1GvBg?RE;eG2|}iSkG4LEobnfAHD8M4i3bQUJy0#V?c0 zMs+A%NsI1?Zt;_47Qpp{E#gJY%9u?X*9GUiZg-ZtC~5jD<>y?w#HG=SYo%BXLZ0L< zx% z$n+mwTq}5Y3Fhvoz?^UIvDk@j-%t+<3cnn*(A8<@}uFg#aKU~#pgPLLh^}Tc#*7o71Buj4pIC15tv6admRlfdy{_&){PPVY%miFCb z>UGkz%w9m&^S!A zbUuU5V@zmD+upC46L{|+yr?k`?ROwZe?a2??vV! zdrxYDY*7AUaLH_3*29g7H>xj~-TXNiD6U-Oe(rRomPwZOV6m^mh9OpOX zfX&x*n*$CR3H$ic<7eP~j`d^xqnm2>yQ!^Tqn*_FlAZkS zFg4>V>sJ1juVW+a!H))h9XlQUUp=)C!f@>mMS&_c4G5QJ(@9qQj zmRxt|qYbrR69UA6s+R*WS(Hln<*nBuZws~GHgL|k3 z6Z>_P%BWY}9fw~dmrx&7{%q8rJg^X7`Um-SY4ZCuyyy*%g$q2A^jFz8HZ2k~`Gdc_ z82NcUNh12s7+znbJ7*@xH%$$TyF;5gm6dRq36cL%nAH2z2z;Um4eDB&`90|y&tI(L zyX5qDcx^}j#Yl`@IUCQ;kE!{P&%aNWP-$c7dgk|&?>w@e8xor zQtTpF>-YpbckuV;X{@gCQqj!&wf;qKV3Xxf{(L(*-wW@WP*0h8OPV>}QxA^#>wdGI zQa0;ycqDo`^oF~XO-I+rNv9Wm!K?e3B5HXkG)eR18G{)0)lKpxP(`$<3x?b<)%W) zt@}B{ucWWxyP|JTy*S>AC=UaDT}!gKedMoI2+_S$Nb1h=x9EvZ`=y7s|NG}g{L+;O zWBA`!T9MtK@(Z$C4x*nC@f*9pcfQ{@lMeZq8m9Z}70<6n`GpASD?IR;!l@?*mEW(~ zTlDBVF8!SSZn6k^CrX3c4)@uE~Vn>{?8pIOYFwXpX-+{bL@)s z-I{Jm&i5@lpE>)+dNW_SdSIN0W%-v$TGIK#h=t-U<#bGH*GbEW{IL6kcgrri#&*HN zphnGQ&VjrzEtw?T{BDb~W3@*zsi`eih)@%Leoh#Mr$-kZJ3C#55*a`5(BmqPzDnNv zW5glZ-1uC|Z!My0=!Zz+{)c}(k3I|3|H-JbPpKt;mskzUZ+N~s@9YTA%y4@)+^|H_ zI4D}=+FW1U_;CAPaE+;djG=-oO4ZC~sgNglf%Ey@9~@n311~}|kBCOzxdt-~wHA>R zF8*;X4)~ngK|{|-@LhZNlxA?V9P%QF;g+l#(c}$eM`NTSJ~t+>`wp9()uFC0e|&2&5Q?odvacFT#s`Pw)7@PiAACC}b`Mc}X{_f30A||Qm-EMy!85UtIIa8%%th8%^02H> z)YR79+ArJP*8=!;VaZdMlq}5e^bmrT? zW9)kMwbHqC+IEtlm?Sx$y|KNv>2vE`WbuV3dq(472g2WkA?cUkg9j!n>77s!C%=L1 z_evpEN0Np2U-63I5iHg$?>tnuQYU}gJZNOKTYS89NiLt%phn0b=wzvhP4A3ee@8(r zhwHrM4_rLBzqq-uFeN;4Hz-}6jjr_M65Ou7+rq3SAiTISD?BD-F@Ue*C4x6R%-ER^ zjia#)0vTNkY9spJIk#l1r+9gpiwoCnn--`&0UX|Z4<K0-ZpDYY z_gw`OL@OXUPo$If!dNO)LO2h~QSSN!H0@6~~}oVmx3pK$u8Kam>%dfRePVVDqO^}^zs>+d5; zcF>Qm51YvDc)rP{tJMdeHgnV%Ze`Rq?lrut3=8uK&NT=K*Ukz(3f;t>$0nk&@oxbi zUznPNzs2uB&LO+bb^86;=4u9~mzch9Q*UQ1dE2beAE(tTQaDg?^Prm8%oVC=aqi`x z1A}yO{6N99<|gB+nA*9+RFtJ)dwPTK`{z%J+u4dCJ9yjgo<%0{y~=1ThFfiX{zs)o zV%PW~i{*J_J+e~nnOXC>%gWzmIhBx%d&CjAPBigR^{~5=%`J^f{$;yugFCsV6JFcBI!#yniA7WB&52$8LqKi( zK(6uN=L0;isnN_)29i7V$9n_JkI<9G@_pI%!>q1wGLXBWh(F^WRg&5C%XX!56G0bo zlMzjah%_eS%a;^0i~*K81Yu?2(4X{Qaf;USIiJ6@>|;{^Xz8(Xg;R`#3zuJ{)@7oM zM7rEi_zxsQtQ08e(k{SEmiryZ{M+%Jg&e-CUsCavu@84lwq{-&pP1dDm+1GR93Hg_ z9)hI0AAH#S7aBME&)b(dSAP!m-UvR9`9d-0cR{z=H}C5;os!K5dl{5P1rwb=A9a*` zQE1dMH{Gv@|81La?$dN#cIf>57JCs-bGrDiPj!7aQmp?R+>44UF1kCoTGPA`SbT#? z5hf&%gNLrYj^GSn^E6UVQ~D|N6L!D*m8pk?l)j0}rG%#sMvgD)-va4-gKp5k_-W9E z!P>TkJL-QIQh^l$iv!@A9yh=2p88?x_lOc7oSyLTO5&U9O*Ypu4z^Zrk3!q$%yjCg z8{OL)amZQXy#*b&b#T7clbtEea9YfpGNL3EinkbeTB>BEIwJRw4e)6AhFtyZ)9Wjv zn*!CKi|}NdPMdWVp>>;rJRek*1G(OSf?xFYSA!5*m(goOjZbO zf@fu#0t<11Ei1t$fb|UNIBZEtktj0K;F|>tIoxRJ%kIsSkLSD+AYRu^x@I@7H&FVI zKj|{+Mo1l{N-wi9j)l(Z)PyXXs$K;oVQB6bZ9hEBQt?{7`X8wkdqNuAeN{IxUUDGL zNnB69ic*8@XseZEmaTn+lsyUgVtf(+z{Bh^LPVy4h`={w zC{}_sc~TUNAtTdOa7_eO{CHkq4h?{gUtNcQNge=Ir%sYQt6-2hQA8ZnNs*rr;`QST z9}e3-s zr~meD=G%vpO%LxEKK~QEGxgx%%-#6i_46X_Hgif}ZioLpg?9dVCtyaqp|Tg#c-;Lu zL7VHP((_7Yos<~whg{Z&P%)AfBahb_i-T;=hqZ< z@G2#Yz2a&?j?-Sr+S_Lh#i5sB{;s!dU_Q6-6|8quYrEV(=-nt?i@dKp}HaT9os<%AYsVAK3GyQz%dMMr%53{&A}rviUV<(Q5N=uJx!C>kWD9x2L1! zbXbJaL#06L@A;}75k&2qR0G`xMSD9`mj_h`nG+;M-Fe&w&m|y=C?Z4hL%a({K@C)ua<|; z6)Z7b z%dfMixj!$E>KY6uh-;jkNdhoALkLS(;E_z#Kya)`bEH1tkdr+GegSS6()xziLn3sA zl@UIdHp?-phkdk^Pg$l-zLnG-;J8q9Hhaepe&B2oKEX3vCXzh|UzDze9Z~dx210Y) zS+ZFGVf<{p<`TNumJ~K*l2D|y3)SUK8gN)u8tc-zKFXhWf!|Q_62J=XG6~*t1mBn_kg*p@B3ndgg<?AhESl4uHfeR3Fj^=Pf6$rZfU7#X*N;E^tjzjP3oakd=7Ey zY$2N$3jAP|Vy}~~M6aV1c?Df5a2{bGi!pnynC|LUF*G+Gf>QJlLc>SFG!T54l9Jm; z87Hg2Cvs_WCSZIDG?y=E1;1e*JX2f)BfZ96=ceki z(mOy9*7BQ$ujO`(v?Pxes06+tn*y9VgV_iY5Nsfx1fM7Zk>KWlKyrVHGMWNS?oI8K zluLd6dJ*Ae1R|~%>v5#?%SqF=vEMMtepFZfd-7?5pj`JjXex<(7SC^G*CWdjbuejO zsnWA^%@qL=Ef{i6I8HdoD#$cd^d>yBe5PmJkG_c#*8&}afvlslvcTa^ge&PDeG?x@ zPk^F&<58g8pbd^rx8xyy;~vFaPyrCZD};7ahH9ZiKz_!EyfEBeCndQrHdi_^O0b1> zkW&aH2HFc8N$5CIM$!9HIA|ag&_l>b*@>9Kp1=+*RfVVV7V2>`rOEcfvIP6si>ui)3lm>BG35K1)r4v0^fkzaABi|HJ38V-0;xh_W&~&(IA(*rXTn1N2 za|&)^D-u}w1QXt=mhQDR!OQ^>TWZZ-cvs*6f zz!vdxgWM^utGZdEIy}Wvd1z?rAt;cD;y#OTHV#b*Ra9S+xNI{DlqJC>2Spq9isrN8HIq1+lAe2Ca5Br6nwgYuEB7iA8XKn$~<9o}RngM_#iMs3_lW8i(pd^3>ikAo* z(Ff_Z=zj{5lXai%WSbm5b1QdH0{-z4a5gZi6vA)Rx8T%CQQ#?34FcxYuHRL0T6XcH zQ(L^aKqCokBdRWxqzqwY9JMc*Ph1NfWGbqPp+O&N1=5>0C&eY?l;Oxu=?4SVZZ3#i z`V_5qaO@#)0rQhpP!ycfk$q1H0WH?yO-PB~s$1@H8_HdGGOkd}C2rxNEhC=Gpr8CMC#S zr~~dvNNIwNNq;-e-3RtGn~7?K>1d(eV=3bm&B)>YNMdBEGOm-t?g7AVFE3L`Y@^DN z1=^~TAd&|s0H`oHCZ@-mhYnWs$l?~)LM@Z~3qad`loUP@NEn9d&PvFo(P(o082om~ z*=yqNA)a+)ha}4De>={J%<;?2X3-srTaE&7bC*}Nzz8MkxG7qT=R~OlBx(ql+ma-v z;WTTj`qCI|Pd2!6`ew|?9;9g@uw!Q3l3q^ix9ow$PF zDb|Ac(I`+NZlupuwuKR%0YF3Zuym-?NbeArDz1lUE~)G^Lh|I51k;*E$Z2ONPDx?s z$-_`Qgzz|;s9Zlv90g)d;Na4=`p) zT_E)o?U9n$CIHy?Xt#AVwS|J<1Z@)p-4z7`S6VujL!5eA$b+k>j8u9bv{Va2g{N!X zRR&X`CCb_q4Ro za6S>x3MqLKaF&Zsa;f8}?jGX>5R^MB0EZ=;ow*W~m{|x~QQ$2CQ|0gjS!dZSY@98A zkRw}qW}MVRZ%LjUD@juTf5ZF~Q;9^xu27OWRnc*LaBfj;2!^_j%P0ixp%Ylgh}T5H zP8vSiXNW-@2iJN@!L%c*dw|GW@y4>0AjzsA>3pp_zZPidIzOxG` z;vUHhmI*g20pZCgnGuCEh$|Y2<4OmQ2&-_5L;O04gKh*491WCxiT};7evZBJ&NqnI zBM~k>U*xsV|Abp;!Vc0)V9UQlh>yJDE(D~i4CX{SB#+uJuk*)c*Ub&q>b*LDAHul0 z0jayKl~quslJ!jRUVY6@5dBcyc6O$rfqdJ}FVsP$5ic03Ve)y!UZ0raXnEV;CIwQj zQe6?M*(N_bs!}bt>rYl`w4!PnYr6R;Pg@mhn3F9>+cb%(-!s?W-nI%1Pw8#5^;-NS+-09&^_xQ{1%K4!EXjMYlx?E+<#_$WlAE`je(`-&ETUPL zO4z5R4Tg9OHaxmhH<(g49IT#ptnauF!y7;r8Q|l2d08jJv6a6pM31W%o-1g*V#I^`cPBXMbtl$HrV(rNos~W@MtieYV(j$$ z(*{!NojyYwb=RuqiK}VXlujSBo`3E~V{meyG3c-dvZOT_J9CaIta4i42sPpR=-+FC ztc-CAJAHrugkk&g1?68NQ{SSyy&Ml3uQ9A0@KQZo2FwXU9%3&NL+oZ5fj)I#vyr|( zyK^XRxd35fwmOg6R(}(8X17|~nCK!Iat*S$5!C4G9@O4w*IqEYXh$2geCC+GW^@yC zFWAp zkV3-+-cJ0-#w6q-P{EW*l$I_xKn1Z3k81d*XG~E2CfsPXvv$UA>HQ88;0j-12#tbS z9;8cu81ZL%8T7(-SnT0h)`3vH^W^sPH9+wB;q+>B|&cwV9Bg z!zBr{E2J``pg|B_=Qb@~05*21BxcIW3;25D$lf4Y`d}uN3OirP)gXMysi9M0F+GmF zFfW?<6l3nl=6_x+_p+6z13sPh483djB%L7IK%7@dZyNE3XX0jWwPx09Qhk&tb{WKl zdO6Wqe`@VwMSZ40?c-K#8@zIkC0go}Y_^b>fykGQGNB~ziN zzHU!#{HauPtJ-#|jg1i9ryUDP^LJ!+#njWtOHfBtJ7i&Y%mF`8XY@S2CTt15Fh}uc zb+}C7^`JhrZQp3Lzx$&8#c4k~&1csem}#XFW1i)Mcy{rNi6^#3bMna?{}wg*QNTp&1|3Rujh3hwMijw{+fg98_FquE03Kn@VFA38Z zHVM+^3j1Vaox+LN3TC~dU1#%T>l63E>(hT8;aC1VB7PNerdprI^>7c)*+lN`Bm4Q9A5XDkn1KfRxcsQTqs5ZZ2b?s>S#jCcGvyauOiDSc(XXB0U4M?94)=CS86l!t#;OMd51IrxE$+ojs! zt}$Ewx3{02Uk--d=0SQ|y*NoApksyxL;vr|lGH-=aV|e#eiKyX95Ar_Exn|$ z@1NuMzxR{IR`CD4d(Zyl`+42_XEnn1r5MAr|JF~{tY26i>#xjs9{cZe6QX-sd`Lr` z?wmaDCR$kzO3z_AmNas{*>73rj?YH3f9km6J6+ck)nf83VS|4vMXB)_p=SuIb$_AFzCfR=1{>CQ zvR3m6VXpUXDKM-3IZmr^z8?FXnQ>)HKV?QK^mOjJ9_GkBP5Lc8B0T$th1p;0LH&U_ zM6;+f$A~=^?^U?VJRr>xVcd4XB3~Q#fG~9+zwH04$&Q0J8hq*57mXn$jx8K z23&lUkz}NQ*Z)A~Cts4o*NP~^RN#8AvCGQBLH_k1N{#)^<+*U+J9fhG<`Vta1DmMn z@;K}C;4M69NU%-1^6s_apQWF>_oqXpujPK-&kYxfSi2oIIg=dTGhjQbau?G1jMqAC zth3%$X^s9z6d?-DzkB!8n)oma@ucOo&Bov_3%h>;yo%pu?S#$-+FLDrs@8@FE}mM%ML}Lmyd}l|L&Z@qVBa2;{@9}e_q7O; zZZ_(W>auJ6aQOW9qWe{515 zkgSZh!K{j|?j(u7IrxP<-=AdYR^s9N1k3Hc@woXSb67ml1{_bgrKT4DH74c~BfuP# zFXtGFH|fJk7(dx`1*!aYJm2`W6K68ucHrOP9>KLEk1b!@_WqK?Y)9lmDuaXpkUW$U z)4a&eKIVrrSqh01*nSf~Ea5z)x@#slS7I2n_=6>J5rB$6uWDRJ!nvbA1*jd4%W|~% zaa8cBKM@3U7nlD4DG8vF_1rpT7;NF*8xX!0X6gCmTorlpc6gazq8tfe-V9@mj@ltS z@$ddZqV`8I%1Ri-6Xentg}Vmg5P)`?@ty1P=e$F#3cgHsIkhf^RqG4oShrnOcK*28 zEUqi2O&gOV^y?iO+Oauu(|yE$sph9dB$J-#sfZ^3Fov8oL9c_Q_nb!`S(FA=k`NrA=I3qE<=pLyw+`x z~K_u|$fZDiM9(w<%CMZFTLV@*P}sGLaJHKuEGqU_#Ff7klQWR5LXjL7S#2w{v*=49u{tBMf3^d>I(#6`R# z%A~_)El$*>Z$fY3*7H?h^`z&0E_`kMZhRL>x!G)7mfWN9@`D zyE}X8lz+$Ln{K1mo#XJ<9pcgJfLLM&GI}1ZIu3*Vb|1oQYwC3((&(<;<-eZ^Q3a35uV0m-Sy+c z<~QvW@?L1U>6YHx>f?92$FFPuiGw`BjE^dgpqScXm*?a|i&$jbmN4D}FBpwWZjkLZm6g4Xs;V8>+UC+%BpK#ue{D#6fu}HaM#ts zp2eF`B-V2(=m!RJ0+CMU&zbPNv&s!mk*%~;WM}&2X)x6~AOn;EiY4!n=-jsM6 zn#m#>-s>-JsD4p4{u@p+>(%_7A<_`fsQhL1_Np|NPuG&&*?kzNo5p$i)Ui3aKzQw} zudtG`pB>G{j_T+0Jj3U?3Bc^lc(RKa(YHB{%{%!jnYL0I^SiY~{~12mtXA|z0(9{R zEvsdO{7pu_K2P;eBksw_EV=#Ym}`fBdspR=kQ#bR_fqujT-l-uof@auAen2S_Pwte zJRJhzxM-%=3}u>|j%pxX&2#l0UL+7(eYs_+${|i|aPL0kAAe~aa9`?YV58(+SIn>I z$r5Y5Tt*oOAF+@!SG(h~*WP8om(N>Ub^2nNP@D{ceO27=Hk!YOgyn@BxQMi;K3iH4 z5Q1iNjW`rBCc*+lV-bH1lj~pg$DEubY}t=Ou6MpViou*3e?y4=Oj+iS)jw42{JUEJ R?@1T&VsB#d;6a|~{{uV5H-`WK delta 11558 zcmYki2T+s27B)=pEl3Fv0we)K6)?omTM`IOLXk*O1Vm6!x-<>F1VRZ&2O$9jDH@s} z7D5kAqM!m6x>!)EiuL2(`~Tm3JM+%jeRg)|Y&*|6yYDUL$yZ}16I0n!IZ`=Oxl*}P zc~W^(`BM2)1yTi5g*t^NJNcacx^E}u2r87!b_zp@E>BIk1&X#Ihu3R3T@@03fHClf zG&lAdI3VWm&9=1tPxgK`HrGTpHa0#3QQDS|aEt^SyE&WF5%B*c7#l|-TjCLL$UFMZ zI7ab*RRXp(Jc-}>d0&u4L>1XgHMYtZ9@>XJ{1&O>pE`34jqvA5Bx3o3PB~EH5X^#x z^6K<*wXIXQoTR2S6TMT%@N~p;iQtCkE;FpAbz>aO>RE`QoYMk!7L1;mnR!0D3-zC! zfmvQsY;Zj=gD24(*f?k7u!#u7jHm#(hP{Tpn#6|SZTgwd#x`SO&c?=*_+PcS%55lt zE)vgOhGp5%{^-u(20feDoK^*Rb&<)vyV(fg2Kq8-)@7ZNzcP^^SA- zf3s}=h5iEz|F8UirGPC3&_oJ$0DQ=F;M&h~5e68ii;bH=1MSPqTomS;Q34;$u(^t43IuZH5zDs`QcDDG{B_f66v!%W0q7R3m((;{THW|DXAv zG!eb9f|}Hd3I1?A?Dx%+4zu9ACuKJ#-dx{)^}|$1%!K$k?C{?f>#xX@pr@N= z(53CVykqJrLyZeB`$#>SGUoHC3@s2Atj;jSmUSBpFd&9G>`Xc&nM;YDM`*d%s*v-9YDhPU&>ck_y=D{~ zUiI4P&6_!?W%`?_>5OZUU!b1h-977F<1E4pX5zZrd~};@NUwcmWz~dNxAqq|sYa%R z)SC@lrg)S8O;3)R`U_VvuQxufH=(*Byxyb8A$ai@TzBqu6WBxAHI-aB7JabU8j5Oy zGC>}To^n2QizSkHug_Z#mRlT&ZV{g4Vxt0q9aQd~AmX=vv7iLm~HLj>6Y85(r3Y-sl9;24%Sn1*V%oTZF`9uM#!M4=g ze?)*kDgy3Ml1xo38zy;RmBc6AjwOlD2YH6i3y)ibOfsYN-Cba-To$fG0qZFB%{pa( zdrTAV*p)_!S@c={;TBzBpA7>DS6+JNayzm;L;WdNk#eV$~mImuBk|U$_2q4 zcBFV-Cd*FBzW8XfkpCa=iBsNSc&cCL0mq#WkcM&zFYjdm$wet` z{ho(!qbe^}J7rf5#3 zB_4A(HoxJjS9!CqrpK$OCLLXCCeK;8b^S|UC&>C4@!sx+URlJpiCKR7{nc-ckEB=U zOg=wz@EYC7jO{BJN910MJs^HKi1~Zh`ob>TL-tHlmB^)oUuWQDHzPj8kKgu2+CLyx zRYzgJ(_P|Z>P44LbC364{lNCo?Ns#Y$6IFcU)Ww=i&T0w_x0>IiDHW~cvwQb*{e^F z-oMa)2m0dhTNmGZ!tn}`?D@mEq3=4*&TpZlE$ZdI%foWP z34e33*q>+Hj16NN$L|KW9f!1avSw_{?h~KPFrN?obLp@<$16z9pJ7gH9zQ%&vEP~P zyd@OlX>+ed8gCc!<>#4qx{LOF(ivYHz_FzseJ|a@E<^X1ng!quG4lAZ`Ugv%pY{vR zIG#NT<(&PhSMW3_T0Zv7GVPU-wCC?|O*t3hrH`Hg-^!dj;lHAZ zRi&}jZ9>K-pZ_S3jffW=`VV6k-F-^7-|G;q&n^6WBE8g1?-9ZE)kRI;D*W)uXeHvt zt?1(u%V8@i^G`lmyygsBpuQPCzHaq; zgUMg$5ch84HTFL&Ue~>ick7<-p26RTJtlftoX->pMCkIW&jcepCCfQEgu|U`5PXiF zQ5TmFPsh9w{%YB2aQEk3x?5&H2gIGrcc=$?$)~IhU3ljFTlLGvpB9g8893gmKBFFn z*gwS4V%AQnUBT?#x}j-ZI(ju`S3*8yBPH85=bYBtb0*=00yC2tgb?Mv04Uh1-%Bm*`PwG6k!kp3uF5FS&T5v-R&#;tJl3Zs}8*e(THQjLuY6Y_mvlXZF z?A?k@*0!tI*!|V<^@r|bzrd@-^WOm{6T0g!C#-4jUgw#-Hj;U!X_>9E9gw#YQh|4U zuPd=Sn*R*EC#&94txuwNTvOQUp;&z;3L;6&RAQ5V?Z5_)t~KP&%$+Xb^{cp#Uw%FK z^8L7Lt-xL8xLb_A#98`djU#JyL;u41Y~ZUA^__#h@PO*u`l26QPtSQ))_4ExF5kC@Tj z*bcE#gRz}a(xuq;BR?RRwaEqfQR7pWT;0O1{9vJQh#4$~!j9-Kw+zZh%qDsmGDkvM z`)|{k&OuJkbJ#B&k>Va3|;E#}6%QLth6# zI^2qCVmH^rmn*L&=&3V;L1$ZEU+H=| za&X9%0ZS%j@~g<$X1Dl$IrI_Ik!%OeD$#{&fiNU0@LKd?D)96ZaRW*FkknH^shGaZkVJ})9$tde^cW;(-O$q!#Je(RD%I0WS zCpG3iFw|$h^P?4C^SS37k|Lvyv%@}vU%0P1+!gnMs11*THA!%$N9){Mt_6U-97o)8 zb~s}%0Am2&|2M3kiZKH>Wa{N_Hr%DZLiUis}oa73Q>cV7T z8k$D}?sjGCE)IG26BdZ@Rynz0jdb;OG4Ig%Y-W$^y4{DE^WB2n`BDPy)~=)W2vTil z3qj7dz~;6pmkZZR2c6Zfs>h9gTm@RYEV@r)G*l5I+IpIw!ZN4b9r4p~h^lwzLYBW6 zoyELq3n7!)gNxt8sZJr3JMEnpE;Y~Wjz>kuOoc~utTjB3i5e!Zpf2~d%Wj8fV>4BB zfr3I^Ru3E^UWD!x8OFs1&UBpu_WH@vk1KBtkTjG@ijD1>pZ8yTUX{3OC|~@3t^lY@ zIu;~Af7V)yKFtE&2@9NVLq_hdqMk?1Z|&txiO7`z#_7Ik{!LEpI=z+`k;8LM&r5-k z-k-=-Rn-HUZo)*J^upHkve)|gb2T;1{2`=f$S^d>jgO8Ygj?yA%4vC zyQ$?skhE{qqM0d23klLdIanz3WShG9s?gWGQYUTYJD&@&JE*QDxr%>c^s1S;6Qr?* zexZ{b({E&7nso?+HS#TDPIrB}Ps>^4w6BP~^Al>XY;j%+-mb}Ax^X$~nXIMvMm$}D z+gY8W#@9TdnL1%-)MIH&VT*TR`*acZ;j@kt;w)YVOB6DQ`-%o)~hu(uP)Q-euStapJ&< zk3XbC^KU!OMeR?ditqI5H*t=IP4ugLz1?KcPigNr0-nFr^X^hk-`?U%nCHD}rm^`* zmy`61haxtqj7nxto_$rDh6R`E>)QL(I{ zI^TZSt$y`-<@L3g?I%KwQ8&$c&mdh>&r>icA!XAazssAQ@4e5^D77rjuLtVYkKa}b zy$z}Bq`^CzUES=RS>?7s{H0q`D2abcugu0{RewGWh##C#P%%yXV?g}2Bf^|7+Tl)` zCwkv$PGsua7i~F{Bq%_CiY-o1g=b(OO^_o^QVfq#lEB+asHX`4B;d{x%KYJ2kpZT@ zzK%}*kNzD|F={j;%QIrZU@*=feGJCEP@AQt<-rUQ5DINdNJu)$Nof)`>Aoa5ecpF?Tod1( zD}jeq6~XiKedKuTDQFz5K%NA{THGT9FDIvwV42j7Y`%M5Olu;)FhCiot!?F-g~UU- zC(87k@Y1k(r7)^pxt;fh!x;=-P%gLtEC4D2pAfjoE33V+nyP)0hINyWy$+Kd;(gwc6*){Yi!P35@ zWsx_I1Le2nYY)A~aU_Jw<}9ByJ_;`!&} zQ(h<4G1Bgex5{r2mwt-`xy8i6o*ZBJ{e4XVk+Ly`+|+)2C-t0xi&)Hk?|+Z++C-0~ zo8_wSz3*l%ZoP{zgk9^xP5o^>bhNyNy;R)zC+{;IZxfa%ZLTtPMSOTm`g2&HuU%fw z^T$;uFV!Rqttp8Wwl36F+qoSVCz>D^L%QcrR*W}X`&UJ@x?^`c(suu*ZN2w{u-W1D z!)ot;)l)y-?!2gbA9Zom5$(u1XCz``R`c_2bH+X4ZzsQWe>S&5>5piwTpUjpx$)8Q z`r~Qql!Q~7+?R8rSQp}vHKJ#};1$n&q)gP~^gr2bs!Se(TU|=C;5xc15I5H+vKb6s zpV)nK4wjlqcraq{z=k9LZUWDV81u9jzt5dqBu4(8mY1})CS2D#73Ff!Wh#8?cGJO} z*;>YZbv^~94kjrAZyvVjNvtB>W>s&wkm~=b!wr$j% zgq?lA-#h(>5xhR3p}Oh1egfYbM1+WhL7+Pi2si@l5A{nCI-f%r2n`KA0#ECVsE_E9 z{0L0lmdmn2ms=V?MdU^B=9mWDE@vBO)Ay5eS3T1*+`vY>$0Lz^g-v#wE0LpL)?v84 z18t0pc&~lq8q4I`t$pIxL*9+o)o5O7Ov}RJbKZPWiYdetq663w&9qvPBEo4I{4iO) zK>EOn*=1`?j=U_akVd10$@AE16Y%ZBS%^S-NL3Qxr<$6|T_pVJThuo}5htp@QkYBhw%QP$nXD9y=kUhxc8> z`I3+XmY;H3b{UMCE2kYAOjibJ61-T{5we^np=NBE#w~-yrSL+8k2}vuD&vcZ)!2cG;yi5(Y(ZR8 z*ns59=;7vBAi&WyKp9+BDU5rC*r(80ID?{u0b7@95_BLoyF$EWK<#MwHr@}fipC{d zV|i>~yT%jc2rt%WL>X;b z>L<&~KyXmi3rrxCn*>b;A_=>&10zH<&W|Tg@BM`=BN#P1nIx)bJT_A+pV#uZ?ZN>n(sV2rpHt|u!5gfDlf2ub+# z`VLhB;Y?U)VTOpxByW(*wef>IIkI{L9c(582!(~_)ec#`)ec^sAMi^x z87fd2$%70P;ThHissYnap)|jYWE^~6p3iIZRq!qI(bRYz1oXLDIZ9ERk0kq?cj;I5P z$7Vs07?5$UJoo}292uOOC5}fM!ZIV#MnqI}WazwglSRZuc9x~_*dAm!rwmTP;vn!s zhP06(yl|4fw1+T6;V}}CxRf-$yfW+eRHbbQZ&wDc{LbA(`D&4Ln{DeJGGCmCEFzFa z?f(umu>+*={)b`dg8|f&Ao7SqCDMx?k#%5h`?97M2AIre|QutxauSzLAlcm~Tz2Q4@y8(%jg|#(=5Q+(Eun?ns|_q*d?S-VTJ(0%AbM*Atlh6 z)Ra{1I3OanN(upJ&w^vKq+#UzSDS@1OVNSb)*?U%4q_80Uo8j#!g2~=F(MOH;$G=E z0azeK4wg+x2#gQ7{&)>-IocF1T$LXO5E;SoT5MW3E*%h&!Qixvd%Y1@dl?;{HESq- z?6p^KfB$@kU%<2ku8>EV;=`~O1Zp0c;|w7XB9yvTe00jFSw{=sabc3c`*5 zWZMt*=RrGaX=&sVl=VIOX90)~UNj860?*CTi8pRAmwn;qSv3aLjWay)fpp%tB{~2myI=K*&gcCIe>WPZ&Bfc6QZ*cjg+Ryh)`C-`y$aKDsAwZ;&B!HDo;uyaf2ALEGm>4!6PV+ z$p%8LCSIWs{`2$7xw$eX$&O6evXox%@*pN#P7gMpjINM21^^B7xQJB}!4xC;3Ml~1 zSH{o~BCQ>bPvI5y$u2fo{tA6-o#J1r6Zkngw6KS1U1sH9P>{I?+dJaM2&~9BjLBj! z{8WUC2z#25J}8{ZxHt^!KrYsclY>(?94A0hh|od?CQB1HGjA;d(hmN7#I%CaP#TSj z$)wM09`(XOT**5Z2L8Zjdwuz<;-2EX9GEXQkF$RtCM9iR6_72B@MqQ10;VdE$l!xq zYv@IRBX46cEk{PvpAgC@#PSNuA@F{_>)En!7A=j5Ah1+&$~AEVxqt#GNG1b(g9>h# zltBa*b}-O+KJiv>tsmx8QT(?x<4TOvfQg&%=amK6_q z!;f~LpRSC2r4b@@IeAnFBxL|XXh!2f;;}`6I98$RMQf2fUO%fei5NJAWF(&-yxcyD z$)Wgc_Ob@-DL9s|LmCsHJe@ui9JoyD2w>_2Vnn31LoJ81fe0M!UT?|_$}3NixDTDl z2E2}PY|4Fi`L*ca_o{c$$%Ko~jmAZOwHTkXxF0*R+_Pr1cFnq0ZN|GjiZ>=^^|3*& zPo%H{|G<&o5E?vB!)C!)h3NxU0UTr?jG6^!wrBPhrsYcbuu@>hKuN?;ZNwmjPS1}VgKtn`euk;Bi zI8!_@n*tf^pwZIC;hRY7QPErlP+l%p@NQ(4MNV`)#X5M4e?dFBioM;$x+bGm9Jh63 z(J;~`8N32;#=u&oUg-P)4i!Alz&c222KQ1%n3_IR2caTd;Sr#RZ(pl~F*=UorXzAp z1Hy^Di8BNSV09su5@Uo6ou7}u!-m0v;6df5@a-dW!%z{>Q5s|T@nR>WjL87XS`|sh zr#BqC$}N~qw#Ev9^azYX?y+7h&3||ZjuZD{%n+8_Z5Vz>-U8Z&A*UA^gCNjpsu!^$ zv|83ty!%Bjc$419EJPRKU@|ZU5nXmdMn?&ck%0^|{IEwvL72p!7nX$;te^iON=UjU z>c0dXjnhcu)vuHLU{c~zPW{#80Qs8m6L)HU$$Br0d*PYr38#76-$F@W!}bk(=6Ihj zyZ7`Pw=~RYG-ZC&SBs3?k(RxaFzsn7W&7YkR9B~6eTBT1v?CxY()PmuvF*CDNO#SH zg(i^@S8s32kDo5tH5Yk%VO}hvy$x)K=7i?58ar^d{w!pcmHO-pTj+}**NmG@xPPE} zVui(l;muu{Q>RjP_WKr8S2hzshJS7#&w}q1?8KtH7Hq9A<|;j#S%l<5%SLa;Kf0yU2FTo-FXF zxM1-yYi&MSg}U3+O`8yxieNivSwCt~!asD`d{tW0?Oa)Rqv48&Z@{eyz!Um0^NY{>}Ghjte@+H=I+bclX0}IY;ZsKhhXx ziJr@PJ?Ax9^Xa`Q=#n6@H9xA;?bCLZW5cR{;eFETlYvuiO{%8dP4X>b?`P$gi*E5k z){7V3n$Hb_x+L$s?hWw_?`;bl>0bA|T)MEAHukFft(-uG=j3X!O59t;yHKlQPdQKF zCW8e^2+W#q>jLeJRRU7yFs+#4}>dw!f`v3rBVLL1yF`+j0QqImNB4#lIRYDOcP!aZ}R~ z!a*;QJ3X564^aXh_4=;&c5t()G}d73N(hcNG<%Q^}3 zP$e{y*iDz!m!;BhLF_78otL@A2t%dC8hE{t5m>qi1TpDGnk~ER>U>($Hc(5) znfo*~+an7?%@o4UDfyDSyN)rso9jHWEc;UUtIpF%>OcriM#}?2a~`hK*K|r}91*Ui zt>TMkEc7!RO_D6<=S$MCk@V0!-Rl|d&?U)!GkY!no9YJ!gmwjj5u&px6bXT-)uXVl zyM>#ONXbqNzE$=`U?)?SME20{aC+v6_aM~!c>;owcNV%4I>Gh#f{|+NbHQUF;tSn8 z1C(ZLb_gIu<<+8}i<%a(fJ_e`3>$X2)ScDk=K}314zIm%?pJwsb`yL-_A`-rsrQ+E>F|f8KBwWq@8g z=Q$iq!}{Fx)nlS@kEw=9V?_yh+r$sM2jLB$GCHTPnMFp%zBZM=_u!=Hn>tDDjl?1w z4s%N!vW_j0!Y2OdjylmG&wbu7^+~F6F05M8x(*z=m!S`fxOGnGf@d1%!l?O*KnyW- zDJ7#pBo}8@NlarLclq$6_D51?TQlVILSx+G@NTsk!fj?K{es5bOjMWjk6Y27{yv(r z?aP)29eK0-tN*lCd!6qZeTgypSYGnv*D3n<>R(6l+-u(Ergh?0_tA~v;#P*Y%Nrio z&uZ6IdkxSxBoa356bv~Ya}8>Ke`b+D+fLFsr@w6ul~ITQCXbk?8*lb-wHiNj5aZc$ zI9GfAW`E0gk`88SLweyU$HNzemkJ-tfJ}OpP%EHQGIkHairb>Mn&w0+a!lZn84kFQ zkL+f5>pmN?FDbAQ4H`EBOA_KNw$#H7%JWLD|4QC$%6?hyo~s}hZ@%s}w~In;Wr!wf zx$UHUpEL4%tT3$5mP@*>`YJTH`+g{@9qV+#A*>Fc7it{Y8*FY|Q5p!<(sK8Z7{ zjlH|}koHZaC?21-nsYNI#UwEnIwlIbM%(CeB!L%MJIZesiD!4pvSKpY!d^%4R=B*H z-FT3v_etbjoPr1Vui;Ja@yh)N$uCcjN8X7Gn^8cU0?+gPGMk;Xo=EMjeT~*Oy=Oc+ zNw$QJ#3mThra^A=BI}NgJqp<_lTDwj%=Mg#x~g2R)IZ5;zFd|KrN1Zuz>*N`&_Fgp zXK{_dDTnU*x5RxD*7$XO2Y1>V;p5=jsBPYW+h<$`MFR}0IM+rOT-Q`nhgUPsKH^&+ zyPi0hVU~m{nYnV7WRlX~>^*zjMwct$o{icrL`jz?D)Nf3yH~v8()#`T2VRZe)Z=6; zU2d6}fv~f&gJF-QC$B~vm}$(#o-KLvPGGxn-_JXGdNrLGTRZGFr+o$4+xEv%ReRXE z$sPRe^D^kXxLfZ-+sdM~a8ujqEv1`Z6Uvsq%OW#;)GRKRh}aJB#;={0%J+w4D>>H* z25~DM#+jdh+te8isa(*izr5?GJ5)TAc_}$f?#h;Ey;w7BSo>%%KUA+kBP7mBqKB=2Z?il1;FZ#yghE*_YC zLTpsp9{z%t^31#qrYL_(~SDFYR zoPm)Eb+EKIsU)%Pn~II8PNYaZH{VIy56b@4o+G#8iER2oign2Y7U!WsXiAON`5WbX z;OrTzT7C{tsn{9L%g;7I&$*%k^1Zqru`3}{DL!fTVw?U8z}ahaBc@5IN>U+75=w*W zbb!Y5vW>blO93@Wi;L&5GRnUc`PBigGJWXvz>e&;#R?_9x}n100`s$Nk;8M{Z-)!( zhho{4N=_^Z1Bl0!j|D|K7jI2R_Xc%-w7)Kt@6vQa^K9EAe?$x=!djDboG*STxkad3 zx?xs-Ur9q`URV{qKWiuC%C)?eSWufO+~S)5?HU^1cwCDd^?+?RG?oPcVs}O@M#VjR zPdf29*r8T9Ko_|TM$HSfuo2bO=asdj80*7I$sX!bZp6pz{7Lb_2W60pNh4i%h4&~N z%r9*xUMY~29&N-lo^n|ELDhaJpy3dD*hNEC@5$}UeJ@_&EV=#_eQII0Jl<2c1S8M@ z+jedIcNMjlUbk3U(>)F;eguGBd||`qA1)3IqIG=qIrU8WTNA2d%hqYI0QFcKqLEMG2+t_DkOr zyxPC!rh;D!D#jMOKQSJNuTxEee0`$y_;=FM`{ZLZp{UWjRj-o~iu4c+GwfJ6Uv#`y z(Ltjp2FtA=i+UHYqGkaN{vqiolzpiv@Y@?>QJdj&e;bL&b* z|GrBl_8{7f&mM2Q(-!?4}CkT)BrH&)|cCM_I1rMI-yC~nhW$>wWz zDdt=cqI)o)L`lQup$M@ZUXHIZTo5tLQkeh~26_d{Iw?*mSkgN-L)!V5-l5!X;hPd~i*zSm zdAZ771kp8ux~iop9-JH$5|wcKm`T-z!E9J}^ks)h*>aMdZjvpYvd7QYhK|eyhalxn zALI^JOivyH9y}AiasJF=YxGFW7|?^g!a-NuDkTo55J}NyG)ub#i^z>fRa^D9?GfUUsEgvV#3HKyB5tyXWaoUHvw~LfZE*lH83qZ&6B@ zTm%PjToiufA$!Ed*uy-X%U`^ECj@kA>7lDzAzh9ed1B>0?XgtfFDEA>Q^HjEif!lSb yAIslXS6&Vw4SFB{xIUX;@q_c1)nUrpyyX#TvoDwK{>$!#R37~op>XDE;{O5elBT!- diff --git a/man/clean_DataSetList.Rd b/man/clean_DataSetList.Rd index b2518a86..05af812f 100644 --- a/man/clean_DataSetList.Rd +++ b/man/clean_DataSetList.Rd @@ -10,7 +10,7 @@ clean_DataSetList(dsList) \item{dsList}{The DataSetList object to clean} } \description{ -Concatenates all DataSets with the same algorith name, function id and dimension +Concatenates all DataSets with the same ID, function id and dimension } \examples{ clean_DataSetList(dsl) diff --git a/man/get_color_scheme.Rd b/man/get_color_scheme.Rd index 45f829be..dadf3e87 100644 --- a/man/get_color_scheme.Rd +++ b/man/get_color_scheme.Rd @@ -4,10 +4,10 @@ \alias{get_color_scheme} \title{Get colors according to the current colorScheme of the IOHanalyzer} \usage{ -get_color_scheme(algnames_in) +get_color_scheme(ids_in) } \arguments{ -\item{algnames_in}{List of algorithms for which to get colors} +\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get colors} } \description{ Get colors according to the current colorScheme of the IOHanalyzer diff --git a/man/get_color_scheme_dt.Rd b/man/get_color_scheme_dt.Rd index e7297103..6fb2c779 100644 --- a/man/get_color_scheme_dt.Rd +++ b/man/get_color_scheme_dt.Rd @@ -7,7 +7,7 @@ get_color_scheme_dt() } \value{ -data.table object with 3 columns: algnames, colors, linestyles +data.table object with 3 columns: ids, colors, linestyles } \description{ Get datatable of current color (and linestyle) scheme to file diff --git a/man/get_id.Rd b/man/get_id.Rd index f64f8922..bff5e0be 100644 --- a/man/get_id.Rd +++ b/man/get_id.Rd @@ -1,21 +1,33 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R +% Please edit documentation in R/DataSet.R, R/DataSetList.R \name{get_id} \alias{get_id} -\title{Get the unique identifiers for each DataSet in the provided DataSetList} +\alias{get_id.DataSet} +\alias{get_id.DataSetList} +\title{Get condensed overview of datasets} \usage{ -get_id(dsl) +get_id(ds, ...) + +\method{get_id}{DataSet}(ds, ...) + +\method{get_id}{DataSetList}(ds, ...) } \arguments{ -\item{dsl}{The DataSetList} +\item{ds}{The DataSetList} + +\item{...}{Arguments passed to other methods} } \value{ The list of unique identiefiers present in dsl } \description{ +Get the unique identifiers for each DataSet in the provided DataSetList +} +\details{ If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility } \examples{ get_id(dsl) +get_id(dsl[[1]]) } diff --git a/man/get_line_style.Rd b/man/get_line_style.Rd index fd4f4b83..beb5adbd 100644 --- a/man/get_line_style.Rd +++ b/man/get_line_style.Rd @@ -4,10 +4,10 @@ \alias{get_line_style} \title{Get line styles according to the current styleScheme of the IOHanalyzer} \usage{ -get_line_style(algnames_in) +get_line_style(ids_in) } \arguments{ -\item{algnames_in}{List of algorithms for which to get linestyles} +\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get linestyles} } \description{ Get line styles according to the current styleScheme of the IOHanalyzer diff --git a/man/get_marg_contrib_ecdf.Rd b/man/get_marg_contrib_ecdf.Rd index be2e725f..55b91023 100644 --- a/man/get_marg_contrib_ecdf.Rd +++ b/man/get_marg_contrib_ecdf.Rd @@ -4,10 +4,10 @@ \alias{get_marg_contrib_ecdf} \title{Get the marginal contribution of an algorithm to a portfolio} \usage{ -get_marg_contrib_ecdf(alg, perm, j, dt) +get_marg_contrib_ecdf(id, perm, j, dt) } \arguments{ -\item{alg}{The result from a call to `get_dsc_omnibus`} +\item{id}{The id for which to get the contribution} \item{perm}{The permutation of algorithms to which is being contributed} @@ -20,5 +20,5 @@ Based on the contribution to the ECDF-curve of the VBS of the portfolio } \examples{ dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) -get_marg_contrib_ecdf(get_algId(dsl)[[1]], get_algId(dsl), 1, dt) +get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) } diff --git a/man/plot_general_data.Rd b/man/plot_general_data.Rd index 965196d7..5a2d2b19 100644 --- a/man/plot_general_data.Rd +++ b/man/plot_general_data.Rd @@ -4,8 +4,8 @@ \alias{plot_general_data} \title{General function for plotting within IOHanalyzer} \usage{ -plot_general_data(df, x_attr = "algId", y_attr = "vals", - type = "violin", legend_attr = "algId", scale.xlog = F, +plot_general_data(df, x_attr = "ID", y_attr = "vals", + type = "violin", legend_attr = "ID", scale.xlog = F, scale.ylog = F, scale.reverse = F, p = NULL, x_title = NULL, y_title = NULL, plot_title = NULL, upper_attr = NULL, lower_attr = NULL, subplot_attr = NULL, show.legend = F, diff --git a/man/set_color_scheme.Rd b/man/set_color_scheme.Rd index cd81b978..51c1da0d 100644 --- a/man/set_color_scheme.Rd +++ b/man/set_color_scheme.Rd @@ -4,7 +4,7 @@ \alias{set_color_scheme} \title{Set the colorScheme of the IOHanalyzer plots} \usage{ -set_color_scheme(schemename, algnames, path = NULL) +set_color_scheme(schemename, ids, path = NULL) } \arguments{ \item{schemename}{Three default colorschemes are implemented: @@ -16,7 +16,7 @@ set_color_scheme(schemename, algnames, path = NULL) } And it is also possible to select "Custom", which allows uploading of a custom set of colors} -\item{algnames}{The names of the algorithms for which to set the colors} +\item{ids}{The names of the algorithms (or custom ids, see `change_id`) for which to set the colors} \item{path}{The path to the file containing the colors to use. Only used if schemename is "Custom"} diff --git a/tests/testthat/test_data_generation.R b/tests/testthat/test_data_generation.R index 1828aa23..66a98b67 100644 --- a/tests/testthat/test_data_generation.R +++ b/tests/testthat/test_data_generation.R @@ -3,12 +3,12 @@ context("Data generation for plotting") test_that("Single-function info",{ dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "target", "ERT", + expect_true(all(c("DIM", "funcId", "ID", "target", "ERT", "lower", "upper", "mean", "median") %in% colnames(dt))) dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "runtime", + expect_true(all(c("DIM", "funcId", "ID", "runtime", "lower", "upper", "mean", "median") %in% colnames(dt))) expect_false("ERT" %in% colnames(dt)) expect_error(generate_data.Single_Function(dsl)) @@ -17,12 +17,12 @@ test_that("Single-function info",{ test_that("PMF-data", { dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "target", "RT") %in% colnames(dt))) + expect_true(all(c("DIM", "funcId", "ID", "target", "RT") %in% colnames(dt))) expect_false("f(x)" %in% colnames(dt)) dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 100, which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "runtime", "f(x)") %in% colnames(dt))) + expect_true(all(c("DIM", "funcId", "ID", "runtime", "f(x)") %in% colnames(dt))) expect_false("RT" %in% colnames(dt)) expect_error(generate_data.PMF(dsl)) }) @@ -30,23 +30,23 @@ test_that("PMF-data", { test_that("Histogram-data", { dt <- generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') expect_true(is.data.table(dt)) - expect_true(all(c("x", "y", "width", "text", "algId") %in% colnames(dt))) + expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) dt <- generate_data.hist(subset(dsl, funcId == 1), target = 100, which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("x", "y", "width", "text", "algId") %in% colnames(dt))) + expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) expect_error(generate_data.hist(dsl)) }) test_that("ECDF-data (single function)", { dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) expect_error(generate_data.ECDF(dsl, c(10, 15, 16))) }) @@ -57,24 +57,24 @@ test_that("ECDF-data (multiple functions, auto-generated targets)", { dt <- generate_data.ECDF(dsl, targets) expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) dt <- generate_data.ECDF(dsl, c(1, 10, 100), which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) }) test_that("AUC-data", { dt <- generate_data.AUC(subset(dsl, funcId == 1), c(10, 13, 16)) expect_true(is.data.table(dt)) - expect_true(all(c("x", "auc", "algId") %in% colnames(dt))) + expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) dt <- generate_data.AUC(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("x", "auc", "algId") %in% colnames(dt))) + expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) dt <- generate_data.AUC(dsl, get_ECDF_targets(dsl)) @@ -86,12 +86,12 @@ test_that("AUC-data", { test_that("Parameter-data", { dt <- generate_data.Parameters(subset(dsl, funcId == 1)) expect_true(is.data.table(dt)) - expect_true(all(c("algId", "runtime", "parId", + expect_true(all(c("ID", "runtime", "parId", "lower", "upper", "mean", "median") %in% colnames(dt))) dt <- generate_data.Parameters(subset(dsl, funcId == 1), which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("algId", "target", "parId", + expect_true(all(c("ID", "target", "parId", "lower", "upper", "mean", "median") %in% colnames(dt))) expect_error(generate_data.Parameters(dsl)) @@ -104,12 +104,12 @@ test_that("Aggregated data for multiple functions / dimensions", { dt <- generate_data.Aggr(dsl, targets = targets) expect_true(is.data.table(dt)) - expect_true(all(c("algId", "target", "rank", "DIM", "funcId", "value", + expect_true(all(c("ID", "target", "rank", "DIM", "funcId", "value", "median") %in% colnames(dt))) dt <- generate_data.Aggr(dsl, which = 'by_FV') expect_true(is.data.table(dt)) - expect_true(all(c("algId", "runtime", "rank", "DIM", "funcId", "value", + expect_true(all(c("ID", "runtime", "rank", "DIM", "funcId", "value", "median") %in% colnames(dt))) expect_error(generate_data.Aggr(dsl, targets = c(12, 16))) From 2d5b983ae6c3eacf269d26c2f42be3e72ad5d4b2 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 22 Jul 2021 13:44:35 +0200 Subject: [PATCH 14/28] change algid to ID attribute in server-functions for GUI --- R/DataSetList.R | 16 ++++++-- R/IOHanalyzer.R | 1 + inst/shiny-server/server/ERTPlot.R | 20 +++++----- inst/shiny-server/server/ERT_aggr_dim_plot.R | 6 +-- inst/shiny-server/server/ERT_aggr_plot.R | 6 +-- inst/shiny-server/server/FCEPDF.R | 8 ++-- inst/shiny-server/server/FCEPlot.R | 20 +++++----- inst/shiny-server/server/FCE_ECDF.R | 6 +-- inst/shiny-server/server/FCE_aggr_plot.R | 8 ++-- inst/shiny-server/server/FV_DSCinterface.R | 8 ++-- inst/shiny-server/server/FV_PAR.R | 2 +- inst/shiny-server/server/RTPMF.R | 8 ++-- inst/shiny-server/server/RT_DSCinterface.R | 8 ++-- inst/shiny-server/server/RT_ECDF.R | 8 ++-- inst/shiny-server/server/RT_Par.R | 2 +- .../server/Shapley_computations.R | 10 ++--- inst/shiny-server/server/fv_stat_tests.R | 14 +++---- inst/shiny-server/server/general_overview.R | 2 +- inst/shiny-server/server/parrallel_coord.R | 8 ++-- .../server/statistical_significance.R | 12 +++--- inst/shiny-server/server/tables_multi_func.R | 8 ++-- inst/shiny-server/server/upload.R | 40 ++++++++++--------- 22 files changed, 116 insertions(+), 105 deletions(-) diff --git a/R/DataSetList.R b/R/DataSetList.R index 56bdb551..ac376c01 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -836,6 +836,7 @@ subset.DataSetList <- function(x, ...) { #' @examples #' change_id(dsl, c('instance')) change_id <- function(dsl, attrs) { + if (length(dsl) == 0) return(dsl) if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(dsl, x)})) colnames(grid) <- attrs @@ -843,11 +844,11 @@ change_id <- function(dsl, attrs) { dsl_new <- DataSetList() attr_vals <- c() for (x in transpose(grid)) { - # conditions <- unlist(lapply(seq(length(attrs)), function(idx) { - # parse(text = paste0(attrs[[idx]], ' == ', x[[idx]])) - # })) + #TODO: on Windows, UTF-8 Characters get converted into characters, which then + #don't interact correctly with the parse used in 'subset' function, leading to empty datasetlist objects. + #I'm not aware of any way to fix this, so UFT-8 characters should be avoided in ID for now. conditions <- paste0(unlist(lapply(seq(length(attrs)), function(idx) { - paste0(attrs[[idx]], ' == ', x[[idx]]) + paste0(attrs[[idx]], ' == "', x[[idx]], '"') })), collapse = " & ") dsl_temp <- subset(dsl, text = conditions) if (length(attrs) == 1) @@ -863,6 +864,13 @@ change_id <- function(dsl, attrs) { for (idx in seq(length(dsl_new))) { attr(dsl_new[[idx]], 'ID') <- attr_vals[[idx]] } + class(dsl_new) <- c("DataSetList", "list") + attr(dsl_new, 'suite') <- attr(dsl, 'suite') + attr(dsl_new, 'maximization') <- attr(dsl, 'maximization') + + attr(dsl_new, 'DIM') <- lapply(dsl_new, function(x) attr(x, 'DIM')) + attr(dsl_new, 'funcId') <- lapply(dsl_new, function(x) attr(x, 'funcId')) + attr(dsl_new, 'algId') <- lapply(dsl_new, function(x) attr(x, 'algId')) dsl_new } diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index e09f46a7..94e9e7e0 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -34,6 +34,7 @@ options(shiny.port = 4242) .onLoad <- function(libname, pkgname) { op <- options() op.IOHanalyzer <- list( + IOHanalyzer.ID_vars = c("algId"), IOHanalyzer.quantiles = c(2, 5, 10, 25, 50, 75, 90, 95, 98) / 100., IOHanalyzer.max_samples = 100, IOHanalyzer.backend = 'plotly', diff --git a/inst/shiny-server/server/ERTPlot.R b/inst/shiny-server/server/ERTPlot.R index 5fb0d35e..f6fa7b16 100644 --- a/inst/shiny-server/server/ERTPlot.R +++ b/inst/shiny-server/server/ERTPlot.R @@ -9,7 +9,7 @@ output$ERT_PER_FUN <- renderPlotly({ get_data_ERT_PER_FUN <- reactive({ req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) selected_algs <- input$ERTPlot.Algs - data <- subset(DATA(), algId %in% input$ERTPlot.Algs) + data <- subset(DATA(), ID %in% input$ERTPlot.Algs) fstart <- input$ERTPlot.Min %>% as.numeric fstop <- input$ERTPlot.Max %>% as.numeric budget <- input$ERTPlot.Additional.Budget %>% as.numeric @@ -26,7 +26,7 @@ render_ert_per_fct <- reactive({ show_legend <- T if (length(y_attrs) > 0) { p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, + type = 'line', legend_attr = 'ID', show.legend = show_legend, scale.ylog = input$ERTPlot.semilogy, scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", y_title = "Function Evaluations", @@ -37,7 +37,7 @@ render_ert_per_fct <- reactive({ p <- NULL if (input$ERTPlot.show.CI) { p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', upper_attr = 'upper', p = p, show.legend = show_legend, scale.ylog = input$ERTPlot.semilogy, scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", @@ -48,7 +48,7 @@ render_ert_per_fct <- reactive({ if (input$ERTPlot.show.Quantiles) { quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'median', - type = 'ribbon', legend_attr = 'algId', lower_attr = quantiles[[1]], + type = 'ribbon', legend_attr = 'ID', lower_attr = quantiles[[1]], upper_attr = quantiles[[length(quantiles)]], p = p, show.legend = show_legend, scale.ylog = input$ERTPlot.semilogy, @@ -60,13 +60,13 @@ render_ert_per_fct <- reactive({ if (input$ERTPlot.show.runs) { fstart <- isolate(input$ERTPlot.Min %>% as.numeric) fstop <- isolate(input$ERTPlot.Max %>% as.numeric) - data <- isolate(subset(DATA(), algId %in% input$ERTPlot.Algs)) + data <- isolate(subset(DATA(), ID %in% input$ERTPlot.Algs)) dt <- get_RT_sample(data, seq_FV(get_funvals(data), from = fstart, to = fstop, length.out = 50, scale = ifelse(isolate(input$ERTPlot.semilogx), 'log', 'linear'))) nr_runs <- ncol(dt) - 4 for (i in seq_len(nr_runs)) { p <- plot_general_data(dt, x_attr = 'target', y_attr = paste0('run.', i), type = 'line', - legend_attr = 'algId', p = p, show.legend = show_legend, + legend_attr = 'ID', p = p, show.legend = show_legend, scale.ylog = input$ERTPlot.semilogy, scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", y_title = "Function Evaluations", @@ -135,7 +135,7 @@ output$ERTPlot.Multi.Plot <- renderPlotly( get_data_ERT_multi_func_bulk <- reactive({ data <- subset(DATA_RAW(), DIM == input$Overall.Dim) - if (length(get_algId(data)) < 20) { #Arbitrary limit for the time being + if (length(get_id(data)) < 20) { #Arbitrary limit for the time being rbindlist(lapply(get_funcId(data), function(fid) { generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, which = 'by_RT') @@ -150,13 +150,13 @@ get_data_ERT_multi_func <- reactive({ input$ERTPlot.Multi.PlotButton data <- subset(DATA_RAW(), DIM == input$Overall.Dim) - if (length(get_algId(data)) < 20) { - get_data_ERT_multi_func_bulk()[algId %in% isolate(input$ERTPlot.Multi.Algs), ] + if (length(get_id(data)) < 20) { + get_data_ERT_multi_func_bulk()[ID %in% isolate(input$ERTPlot.Multi.Algs), ] } else { selected_algs <- isolate(input$ERTPlot.Multi.Algs) data <- subset(DATA_RAW(), - algId %in% selected_algs, + ID %in% selected_algs, DIM == input$Overall.Dim) rbindlist(lapply(get_funcId(data), function(fid) { generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, diff --git a/inst/shiny-server/server/ERT_aggr_dim_plot.R b/inst/shiny-server/server/ERT_aggr_dim_plot.R index f2b9bebe..f3571304 100644 --- a/inst/shiny-server/server/ERT_aggr_dim_plot.R +++ b/inst/shiny-server/server/ERT_aggr_dim_plot.R @@ -29,7 +29,7 @@ render_ERTPlot_aggr_plot_dim <- reactive({ ### Gather relevant datasetlist ERTPlot.Aggr_Dim.data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$ERTPlot.Aggr_Dim.Algs)) + data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr_Dim.Algs)) if (length(data) == 0) return(NULL) data <- subset(data, funcId == input$Overall.Funcid) @@ -39,7 +39,7 @@ ERTPlot.Aggr_Dim.data <- function() { return(NULL) } - if (length(unique(get_algId(data))) <= 1) { + if (length(unique(get_id(data))) <= 1) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected function") return(NULL) @@ -50,7 +50,7 @@ ERTPlot.Aggr_Dim.data <- function() { ### format table for display ert_multi_dim <- function(){ dt <- get_data_ERT_aggr_dim() - dt <- dcast(dt, DIM~algId, value.var = 'value') + dt <- dcast(dt, DIM~ID, value.var = 'value') format(dt, digits = 4) } diff --git a/inst/shiny-server/server/ERT_aggr_plot.R b/inst/shiny-server/server/ERT_aggr_plot.R index b6d82476..faae2298 100644 --- a/inst/shiny-server/server/ERT_aggr_plot.R +++ b/inst/shiny-server/server/ERT_aggr_plot.R @@ -29,7 +29,7 @@ render_ERTPlot_aggr_plot <- reactive({ ### Gather relevant datasetlist ERTPlot.Aggr.data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$ERTPlot.Aggr.Algs)) + data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr.Algs)) if (length(data) == 0) return(NULL) data <- subset(data, DIM == input$Overall.Dim) @@ -39,7 +39,7 @@ ERTPlot.Aggr.data <- function() { return(NULL) } - if (length(unique(get_algId(data))) == 1) { + if (length(unique(get_id(data))) == 1) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected dimension.") return(NULL) @@ -50,7 +50,7 @@ ERTPlot.Aggr.data <- function() { ### format table for display ert_multi_function <- function() { dt <- ERTPlot.Aggr.ERTs_obj() - dt <- dcast(dt, funcId~algId, value.var = 'value') + dt <- dcast(dt, funcId~ID, value.var = 'value') format(dt, digits = 4) } diff --git a/inst/shiny-server/server/FCEPDF.R b/inst/shiny-server/server/FCEPDF.R index eadb2523..a99295b4 100644 --- a/inst/shiny-server/server/FCEPDF.R +++ b/inst/shiny-server/server/FCEPDF.R @@ -1,13 +1,13 @@ # empirical p.d.f. of the target value get_data_FV_PMF <- reactive({ ftarget <- input$FCEPDF.Bar.Runtime %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEPDF.Bar.Algs) + data <- subset(DATA(), ID %in% input$FCEPDF.Bar.Algs) generate_data.PMF(data, ftarget, 'by_FV') }) render_FV_PDF <- reactive({ withProgress({ - plot_general_data(get_data_FV_PMF(), 'algId', 'f(x)', scale.ylog = input$FCEPDF.Bar.Logy, + plot_general_data(get_data_FV_PMF(), 'ID', 'f(x)', scale.ylog = input$FCEPDF.Bar.Logy, x_title = "Algorithm", y_title = "Target Value") # ftarget <- input$RTPMF.Bar.Target %>% as.numeric # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) @@ -36,7 +36,7 @@ output$FCE_PDF <- renderPlotly({ get_data_FV_HIST <- reactive({ ftarget <- input$FCEPDF.Hist.Runtime %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEPDF.Hist.Algs) + data <- subset(DATA(), ID %in% input$FCEPDF.Hist.Algs) generate_data.hist(data, ftarget, input$FCEPDF.Hist.Equal, 'by_FV') }) @@ -44,7 +44,7 @@ get_data_FV_HIST <- reactive({ render_FV_HIST <- reactive({ req(input$FCEPDF.Hist.Runtime != "", length(DATA()) > 0) # require non-empty input withProgress({ - subplot_attr <- if (input$FCEPDF.Hist.Mode == 'subplot') 'algId' else NULL + subplot_attr <- if (input$FCEPDF.Hist.Mode == 'subplot') 'ID' else NULL plot_general_data(get_data_FV_HIST(), 'x', 'y', width = 'width', type = 'hist', subplot_attr = subplot_attr, x_title = "Target Values", y_title = "Runs") diff --git a/inst/shiny-server/server/FCEPlot.R b/inst/shiny-server/server/FCEPlot.R index c641c4ec..f564d17a 100644 --- a/inst/shiny-server/server/FCEPlot.R +++ b/inst/shiny-server/server/FCEPlot.R @@ -6,7 +6,7 @@ output$FCE_PER_FUN <- renderPlotly({ get_data_FCE_PER_FUN <- reactive({ req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) - data <- subset(DATA(), algId %in% input$FCEPlot.Algs) + data <- subset(DATA(), ID %in% input$FCEPlot.Algs) fstart <- input$FCEPlot.Min %>% as.numeric fstop <- input$FCEPlot.Max %>% as.numeric generate_data.Single_Function(data, fstart, fstop, input$FCEPlot.semilogx, @@ -21,7 +21,7 @@ render_FV_PER_FUN <- reactive({ show_legend <- T if (length(y_attrs) > 0) { p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, + type = 'line', legend_attr = 'ID', show.legend = show_legend, scale.ylog = isolate(input$FCEPlot.semilogy), scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", y_title = "Best-so-far f(x)-value") @@ -31,7 +31,7 @@ render_FV_PER_FUN <- reactive({ p <- NULL if (input$FCEPlot.show.CI) { p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', upper_attr = 'upper', p = p, show.legend = show_legend) show_legend <- F } @@ -40,7 +40,7 @@ render_FV_PER_FUN <- reactive({ IOHanalyzer.quantiles.bk <- getOption("IOHanalyzer.quantiles") options(IOHanalyzer.quantiles = c(0.25, 0.75)) p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = '25%', + type = 'ribbon', legend_attr = 'ID', lower_attr = '25%', upper_attr = '75%', p = p, show.legend = show_legend) show_legend <- F options(IOHanalyzer.quantiles = IOHanalyzer.quantiles.bk) @@ -48,13 +48,13 @@ render_FV_PER_FUN <- reactive({ if (input$FCEPlot.show.runs) { fstart <- isolate(input$FCEPlot.Min %>% as.numeric) fstop <- isolate(input$FCEPlot.Max %>% as.numeric) - data <- isolate(subset(DATA(), algId %in% input$FCEPlot.Algs)) + data <- isolate(subset(DATA(), ID %in% input$FCEPlot.Algs)) dt <- get_FV_sample(data, seq_RT(c(fstart, fstop), from = fstart, to = fstop, length.out = 50, scale = ifelse(isolate(input$FCEPlot.semilogx), 'log', 'linear'))) nr_runs <- ncol(dt) - 4 for (i in seq_len(nr_runs)) { p <- plot_general_data(dt, x_attr = 'runtime', y_attr = paste0('run.', i), type = 'line', - legend_attr = 'algId', p = p, show.legend = show_legend, + legend_attr = 'ID', p = p, show.legend = show_legend, scale.ylog = input$FCEPlot.semilogy, scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", y_title = "Best-so-far f(x)-value") @@ -91,7 +91,7 @@ output$FCEPlot.Multi.Plot <- renderPlotly( get_data_FCE_multi_func_bulk <- reactive({ data <- subset(DATA_RAW(), DIM == input$Overall.Dim) - if (length(get_algId(data)) < 20) { #Arbitrary limit for the time being + if (length(get_id(data)) < 20) { #Arbitrary limit for the time being rbindlist(lapply(get_funcId(data), function(fid) { generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, which = 'by_FV') @@ -104,12 +104,12 @@ get_data_FCE_multi_func_bulk <- reactive({ get_data_FCEPlot_multi <- reactive({ req(isolate(input$FCEPlot.Multi.Algs)) input$FCEPlot.Multi.PlotButton - if (length(get_algId(data)) < 20) { - get_data_FCE_multi_func_bulk()[algId %in% isolate(input$FCEPlot.Multi.Algs), ] + if (length(get_id(data)) < 20) { + get_data_FCE_multi_func_bulk()[ID %in% isolate(input$FCEPlot.Multi.Algs), ] } else { data <- subset(DATA_RAW(), - algId %in% isolate(input$FCEPlot.Multi.Algs), + ID %in% isolate(input$FCEPlot.Multi.Algs), DIM == input$Overall.Dim) rbindlist(lapply(get_funcId(data), function(fid) { generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, diff --git a/inst/shiny-server/server/FCE_ECDF.R b/inst/shiny-server/server/FCE_ECDF.R index 1c4d267d..8dacc421 100644 --- a/inst/shiny-server/server/FCE_ECDF.R +++ b/inst/shiny-server/server/FCE_ECDF.R @@ -6,7 +6,7 @@ output$FCE_ECDF_PER_TARGET <- renderPlotly({ get_data_FV_ECDF_Single <- reactive({ req(input$FCEECDF.Single.Target) ftargets <- as.numeric(format_FV(input$FCEECDF.Single.Target)) - data <- subset(DATA(), algId %in% input$FCEECDF.Single.Algs) + data <- subset(DATA(), ID %in% input$FCEECDF.Single.Algs) generate_data.ECDF(data, ftargets, input$FCEECDF.Single.Logx, which = 'by_FV') }) @@ -40,7 +40,7 @@ get_data_FV_ECDF_AGGR <- reactive({ fstart <- format_FV(input$FCEECDF.Mult.Min) %>% as.numeric fstop <- format_FV(input$FCEECDF.Mult.Max) %>% as.numeric fstep <- format_FV(input$FCEECDF.Mult.Step) %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEECDF.Mult.Algs) + data <- subset(DATA(), ID %in% input$FCEECDF.Mult.Algs) targets <- seq_RT(get_funvals(data), fstart, fstop, fstep) generate_data.ECDF(data, targets, input$FCEECDF.Mult.Logx, which = 'by_FV') }) @@ -86,7 +86,7 @@ get_data_FV_AUC <- reactive({ rt_min <- input$FCEECDF.AUC.Min %>% as.numeric rt_max <- input$FCEECDF.AUC.Max %>% as.numeric rt_step <- input$FCEECDF.AUC.Step %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEECDF.AUC.Algs) + data <- subset(DATA(), ID %in% input$FCEECDF.AUC.Algs) targets <- seq_RT(get_runtimes(data), rt_min, rt_max, rt_step, length.out = 10) generate_data.AUC(data, targets, which = 'by_FV') }) diff --git a/inst/shiny-server/server/FCE_aggr_plot.R b/inst/shiny-server/server/FCE_aggr_plot.R index 6a7e280c..e670d769 100644 --- a/inst/shiny-server/server/FCE_aggr_plot.R +++ b/inst/shiny-server/server/FCE_aggr_plot.R @@ -27,7 +27,7 @@ render_FCEPlot_aggr_plot <- reactive({ ### Gather relevant datasetlist FCEPlot.Aggr.data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$FCEPlot.Aggr.Algs)) + data <- subset(DATA_RAW(), ID %in% isolate(input$FCEPlot.Aggr.Algs)) if (length(data) == 0) return(NULL) if (input$FCEPlot.Aggr.Aggregator == 'Functions') { data <- subset(data, DIM == input$Overall.Dim) @@ -43,7 +43,7 @@ FCEPlot.Aggr.data <- function() { return(NULL) } } - if (length(unique(get_algId(data))) == 1) { + if (length(unique(get_id(data))) == 1) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected dimension.") return(NULL) } @@ -55,9 +55,9 @@ FCEPlot.Aggr.data <- function() { FCE_multi_function <- function() { dt <- FCEPlot.Aggr.FCEs_obj() if (input$FCEPlot.Aggr.Aggregator == 'Functions') - dt <- dcast(dt, funcId~algId, value.var = 'value') + dt <- dcast(dt, funcId~ID, value.var = 'value') else - dt <- dcast(dt, DIM~algId, value.var = 'value') + dt <- dcast(dt, DIM~ID, value.var = 'value') dt } diff --git a/inst/shiny-server/server/FV_DSCinterface.R b/inst/shiny-server/server/FV_DSCinterface.R index 8009808f..078711c0 100644 --- a/inst/shiny-server/server/FV_DSCinterface.R +++ b/inst/shiny-server/server/FV_DSCinterface.R @@ -103,8 +103,8 @@ FV_DSC_posthoc_result <- reactive({ req(omni_res) data <- FV_DSC_data() req(length(data) > 0) - df_posthoc <- rbindlist(lapply(get_algId(data), function(algname){ - posthoc_res <- get_dsc_posthoc(omni_res, length(get_algId(data)), + df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ + posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), nrow(FV_stats_DSC_targets_obj), alpha = input$FV_Stats.DSC.Alpha_posthoc, base_algorithm = algname, @@ -183,11 +183,11 @@ output$FV_Stats.DSC.Download <- downloadHandler( FV_DSC_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$FV_Stats.DSC.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.DSC.Algid)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$FV_Stats.DSC.Dim) data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) - if (length(unique(get_algId(data))) < 2) { + if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected functions and dimensions.") return(NULL) diff --git a/inst/shiny-server/server/FV_PAR.R b/inst/shiny-server/server/FV_PAR.R index cfb262f9..1010abbf 100644 --- a/inst/shiny-server/server/FV_PAR.R +++ b/inst/shiny-server/server/FV_PAR.R @@ -1,6 +1,6 @@ # Expected Evolution of parameters in the algorithm get_data_FV_PAR_PER_FUN <- reactive({ - data <- subset(DATA(), algId %in% input$FV_PAR.Plot.Algs) + data <- subset(DATA(), ID %in% input$FV_PAR.Plot.Algs) generate_data.Parameters(data, scale_log = input$FV_PAR.Plot.Logx, which = 'by_RT') }) diff --git a/inst/shiny-server/server/RTPMF.R b/inst/shiny-server/server/RTPMF.R index 93bb973a..df5ce7c2 100644 --- a/inst/shiny-server/server/RTPMF.R +++ b/inst/shiny-server/server/RTPMF.R @@ -15,13 +15,13 @@ output$RTPMF.Bar.Download <- downloadHandler( get_data_RT_PMF <- reactive({ ftarget <- input$RTPMF.Bar.Target %>% as.numeric - data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) + data <- subset(DATA(), ID %in% input$RTPMF.Bar.Algs) generate_data.PMF(data, ftarget, 'by_RT') }) render_RT_PMF <- reactive({ withProgress({ - plot_general_data(get_data_RT_PMF(), 'algId', 'RT', scale.ylog = input$RTPMF.Bar.Logy, + plot_general_data(get_data_RT_PMF(), 'ID', 'RT', scale.ylog = input$RTPMF.Bar.Logy, x_title = "Algorithm", y_title = "Function Evaluations") # ftarget <- input$RTPMF.Bar.Target %>% as.numeric # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) @@ -49,7 +49,7 @@ output$RTPMF.Hist.Download <- downloadHandler( get_data_RT_HIST <- reactive({ ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric - data <- subset(DATA(), algId %in% input$RTPMF.Hist.Algs) + data <- subset(DATA(), ID %in% input$RTPMF.Hist.Algs) generate_data.hist(data, ftarget, input$RTPMF.Hist.Equal, 'by_RT') }) @@ -64,7 +64,7 @@ render_RT_HIST <- reactive({ # # Plot.RT.Histogram(data, ftarget, plot_mode = input$RTPMF.Hist.Mode, # use.equal.bins = input$RTPMF.Hist.Equal) - subplot_attr <- if (input$RTPMF.Hist.Mode == 'subplot') 'algId' else NULL + subplot_attr <- if (input$RTPMF.Hist.Mode == 'subplot') 'ID' else NULL plot_general_data(get_data_RT_HIST(), 'x', 'y', width = 'width', type = 'hist', subplot_attr = subplot_attr, x_title = "Function Evaluations", y_title = "Runs") diff --git a/inst/shiny-server/server/RT_DSCinterface.R b/inst/shiny-server/server/RT_DSCinterface.R index dbb4c909..e564032a 100644 --- a/inst/shiny-server/server/RT_DSCinterface.R +++ b/inst/shiny-server/server/RT_DSCinterface.R @@ -102,8 +102,8 @@ RT_DSC_posthoc_result <- reactive({ req(omni_res) data <- RT_DSC_data() req(length(data) > 0) - df_posthoc <- rbindlist(lapply(get_algId(data), function(algname){ - posthoc_res <- get_dsc_posthoc(omni_res, length(get_algId(data)), + df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ + posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), nrow(RT_stats_DSC_targets_obj), alpha = input$RT_Stats.DSC.Alpha_posthoc, base_algorithm = algname, @@ -182,11 +182,11 @@ output$RT_Stats.DSC.Download <- downloadHandler( RT_DSC_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$RT_Stats.DSC.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.DSC.Algid)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$RT_Stats.DSC.Dim) data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) - if (length(unique(get_algId(data))) < 2) { + if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected functions and dimensions.") return(NULL) diff --git a/inst/shiny-server/server/RT_ECDF.R b/inst/shiny-server/server/RT_ECDF.R index 97990789..23414f92 100644 --- a/inst/shiny-server/server/RT_ECDF.R +++ b/inst/shiny-server/server/RT_ECDF.R @@ -6,7 +6,7 @@ output$RT_ECDF_MULT <- renderPlotly({ get_data_RT_ECDF_MULT <- reactive({ req(input$RTECDF.Aggr.Func || input$RTECDF.Aggr.Dim) input$RTECDF.Aggr.Refresh - dsList <- subset(DATA_RAW(), algId %in% input$RTECDF.Aggr.Algs) + dsList <- subset(DATA_RAW(), ID %in% input$RTECDF.Aggr.Algs) if (!input$RTECDF.Aggr.Func) dsList <- subset(dsList, funcId == input$Overall.Funcid) @@ -65,7 +65,7 @@ observe({ req(dim) # req(data$suite != NEVERGRAD) - dsList <- subset(data, algId %in% alg) + dsList <- subset(data, ID %in% alg) req(length(dsList) > 0) if (!input$RTECDF.Aggr.Func) @@ -182,7 +182,7 @@ output$RTECDF.Aggr.Table.Download <- downloadHandler( get_data_RT_ECDF_Single <- reactive({ ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) - data <- subset(DATA(), algId %in% input$RTECDF.Single.Algs) + data <- subset(DATA(), ID %in% input$RTECDF.Single.Algs) generate_data.ECDF(data, ftargets, input$RTECDF.Single.Logx) }) @@ -241,7 +241,7 @@ get_data_RT_ECDF_AGGR <- reactive({ fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric - data <- subset(DATA(), algId %in% input$RTECDF.Multi.Algs) + data <- subset(DATA(), ID %in% input$RTECDF.Multi.Algs) targets <- seq_FV(get_funvals(data), fstart, fstop, fstep) generate_data.ECDF(data, targets, input$RTECDF.Multi.Logx) }) diff --git a/inst/shiny-server/server/RT_Par.R b/inst/shiny-server/server/RT_Par.R index b92d2aef..5287c06d 100644 --- a/inst/shiny-server/server/RT_Par.R +++ b/inst/shiny-server/server/RT_Par.R @@ -1,6 +1,6 @@ get_data_RT_PAR_PER_FUN <- reactive({ - data <- subset(DATA(), algId %in% input$RT_PAR.Plot.Algs) + data <- subset(DATA(), ID %in% input$RT_PAR.Plot.Algs) generate_data.Parameters(data, scale_log = input$RT_PAR.Plot.Logx, which = 'by_FV') }) diff --git a/inst/shiny-server/server/Shapley_computations.R b/inst/shiny-server/server/Shapley_computations.R index c4766685..ef546e09 100644 --- a/inst/shiny-server/server/Shapley_computations.R +++ b/inst/shiny-server/server/Shapley_computations.R @@ -5,12 +5,12 @@ output$RT_SHAPLEY <- renderPlotly({ get_data_RT_SHAPLEY <- reactive({ input$RTportfolio.Shapley.Refresh - dsList <- subset(DATA_RAW(), algId %in% input$RTportfolio.Shapley.Algs & + dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & funcId %in% input$RTportfolio.Shapley.Funcs & DIM %in% input$RTportfolio.Shapley.Dims) - if (length(get_algId(dsList)) <= 1) { + if (length(get_id(dsList)) <= 1) { shinyjs::alert("This is an invalid configuration for this plot. \n Please ensure that the dataset contains multiple algorithms.") return(NULL) @@ -26,8 +26,8 @@ get_data_RT_SHAPLEY <- reactive({ render_RT_SHAPLEY <- reactive({ withProgress({ dt <- get_data_RT_SHAPLEY() - plot_general_data(dt, x_attr = 'algId', y_attr = 'shapley', type = 'bar', - legend_attr = 'algId', show.legend = T) + plot_general_data(dt, x_attr = 'ID', y_attr = 'shapley', type = 'bar', + legend_attr = 'ID', show.legend = T) }, message = "Creating plot") }) @@ -48,7 +48,7 @@ proxy <- dataTableProxy('RT_SHAPLEY_TARGETS_GENERATED') observe({ req(length(DATA_RAW()) > 0) - dsList <- subset(DATA_RAW(), algId %in% input$RTportfolio.Shapley.Algs & + dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & funcId %in% input$RTportfolio.Shapley.Funcs & DIM %in% input$RTportfolio.Shapley.Dims) req(length(dsList) > 0) diff --git a/inst/shiny-server/server/fv_stat_tests.R b/inst/shiny-server/server/fv_stat_tests.R index 0da88088..19c12461 100644 --- a/inst/shiny-server/server/fv_stat_tests.R +++ b/inst/shiny-server/server/fv_stat_tests.R @@ -2,7 +2,7 @@ fv_render_heatmap <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$FV_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.Algid) Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), bootstrap.size = 0, which = 'by_RT') }, @@ -16,8 +16,8 @@ output$FV_Stats.Overview.Heatmap <- renderPlotly( fv_create_stats_table <- reactive({ req(length(DATA()) > 0) - req(length(get_algId(DATA())) > 1) - data <- subset(DATA(), algId %in% input$FV_Stats.Overview.Algid) + req(length(get_id(DATA())) > 1) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.Algid) target <- as.numeric(input$FV_Stats.Overview.Target) df <- pairwise.test(data, target, bootstrap.size = 0, which = 'by_RT') df <- format(df, digits = 3) @@ -36,7 +36,7 @@ fv_render_graph <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$FV_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.Algid) Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), bootstrap.size = 0, which = 'by_RT') }, @@ -69,7 +69,7 @@ fv_data_table_glicko2 <- reactive({ isolate({ withProgress({ data <- FV_glicko_data() - req(length(data) > 0 && length(get_algId(data)) > 0) + req(length(data) > 0 && length(get_id(data)) > 0) nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) df <- glicko2_ranking(data, nr_games, which = 'by_RT', target_dt = FV_stats_glicko_targets_obj)$ratings @@ -119,11 +119,11 @@ output$FV_Stats.Glicko.Download <- downloadHandler( FV_glicko_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$FV_Stats.Glicko.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.Glicko.Algid)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$FV_Stats.Glicko.Dim) data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) - if (length(unique(get_algId(data))) < 2) { + if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected functions and dimensions.") return(NULL) diff --git a/inst/shiny-server/server/general_overview.R b/inst/shiny-server/server/general_overview.R index 89a1cd60..ebea5b9d 100644 --- a/inst/shiny-server/server/general_overview.R +++ b/inst/shiny-server/server/general_overview.R @@ -1,7 +1,7 @@ overview_table_single <- reactive({ data <- DATA() req(length(data) > 0) - df <- get_overview(subset(data, algId %in% input$Overview.Single.Algid)) + df <- get_overview(subset(data, ID %in% input$Overview.Single.Algid)) df$budget %<>% as.numeric df$runs %<>% as.integer diff --git a/inst/shiny-server/server/parrallel_coord.R b/inst/shiny-server/server/parrallel_coord.R index 71a5d0ac..1570f497 100644 --- a/inst/shiny-server/server/parrallel_coord.R +++ b/inst/shiny-server/server/parrallel_coord.R @@ -1,7 +1,7 @@ render_parallel_coord <- reactive({ data <- DATA() req(attr(data, 'suite') == "SOS") - data <- subset(data, algId == input$ParCoordPlot.Algs) + data <- subset(data, ID == input$ParCoordPlot.Algs) withProgress({ p <- IOH_plot_ly_default("Parallel coordinate plot", "Coordinate", "Value") @@ -9,17 +9,17 @@ render_parallel_coord <- reactive({ final_pos_dt <- rbindlist(lapply(ds$PAR$final_position, function(pos) {dt <- as.data.table(t(pos))})) colnames(final_pos_dt) <- as.character(seq_len(ncol(final_pos_dt))) - final_pos_dt[, algId := attr(ds, 'algId')] + final_pos_dt[, ID := attr(ds, 'ID')] final_pos_dt[, funcId := attr(ds, 'funcId')] final_pos_dt[, fval_log := log10(attr(ds, 'finalFV'))] })) - pos2 <- pos %>% melt(id.vars = c('fval_log', 'funcId', 'algId')) + pos2 <- pos %>% melt(id.vars = c('fval_log', 'funcId', 'ID')) p %>% add_trace(data = pos2, x = ~variable, y = ~value, type = 'scatter', mode = 'markers', marker = list(color = ~fval_log, colorscale = 'Viridis', colorbar = list(title = 'Log best f(x)'), symbol = 'cross'), - legendgroup = ~algId, opacity = 0.9, showlegend = F) %>% + legendgroup = ~ID, opacity = 0.9, showlegend = F) %>% layout(yaxis = list(range = c(-0.02,1.02)), xaxis = list(range = c(-0.5, get_dim(dsl) + 0.5))) }) }) diff --git a/inst/shiny-server/server/statistical_significance.R b/inst/shiny-server/server/statistical_significance.R index ed6c6dca..bd422e68 100644 --- a/inst/shiny-server/server/statistical_significance.R +++ b/inst/shiny-server/server/statistical_significance.R @@ -2,7 +2,7 @@ render_heatmap <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$RT_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.Algid) Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), bootstrap.size = input$RT_Stats.Overview.Samples) }, @@ -16,8 +16,8 @@ output$RT_Stats.Overview.Heatmap <- renderPlotly( create_stats_table <- reactive({ req(length(DATA()) > 0) - req(length(get_algId(DATA())) > 1) - data <- subset(DATA(), algId %in% input$RT_Stats.Overview.Algid) + req(length(get_id(DATA())) > 1) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.Algid) target <- as.numeric(input$RT_Stats.Overview.Target) df <- pairwise.test(data, target, bootstrap.size = input$RT_Stats.Overview.Samples) df <- format(df, digits = 3) @@ -36,7 +36,7 @@ render_graph <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$RT_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.Algid) Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), bootstrap.size = input$RT_Stats.Overview.Samples) }, @@ -117,11 +117,11 @@ output$RT_Stats.Glicko.Download <- downloadHandler( RT_glicko_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$RT_Stats.Glicko.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.Glicko.Algid)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$RT_Stats.Glicko.Dim) data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) - if (length(unique(get_algId(data))) < 2) { + if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected functions and dimensions.") return(NULL) diff --git a/inst/shiny-server/server/tables_multi_func.R b/inst/shiny-server/server/tables_multi_func.R index bdb5627f..79122195 100644 --- a/inst/shiny-server/server/tables_multi_func.R +++ b/inst/shiny-server/server/tables_multi_func.R @@ -1,7 +1,7 @@ multi_function_data_summary <- reactive({ data <- DATA_RAW() req(length(data) > 0) - data <- subset(data, algId %in% input$RT.MultiERT.AlgId & + data <- subset(data, ID %in% input$RT.MultiERT.AlgId & funcId %in% input$RT.MultiERT.FuncId & DIM %in% input$RT.MultiERT.DIM) @@ -26,7 +26,7 @@ output$RT.MultiERT.Download <- downloadHandler( multi_function_data_sample <- reactive({ data <- DATA_RAW() req(length(data) > 0) - data <- subset(data, algId %in% input$RT.Multisample.AlgId & + data <- subset(data, ID %in% input$RT.Multisample.AlgId & funcId %in% input$RT.Multisample.FuncId & DIM %in% input$RT.Multisample.DIM) @@ -53,7 +53,7 @@ multi_function_data_summary_FV <- reactive({ data <- DATA_RAW() req(length(data) > 0) req(as.numeric(input$FV.MultiFV.Target) > 0) - data <- subset(data, algId %in% input$FV.MultiFV.AlgId & + data <- subset(data, ID %in% input$FV.MultiFV.AlgId & funcId %in% input$FV.MultiFV.FuncId & DIM %in% input$FV.MultiFV.DIM) @@ -78,7 +78,7 @@ output$FV.MultiFV.Download <- downloadHandler( multi_function_data_sample_FV <- reactive({ data <- DATA_RAW() req(length(data) > 0) - data <- subset(data, algId %in% input$FV.Multisample.AlgId & + data <- subset(data, ID %in% input$FV.Multisample.AlgId & funcId %in% input$FV.Multisample.FuncId & DIM %in% input$FV.Multisample.DIM) diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index 4f817040..1b72566a 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -85,11 +85,12 @@ observeEvent(input$repository.load_button, { } DataList$data <- c(DataList$data, data) + DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) update_menu_visibility(attr(DataList$data, 'suite')) # set_format_func(attr(DataList$data, 'suite')) - algids <- get_algId(DataList$data) - if (!all(algids %in% get_color_scheme_dt()[['algnames']])) { - set_color_scheme("Default", algids) + IDs <- get_id(DataList$data) + if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { + set_color_scheme("Default", IDs) } }) @@ -237,6 +238,7 @@ observeEvent(selected_folders(), { } DataList$data <- clean_DataSetList(DataList$data) + DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) if (is.null(DataList$data)) { shinyjs::alert("An error occurred when processing the uploaded data. Please ensure the data is not corrupted.") @@ -245,9 +247,9 @@ observeEvent(selected_folders(), { update_menu_visibility(attr(DataList$data, 'suite')) # set_format_func(attr(DataList$data, 'suite')) - algids <- get_algId(DataList$data) - if (!all(algids %in% get_color_scheme_dt()[['algnames']])) { - set_color_scheme("Default", algids) + IDs <- get_id(DataList$data) + if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { + set_color_scheme("Default", IDs) } }, message = "Processing data, this might take some time") }) @@ -328,8 +330,8 @@ observe({ return() # TODO: create reactive values for them - algIds_ <- get_algId(data) - algIds <- c(algIds_, 'all') + algIds_ <- get_id(data) + # algIds <- c(algIds_, 'all') parIds_ <- get_parId(data) parIds <- c(parIds_, 'all') funcIds <- get_funcId(data) @@ -338,7 +340,7 @@ observe({ selected_ds <- data[[1]] selected_f <- attr(selected_ds,'funcId') selected_dim <- attr(selected_ds, 'DIM') - selected_alg <- attr(selected_ds, 'algId') + selected_alg <- get_id(selected_ds) updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) @@ -474,18 +476,18 @@ observe({ updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) updateSelectInput(session, 'FV_Stats.Overview.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTSummary.Statistics.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RTSummary.Overview.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FCESummary.Overview.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RTSummary.Sample.Algid', choices = algIds, selected = 'all') + updateSelectInput(session, 'RTSummary.Statistics.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RTSummary.Overview.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FCESummary.Overview.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RTSummary.Sample.Algid', choices = algIds_, selected = algIds_) updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = algIds_, selected = algIds_) updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCESummary.Statistics.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FCESummary.Sample.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Summary.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Sample.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Summary.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Sample.Algid', choices = algIds, selected = 'all') + updateSelectInput(session, 'FCESummary.Statistics.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FCESummary.Sample.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV_PAR.Summary.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV_PAR.Sample.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT_PAR.Summary.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT_PAR.Sample.Algid', choices = algIds_, selected = algIds_) updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = algIds_, selected = selected_alg) updateSelectInput(session, 'ERTPlot.Algs', choices = algIds_, selected = algIds_) updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = algIds_, selected = algIds_) From 2895e57e82a7a065d0f59402125f439bfb5ab44c Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 22 Jul 2021 14:57:48 +0200 Subject: [PATCH 15/28] Add option to change ID variables to settings page --- R/DataSet.R | 4 ++-- inst/shiny-server/server/settings_page.R | 15 ++++++++++++--- inst/shiny-server/server/upload.R | 4 ++++ inst/shiny-server/ui/settings.R | 4 ++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/R/DataSet.R b/R/DataSet.R index 46fa4722..f9d248a9 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -294,8 +294,8 @@ subset.DataSet <- function(x, mask, ...) { format <- info[['format']] - RT <- x$RT[, mask] - FV <- x$FV[, mask] + RT <- as.matrix(x$RT[, mask]) + FV <- as.matrix(x$FV[, mask]) PAR <- list( 'by_FV' = ifelse(ncol(x$PAR$by_FV) == length(mask), x$PAR$by_FV[, mask], NULL), diff --git a/inst/shiny-server/server/settings_page.R b/inst/shiny-server/server/settings_page.R index 7cc884c5..d15ec1f7 100644 --- a/inst/shiny-server/server/settings_page.R +++ b/inst/shiny-server/server/settings_page.R @@ -1,6 +1,6 @@ observe({ if (input$Settings.Color.Scheme != "Custom") { - set_color_scheme(input$Settings.Color.Scheme, get_algId(DATA()), NULL) + set_color_scheme(input$Settings.Color.Scheme, get_id(DATA()), NULL) } }) @@ -33,7 +33,7 @@ plot_color_example <- function(){ ) if (any(is.null(curr_settings))) return(NULL) if (length(DATA_RAW()) > 0) { - algnames <- get_algId(DATA_RAW()) + algnames <- get_id(DATA_RAW()) } else algnames <- c("Alg 1", "Alg 2", "Alg 3", "Alg 4", "Alg 5") colors <- get_color_scheme(algnames) @@ -44,7 +44,7 @@ plot_color_example <- function(){ x <- c(rep(1, length(algnames)),rep(2, length(algnames))) y <- seq_len(length(algnames)) - dt <- data.table(algId = rep(algnames, 2), x, y) + dt <- data.table(ID = rep(algnames, 2), x, y) p <- plot_general_data(dt, 'x', 'y', 'line', show.legend = T, x_title = 'X-title', y_title = 'Y-title', plot_title = 'Plot Title') @@ -148,6 +148,15 @@ observe({ options("IOHanalyzer.precision" = input$Settings.General.Precision) }) +observe({ + req(input$Settings.ID.Variables) + id_vars <- input$Settings.ID.Variables + if (!setequal(id_vars,getOption('IOHanalyzer.ID_vars', c('algId')))) { + options("IOHanalyzer.ID_vars" = input$Settings.ID.Variables) + DataList$data <- change_id(DataList$data, input$Settings.ID.Variables) + } +}) + observe({ setting_preset <- input$Settings.Download.Preset if (setting_preset == "Default") { diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index 1b72566a..47c13377 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -513,6 +513,10 @@ observe({ updateSelectInput(session, 'ParCoordPlot.Algs', choices = algIds_, selected = algIds_[[1]]) updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) + + + updateSelectInput(session, 'Settings.ID.Variables', choices = get_static_attributes(data), + selected = attr(data, 'ID_attributes')) }) # update (filter) according to users selection DataSets diff --git a/inst/shiny-server/ui/settings.R b/inst/shiny-server/ui/settings.R index 33ade29a..60486642 100644 --- a/inst/shiny-server/ui/settings.R +++ b/inst/shiny-server/ui/settings.R @@ -59,6 +59,10 @@ general_settings_box <- function(width=12, collapsible = T, collapsed = F) { numericInput("Settings.General.Precision", label = "Function value precision (digits)", value = 2), hr(), + HTML_P("ID variables"), + selectInput("Settings.ID.Variables", label = "Attributes used to create IDs", + multiple = T, selected = NULL, choices = NULL), + hr(), downloadButton("Settings.Download", "Download current general settings file"), fileInput("Settings.Upload", "Upload a settings file", accept = "rds") ), From 0ef9f9e81538563841f873d8745fa7c59b936536 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 23 Jul 2021 14:39:18 +0200 Subject: [PATCH 16/28] Add ID to SOS-based data and fix its final-pos detection --- R/readFiles.R | 52 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/R/readFiles.R b/R/readFiles.R index 9b91b35a..286d342e 100644 --- a/R/readFiles.R +++ b/R/readFiles.R @@ -743,7 +743,12 @@ read_single_file_SOS <- function(file) { } dim <- as.numeric(info$DIM) - + #Hardcoded fix for SB-related data + if (is.null(dim) || length(dim) == 0) { + warning("Dimension not explicitly defined, setting as 30 by default") + dim <- 30 + info$DIM <- dim + } RT_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] names(RT_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] @@ -761,24 +766,31 @@ read_single_file_SOS <- function(file) { maxRT <- max(RT) finalFV <- min(FV) - if (sum(FV == finalFV) > 1) { - #Reconstruct population to determine which best solution is final position - ids_min <- dt[FV_raw == finalFV, V1] - replaced_idxs <- dt[[colnames(dt)[[ncol(dt) - dim]]]] - #If none, take the last one added - pos_idx <- max(ids_min) - for (i in ids_min) { - if (all(replaced_idxs != i)) { - #If multiple, take the first one added - pos_idx <- i - break - } - } - final_pos <- as.numeric(pos[pos_idx, ]) - } - else { - final_pos <- as.numeric(pos[which.min(FV), ]) - } + idxs_avail <- dt[['V1']] + idxs_replaced <- dt[['V6']] + + idxs_final <- setdiff(idxs_avail, idxs_replaced) + + idx_final_best <- idxs_final[[which.min(FV[idxs_final])]] + final_pos <- as.numeric(pos[idx_final_best, ]) + # if (sum(FV == finalFV) > 1) { + # #Reconstruct population to determine which best solution is final position + # ids_min <- dt[FV_raw == finalFV, V1] + # replaced_idxs <- dt[[colnames(dt)[[ncol(dt) - dim]]]] + # #If none, take the last one added + # pos_idx <- max(ids_min) + # for (i in ids_min) { + # if (all(replaced_idxs != i)) { + # #If multiple, take the first one added + # pos_idx <- i + # break + # } + # } + # final_pos <- as.numeric(pos[pos_idx, ]) + # } + # else { + # final_pos <- as.numeric(pos[which.min(FV), ]) + # } PAR <- list( # 'position' = list(pos), @@ -802,6 +814,7 @@ read_single_file_SOS <- function(file) { for (i in seq_along(info)) { attr(object, names(info)[[i]]) <- type.convert(info[[i]], as.is = T) } + attr(object, 'ID') <- attr(object, 'algId') object } @@ -863,6 +876,7 @@ read_datasetlist_SOS <- function(dir, corrections_files = NULL) { attr(res, 'DIM') <- dims attr(res, 'funcId') <- funcIds attr(res, 'algId') <- algIds + attr(res, 'ID_attributes') <- c('algId') suite <- unique(suites) maximization <- unique(maximizations) From 036036a7775fed737f9184b2c30b0256e47520dd Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 23 Jul 2021 16:24:05 +0200 Subject: [PATCH 17/28] Add filtering on AlgId if not used in ID --- inst/shiny-server/server/settings_page.R | 6 + inst/shiny-server/server/upload.R | 159 ++++++++++++----------- inst/shiny-server/ui.R | 40 +++--- inst/shiny-server/ui/DIM_fID_panel.R | 17 ++- 4 files changed, 124 insertions(+), 98 deletions(-) diff --git a/inst/shiny-server/server/settings_page.R b/inst/shiny-server/server/settings_page.R index d15ec1f7..0de5fb7d 100644 --- a/inst/shiny-server/server/settings_page.R +++ b/inst/shiny-server/server/settings_page.R @@ -150,11 +150,17 @@ observe({ observe({ req(input$Settings.ID.Variables) + withProgress({ id_vars <- input$Settings.ID.Variables if (!setequal(id_vars,getOption('IOHanalyzer.ID_vars', c('algId')))) { options("IOHanalyzer.ID_vars" = input$Settings.ID.Variables) DataList$data <- change_id(DataList$data, input$Settings.ID.Variables) } + if ('algId' %in% id_vars) + shinyjs::hide(id = "overall_algid_box") + else + shinyjs::show(id = "overall_algid_box") +}, message = "Processing IDs") }) observe({ diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index 47c13377..8711223c 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -330,187 +330,193 @@ observe({ return() # TODO: create reactive values for them - algIds_ <- get_id(data) - # algIds <- c(algIds_, 'all') + IDs <- get_id(data) + # algIds <- c(IDs, 'all') parIds_ <- get_parId(data) parIds <- c(parIds_, 'all') funcIds <- get_funcId(data) DIMs <- get_dim(data) - + algIds <- get_algId(data) + selected_ds <- data[[1]] selected_f <- attr(selected_ds,'funcId') selected_dim <- attr(selected_ds, 'DIM') - selected_alg <- get_id(selected_ds) - + selected_alg <- attr(selected_ds, 'algId') + selected_ID <- get_id(selected_ds) + updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) - updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) + if ('algId' %in% input$Settings.ID.Variables) + updateSelectInput(session, 'Overall.AlgId', choices = NULL, selected = NULL) + else + updateSelectInput(session, 'Overall.AlgId', choices = algIds, selected = selected_alg) - updateSelectInput(session, 'Overview.Single.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'Overview.Single.Algid', choices = IDs, selected = IDs) # updateSelectInput(session, 'Report.RT.Overview-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.Overview-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Overview-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.Overview-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.Statistics-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Statistics-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.Statistics-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.Single_ERT-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.Single_ERT-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Single_ERT-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.Single_ERT-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.Multi_ERT-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Multi_ERT-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.Multi_ERT-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.Rank-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Rank-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.Rank-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.Histogram-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.Histogram-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Histogram-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.Histogram-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.PMF-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.PMF-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.PMF-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.PMF-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) # - # updateSelectInput(session, 'Report.RT.ECDF_Aggregated-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.RT.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_AUC-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.RT.ECDF_AUC-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.Overview-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.Overview-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Overview-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.Overview-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.Statistics-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Statistics-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.Statistics-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.Single_FCE-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.Single_FCE-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Single_FCE-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.Single_FCE-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.Multi_FCE-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Multi_FCE-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.Multi_FCE-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.Rank-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Rank-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.Rank-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.Histogram-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.Histogram-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Histogram-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.Histogram-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.PMF-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.PMF-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.PMF-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.PMF-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) # - # updateSelectInput(session, 'Report.FV.ECDF_Aggregated-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.FV.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.FV.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_AUC-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.FV.ECDF_AUC-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.Param.Plot-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.Param.Plot-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.Param.Plot-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.Param.Plot-Alg', choices = IDs, selected = IDs) # # updateSelectInput(session, 'Report.Param.Statistics-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.Param.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.Param.Statistics-Alg', choices = algIds_, selected = algIds_) + # updateSelectInput(session, 'Report.Param.Statistics-Alg', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTportfolio.Shapley.Algs', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RTportfolio.Shapley.Algs', choices = IDs, selected = IDs) updateSelectInput(session, 'RTportfolio.Shapley.Funcs', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RTportfolio.Shapley.Dims', choices = DIMs, selected = DIMs) - updateNumericInput(session, 'RTportfolio.Shapley.Permsize', min = 2, max = length(algIds_), - value = min(10, length(algIds_))) + updateNumericInput(session, 'RTportfolio.Shapley.Permsize', min = 2, max = length(IDs), + value = min(10, length(IDs))) - updateSelectInput(session, 'RT.MultiERT.AlgId', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT.MultiERT.AlgId', choices = IDs, selected = IDs) updateSelectInput(session, 'RT.MultiERT.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RT.MultiERT.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT.Multisample.AlgId', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT.Multisample.AlgId', choices = IDs, selected = IDs) updateSelectInput(session, 'RT.Multisample.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RT.Multisample.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV.MultiFV.AlgId', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV.MultiFV.AlgId', choices = IDs, selected = IDs) updateSelectInput(session, 'FV.MultiFV.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'FV.MultiFV.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV.Multisample.AlgId', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV.Multisample.AlgId', choices = IDs, selected = IDs) updateSelectInput(session, 'FV.Multisample.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'FV.Multisample.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT_Stats.Glicko.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT_Stats.Glicko.Algid', choices = IDs, selected = IDs) updateSelectInput(session, 'RT_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) updateSelectInput(session, 'RT_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT_Stats.DSC.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT_Stats.DSC.Algid', choices = IDs, selected = IDs) updateSelectInput(session, 'RT_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RT_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - updateSelectInput(session, 'FV_Stats.DSC.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV_Stats.DSC.Algid', choices = IDs, selected = IDs) updateSelectInput(session, 'FV_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) updateSelectInput(session, 'FV_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - updateSelectInput(session, 'RT_Stats.Overview.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'RT_Stats.Overview.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_Stats.Glicko.Algid', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV_Stats.Glicko.Algid', choices = IDs, selected = IDs) updateSelectInput(session, 'FV_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV_Stats.Overview.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTSummary.Statistics.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTSummary.Overview.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCESummary.Overview.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTSummary.Sample.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCESummary.Statistics.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCESummary.Sample.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV_PAR.Summary.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV_PAR.Sample.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT_PAR.Summary.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT_PAR.Sample.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = algIds_, selected = selected_alg) - updateSelectInput(session, 'ERTPlot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ERTPlot.Aggr_Dim.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPlot.Multi.Algs', choices = algIds_, selected = selected_alg) - updateSelectInput(session, 'FCEPlot.Aggr.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPlot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPDF.Bar.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPDF.Hist.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTPMF.Bar.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTPMF.Hist.Algs', choices = algIds_, selected = algIds_) + updateSelectInput(session, 'FV_Stats.Overview.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Statistics.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Overview.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Overview.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Sample.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Statistics.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Sample.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Summary.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Sample.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Summary.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Sample.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = IDs, selected = selected_ID) + updateSelectInput(session, 'ERTPlot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Aggr_Dim.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPlot.Multi.Algs', choices = IDs, selected = selected_ID) + updateSelectInput(session, 'FCEPlot.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPlot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPDF.Bar.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPDF.Hist.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTPMF.Bar.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTPMF.Hist.Algs', choices = IDs, selected = IDs) updateSelectInput(session, 'FV_PAR.Summary.Param', choices = parIds, selected = 'all') updateSelectInput(session, 'FV_PAR.Sample.Param', choices = parIds, selected = 'all') updateSelectInput(session, 'RT_PAR.Summary.Param', choices = parIds, selected = 'all') updateSelectInput(session, 'RT_PAR.Sample.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RTECDF.Single.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTECDF.Aggr.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTECDF.AUC.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTECDF.Multi.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEECDF.Single.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEECDF.Mult.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEECDF.AUC.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ParCoordPlot.Algs', choices = algIds_, selected = algIds_[[1]]) + updateSelectInput(session, 'RTECDF.Single.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.AUC.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.Multi.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.Single.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.Mult.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.AUC.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ParCoordPlot.Algs', choices = IDs, selected = IDs[[1]]) updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) @@ -525,9 +531,12 @@ DATA <- reactive({ id <- input$Overall.Funcid req(dim, id) + algid <- input$Overall.AlgId + if (length(DataList$data) == 0) return(NULL) d <- subset(DataList$data, DIM == dim, funcId == id) + if (!is.null(algid)) d <- subset(d, algId == algid) if (length(d) == 0 && dim != "" && id != "") { showNotification("There is no data available for this (dimension,function)-pair") } diff --git a/inst/shiny-server/ui.R b/inst/shiny-server/ui.R index ad64ee12..b6f525b2 100644 --- a/inst/shiny-server/ui.R +++ b/inst/shiny-server/ui.R @@ -93,24 +93,24 @@ tagAssert <- function(tag, type = NULL, class = NULL, allowUI = TRUE) { ), # select inputs for dimension and function/problem ID - HTML(' -
-
- - - - - -
- Dimension: - - - Problem ID: - -
' - ), + # HTML(' + #
+ #
+ # + # + # + # + # + #
+ # Dimension: + # + # + # Problem ID: + # + #
' + # ), div( class = "navbar-custom-menu", @@ -126,8 +126,8 @@ header <- .dashboardHeader(title = HTML('
IOHanalyzer< sidebar <- dashboardSidebar( useShinyjs(), sidebar_menu(), - hr() - # DIM_fID_panel() + hr(), + DIM_fID_panel() ) body <- dashboardBody( diff --git a/inst/shiny-server/ui/DIM_fID_panel.R b/inst/shiny-server/ui/DIM_fID_panel.R index 10d9e963..fa2227a3 100644 --- a/inst/shiny-server/ui/DIM_fID_panel.R +++ b/inst/shiny-server/ui/DIM_fID_panel.R @@ -9,7 +9,7 @@ DIM_fID_panel <- function() { width = 11, selectInput( "Overall.Funcid", - label = HTML('

Select the function ID

'), + label = HTML('

Function ID:

'), choices = NULL, selected = NULL) ) ) @@ -21,11 +21,22 @@ DIM_fID_panel <- function() { width = 11, selectInput( "Overall.Dim", - label = HTML('

Select the dimension

'), + label = HTML('

Dimension:

'), choices = NULL, selected = NULL) ) ) - ) + ), + div(id = 'overall_algid_box', + fluidRow( + column( + width = 11, + selectInput( + "Overall.AlgId", + label = HTML('

Algorithm:

'), + choices = NULL, selected = NULL) + ) + ) + )#, ) ) } From a6f4cca1cda293362d2dfb9c5f33c8e0dda0532c Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Fri, 23 Jul 2021 19:42:08 +0200 Subject: [PATCH 18/28] Update mentions of ID in GUI; fix conditional algid selection --- inst/shiny-server/server/upload.R | 13 ++++++++----- inst/shiny-server/ui/ERT_agg_box.R | 4 ++-- inst/shiny-server/ui/ERT_box.R | 4 ++-- inst/shiny-server/ui/ERT_comparison_box.R | 8 ++++---- inst/shiny-server/ui/fv_box.R | 12 ++++++------ inst/shiny-server/ui/fv_dist_box.R | 8 ++++---- inst/shiny-server/ui/fv_ecdf_box.R | 12 ++++++------ inst/shiny-server/ui/fv_par_box.R | 4 ++-- inst/shiny-server/ui/fv_summary_box.R | 2 +- inst/shiny-server/ui/overview_box.R | 2 +- inst/shiny-server/ui/rt_dist_box.R | 6 +++--- inst/shiny-server/ui/rt_ecdf_box.R | 16 ++++++++-------- inst/shiny-server/ui/rt_par_box.R | 4 ++-- inst/shiny-server/ui/rt_summary_box.R | 2 +- inst/shiny-server/ui/tables_multi_func_box.R | 8 ++++---- inst/shiny-server/ui/variable.R | 5 +++-- 16 files changed, 57 insertions(+), 53 deletions(-) diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index 8711223c..6fa9d902 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -530,13 +530,16 @@ DATA <- reactive({ dim <- input$Overall.Dim id <- input$Overall.Funcid req(dim, id) - - algid <- input$Overall.AlgId - if (length(DataList$data) == 0) return(NULL) - d <- subset(DataList$data, DIM == dim, funcId == id) - if (!is.null(algid)) d <- subset(d, algId == algid) + + if (!'algId' %in% input$Settings.ID.Variables) { + algid <- input$Overall.AlgId + if (!is.null(algid)) d <- subset(d, algId == algid) + } + + if (length(DataList$data) == 0) return(NULL) + if (length(d) == 0 && dim != "" && id != "") { showNotification("There is no data available for this (dimension,function)-pair") } diff --git a/inst/shiny-server/ui/ERT_agg_box.R b/inst/shiny-server/ui/ERT_agg_box.R index 558ccb61..02f5de4d 100644 --- a/inst/shiny-server/ui/ERT_agg_box.R +++ b/inst/shiny-server/ui/ERT_agg_box.R @@ -7,11 +7,11 @@ ERT_agg_box <- function(width = 12, height = '600px', collapsible = T, sidebarLayout( sidebarPanel( width = 2, - selectInput('ERTPlot.Multi.Algs', label = 'Select which algorithms to plot:', + selectInput('ERTPlot.Multi.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/ERT_box.R b/inst/shiny-server/ui/ERT_box.R index 819fe04f..5c585e85 100644 --- a/inst/shiny-server/ui/ERT_box.R +++ b/inst/shiny-server/ui/ERT_box.R @@ -6,11 +6,11 @@ ERT_box <- function(width = 12, collapsible = T, collapsed = T) { sidebarPanel( width = 3, - selectInput('ERTPlot.Algs', label = 'Select which algorithms to plot:', + selectInput('ERTPlot.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/ERT_comparison_box.R b/inst/shiny-server/ui/ERT_comparison_box.R index fa4c5974..0aa2d4fc 100644 --- a/inst/shiny-server/ui/ERT_comparison_box.R +++ b/inst/shiny-server/ui/ERT_comparison_box.R @@ -5,11 +5,11 @@ ERT_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { sidebarLayout( sidebarPanel( width = 2, - selectInput('ERTPlot.Aggr.Algs', label = 'Select which algorithms to plot:', + selectInput('ERTPlot.Aggr.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -72,11 +72,11 @@ ERT_comparison_box_dim <- function(width = 12, collapsible = T, collapsed = T) { sidebarLayout( sidebarPanel( width = 2, - selectInput('ERTPlot.Aggr_Dim.Algs', label = 'Select which algorithms to plot:', + selectInput('ERTPlot.Aggr_Dim.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/fv_box.R b/inst/shiny-server/ui/fv_box.R index 4252fc72..ca6010d3 100644 --- a/inst/shiny-server/ui/fv_box.R +++ b/inst/shiny-server/ui/fv_box.R @@ -9,11 +9,11 @@ fv_per_fct_box <- function(width = 12, collapsible = T, collapsed = T) { textInput('FCEPlot.Min', label = RT_MIN_LABEL, value = ''), textInput('FCEPlot.Max', label = RT_MAX_LABEL, value = ''), - selectInput('FCEPlot.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEPlot.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -80,11 +80,11 @@ fv_agg_box <- function(width = 12, height = '600px', collapsible = T, collapsed sidebarLayout( sidebarPanel( width = 2, - selectInput('FCEPlot.Multi.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEPlot.Multi.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -123,11 +123,11 @@ fv_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { sidebarLayout( sidebarPanel( width = 3, - selectInput('FCEPlot.Aggr.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEPlot.Aggr.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/fv_dist_box.R b/inst/shiny-server/ui/fv_dist_box.R index 5c635126..383a8452 100644 --- a/inst/shiny-server/ui/fv_dist_box.R +++ b/inst/shiny-server/ui/fv_dist_box.R @@ -7,11 +7,11 @@ fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { width = 2, textInput('FCEPDF.Hist.Runtime', label = HTML('Select the budget value'), value = ''), - selectInput('FCEPDF.Hist.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEPDF.Hist.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -60,11 +60,11 @@ fv_pdf_box <- function(width = 12, collapsible = T, collapsed = T) { HTML('Select the budget for which the distribution of best-so-far function values is shown'), textInput('FCEPDF.Bar.Runtime', label = '', value = ''), - selectInput('FCEPDF.Bar.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEPDF.Bar.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/fv_ecdf_box.R b/inst/shiny-server/ui/fv_ecdf_box.R index e9fc1214..68b330a1 100644 --- a/inst/shiny-server/ui/fv_ecdf_box.R +++ b/inst/shiny-server/ui/fv_ecdf_box.R @@ -7,11 +7,11 @@ fv_ecdf_single_budget_box <- function(width = 12, collapsible = T, collapsed = T sidebarLayout( sidebarPanel( width = 3, - selectInput('FCEECDF.Single.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEECDF.Single.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -44,11 +44,11 @@ fv_ecdf_agg_budgets_box <- function(width = 12, collapsible = T, collapsed = T) solidHeader = T, status = "primary", sidebarPanel( width = 3, - selectInput('FCEECDF.Mult.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEECDF.Mult.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -99,11 +99,11 @@ fv_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { solidHeader = T, status = "primary", sidebarPanel( width = 3, - selectInput('FCEECDF.AUC.Algs', label = 'Select which algorithms to plot:', + selectInput('FCEECDF.AUC.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/fv_par_box.R b/inst/shiny-server/ui/fv_par_box.R index 4c816a13..716c9c91 100644 --- a/inst/shiny-server/ui/fv_par_box.R +++ b/inst/shiny-server/ui/fv_par_box.R @@ -11,10 +11,10 @@ fv_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T textInput('FV_PAR.Plot.Max', label = RT_MAX_LABEL, value = ''), selectInput('FV_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), - selectInput('FV_PAR.Plot.Algs', 'Select which algorithms to plot:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( + selectInput('FV_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/fv_summary_box.R b/inst/shiny-server/ui/fv_summary_box.R index bd523377..1b7207c5 100644 --- a/inst/shiny-server/ui/fv_summary_box.R +++ b/inst/shiny-server/ui/fv_summary_box.R @@ -4,7 +4,7 @@ fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('FCESummary.Overview.Algid', 'Algorithms', choices = NULL, selected = NULL), #TODO: implement this button diff --git a/inst/shiny-server/ui/overview_box.R b/inst/shiny-server/ui/overview_box.R index 2779b482..d08f4771 100644 --- a/inst/shiny-server/ui/overview_box.R +++ b/inst/shiny-server/ui/overview_box.R @@ -4,7 +4,7 @@ general_overview_box_single <- function(width = 12, collapsible = T, collapsed = collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('Overview.Single.Algid', 'Algorithms', choices = NULL, selected = NULL, multiple = T), hr(), diff --git a/inst/shiny-server/ui/rt_dist_box.R b/inst/shiny-server/ui/rt_dist_box.R index d40aca2f..e2aea432 100644 --- a/inst/shiny-server/ui/rt_dist_box.R +++ b/inst/shiny-server/ui/rt_dist_box.R @@ -16,7 +16,7 @@ rt_histogram_box <- function(width = 12, collapsed = T, collapsible = T) { shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", + title = "ID selection", content = alg_select_info, placement = "auto" ) @@ -64,7 +64,7 @@ rt_pmf_box <- function(width = 12, collapsed = T, collapsible = T) { selectInput( 'RTPMF.Bar.Algs', - label = 'Select which algorithms to plot:', + label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL @@ -72,7 +72,7 @@ rt_pmf_box <- function(width = 12, collapsed = T, collapsible = T) { shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", + title = "ID selection", content = alg_select_info, placement = "auto" ) diff --git a/inst/shiny-server/ui/rt_ecdf_box.R b/inst/shiny-server/ui/rt_ecdf_box.R index cd33a5d6..2e7eb064 100644 --- a/inst/shiny-server/ui/rt_ecdf_box.R +++ b/inst/shiny-server/ui/rt_ecdf_box.R @@ -5,11 +5,11 @@ rt_ecdf_single_target_box <- function(width = 12, collapsible = T, collapsed = T sidebarLayout( sidebarPanel( width = 3, - selectInput('RTECDF.Single.Algs', label = 'Select which algorithms to plot:', + selectInput('RTECDF.Single.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -48,11 +48,11 @@ rt_ecdf_agg_targets_box <- function(width = 12, collapsible = T, collapsed = T) solidHeader = T, status = "primary", sidebarPanel( width = 3, - selectInput('RTECDF.Multi.Algs', label = 'Select which algorithms to plot:', + selectInput('RTECDF.Multi.Algs', label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -106,14 +106,14 @@ rt_ecdf_agg_fct_box <- function(width = 12, collapsible = T, collapsed = T) { width = 3, selectInput( 'RTECDF.Aggr.Algs', - label = 'Select which algorithms to plot:', + label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL ) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), @@ -218,11 +218,11 @@ rt_ecdf_agg_fct_box <- function(width = 12, collapsible = T, collapsed = T) { # solidHeader = T, status = "primary", # sidebarPanel( # width = 3, -# selectInput('RTECDF.AUC.Algs', label = 'Select which algorithms to plot:', +# selectInput('RTECDF.AUC.Algs', label = 'Select which IDs to include:', # multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( # custom_icon() %>% # bs_embed_popover( -# title = "Algorithm selection", content = alg_select_info, +# title = "ID selection", content = alg_select_info, # placement = "auto" # ) # ), diff --git a/inst/shiny-server/ui/rt_par_box.R b/inst/shiny-server/ui/rt_par_box.R index 0345e581..34510870 100644 --- a/inst/shiny-server/ui/rt_par_box.R +++ b/inst/shiny-server/ui/rt_par_box.R @@ -11,10 +11,10 @@ rt_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T textInput('RT_PAR.Plot.Max', label = F_MAX_LABEL, value = ''), selectInput('RT_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), - selectInput('RT_PAR.Plot.Algs', 'Select which algorithms to plot:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( + selectInput('RT_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( custom_icon() %>% bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, + title = "ID selection", content = alg_select_info, placement = "auto" ) ), diff --git a/inst/shiny-server/ui/rt_summary_box.R b/inst/shiny-server/ui/rt_summary_box.R index 8ed1ddb9..6ebc5cb6 100644 --- a/inst/shiny-server/ui/rt_summary_box.R +++ b/inst/shiny-server/ui/rt_summary_box.R @@ -76,7 +76,7 @@ rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('RTSummary.Overview.Algid', 'Algorithms', choices = NULL, selected = NULL), hr(), diff --git a/inst/shiny-server/ui/tables_multi_func_box.R b/inst/shiny-server/ui/tables_multi_func_box.R index 7402a11f..09a7c10b 100644 --- a/inst/shiny-server/ui/tables_multi_func_box.R +++ b/inst/shiny-server/ui/tables_multi_func_box.R @@ -4,7 +4,7 @@ multi_function_ert_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('RT.MultiERT.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('RT.MultiERT.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), @@ -30,7 +30,7 @@ multi_function_sample_box <- function(width = 12, collapsible = T, collapsed = T collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('RT.Multisample.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('RT.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), @@ -57,7 +57,7 @@ multi_function_fv_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('FV.MultiFV.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('FV.MultiFV.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), @@ -83,7 +83,7 @@ multi_function_sample_box_fv <- function(width = 12, collapsible = T, collapsed collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which algorithms to show.

'), + HTML('

Select which IDs to include:

'), selectInput('FV.Multisample.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('FV.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), diff --git a/inst/shiny-server/ui/variable.R b/inst/shiny-server/ui/variable.R index 9b303446..b77fdb6a 100644 --- a/inst/shiny-server/ui/variable.R +++ b/inst/shiny-server/ui/variable.R @@ -27,9 +27,10 @@ Pdsc_mc_info <- "Practical Deep Comparison uses this HTML_P <- function(s) HTML(paste0('

', s, '

')) -alg_select_info <- "Use this option to select which algorithms to plot. +alg_select_info <- "Use this option to select which IDs to plot. This will hava an effect the dowloaded plot, - as opposed to using the legend-entries to show or hide algorithms " + as opposed to using the legend-entries to show or hide algorithms. + The attributes which define the ID can be changed in the settings." custom_icon <- function(name = NULL){ if (is.null(name)) name <- "info-circle" From b5ef8fbf0898a3af576787a7a61ef99fb2a72654 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Mon, 26 Jul 2021 11:21:52 +0200 Subject: [PATCH 19/28] Add option to use funcname instead of funcid if available --- NAMESPACE | 1 + R/DataSetList.R | 12 +++++++ inst/shiny-server/server/settings_page.R | 13 ++++++++ inst/shiny-server/server/upload.R | 40 +++++++++++++++++++++--- inst/shiny-server/ui/DIM_fID_panel.R | 14 ++++++++- inst/shiny-server/ui/settings.R | 1 + man/get_funcName.Rd | 20 ++++++++++++ 7 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 man/get_funcName.Rd diff --git a/NAMESPACE b/NAMESPACE index b352ec66..d1190bcd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -131,6 +131,7 @@ export(get_dsc_omnibus) export(get_dsc_posthoc) export(get_dsc_rank) export(get_funcId) +export(get_funcName) export(get_funvals) export(get_id) export(get_line_style) diff --git a/R/DataSetList.R b/R/DataSetList.R index ac376c01..914c64e8 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -637,6 +637,18 @@ get_funcId <- function(dsList) { sort(lli) } +#' Get all function names present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list of all unique function names which occur in the DataSetList +#' @export +#' @examples +#' get_funcName(dsl) +get_funcName <- function(dsList) { + unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcName'))))) +} + #' Get all algorithm ids present in a DataSetList #' #' @param dsList The DataSetLsit diff --git a/inst/shiny-server/server/settings_page.R b/inst/shiny-server/server/settings_page.R index 0de5fb7d..52c17d1c 100644 --- a/inst/shiny-server/server/settings_page.R +++ b/inst/shiny-server/server/settings_page.R @@ -163,6 +163,19 @@ observe({ }, message = "Processing IDs") }) +observe({ + if (input$Settings.Use_Funcname) { + shinyjs::show(id = "overall_funcname_box") + shinyjs::hide(id = "overall_funcid_box") + options('IOHanalyzer.function_representation' = 'funcname') + } + else { + shinyjs::hide(id = "overall_funcname_box") + shinyjs::show(id = "overall_funcid_box") + options('IOHanalyzer.function_representation' = 'funcId') + } +}) + observe({ setting_preset <- input$Settings.Download.Preset if (setting_preset == "Default") { diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index 6fa9d902..fa015969 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -274,6 +274,10 @@ update_menu_visibility <- function(suite){ else { session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "Positions")) } + if (!is.null(get_funcName(DataList$data))) + shinyjs::enable("Settings.Use_Funcname") + else + shinyjs::disable("Settings.Use_Funcname") } observeEvent(input$Upload.Add_to_repo, { @@ -307,7 +311,9 @@ observeEvent(input$upload.remove_data, { updateSelectInput(session, 'Overall.Dim', choices = c(), selected = '') updateSelectInput(session, 'Overall.Funcid', choices = c(), selected = '') - + updateSelectInput(session, 'Overall.Funcname', choices = c(), selected = '') + updateSelectInput(session, 'Overall.Algid', choices = c(), selected = '') + print_html('

all data are removed!

') print_html('', 'upload_data_promt') } @@ -346,6 +352,9 @@ observe({ updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) + if (input$Settings.Use_Funcname) + updateSelectInput(session, 'Overall.Funcname', choices = get_funcName(data), selected = get_funcName(data[1])) + if ('algId' %in% input$Settings.ID.Variables) updateSelectInput(session, 'Overall.AlgId', choices = NULL, selected = NULL) else @@ -525,22 +534,32 @@ observe({ selected = attr(data, 'ID_attributes')) }) + + # update (filter) according to users selection DataSets DATA <- reactive({ dim <- input$Overall.Dim - id <- input$Overall.Funcid - req(dim, id) + req(dim) - d <- subset(DataList$data, DIM == dim, funcId == id) + d <- subset(DataList$data, DIM == dim) if (!'algId' %in% input$Settings.ID.Variables) { algid <- input$Overall.AlgId if (!is.null(algid)) d <- subset(d, algId == algid) } + + if (input$Settings.Use_Funcname) { + fname <- input$Overall.Funcname + if (!is.null(fname)) d <- subset(d, funcname == fname) + } + else { + fid <- input$Overall.Funcid + if (!is.null(fid)) d <- subset(d, funcId == fid) + } if (length(DataList$data) == 0) return(NULL) - if (length(d) == 0 && dim != "" && id != "") { + if (length(d) == 0 && dim != "") { showNotification("There is no data available for this (dimension,function)-pair") } d @@ -551,6 +570,17 @@ DATA_RAW <- reactive({ DataList$data }) +# This observe statement tries to match funcid and funcname seletions so funcId can still be used internally. +# TODO: think of a better solution to ensure this matching doesn't break. +observe({ + fname <- input$Overall.Funcname + dsl_sub <- subset(DATA_RAW(), funcName == fname) + fids <- get_funcId(dsl_sub) + if (length(fids) == 1) { + updateSelectInput(session, 'Overall.Funcid', selected = fids) + } +}) + MAX_ERTS_FUNC <- reactive({ dim <- input$Overall.Dim data <- subset(DataList$data, DIM == dim) diff --git a/inst/shiny-server/ui/DIM_fID_panel.R b/inst/shiny-server/ui/DIM_fID_panel.R index fa2227a3..93e82223 100644 --- a/inst/shiny-server/ui/DIM_fID_panel.R +++ b/inst/shiny-server/ui/DIM_fID_panel.R @@ -2,7 +2,7 @@ DIM_fID_panel <- function() { conditionalPanel( "input.tabs!='Report' && input.tabs!='upload' && input.tabs!='readme' && input.tabs!='about' && input.tabs!='Settings'", column(12, offset = 0, - div( + div(id = 'overall_funcid_box', # style = "padding: 0px 0px; margin-top:-5em; margin:0%", fluidRow( column( @@ -14,6 +14,18 @@ DIM_fID_panel <- function() { ) ) ), + div(id = 'overall_funcname_box', + # style = "padding: 0px 0px; margin-top:-5em; margin:0%", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Funcname", + label = HTML('

Function Name:

'), + choices = NULL, selected = NULL) + ) + ) + ), div( # style = "padding: 0px 0px; margin-top:-50em;", fluidRow( diff --git a/inst/shiny-server/ui/settings.R b/inst/shiny-server/ui/settings.R index 60486642..1d0dadf0 100644 --- a/inst/shiny-server/ui/settings.R +++ b/inst/shiny-server/ui/settings.R @@ -62,6 +62,7 @@ general_settings_box <- function(width=12, collapsible = T, collapsed = F) { HTML_P("ID variables"), selectInput("Settings.ID.Variables", label = "Attributes used to create IDs", multiple = T, selected = NULL, choices = NULL), + checkboxInput("Settings.Use_Funcname", "Use function names instead of IDs"), hr(), downloadButton("Settings.Download", "Download current general settings file"), fileInput("Settings.Upload", "Upload a settings file", accept = "rds") diff --git a/man/get_funcName.Rd b/man/get_funcName.Rd new file mode 100644 index 00000000..415d15cd --- /dev/null +++ b/man/get_funcName.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funcName} +\alias{get_funcName} +\title{Get all function names present in a DataSetList} +\usage{ +get_funcName(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list of all unique function names which occur in the DataSetList +} +\description{ +Get all function names present in a DataSetList +} +\examples{ +get_funcName(dsl) +} From 020a7e84f0a0ce68ce992c38c61cc99758b57768 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Mon, 26 Jul 2021 13:35:57 +0200 Subject: [PATCH 20/28] Update data summary GUIs --- R/stats.R | 1 - inst/shiny-server/server/ERT_summary.R | 13 +++++----- inst/shiny-server/server/FCESummary.R | 35 +++++++------------------- inst/shiny-server/ui/fv_summary_box.R | 10 +++----- inst/shiny-server/ui/rt_summary_box.R | 10 +++----- 5 files changed, 24 insertions(+), 45 deletions(-) diff --git a/R/stats.R b/R/stats.R index 57b7c013..d46092c3 100644 --- a/R/stats.R +++ b/R/stats.R @@ -12,7 +12,6 @@ SP <- function(data, max_runtime) { N <- ncol(data) if (length(max_runtime) > 1) { - warning("Argument 'max_runtime' will only be available to use as a single numeric value in the next release of IOHanalyzer.") max_runtime <- max(max_runtime) } idx <- is.na(data) | data > max_runtime diff --git a/inst/shiny-server/server/ERT_summary.R b/inst/shiny-server/server/ERT_summary.R index 5fa30543..02fb58e1 100644 --- a/inst/shiny-server/server/ERT_summary.R +++ b/inst/shiny-server/server/ERT_summary.R @@ -2,8 +2,9 @@ runtime_summary_condensed <- reactive({ data <- DATA() req(length(data) > 0) - fall <- get_funvals(data) - df <- get_FV_overview(data, algorithm = input$RTSummary.Overview.Algid) + # fall <- get_funvals(data) + data <- subset(data, ID %in% input$RTSummary.Overview.Algid) + df <- get_FV_overview(data) df$budget %<>% as.numeric df$runs %<>% as.integer df$funcId %<>% as.integer @@ -44,7 +45,7 @@ runtime_summary <- reactive({ fstop <- format_FV(input$RTSummary.Statistics.Max) fstep <- format_FV(input$RTSummary.Statistics.Step) data <- DATA() - + data <- subset(data, ID %in% input$RTSummary.Statistics.Algid) if (!input$RTSummary.Statistics.Single) { req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) fall <- get_funvals(data) @@ -54,7 +55,7 @@ runtime_summary <- reactive({ fseq <- fstart } - df <- get_RT_summary(data, fseq, algorithm = input$RTSummary.Statistics.Algid) + df <- get_RT_summary(data, fseq) df <- df[, c('DIM', 'funcId') := NULL] df$target <- format_FV(df$target) %>% as.numeric @@ -98,7 +99,7 @@ get_RT <- reactive({ fstop <- format_FV(input$RTSummary.Sample.Max) fstep <- format_FV(input$RTSummary.Sample.Step) data <- DATA() - + data <- subset(data, ID %in% input$RTSummary.Sample.Algid) if (!input$RTSummary.Sample.Single) { req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) fall <- get_funvals(data) @@ -109,7 +110,7 @@ get_RT <- reactive({ fseq <- fstart } - df <- get_RT_sample(data, ftarget = fseq, algorithm = input$RTSummary.Sample.Algid, + df <- get_RT_sample(data, ftarget = fseq, output = input$RTSummary.Sample.DownloadFormat) df$target <- format_FV(df$target) df[is.na(df)] <- 'NA' diff --git a/inst/shiny-server/server/FCESummary.R b/inst/shiny-server/server/FCESummary.R index e7f4da1c..f0711121 100644 --- a/inst/shiny-server/server/FCESummary.R +++ b/inst/shiny-server/server/FCESummary.R @@ -2,8 +2,9 @@ # Data summary for Fixed-Budget target (FCE) -------------- FCE_runtime_summary_condensed <- reactive({ data <- DATA() - fall <- get_funvals(data) - df <- get_RT_overview(data, algorithm = input$FCESummary.Overview.Algid) + data <- subset(data, ID %in% input$FCESummary.Overview.Algid) + # fall <- get_funvals(data) + df <- get_RT_overview(data) df$"runs" %<>% as.integer df$"Budget" %<>% as.numeric df$"miminal runtime" %<>% as.numeric @@ -37,9 +38,10 @@ get_FCE_summary <- reactive({ rt_step <- input$FCESummary.Statistics.Step %>% as.numeric data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Statistics.Algid) - if (!input$FCESummary.Statistics.Single){ + if (!input$FCESummary.Statistics.Single) { req(rt_min <= rt_max, rt_step <= rt_max - rt_min) rt <- get_runtimes(data) rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) @@ -49,7 +51,7 @@ get_FCE_summary <- reactive({ rt_seq <- rt_min } - df <- get_FV_summary(data, rt_seq, algorithm = input$FCESummary.Statistics.Algid)[ + df <- get_FV_summary(data, rt_seq)[ , c('DIM', 'funcId') := NULL ] df$runs %<>% as.integer @@ -91,8 +93,9 @@ get_FCE <- reactive({ rt_step <- input$FCESummary.Sample.Step %>% as.numeric data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Sample.Algid) - if (!input$FCESummary.Sample.Single){ + if (!input$FCESummary.Sample.Single) { req(rt_min <= rt_max, rt_step <= rt_max - rt_min) rt <- get_runtimes(data) rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) @@ -102,28 +105,8 @@ get_FCE <- reactive({ rt_seq <- rt_min } - get_FV_sample(data, rt_seq, algorithm = input$FCESummary.Sample.Algid, + get_FV_sample(data, rt_seq, output = input$FCESummary.Sample.Format) - - # res <- list() - # n_runs_max <- sapply(data, function(x) length(attr(x, 'instance'))) %>% max - # - # for (i in seq_along(data)) { - # ds <- data[[i]] - # algId <- attr(ds, 'algId') - # if (input$FCESummary.Sample.Algid != 'all' && algId != input$FCESummary.Sample.Algid) - # next - # - # rt <- get_FV_sample(ds, rt_seq, output = input$FCESummary.Sample.Format) - # if (input$FCESummary.Sample.Format == 'wide') { - # # impute the missing records - # n <- ncol(rt) - 2 - # if (n < n_runs_max) - # rt %<>% cbind(., matrix(NA, nrow(.), n_runs_max - n)) - # } - # res[[i]] <- rt - # } - # do.call(rbind, res) }) output$FCESummary.Sample.Download <- downloadHandler( diff --git a/inst/shiny-server/ui/fv_summary_box.R b/inst/shiny-server/ui/fv_summary_box.R index 1b7207c5..bc1e2976 100644 --- a/inst/shiny-server/ui/fv_summary_box.R +++ b/inst/shiny-server/ui/fv_summary_box.R @@ -4,9 +4,7 @@ fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('FCESummary.Overview.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('FCESummary.Overview.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), #TODO: implement this button hr(), selectInput('FCESummary.Overview.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), @@ -34,7 +32,7 @@ fv_stats_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('FCESummary.Statistics.Single', label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? Once toggled, only \\(B_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Statistics.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('FCESummary.Statistics.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('FCESummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("FCESummary.Statistics.Download", "Save this table") @@ -66,8 +64,8 @@ fv_sample_box <- function(width = 12, collapsible = T, collapsed = T) { label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? Once toggled, only \\(B_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Sample.Algid', 'Algorithms', - choices = NULL, selected = NULL), + selectInput('FCESummary.Sample.Algid', 'Select which IDs to include:', + choices = NULL, selected = NULL, multiple = T), selectInput('FCESummary.Sample.Format', 'Format of the table', choices = c('long', 'wide'), selected = 'wide'), diff --git a/inst/shiny-server/ui/rt_summary_box.R b/inst/shiny-server/ui/rt_summary_box.R index 6ebc5cb6..12d58517 100644 --- a/inst/shiny-server/ui/rt_summary_box.R +++ b/inst/shiny-server/ui/rt_summary_box.R @@ -13,7 +13,7 @@ rt_stats_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('RTSummary.Statistics.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('RTSummary.Statistics.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('RTSummary.Statistics.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('RTSummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("RTSummary.Statistics.Download", "Save this table") @@ -47,8 +47,8 @@ rt_sample_box <- function(width = 12, collapsible = T, collapsed = T) { # TODO: do we need this log scaling? # checkboxInput('F_LOGSPACE_DATA_SUMMARY', # label = HTML('Evenly space target values in \\(log_{10}\\) space')), - selectInput('RTSummary.Sample.Algid', 'Algorithms', - choices = NULL, selected = NULL), + selectInput('RTSummary.Sample.Algid', 'Select which IDs to include:', + choices = NULL, selected = NULL, multiple = T), hr(), selectInput('RTSummary.Sample.DownloadFormat', 'Format of the table', @@ -76,9 +76,7 @@ rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('RTSummary.Overview.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('RTSummary.Overview.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('RTSummary.Overview.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("RTSummary.Overview.Download", "Save this table") From 4e93e33a8f64e055d28e3d8b9f171f15f929d37f Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Mon, 26 Jul 2021 14:01:09 +0200 Subject: [PATCH 21/28] more GUI updates regarding ID selection --- inst/shiny-server/server/ERT_aggr_dim_plot.R | 4 ++-- inst/shiny-server/server/ERT_aggr_plot.R | 2 +- inst/shiny-server/server/FCE_aggr_plot.R | 2 +- inst/shiny-server/server/FV_DSCinterface.R | 2 +- inst/shiny-server/server/FV_PAR.R | 8 ++++---- inst/shiny-server/server/RT_DSCinterface.R | 2 +- inst/shiny-server/server/RT_Par.R | 5 +++-- inst/shiny-server/server/Shapley_computations.R | 2 +- inst/shiny-server/server/fv_stat_tests.R | 2 +- inst/shiny-server/server/statistical_significance.R | 2 +- inst/shiny-server/ui/DSC_interface.R | 2 +- inst/shiny-server/ui/fv_dist_box.R | 6 +++--- inst/shiny-server/ui/fv_ecdf_box.R | 2 +- inst/shiny-server/ui/fv_par_box.R | 4 ++-- inst/shiny-server/ui/overview_box.R | 2 +- inst/shiny-server/ui/rt_dist_box.R | 2 +- inst/shiny-server/ui/rt_par_box.R | 4 ++-- 17 files changed, 27 insertions(+), 26 deletions(-) diff --git a/inst/shiny-server/server/ERT_aggr_dim_plot.R b/inst/shiny-server/server/ERT_aggr_dim_plot.R index f3571304..3e26b77e 100644 --- a/inst/shiny-server/server/ERT_aggr_dim_plot.R +++ b/inst/shiny-server/server/ERT_aggr_dim_plot.R @@ -41,7 +41,7 @@ ERTPlot.Aggr_Dim.data <- function() { if (length(unique(get_id(data))) <= 1) { shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected function") + multiple IDs for the selected function") return(NULL) } data @@ -58,7 +58,7 @@ ert_multi_dim <- function(){ default_targets_table_dim <- reactive({ data <- ERTPlot.Aggr_Dim.data() if (is.null(data)) return(NULL) - targets <- get_target_dt(data) + targets <- get_target_dt(data, 'by_FV') targets <- targets[, c('DIM','target')] }) diff --git a/inst/shiny-server/server/ERT_aggr_plot.R b/inst/shiny-server/server/ERT_aggr_plot.R index faae2298..9d8a59b8 100644 --- a/inst/shiny-server/server/ERT_aggr_plot.R +++ b/inst/shiny-server/server/ERT_aggr_plot.R @@ -41,7 +41,7 @@ ERTPlot.Aggr.data <- function() { if (length(unique(get_id(data))) == 1) { shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected dimension.") + multiple IDs for the selected dimension.") return(NULL) } data diff --git a/inst/shiny-server/server/FCE_aggr_plot.R b/inst/shiny-server/server/FCE_aggr_plot.R index e670d769..58e741d7 100644 --- a/inst/shiny-server/server/FCE_aggr_plot.R +++ b/inst/shiny-server/server/FCE_aggr_plot.R @@ -44,7 +44,7 @@ FCEPlot.Aggr.data <- function() { } } if (length(unique(get_id(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected dimension.") + shinyjs::alert("This plot is only available when the dataset contains multiple IDs for the selected dimension.") return(NULL) } data diff --git a/inst/shiny-server/server/FV_DSCinterface.R b/inst/shiny-server/server/FV_DSCinterface.R index 078711c0..3cf3f9cd 100644 --- a/inst/shiny-server/server/FV_DSCinterface.R +++ b/inst/shiny-server/server/FV_DSCinterface.R @@ -189,7 +189,7 @@ FV_DSC_data <- function() { data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") + multiple IDs for the selected functions and dimensions.") return(NULL) } data diff --git a/inst/shiny-server/server/FV_PAR.R b/inst/shiny-server/server/FV_PAR.R index 1010abbf..0b0b8bbd 100644 --- a/inst/shiny-server/server/FV_PAR.R +++ b/inst/shiny-server/server/FV_PAR.R @@ -87,6 +87,7 @@ fv_parameter_summary <- reactive({ rtstop <- as.numeric(input$FV_PAR.Summary.Max) rtstep <- as.numeric(input$FV_PAR.Summary.Step) data <- DATA() + data <- subset(data, ID %in% input$FV_PAR.Summary.Algid) if (!input$FV_PAR.Summary.Single) { req(rtstart <= rtstop, rtstep <= rtstop - rtstart) @@ -97,8 +98,7 @@ fv_parameter_summary <- reactive({ else rtseq <- rtstart - dt <- get_PAR_summary(data, rtseq, input$FV_PAR.Summary.Algid, - input$FV_PAR.Summary.Param, which = 'by_RT') + dt <- get_PAR_summary(data, rtseq, parId = input$FV_PAR.Summary.Param, which = 'by_RT') req(length(dt) != 0) dt$runs %<>% as.integer @@ -124,6 +124,7 @@ fv_parameter_sample <- reactive({ rtstop <- as.numeric(input$FV_PAR.Sample.Max) rtstep <- as.numeric(input$FV_PAR.Sample.Step) data <- DATA() + data <- subset(data, ID %in% input$FV_PAR.Sample.Algid) if (!input$FV_PAR.Sample.Single) { req(rtstart <= rtstop, rtstep <= rtstop - rtstart) @@ -134,8 +135,7 @@ fv_parameter_sample <- reactive({ else rtseq <- rtstart - df <- get_PAR_sample(data, idxValue = rtseq, - algorithm = input$FV_PAR.Sample.Algid, + df <- get_PAR_sample(data, idxValue = rtseq, parId = input$FV_PAR.Sample.Param, output = input$FV_PAR.Sample.Format, which = 'by_RT') diff --git a/inst/shiny-server/server/RT_DSCinterface.R b/inst/shiny-server/server/RT_DSCinterface.R index e564032a..8ec905f9 100644 --- a/inst/shiny-server/server/RT_DSCinterface.R +++ b/inst/shiny-server/server/RT_DSCinterface.R @@ -188,7 +188,7 @@ RT_DSC_data <- function() { data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") + multiple IDs for the selected functions and dimensions.") return(NULL) } data diff --git a/inst/shiny-server/server/RT_Par.R b/inst/shiny-server/server/RT_Par.R index 5287c06d..040072a6 100644 --- a/inst/shiny-server/server/RT_Par.R +++ b/inst/shiny-server/server/RT_Par.R @@ -87,6 +87,7 @@ rt_parameter_summary <- reactive({ fstop <- format_FV(input$RT_PAR.Summary.Max) %>% as.numeric fstep <- format_FV(input$RT_PAR.Summary.Step) %>% as.numeric data <- DATA() + data <- subset(data, ID %in% input$RT_PAR.Summary.Algid) if (!input$RT_PAR.Summary.Single) { req(fstart <= fstop, fstep <= fstop - fstart) @@ -97,7 +98,7 @@ rt_parameter_summary <- reactive({ else fseq <- fstart - dt <- get_PAR_summary(data, fseq, input$RT_PAR.Summary.Algid, input$RT_PAR.Summary.Param) + dt <- get_PAR_summary(data, fseq, parId = input$RT_PAR.Summary.Param) req(length(dt) != 0) dt$runs %<>% as.integer @@ -123,6 +124,7 @@ rt_parameter_sample <- reactive({ fstop <- format_FV(input$RT_PAR.Sample.Max) %>% as.numeric fstep <- format_FV(input$RT_PAR.Sample.Step) %>% as.numeric data <- DATA() + data <- subset(data, ID %in% input$RT_PAR.Sample.Algid) if (!input$RT_PAR.Sample.Single) { req(fstart <= fstop, fstep <= fstop - fstart) @@ -134,7 +136,6 @@ rt_parameter_sample <- reactive({ fseq <- fstart df <- get_PAR_sample(data, idxValue = fseq, - algorithm = input$RT_PAR.Sample.Algid, parId = input$RT_PAR.Sample.Param, output = input$RT_PAR.Sample.Format) diff --git a/inst/shiny-server/server/Shapley_computations.R b/inst/shiny-server/server/Shapley_computations.R index ef546e09..2e9c65a5 100644 --- a/inst/shiny-server/server/Shapley_computations.R +++ b/inst/shiny-server/server/Shapley_computations.R @@ -12,7 +12,7 @@ get_data_RT_SHAPLEY <- reactive({ if (length(get_id(dsList)) <= 1) { shinyjs::alert("This is an invalid configuration for this plot. \n - Please ensure that the dataset contains multiple algorithms.") + Please ensure that the dataset contains multiple IDs.") return(NULL) } diff --git a/inst/shiny-server/server/fv_stat_tests.R b/inst/shiny-server/server/fv_stat_tests.R index 19c12461..e59c1810 100644 --- a/inst/shiny-server/server/fv_stat_tests.R +++ b/inst/shiny-server/server/fv_stat_tests.R @@ -125,7 +125,7 @@ FV_glicko_data <- function() { data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") + multiple IDs for the selected functions and dimensions.") return(NULL) } data diff --git a/inst/shiny-server/server/statistical_significance.R b/inst/shiny-server/server/statistical_significance.R index bd422e68..43c15a80 100644 --- a/inst/shiny-server/server/statistical_significance.R +++ b/inst/shiny-server/server/statistical_significance.R @@ -123,7 +123,7 @@ RT_glicko_data <- function() { data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) if (length(unique(get_id(data))) < 2) { shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") + multiple IDs for the selected functions and dimensions.") return(NULL) } data diff --git a/inst/shiny-server/ui/DSC_interface.R b/inst/shiny-server/ui/DSC_interface.R index 494aaf67..404aeb71 100644 --- a/inst/shiny-server/ui/DSC_interface.R +++ b/inst/shiny-server/ui/DSC_interface.R @@ -5,7 +5,7 @@ fv_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('FV_Stats.DSC.Algid', 'Algorithms to compare', choices = NULL, + selectInput('FV_Stats.DSC.Algid', 'IDs to compare', choices = NULL, selected = NULL, multiple = T), selectInput('FV_Stats.DSC.Funcid', 'Functions to use', choices = NULL, selected = NULL, multiple = T), diff --git a/inst/shiny-server/ui/fv_dist_box.R b/inst/shiny-server/ui/fv_dist_box.R index 383a8452..b9787b29 100644 --- a/inst/shiny-server/ui/fv_dist_box.R +++ b/inst/shiny-server/ui/fv_dist_box.R @@ -20,7 +20,7 @@ fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { selectInput('FCEPDF.Hist.Mode', '', choices = c("overlay", "subplot"), selected = 'subplot'), - checkboxInput("FCEPDF.Hist.Equal", "Use equal bins for all algorithms", F), + checkboxInput("FCEPDF.Hist.Equal", "Use equal bins for all IDs", F), hr(), selectInput('FCEPDF.Hist.Format', label = 'Select the figure format', @@ -41,7 +41,7 @@ fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { according to the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. - The displayed algorithms can be selected by clicking on the legend on the right. + The displayed IDs can be selected by clicking on the legend on the right. A tooltip and toolbar appears when hovering over the figure.

'), plotlyOutput.IOHanalyzer('FCE_HIST') ) @@ -85,7 +85,7 @@ fv_pdf_box <- function(width = 12, collapsible = T, collapsed = T) { HTML('

The plot shows, for the budget selected on the left, the distribution of the best-so-far function values of the individual runs (dots), and an estimated distribution of the probability mass function. - The displayed algorithms can be selected by clicking on the legend on the right. A tooltip and toolbar + The displayed IDs can be selected by clicking on the legend on the right. A tooltip and toolbar appear when hovering over the figure. A csv file with the runtime data can be downloaded from the Data Summary tab.'), plotlyOutput.IOHanalyzer('FCE_PDF') diff --git a/inst/shiny-server/ui/fv_ecdf_box.R b/inst/shiny-server/ui/fv_ecdf_box.R index 68b330a1..09042e32 100644 --- a/inst/shiny-server/ui/fv_ecdf_box.R +++ b/inst/shiny-server/ui/fv_ecdf_box.R @@ -127,7 +127,7 @@ fv_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { caculated for the sequence of budget values specified on the left. The displayed values are normalized against the maximal target value recorded for each algorithm. Intuitively, the smaller the area, the better the algorithm. - The displayed algorithms can be selected by clicking on the legend on the right. + The displayed IDs can be selected by clicking on the legend on the right. A tooltip and toolbar appears when hovering over the figure.'), plotlyOutput.IOHanalyzer("FCE_AUC") ) diff --git a/inst/shiny-server/ui/fv_par_box.R b/inst/shiny-server/ui/fv_par_box.R index 716c9c91..f2c5a163 100644 --- a/inst/shiny-server/ui/fv_par_box.R +++ b/inst/shiny-server/ui/fv_par_box.R @@ -82,7 +82,7 @@ fv_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('FV_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Summary.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('FV_PAR.Summary.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('FV_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('FV_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), @@ -114,7 +114,7 @@ fv_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('FV_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Sample.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('FV_PAR.Sample.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('FV_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('FV_PAR.Sample.Format', 'Format of the table', diff --git a/inst/shiny-server/ui/overview_box.R b/inst/shiny-server/ui/overview_box.R index d08f4771..ee65133b 100644 --- a/inst/shiny-server/ui/overview_box.R +++ b/inst/shiny-server/ui/overview_box.R @@ -6,7 +6,7 @@ general_overview_box_single <- function(width = 12, collapsible = T, collapsed = width = 3, HTML('

Select which IDs to include:

'), - selectInput('Overview.Single.Algid', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('Overview.Single.Algid', 'IDs:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('Overview.Single.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("Overview.Single.Download", "Save this table") diff --git a/inst/shiny-server/ui/rt_dist_box.R b/inst/shiny-server/ui/rt_dist_box.R index e2aea432..f7efbc39 100644 --- a/inst/shiny-server/ui/rt_dist_box.R +++ b/inst/shiny-server/ui/rt_dist_box.R @@ -8,7 +8,7 @@ rt_histogram_box <- function(width = 12, collapsed = T, collapsible = T) { value = ''), selectInput( 'RTPMF.Hist.Algs', - label = 'Select which algorithms to plot', + label = 'Select which IDs to include:', multiple = T, selected = NULL, choices = NULL diff --git a/inst/shiny-server/ui/rt_par_box.R b/inst/shiny-server/ui/rt_par_box.R index 34510870..b1325233 100644 --- a/inst/shiny-server/ui/rt_par_box.R +++ b/inst/shiny-server/ui/rt_par_box.R @@ -82,7 +82,7 @@ rt_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('RT_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Summary.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('RT_PAR.Summary.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('RT_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('RT_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), @@ -114,7 +114,7 @@ rt_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('RT_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Sample.Algid', 'Algorithms', choices = NULL, selected = NULL), + selectInput('RT_PAR.Sample.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('RT_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('RT_PAR.Sample.Format', 'Format of the table', From e8ac02a017a1a639eeea21e167eb5a797433f2a6 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Mon, 26 Jul 2021 14:03:45 +0200 Subject: [PATCH 22/28] Change default multi-function ERT plot --- inst/shiny-server/ui.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inst/shiny-server/ui.R b/inst/shiny-server/ui.R index b6f525b2..e23ff8c6 100644 --- a/inst/shiny-server/ui.R +++ b/inst/shiny-server/ui.R @@ -376,8 +376,8 @@ body <- dashboardBody( fluidRow( column( width = 12, - ERT_comparison_box(collapsed = F), - ERT_agg_box(height = '800px') + ERT_agg_box(collapsed = F), + ERT_comparison_box(collapsed = T) ) ) ), From bc48bdf7e21f88ff86b9131116e21674931426db Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Tue, 27 Jul 2021 11:31:14 +0200 Subject: [PATCH 23/28] Remove some attributes from ID selection --- inst/shiny-server/server/upload.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index fa015969..a78cdf3b 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -529,8 +529,9 @@ observe({ updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) - - updateSelectInput(session, 'Settings.ID.Variables', choices = get_static_attributes(data), + attr_choices <- get_static_attributes(data) + invalid_choices <- c('funcId', 'DIM', 'ID') + updateSelectInput(session, 'Settings.ID.Variables', choices = attr_choices[!attr_choices %in% invalid_choices], selected = attr(data, 'ID_attributes')) }) From 409b120fd52d8765f3830f6e66158771a4de7668 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Thu, 12 Aug 2021 11:22:21 +0200 Subject: [PATCH 24/28] Implement review feedback + fix funcname selection --- R/DataSet.R | 102 +++++++++++------- inst/shiny-server/server/ERT_summary.R | 8 +- inst/shiny-server/server/FCESummary.R | 8 +- inst/shiny-server/server/FV_DSCinterface.R | 2 +- inst/shiny-server/server/FV_PAR.R | 6 +- inst/shiny-server/server/RT_DSCinterface.R | 2 +- inst/shiny-server/server/RT_Par.R | 6 +- inst/shiny-server/server/fv_stat_tests.R | 8 +- inst/shiny-server/server/general_overview.R | 4 +- .../server/statistical_significance.R | 8 +- inst/shiny-server/server/tables_multi_func.R | 16 +-- inst/shiny-server/server/upload.R | 57 +++++----- inst/shiny-server/ui/DIM_fID_panel.R | 2 +- inst/shiny-server/ui/DSC_interface.R | 4 +- inst/shiny-server/ui/fv_par_box.R | 4 +- inst/shiny-server/ui/fv_stats_box.R | 4 +- inst/shiny-server/ui/fv_summary_box.R | 6 +- inst/shiny-server/ui/overview_box.R | 2 +- inst/shiny-server/ui/rt_par_box.R | 4 +- inst/shiny-server/ui/rt_summary_box.R | 6 +- inst/shiny-server/ui/stats_box.R | 4 +- inst/shiny-server/ui/tables_multi_func_box.R | 8 +- inst/shiny-server/ui/upload_box.R | 2 +- inst/shiny-server/ui/variable.R | 2 +- man/subset.DataSet.Rd | 4 +- 25 files changed, 153 insertions(+), 126 deletions(-) diff --git a/R/DataSet.R b/R/DataSet.R index f9d248a9..9232c820 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -184,35 +184,49 @@ DataSet <- function(info, verbose = F, maximization = NULL, format = IOHprofiler c.DataSet <- function(...) { dsl <- list(...) - if (length(dsl) == 1) dsl <- dsl[[1]] + if (length(dsl) == 1) + dsl <- dsl[[1]] dsl <- dsl[sapply(dsl, length) != 0] - if (length(dsl) == 0) return() - if (length(dsl) == 1) return(dsl[[1]]) + if (length(dsl) == 0) + return() + if (length(dsl) == 1) + return(dsl[[1]]) for (ds in dsl) { if (!any((class(ds)) == 'DataSet')) stop("Operation only possible when all arguments are DataSets") } - fixed_attrs <- c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') + fixed_attrs <- + c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') info <- list() for (attr_str in fixed_attrs) { - temp <- unique(unlist(lapply(dsl, function(x) attr(x, attr_str)))) + temp <- unique(unlist(lapply(dsl, function(x) + attr(x, attr_str)))) if (length(temp) > 1) { - stop(paste0("Attempted to add datasets with different ", attr_str, - "-attributes! Tis is not allowed, please keep them as separate DataSets!")) + stop( + paste0( + "Attempted to add datasets with different ", + attr_str, + "-attributes! This is not supported, please keep them as separate DataSets!" + ) + ) } - info <- c(info,temp) + info <- c(info, temp) } names(info) <- fixed_attrs #Record number of runs to make masks of static attributes - nr_runs <- unlist(lapply(dsl, function(x) ncol(x$FV))) + nr_runs <- sapply(dsl, function(x) + ncol(x$FV)) for (attr_str in names(attributes(dsl[[1]]))) { - if (attr_str %in% fixed_attrs || attr_str %in% c("names", "class")) next - temp <- unlist(lapply(dsl, function(x) attr(x, attr_str))) - if (length(unique(temp)) == 1) + if (attr_str %in% fixed_attrs || + attr_str %in% c("names", "class")) + next + temp <- unlist(lapply(dsl, function(x) + attr(x, attr_str))) + if (length(unique(temp)) == 1) temp <- unique(temp) else { if (length(temp) == length(nr_runs)) @@ -233,58 +247,68 @@ c.DataSet <- function(...) { }) }), recursive = F) - RT <- align_running_time(RT_raw, format = "TWO_COL", maximization = info$maximization)$RT + RT <- + align_running_time(RT_raw, + format = "TWO_COL", + maximization = info$maximization)$RT FV <- align_function_value(RT_raw, format = "TWO_COL")$FV # TODO: to deal with cases where aligned parameters are present in original DataSets - PAR <- list( - 'by_FV' = RT[names(RT) != 'RT'], - 'by_RT' = FV[names(FV) != 'FV'] - ) + PAR <- list('by_FV' = RT[names(RT) != 'RT'], + 'by_RT' = FV[names(FV) != 'FV']) # Unaligned parameters for (par_name in names(dsl[[1]]$PAR)) { if (!par_name %in% c('by_FV', 'by_RT')) - PAR[[par_name]] <- unlist(lapply(dsl, function(x) {x$PAR[[par_name]]}), recursive = F) + PAR[[par_name]] <- + unlist(lapply(dsl, function(x) { + x$PAR[[par_name]] + }), recursive = F) } - do.call( - function(...) - structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), - c(info) - ) + do.call(function(...) + structure(list( + RT = RT, FV = FV, PAR = PAR + ), class = c('DataSet', 'list'), ...), + c(info)) } #' S3 subset function for DataSet #' #' @description Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet -#' and turned into a new DataSet object. +#' and turned into a new DataSet object. #' #' @param x The DataSet from which to get a subset -#' @param mask The mask to use when subsetting. The length should be equal to the number of runs present in the -#' provided dataset object x. +#' @param mask The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs +#' present in the provided dataset object x. #' @param ... Arguments passed to underlying subset method (not yet supported) -#' +#' #' @return A new DataSet #' @export -#' @examples +#' @examples #' subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) subset.DataSet <- function(x, mask, ...) { - if (length(mask) != ncol(x$FV)) - stop("Operation only possible when maks is the same length as the number of runs in the DataSet") + stop(paste("The input DataSet has", ncol(x$FV), "runs while the input mask array has length", length(mask))) info <- list() for (attr_str in names(attributes(x))) { - if (attr_str %in% c('names', 'class')) next + if (attr_str %in% c('names', 'class')) + next temp <- attr(x, attr_str) - if (length(unique(temp)) == 1) + if (length(unique(temp)) == 1) temp <- unique(temp) else { if (length(temp) == length(mask)) temp <- list(temp[mask]) else{ - warning(paste0("Attribute detected (", attr_str, ") with incorrect length for the mask-based subsetting!")) + warning( + paste0( + "Attribute detected (", + attr_str, + ") with incorrect length for the mask-based subsetting!" + ) + ) next } } @@ -292,7 +316,7 @@ subset.DataSet <- function(x, mask, ...) { info <- c(info, temp) } - format <- info[['format']] + format <- info[['format']] RT <- as.matrix(x$RT[, mask]) FV <- as.matrix(x$FV[, mask]) @@ -302,11 +326,11 @@ subset.DataSet <- function(x, mask, ...) { 'by_RT' = ifelse(ncol(x$PAR$by_RT) == length(mask), x$PAR$by_RT[, mask], NULL) ) - do.call( - function(...) - structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), - c(info) - ) + do.call(function(...) + structure(list( + RT = RT, FV = FV, PAR = PAR + ), class = c('DataSet', 'list'), ...), + c(info)) } #' S3 generic print operator for DataSet diff --git a/inst/shiny-server/server/ERT_summary.R b/inst/shiny-server/server/ERT_summary.R index 02fb58e1..a662e7e2 100644 --- a/inst/shiny-server/server/ERT_summary.R +++ b/inst/shiny-server/server/ERT_summary.R @@ -3,7 +3,7 @@ runtime_summary_condensed <- reactive({ data <- DATA() req(length(data) > 0) # fall <- get_funvals(data) - data <- subset(data, ID %in% input$RTSummary.Overview.Algid) + data <- subset(data, ID %in% input$RTSummary.Overview.ID) df <- get_FV_overview(data) df$budget %<>% as.numeric df$runs %<>% as.integer @@ -19,7 +19,7 @@ runtime_summary_condensed <- reactive({ }) output$table_RT_overview <- DT::renderDataTable({ - req(input$RTSummary.Overview.Algid) + req(input$RTSummary.Overview.ID) runtime_summary_condensed() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) @@ -45,7 +45,7 @@ runtime_summary <- reactive({ fstop <- format_FV(input$RTSummary.Statistics.Max) fstep <- format_FV(input$RTSummary.Statistics.Step) data <- DATA() - data <- subset(data, ID %in% input$RTSummary.Statistics.Algid) + data <- subset(data, ID %in% input$RTSummary.Statistics.ID) if (!input$RTSummary.Statistics.Single) { req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) fall <- get_funvals(data) @@ -99,7 +99,7 @@ get_RT <- reactive({ fstop <- format_FV(input$RTSummary.Sample.Max) fstep <- format_FV(input$RTSummary.Sample.Step) data <- DATA() - data <- subset(data, ID %in% input$RTSummary.Sample.Algid) + data <- subset(data, ID %in% input$RTSummary.Sample.ID) if (!input$RTSummary.Sample.Single) { req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) fall <- get_funvals(data) diff --git a/inst/shiny-server/server/FCESummary.R b/inst/shiny-server/server/FCESummary.R index f0711121..0dc65bd5 100644 --- a/inst/shiny-server/server/FCESummary.R +++ b/inst/shiny-server/server/FCESummary.R @@ -2,7 +2,7 @@ # Data summary for Fixed-Budget target (FCE) -------------- FCE_runtime_summary_condensed <- reactive({ data <- DATA() - data <- subset(data, ID %in% input$FCESummary.Overview.Algid) + data <- subset(data, ID %in% input$FCESummary.Overview.ID) # fall <- get_funvals(data) df <- get_RT_overview(data) df$"runs" %<>% as.integer @@ -14,7 +14,7 @@ FCE_runtime_summary_condensed <- reactive({ }) output$table_FV_overview <- DT::renderDataTable({ - req(input$FCESummary.Overview.Algid) + req(input$FCESummary.Overview.ID) FCE_runtime_summary_condensed() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) @@ -38,7 +38,7 @@ get_FCE_summary <- reactive({ rt_step <- input$FCESummary.Statistics.Step %>% as.numeric data <- DATA() - data <- subset(data, ID %in% input$FCESummary.Statistics.Algid) + data <- subset(data, ID %in% input$FCESummary.Statistics.ID) if (!input$FCESummary.Statistics.Single) { @@ -93,7 +93,7 @@ get_FCE <- reactive({ rt_step <- input$FCESummary.Sample.Step %>% as.numeric data <- DATA() - data <- subset(data, ID %in% input$FCESummary.Sample.Algid) + data <- subset(data, ID %in% input$FCESummary.Sample.ID) if (!input$FCESummary.Sample.Single) { req(rt_min <= rt_max, rt_step <= rt_max - rt_min) diff --git a/inst/shiny-server/server/FV_DSCinterface.R b/inst/shiny-server/server/FV_DSCinterface.R index 3cf3f9cd..944dea5a 100644 --- a/inst/shiny-server/server/FV_DSCinterface.R +++ b/inst/shiny-server/server/FV_DSCinterface.R @@ -183,7 +183,7 @@ output$FV_Stats.DSC.Download <- downloadHandler( FV_DSC_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.DSC.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.DSC.ID)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$FV_Stats.DSC.Dim) data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) diff --git a/inst/shiny-server/server/FV_PAR.R b/inst/shiny-server/server/FV_PAR.R index 0b0b8bbd..a59d9b6b 100644 --- a/inst/shiny-server/server/FV_PAR.R +++ b/inst/shiny-server/server/FV_PAR.R @@ -87,7 +87,7 @@ fv_parameter_summary <- reactive({ rtstop <- as.numeric(input$FV_PAR.Summary.Max) rtstep <- as.numeric(input$FV_PAR.Summary.Step) data <- DATA() - data <- subset(data, ID %in% input$FV_PAR.Summary.Algid) + data <- subset(data, ID %in% input$FV_PAR.Summary.ID) if (!input$FV_PAR.Summary.Single) { req(rtstart <= rtstop, rtstep <= rtstop - rtstart) @@ -116,7 +116,7 @@ fv_parameter_summary <- reactive({ }) fv_parameter_sample <- reactive({ - req(input$FV_PAR.Sample.Algid, input$FV_PAR.Sample.Max, + req(input$FV_PAR.Sample.ID, input$FV_PAR.Sample.Max, input$FV_PAR.Sample.Step, input$FV_PAR.Sample.Min, input$FV_PAR.Sample.Param) @@ -124,7 +124,7 @@ fv_parameter_sample <- reactive({ rtstop <- as.numeric(input$FV_PAR.Sample.Max) rtstep <- as.numeric(input$FV_PAR.Sample.Step) data <- DATA() - data <- subset(data, ID %in% input$FV_PAR.Sample.Algid) + data <- subset(data, ID %in% input$FV_PAR.Sample.ID) if (!input$FV_PAR.Sample.Single) { req(rtstart <= rtstop, rtstep <= rtstop - rtstart) diff --git a/inst/shiny-server/server/RT_DSCinterface.R b/inst/shiny-server/server/RT_DSCinterface.R index 8ec905f9..d188cb08 100644 --- a/inst/shiny-server/server/RT_DSCinterface.R +++ b/inst/shiny-server/server/RT_DSCinterface.R @@ -182,7 +182,7 @@ output$RT_Stats.DSC.Download <- downloadHandler( RT_DSC_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.DSC.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.DSC.ID)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$RT_Stats.DSC.Dim) data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) diff --git a/inst/shiny-server/server/RT_Par.R b/inst/shiny-server/server/RT_Par.R index 040072a6..47cd746d 100644 --- a/inst/shiny-server/server/RT_Par.R +++ b/inst/shiny-server/server/RT_Par.R @@ -87,7 +87,7 @@ rt_parameter_summary <- reactive({ fstop <- format_FV(input$RT_PAR.Summary.Max) %>% as.numeric fstep <- format_FV(input$RT_PAR.Summary.Step) %>% as.numeric data <- DATA() - data <- subset(data, ID %in% input$RT_PAR.Summary.Algid) + data <- subset(data, ID %in% input$RT_PAR.Summary.ID) if (!input$RT_PAR.Summary.Single) { req(fstart <= fstop, fstep <= fstop - fstart) @@ -116,7 +116,7 @@ rt_parameter_summary <- reactive({ }) rt_parameter_sample <- reactive({ - req(input$RT_PAR.Sample.Algid, input$RT_PAR.Sample.Max, + req(input$RT_PAR.Sample.ID, input$RT_PAR.Sample.Max, input$RT_PAR.Sample.Step, input$RT_PAR.Sample.Min, input$RT_PAR.Sample.Param) @@ -124,7 +124,7 @@ rt_parameter_sample <- reactive({ fstop <- format_FV(input$RT_PAR.Sample.Max) %>% as.numeric fstep <- format_FV(input$RT_PAR.Sample.Step) %>% as.numeric data <- DATA() - data <- subset(data, ID %in% input$RT_PAR.Sample.Algid) + data <- subset(data, ID %in% input$RT_PAR.Sample.ID) if (!input$RT_PAR.Sample.Single) { req(fstart <= fstop, fstep <= fstop - fstart) diff --git a/inst/shiny-server/server/fv_stat_tests.R b/inst/shiny-server/server/fv_stat_tests.R index e59c1810..6e6cf72b 100644 --- a/inst/shiny-server/server/fv_stat_tests.R +++ b/inst/shiny-server/server/fv_stat_tests.R @@ -2,7 +2,7 @@ fv_render_heatmap <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$FV_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), bootstrap.size = 0, which = 'by_RT') }, @@ -17,7 +17,7 @@ output$FV_Stats.Overview.Heatmap <- renderPlotly( fv_create_stats_table <- reactive({ req(length(DATA()) > 0) req(length(get_id(DATA())) > 1) - data <- subset(DATA(), ID %in% input$FV_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) target <- as.numeric(input$FV_Stats.Overview.Target) df <- pairwise.test(data, target, bootstrap.size = 0, which = 'by_RT') df <- format(df, digits = 3) @@ -36,7 +36,7 @@ fv_render_graph <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$FV_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), bootstrap.size = 0, which = 'by_RT') }, @@ -119,7 +119,7 @@ output$FV_Stats.Glicko.Download <- downloadHandler( FV_glicko_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.Glicko.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.Glicko.ID)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$FV_Stats.Glicko.Dim) data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) diff --git a/inst/shiny-server/server/general_overview.R b/inst/shiny-server/server/general_overview.R index ebea5b9d..12a80da5 100644 --- a/inst/shiny-server/server/general_overview.R +++ b/inst/shiny-server/server/general_overview.R @@ -1,7 +1,7 @@ overview_table_single <- reactive({ data <- DATA() req(length(data) > 0) - df <- get_overview(subset(data, ID %in% input$Overview.Single.Algid)) + df <- get_overview(subset(data, ID %in% input$Overview.Single.ID)) df$budget %<>% as.numeric df$runs %<>% as.integer @@ -18,7 +18,7 @@ overview_table_single <- reactive({ }) output$Overview.Single.Table <- DT::renderDataTable({ - req(input$Overview.Single.Algid) + req(input$Overview.Single.ID) overview_table_single() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) diff --git a/inst/shiny-server/server/statistical_significance.R b/inst/shiny-server/server/statistical_significance.R index 43c15a80..eb08f95a 100644 --- a/inst/shiny-server/server/statistical_significance.R +++ b/inst/shiny-server/server/statistical_significance.R @@ -2,7 +2,7 @@ render_heatmap <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$RT_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), bootstrap.size = input$RT_Stats.Overview.Samples) }, @@ -17,7 +17,7 @@ output$RT_Stats.Overview.Heatmap <- renderPlotly( create_stats_table <- reactive({ req(length(DATA()) > 0) req(length(get_id(DATA())) > 1) - data <- subset(DATA(), ID %in% input$RT_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) target <- as.numeric(input$RT_Stats.Overview.Target) df <- pairwise.test(data, target, bootstrap.size = input$RT_Stats.Overview.Samples) df <- format(df, digits = 3) @@ -36,7 +36,7 @@ render_graph <- reactive({ req(length(DATA()) > 0) withProgress({ target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$RT_Stats.Overview.Algid) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), bootstrap.size = input$RT_Stats.Overview.Samples) }, @@ -117,7 +117,7 @@ output$RT_Stats.Glicko.Download <- downloadHandler( RT_glicko_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.Glicko.Algid)) + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.Glicko.ID)) if (length(data) == 0) return(NULL) data <- subset(data, DIM %in% input$RT_Stats.Glicko.Dim) data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) diff --git a/inst/shiny-server/server/tables_multi_func.R b/inst/shiny-server/server/tables_multi_func.R index 79122195..7c745c8a 100644 --- a/inst/shiny-server/server/tables_multi_func.R +++ b/inst/shiny-server/server/tables_multi_func.R @@ -1,7 +1,7 @@ multi_function_data_summary <- reactive({ data <- DATA_RAW() req(length(data) > 0) - data <- subset(data, ID %in% input$RT.MultiERT.AlgId & + data <- subset(data, ID %in% input$RT.MultiERT.ID & funcId %in% input$RT.MultiERT.FuncId & DIM %in% input$RT.MultiERT.DIM) @@ -9,7 +9,7 @@ multi_function_data_summary <- reactive({ }) output$RT.MultiERT.Table <- DT::renderDataTable({ - req(input$RT.MultiERT.AlgId) + req(input$RT.MultiERT.ID) multi_function_data_summary() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) @@ -26,7 +26,7 @@ output$RT.MultiERT.Download <- downloadHandler( multi_function_data_sample <- reactive({ data <- DATA_RAW() req(length(data) > 0) - data <- subset(data, ID %in% input$RT.Multisample.AlgId & + data <- subset(data, ID %in% input$RT.Multisample.ID & funcId %in% input$RT.Multisample.FuncId & DIM %in% input$RT.Multisample.DIM) @@ -34,7 +34,7 @@ multi_function_data_sample <- reactive({ }) output$RT.Multisample.Table <- DT::renderDataTable({ - req(input$RT.Multisample.AlgId) + req(input$RT.Multisample.ID) multi_function_data_sample() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) @@ -53,7 +53,7 @@ multi_function_data_summary_FV <- reactive({ data <- DATA_RAW() req(length(data) > 0) req(as.numeric(input$FV.MultiFV.Target) > 0) - data <- subset(data, ID %in% input$FV.MultiFV.AlgId & + data <- subset(data, ID %in% input$FV.MultiFV.ID & funcId %in% input$FV.MultiFV.FuncId & DIM %in% input$FV.MultiFV.DIM) @@ -61,7 +61,7 @@ multi_function_data_summary_FV <- reactive({ }) output$FV.MultiFV.Table <- DT::renderDataTable({ - req(input$FV.MultiFV.AlgId) + req(input$FV.MultiFV.ID) multi_function_data_summary_FV() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) @@ -78,7 +78,7 @@ output$FV.MultiFV.Download <- downloadHandler( multi_function_data_sample_FV <- reactive({ data <- DATA_RAW() req(length(data) > 0) - data <- subset(data, ID %in% input$FV.Multisample.AlgId & + data <- subset(data, ID %in% input$FV.Multisample.ID & funcId %in% input$FV.Multisample.FuncId & DIM %in% input$FV.Multisample.DIM) @@ -86,7 +86,7 @@ multi_function_data_sample_FV <- reactive({ }) output$FV.Multisample.Table <- DT::renderDataTable({ - req(input$FV.Multisample.AlgId) + req(input$FV.Multisample.ID) multi_function_data_sample_FV() }, filter = list(position = 'top', clear = FALSE), options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index a78cdf3b..a32ba10e 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -59,7 +59,7 @@ observeEvent(input$repository.dataset, { dims <- unique(dims) funcs <- unique(funcs) - updateSelectInput(session, 'repository.algId', choices = algs, selected = algs) + updateSelectInput(session, 'repository.ID', choices = algs, selected = algs) updateSelectInput(session, 'repository.dim', choices = dims, selected = dims) updateSelectInput(session, 'repository.funcId', choices = funcs, selected = funcs) shinyjs::enable('repository.load_button') @@ -75,7 +75,7 @@ observeEvent(input$repository.load_button, { } data <- subset(data, funcId %in% input$repository.funcId) data <- subset(data, DIM %in% input$repository.dim) - data <- subset(data, algId %in% input$repository.algId) + data <- subset(data, algId %in% input$repository.ID) if (length(DataList$data) > 0 && attr(data, 'maximization') != attr(DataList$data, 'maximization')) { shinyjs::alert(paste0("Attempting to add data from a different optimization type to the currently", @@ -312,7 +312,7 @@ observeEvent(input$upload.remove_data, { updateSelectInput(session, 'Overall.Dim', choices = c(), selected = '') updateSelectInput(session, 'Overall.Funcid', choices = c(), selected = '') updateSelectInput(session, 'Overall.Funcname', choices = c(), selected = '') - updateSelectInput(session, 'Overall.Algid', choices = c(), selected = '') + updateSelectInput(session, 'Overall.ID', choices = c(), selected = '') print_html('

all data are removed!

') print_html('', 'upload_data_promt') @@ -356,12 +356,12 @@ observe({ updateSelectInput(session, 'Overall.Funcname', choices = get_funcName(data), selected = get_funcName(data[1])) if ('algId' %in% input$Settings.ID.Variables) - updateSelectInput(session, 'Overall.AlgId', choices = NULL, selected = NULL) + updateSelectInput(session, 'Overall.ID', choices = NULL, selected = NULL) else - updateSelectInput(session, 'Overall.AlgId', choices = algIds, selected = selected_alg) + updateSelectInput(session, 'Overall.ID', choices = algIds, selected = selected_alg) updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'Overview.Single.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'Overview.Single.ID', choices = IDs, selected = IDs) # updateSelectInput(session, 'Report.RT.Overview-FuncId', choices = funcIds, selected = selected_f) # updateSelectInput(session, 'Report.RT.Overview-DIM', choices = DIMs, selected = selected_dim) @@ -458,51 +458,51 @@ observe({ value = min(10, length(IDs))) - updateSelectInput(session, 'RT.MultiERT.AlgId', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT.MultiERT.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'RT.MultiERT.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RT.MultiERT.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT.Multisample.AlgId', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT.Multisample.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'RT.Multisample.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RT.Multisample.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV.MultiFV.AlgId', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV.MultiFV.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'FV.MultiFV.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'FV.MultiFV.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV.Multisample.AlgId', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV.Multisample.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'FV.Multisample.FuncId', choices = funcIds, selected = funcIds) updateSelectInput(session, 'FV.Multisample.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT_Stats.Glicko.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.Glicko.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'RT_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) updateSelectInput(session, 'RT_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT_Stats.DSC.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.DSC.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'RT_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) updateSelectInput(session, 'RT_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - updateSelectInput(session, 'FV_Stats.DSC.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.DSC.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'FV_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) updateSelectInput(session, 'FV_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - updateSelectInput(session, 'RT_Stats.Overview.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.Overview.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_Stats.Glicko.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.Glicko.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'FV_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV_Stats.Overview.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTSummary.Statistics.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTSummary.Overview.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCESummary.Overview.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTSummary.Sample.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Statistics.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Sample.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = IDs, selected = IDs) updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCESummary.Statistics.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCESummary.Sample.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_PAR.Summary.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_PAR.Sample.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_PAR.Summary.Algid', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_PAR.Sample.Algid', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Statistics.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Summary.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Summary.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Sample.ID', choices = IDs, selected = IDs) updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = IDs, selected = selected_ID) updateSelectInput(session, 'ERTPlot.Algs', choices = IDs, selected = IDs) updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = IDs, selected = IDs) @@ -545,7 +545,7 @@ DATA <- reactive({ d <- subset(DataList$data, DIM == dim) if (!'algId' %in% input$Settings.ID.Variables) { - algid <- input$Overall.AlgId + algid <- input$Overall.ID if (!is.null(algid)) d <- subset(d, algId == algid) } @@ -574,7 +574,10 @@ DATA_RAW <- reactive({ # This observe statement tries to match funcid and funcname seletions so funcId can still be used internally. # TODO: think of a better solution to ensure this matching doesn't break. observe({ + req(length(DATA_RAW()) > 0) fname <- input$Overall.Funcname + req(fname) + req(getOption('IOHanalyzer.function_representation', 'funcId') == 'funcname') dsl_sub <- subset(DATA_RAW(), funcName == fname) fids <- get_funcId(dsl_sub) if (length(fids) == 1) { diff --git a/inst/shiny-server/ui/DIM_fID_panel.R b/inst/shiny-server/ui/DIM_fID_panel.R index 93e82223..363f0bab 100644 --- a/inst/shiny-server/ui/DIM_fID_panel.R +++ b/inst/shiny-server/ui/DIM_fID_panel.R @@ -43,7 +43,7 @@ DIM_fID_panel <- function() { column( width = 11, selectInput( - "Overall.AlgId", + "Overall.ID", label = HTML('

Algorithm:

'), choices = NULL, selected = NULL) ) diff --git a/inst/shiny-server/ui/DSC_interface.R b/inst/shiny-server/ui/DSC_interface.R index 404aeb71..8b0410fb 100644 --- a/inst/shiny-server/ui/DSC_interface.R +++ b/inst/shiny-server/ui/DSC_interface.R @@ -5,7 +5,7 @@ fv_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('FV_Stats.DSC.Algid', 'IDs to compare', choices = NULL, + selectInput('FV_Stats.DSC.ID', 'IDs to compare', choices = NULL, selected = NULL, multiple = T), selectInput('FV_Stats.DSC.Funcid', 'Functions to use', choices = NULL, selected = NULL, multiple = T), @@ -145,7 +145,7 @@ rt_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('RT_Stats.DSC.Algid', 'Algorithms to compare', choices = NULL, + selectInput('RT_Stats.DSC.ID', 'Algorithms to compare', choices = NULL, selected = NULL, multiple = T), selectInput('RT_Stats.DSC.Funcid', 'Functions to use', choices = NULL, selected = NULL, multiple = T), diff --git a/inst/shiny-server/ui/fv_par_box.R b/inst/shiny-server/ui/fv_par_box.R index f2c5a163..7f26c8eb 100644 --- a/inst/shiny-server/ui/fv_par_box.R +++ b/inst/shiny-server/ui/fv_par_box.R @@ -82,7 +82,7 @@ fv_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('FV_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Summary.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FV_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('FV_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('FV_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), @@ -114,7 +114,7 @@ fv_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('FV_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Sample.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FV_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('FV_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('FV_PAR.Sample.Format', 'Format of the table', diff --git a/inst/shiny-server/ui/fv_stats_box.R b/inst/shiny-server/ui/fv_stats_box.R index fc7a9d26..0463a387 100644 --- a/inst/shiny-server/ui/fv_stats_box.R +++ b/inst/shiny-server/ui/fv_stats_box.R @@ -4,7 +4,7 @@ fv_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('FV_Stats.Overview.Algid', 'Algorithms to compare', choices = NULL, + selectInput('FV_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, selected = NULL, multiple = T), textInput('FV_Stats.Overview.Target', label = RT_TAR_LABEL), textInput('FV_Stats.Overview.Alpha', @@ -78,7 +78,7 @@ fv_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('FV_Stats.Glicko.Algid', 'Algorithms to compare', choices = NULL, + selectInput('FV_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, selected = NULL, multiple = T), selectInput('FV_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, selected = NULL, multiple = T), diff --git a/inst/shiny-server/ui/fv_summary_box.R b/inst/shiny-server/ui/fv_summary_box.R index bc1e2976..5efd1cc7 100644 --- a/inst/shiny-server/ui/fv_summary_box.R +++ b/inst/shiny-server/ui/fv_summary_box.R @@ -4,7 +4,7 @@ fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('FCESummary.Overview.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FCESummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), #TODO: implement this button hr(), selectInput('FCESummary.Overview.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), @@ -32,7 +32,7 @@ fv_stats_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('FCESummary.Statistics.Single', label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? Once toggled, only \\(B_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Statistics.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FCESummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('FCESummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("FCESummary.Statistics.Download", "Save this table") @@ -64,7 +64,7 @@ fv_sample_box <- function(width = 12, collapsible = T, collapsed = T) { label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? Once toggled, only \\(B_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Sample.Algid', 'Select which IDs to include:', + selectInput('FCESummary.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('FCESummary.Sample.Format', 'Format of the table', diff --git a/inst/shiny-server/ui/overview_box.R b/inst/shiny-server/ui/overview_box.R index ee65133b..a8869c8d 100644 --- a/inst/shiny-server/ui/overview_box.R +++ b/inst/shiny-server/ui/overview_box.R @@ -6,7 +6,7 @@ general_overview_box_single <- function(width = 12, collapsible = T, collapsed = width = 3, HTML('

Select which IDs to include:

'), - selectInput('Overview.Single.Algid', 'IDs:', choices = NULL, selected = NULL, multiple = T), + selectInput('Overview.Single.ID', 'IDs:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('Overview.Single.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("Overview.Single.Download", "Save this table") diff --git a/inst/shiny-server/ui/rt_par_box.R b/inst/shiny-server/ui/rt_par_box.R index b1325233..21b964a7 100644 --- a/inst/shiny-server/ui/rt_par_box.R +++ b/inst/shiny-server/ui/rt_par_box.R @@ -82,7 +82,7 @@ rt_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('RT_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Summary.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RT_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('RT_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('RT_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), @@ -114,7 +114,7 @@ rt_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('RT_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Sample.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RT_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), selectInput('RT_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), hr(), selectInput('RT_PAR.Sample.Format', 'Format of the table', diff --git a/inst/shiny-server/ui/rt_summary_box.R b/inst/shiny-server/ui/rt_summary_box.R index 12d58517..75d8e671 100644 --- a/inst/shiny-server/ui/rt_summary_box.R +++ b/inst/shiny-server/ui/rt_summary_box.R @@ -13,7 +13,7 @@ rt_stats_box <- function(width = 12, collapsible = T, collapsed = T) { checkboxInput('RTSummary.Statistics.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? Once toggled, only \\(f_{\\text{min}}\\) is used to generate the table on the right.

'), value = FALSE), - selectInput('RTSummary.Statistics.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RTSummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('RTSummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("RTSummary.Statistics.Download", "Save this table") @@ -47,7 +47,7 @@ rt_sample_box <- function(width = 12, collapsible = T, collapsed = T) { # TODO: do we need this log scaling? # checkboxInput('F_LOGSPACE_DATA_SUMMARY', # label = HTML('Evenly space target values in \\(log_{10}\\) space')), - selectInput('RTSummary.Sample.Algid', 'Select which IDs to include:', + selectInput('RTSummary.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), @@ -76,7 +76,7 @@ rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('RTSummary.Overview.Algid', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RTSummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), hr(), selectInput('RTSummary.Overview.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), downloadButton("RTSummary.Overview.Download", "Save this table") diff --git a/inst/shiny-server/ui/stats_box.R b/inst/shiny-server/ui/stats_box.R index 5ae0e15f..3190bbab 100644 --- a/inst/shiny-server/ui/stats_box.R +++ b/inst/shiny-server/ui/stats_box.R @@ -4,7 +4,7 @@ rt_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('RT_Stats.Overview.Algid', 'Algorithms to compare', choices = NULL, + selectInput('RT_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, selected = NULL, multiple = T), textInput('RT_Stats.Overview.Target', label = F_TAR_LABEL), textInput('RT_Stats.Overview.Alpha', @@ -85,7 +85,7 @@ rt_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { collapsible = collapsible, collapsed = collapsed, sidebarPanel( width = 3, - selectInput('RT_Stats.Glicko.Algid', 'Algorithms to compare', choices = NULL, + selectInput('RT_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, selected = NULL, multiple = T), selectInput('RT_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, selected = NULL, multiple = T), diff --git a/inst/shiny-server/ui/tables_multi_func_box.R b/inst/shiny-server/ui/tables_multi_func_box.R index 09a7c10b..cbb24c1b 100644 --- a/inst/shiny-server/ui/tables_multi_func_box.R +++ b/inst/shiny-server/ui/tables_multi_func_box.R @@ -6,7 +6,7 @@ multi_function_ert_box <- function(width = 12, collapsible = T, collapsed = T) { width = 3, HTML('

Select which IDs to include:

'), - selectInput('RT.MultiERT.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.MultiERT.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('RT.MultiERT.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), selectInput('RT.MultiERT.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), hr(), @@ -32,7 +32,7 @@ multi_function_sample_box <- function(width = 12, collapsible = T, collapsed = T width = 3, HTML('

Select which IDs to include:

'), - selectInput('RT.Multisample.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('RT.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), selectInput('RT.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), hr(), @@ -59,7 +59,7 @@ multi_function_fv_box <- function(width = 12, collapsible = T, collapsed = T) { width = 3, HTML('

Select which IDs to include:

'), - selectInput('FV.MultiFV.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.MultiFV.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('FV.MultiFV.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), selectInput('FV.MultiFV.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), hr(), @@ -85,7 +85,7 @@ multi_function_sample_box_fv <- function(width = 12, collapsible = T, collapsed width = 3, HTML('

Select which IDs to include:

'), - selectInput('FV.Multisample.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), selectInput('FV.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), selectInput('FV.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), hr(), diff --git a/inst/shiny-server/ui/upload_box.R b/inst/shiny-server/ui/upload_box.R index eb836cb1..59a621b7 100644 --- a/inst/shiny-server/ui/upload_box.R +++ b/inst/shiny-server/ui/upload_box.R @@ -117,7 +117,7 @@ repository_box <- function(width = 12, collapsible = F, collapsed = T, label = "Please choose the dimension", choices = NULL, selected = NULL, width = '50%', multiple = T), - selectInput('repository.algId', + selectInput('repository.ID', label = "Please choose the algorithm", choices = NULL, selected = NULL, width = '50%', multiple = T), diff --git a/inst/shiny-server/ui/variable.R b/inst/shiny-server/ui/variable.R index b77fdb6a..f69f6439 100644 --- a/inst/shiny-server/ui/variable.R +++ b/inst/shiny-server/ui/variable.R @@ -28,7 +28,7 @@ Pdsc_mc_info <- "Practical Deep Comparison uses this HTML_P <- function(s) HTML(paste0('

', s, '

')) alg_select_info <- "Use this option to select which IDs to plot. - This will hava an effect the dowloaded plot, + This will have an effect the dowloaded plot, as opposed to using the legend-entries to show or hide algorithms. The attributes which define the ID can be changed in the settings." diff --git a/man/subset.DataSet.Rd b/man/subset.DataSet.Rd index b30225d1..1d225720 100644 --- a/man/subset.DataSet.Rd +++ b/man/subset.DataSet.Rd @@ -9,8 +9,8 @@ \arguments{ \item{x}{The DataSet from which to get a subset} -\item{mask}{The mask to use when subsetting. The length should be equal to the number of runs present in the -provided dataset object x.} +\item{mask}{The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs +present in the provided dataset object x.} \item{...}{Arguments passed to underlying subset method (not yet supported)} } From e8ddc330fb1f8411ace58e54f5dab473d9e41767 Mon Sep 17 00:00:00 2001 From: Diederick Vermetten Date: Tue, 17 Aug 2021 14:37:00 +0200 Subject: [PATCH 25/28] Add reading of json-based info files --- DESCRIPTION | 3 +- NAMESPACE | 1 + R/IOHanalyzer.R | 1 + R/readFiles.R | 71 ++++++++++++++++++++++++++++++++++++------ man/scan_index_file.Rd | 4 +-- 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3e03d747..0fb8dbd0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -40,7 +40,8 @@ Imports: stringi, httr, knitr, - methods + methods, + rjson LinkingTo: Rcpp SystemRequirements: C++11 RoxygenNote: 7.0.0 diff --git a/NAMESPACE b/NAMESPACE index d1190bcd..6ab7daf8 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -216,6 +216,7 @@ importFrom(plotly,plot_ly) importFrom(plotly,rename_) importFrom(plotly,subplot) importFrom(reshape2,acast) +importFrom(rjson,fromJSON) importFrom(shiny,req) importFrom(stats,dt) importFrom(stats,ecdf) diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index 94e9e7e0..e2b54eb0 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -20,6 +20,7 @@ #' @importFrom reshape2 acast #' @importFrom knitr kable #' @importFrom methods hasArg +#' @importFrom rjson fromJSON #' @useDynLib IOHanalyzer NULL # Ugly hack, but appears to be required to appease CRAN diff --git a/R/readFiles.R b/R/readFiles.R index 286d342e..07f9d45a 100644 --- a/R/readFiles.R +++ b/R/readFiles.R @@ -17,8 +17,8 @@ limit.data <- function(df, n) { #' Scan *.info files for IOHProfiler or COCO #' -#' @param folder The folder containing the .info files -#' @return The paths to all found .info-files +#' @param folder The folder containing the .info and .json files +#' @return The paths to all found .info and .json-files #' @export #' @note This automatically filetrs our files of size 0 #' @examples @@ -26,7 +26,7 @@ limit.data <- function(df, n) { #' scan_index_file(path) scan_index_file <- function(folder) { folder <- trimws(folder) - files <- list.files(folder, pattern = '.info$', recursive = T, full.names = T) + files <- list.files(folder, pattern = '.(info|json)$', recursive = T, full.names = T) files[file.size(files) > 0] } @@ -39,12 +39,65 @@ scan_index_file <- function(folder) { #' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") #' info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) read_index_file <- function(fname) { - tryCatch( - read_index_file__IOH(fname), - warning = function(e) read_index_file__COCO(fname), - error = function(e) read_index_file__COCO(fname), - finally = function(e) stop(paste0('Error in reading .info files ', e)) - ) + format <- tools::file_ext(file) + if (format == 'json') + read_index_file__json(fname) + else { + tryCatch( + read_index_file__IOH(fname), + warning = function(e) read_index_file__COCO(fname), + error = function(e) read_index_file__COCO(fname), + finally = function(e) stop(paste0('Error in reading .info files ', e)) + ) + } +} + +#' Read IOHprofiler-based .json files and extract information +#' +#' @param fname The path to the json info-file +#' @return The data contained in the json info-file +#' @noRd +read_index_file__json <- function(fname) { + + json_data <- fromJSON(file = fname) + attribute_names <- attributes(json_data) + + exp_attrs <- json_data$experiment_attributes + + data <- list() + + tryCatch({ + fid <- json_data$function_id + fname <- json_data$function_name + suite <- json_data$suite + maximization <- json_data$maximization + algid <- json_data$algorithm$name + }, error = function(e) {return(NULL)}) + + data <- lapply(json_data$dimensions, function(scenario) { + run_attrs <- list() + + for (run_attr in json_data$run_attributes) { + attr(run_attrs, run_attr) <- sapply(scenario$runs, function(x) x$run_attr) + } + + temp <- c(list( + funcId = fid, + funcName = fname, + suite = suite, + maximization = maximization, + algId = algid, + DIM = scenario$dimension, + datafile = scenario$path, + instance = sapply(scenario$runs, function(x) x$instance), + maxRT = sapply(scenario$runs, function(x) x$evals), + finalFV = sapply(scenario$runs, function(x) x$best$y), + final_pos = sapply(scenario$runs, function(x) x$best$x) + ), exp_attrs, + run_attrs) + + }) + data } #' Read IOHprofiler-based .info files and extract information diff --git a/man/scan_index_file.Rd b/man/scan_index_file.Rd index 441ba077..33d04866 100644 --- a/man/scan_index_file.Rd +++ b/man/scan_index_file.Rd @@ -7,10 +7,10 @@ scan_index_file(folder) } \arguments{ -\item{folder}{The folder containing the .info files} +\item{folder}{The folder containing the .info and .json files} } \value{ -The paths to all found .info-files +The paths to all found .info and .json-files } \description{ Scan *.info files for IOHProfiler or COCO From 1a3101cadcd2572696b2ea3027ffe3cee00b0c4f Mon Sep 17 00:00:00 2001 From: Dvermetten Date: Tue, 17 Aug 2021 15:00:11 +0200 Subject: [PATCH 26/28] Fix file extension detection --- .Rbuildignore | 22 +- .github/workflows/R-CMD-check.yaml | 182 +- .gitignore | 52 +- DESCRIPTION | 118 +- LICENSE | 4 +- LICENSE.md | 66 +- NAMESPACE | 482 +-- NEWS.md | 4 +- R/DataSet.R | 2106 ++++++------ R/DataSetList.R | 2880 ++++++++--------- R/IOHanalyzer-deprecated.R | 390 +-- R/IOHanalyzer.R | 256 +- R/RcppExports.R | 62 +- R/data.R | 32 +- R/plot.R | 738 ++--- R/plotDataSetList.R | 2834 ++++++++-------- R/readFiles.R | 1912 +++++------ R/runServer.R | 36 +- R/stats.R | 2048 ++++++------ README.md | 336 +- cran-comments.md | 32 +- .../ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info | 4 +- .../data_f1/IOHprofiler_f1_DIM100_i1.dat | 944 +++--- .../IOHprofiler_fbla_i1.info | 6 +- .../IOHprofiler_fblubb_i1.info | 6 +- .../data_fbla/IOHprofiler_fbla_DIM100_i1.dat | 944 +++--- .../IOHprofiler_fblubb_DIM100_i1.dat | 944 +++--- inst/shiny-server/LICENSE.txt | 58 +- inst/shiny-server/global.R | 524 +-- .../markdown/FV_PAR_SUMMARY_TABLE.Rmd | 18 +- inst/shiny-server/markdown/INSTALL.Rmd | 78 +- .../markdown/PAR_SUMMARY_TABLE.Rmd | 18 +- .../markdown/RT_OVERVIEW_TABLE.Rmd | 20 +- .../markdown/RT_SUMMARY_TABLE.Rmd | 18 +- .../shiny-server/markdown/Report/ECDF_AUC.Rmd | 80 +- .../markdown/Report/ECDF_Aggregated.Rmd | 54 +- .../markdown/Report/ECDF_Single_Function.Rmd | 80 +- .../markdown/Report/ECDF_Single_Target.Rmd | 62 +- .../markdown/Report/FV_ECDF_AUC.Rmd | 80 +- .../Report/FV_ECDF_Single_Function.Rmd | 84 +- .../markdown/Report/FV_ECDF_Single_Target.Rmd | 64 +- .../markdown/Report/FV_Histogram.Rmd | 64 +- inst/shiny-server/markdown/Report/FV_PMF.Rmd | 64 +- inst/shiny-server/markdown/Report/FV_Rank.Rmd | 44 +- .../markdown/Report/Multi_ERT.Rmd | 44 +- .../markdown/Report/Multi_FCE.Rmd | 42 +- .../markdown/Report/Param_plot.Rmd | 78 +- .../markdown/Report/RT_Histogram.Rmd | 64 +- inst/shiny-server/markdown/Report/RT_PMF.Rmd | 64 +- inst/shiny-server/markdown/Report/RT_Rank.Rmd | 46 +- .../markdown/Report/Single_ERT.Rmd | 80 +- .../markdown/Report/Single_FCE.Rmd | 70 +- .../markdown/Report/Template_test.Rmd | 930 +++--- .../markdown/Report/bibliography.bib | 20 +- .../markdown/TAR_SUMMARY_TABLE.Rmd | 14 +- inst/shiny-server/markdown/about.md | 194 +- inst/shiny-server/markdown/dataformat.md | 210 +- inst/shiny-server/markdown/docs.Rmd | 52 +- inst/shiny-server/markdown/quantile.Rmd | 18 +- inst/shiny-server/markdown/welcome.md | 90 +- inst/shiny-server/server.R | 130 +- inst/shiny-server/server/ERTPlot.R | 390 +-- inst/shiny-server/server/ERT_aggr_dim_plot.R | 266 +- inst/shiny-server/server/ERT_aggr_plot.R | 268 +- inst/shiny-server/server/ERT_summary.R | 272 +- inst/shiny-server/server/FCEPDF.R | 140 +- inst/shiny-server/server/FCEPlot.R | 280 +- inst/shiny-server/server/FCESummary.R | 256 +- inst/shiny-server/server/FCE_ECDF.R | 242 +- inst/shiny-server/server/FCE_aggr_plot.R | 294 +- inst/shiny-server/server/FV_DSCinterface.R | 448 +-- inst/shiny-server/server/FV_PAR.R | 362 +-- inst/shiny-server/server/RTPMF.R | 146 +- inst/shiny-server/server/RT_DSCinterface.R | 446 +-- inst/shiny-server/server/RT_ECDF.R | 606 ++-- inst/shiny-server/server/RT_Par.R | 360 +-- .../server/Shapley_computations.R | 244 +- inst/shiny-server/server/fv_stat_tests.R | 320 +- inst/shiny-server/server/general_overview.R | 122 +- inst/shiny-server/server/parrallel_coord.R | 76 +- inst/shiny-server/server/report_generation.R | 78 +- inst/shiny-server/server/settings_page.R | 460 +-- .../server/statistical_significance.R | 318 +- inst/shiny-server/server/tables_multi_func.R | 204 +- inst/shiny-server/server/upload.R | 1468 ++++----- inst/shiny-server/theme.R | 200 +- inst/shiny-server/ui.R | 1242 +++---- inst/shiny-server/ui/DIM_fID_panel.R | 108 +- inst/shiny-server/ui/DSC_interface.R | 560 ++-- inst/shiny-server/ui/ERT_agg_box.R | 92 +- inst/shiny-server/ui/ERT_box.R | 266 +- inst/shiny-server/ui/ERT_comparison_box.R | 254 +- inst/shiny-server/ui/Parrallel_coord_box.R | 66 +- inst/shiny-server/ui/fv_box.R | 350 +- inst/shiny-server/ui/fv_dist_box.R | 192 +- inst/shiny-server/ui/fv_ecdf_box.R | 272 +- inst/shiny-server/ui/fv_par_box.R | 268 +- inst/shiny-server/ui/fv_stats_box.R | 234 +- inst/shiny-server/ui/fv_summary_box.R | 176 +- inst/shiny-server/ui/overview_box.R | 72 +- inst/shiny-server/ui/report_generation_box.R | 236 +- inst/shiny-server/ui/rt_dist_box.R | 228 +- inst/shiny-server/ui/rt_ecdf_box.R | 512 +-- inst/shiny-server/ui/rt_par_box.R | 268 +- inst/shiny-server/ui/rt_portfolio_box.R | 192 +- inst/shiny-server/ui/rt_summary_box.R | 184 +- inst/shiny-server/ui/settings.R | 182 +- inst/shiny-server/ui/sidebar_menu.R | 98 +- inst/shiny-server/ui/stats_box.R | 250 +- inst/shiny-server/ui/tables_multi_func_box.R | 206 +- inst/shiny-server/ui/upload_box.R | 298 +- inst/shiny-server/ui/variable.R | 74 +- man/AUC.Rd | 56 +- man/DataSet.Rd | 82 +- man/DataSetList.Rd | 100 +- man/ECDF.Rd | 62 +- man/IOH_plot_ly_default.Rd | 42 +- man/IOHanalyzer.Rd | 72 +- man/IOHanlyzer-deprecated.Rd | 22 +- man/Plot.FV.Aggregated.Rd | 84 +- man/Plot.FV.ECDF_AUC.Rd | 62 +- man/Plot.FV.ECDF_Per_Target.Rd | 68 +- man/Plot.FV.ECDF_Single_Func.Rd | 84 +- man/Plot.FV.Histogram.Rd | 68 +- man/Plot.FV.Multi_Func.Rd | 62 +- man/Plot.FV.PDF.Rd | 66 +- man/Plot.FV.Parameters.Rd | 92 +- man/Plot.FV.Single_Func.Rd | 92 +- man/Plot.Performviz.Rd | 44 +- man/Plot.RT.Aggregated.Rd | 92 +- man/Plot.RT.ECDF_AUC.Rd | 66 +- man/Plot.RT.ECDF_Multi_Func.Rd | 64 +- man/Plot.RT.ECDF_Per_Target.Rd | 62 +- man/Plot.RT.ECDF_Single_Func.Rd | 76 +- man/Plot.RT.Histogram.Rd | 68 +- man/Plot.RT.Multi_Func.Rd | 66 +- man/Plot.RT.PMF.Rd | 68 +- man/Plot.RT.Parameters.Rd | 92 +- man/Plot.RT.Single_Func.Rd | 106 +- man/Plot.Stats.Glicko2_Candlestick.Rd | 62 +- man/Plot.Stats.Significance_Graph.Rd | 66 +- man/Plot.Stats.Significance_Heatmap.Rd | 66 +- man/SP.Rd | 48 +- man/arrange.Rd | 48 +- man/as.character.DataSet.Rd | 48 +- man/bootstrap_RT.Rd | 56 +- man/c.DataSet.Rd | 44 +- man/c.DataSetList.Rd | 40 +- man/cat.DataSet.Rd | 40 +- man/change_id.Rd | 50 +- man/check_dsc_configured.Rd | 40 +- man/check_format.Rd | 44 +- man/clean_DataSetList.Rd | 34 +- man/dsl.Rd | 34 +- man/dsl_large.Rd | 34 +- man/equals-.DataSet.Rd | 46 +- man/fast_RT_samples.Rd | 36 +- man/generate_data.AUC.Rd | 62 +- man/generate_data.Aggr.Rd | 48 +- man/generate_data.ECDF.Rd | 56 +- man/generate_data.ECDF_raw.Rd | 44 +- man/generate_data.PMF.Rd | 42 +- man/generate_data.Parameters.Rd | 42 +- man/generate_data.Single_Function.Rd | 64 +- man/generate_data.hist.Rd | 46 +- man/get_ECDF_targets.Rd | 50 +- man/get_ERT.Rd | 74 +- man/get_FV_overview.Rd | 66 +- man/get_FV_sample.Rd | 72 +- man/get_FV_summary.Rd | 68 +- man/get_PAR_name.Rd | 54 +- man/get_PAR_sample.Rd | 90 +- man/get_PAR_summary.Rd | 86 +- man/get_RT_overview.Rd | 64 +- man/get_RT_sample.Rd | 72 +- man/get_RT_summary.Rd | 68 +- man/get_algId.Rd | 40 +- man/get_color_scheme.Rd | 34 +- man/get_color_scheme_dt.Rd | 34 +- man/get_default_ECDF_targets.Rd | 44 +- man/get_dim.Rd | 40 +- man/get_dsc_omnibus.Rd | 52 +- man/get_dsc_posthoc.Rd | 72 +- man/get_dsc_rank.Rd | 94 +- man/get_funcId.Rd | 40 +- man/get_funcName.Rd | 40 +- man/get_funvals.Rd | 40 +- man/get_id.Rd | 66 +- man/get_line_style.Rd | 34 +- man/get_marg_contrib_ecdf.Rd | 48 +- man/get_maxRT.Rd | 68 +- man/get_overview.Rd | 58 +- man/get_parId.Rd | 48 +- man/get_runtimes.Rd | 40 +- man/get_shapley_values.Rd | 58 +- man/get_static_attribute_values.Rd | 46 +- man/get_static_attributes.Rd | 40 +- man/get_target_dt.Rd | 46 +- man/glicko2_ranking.Rd | 66 +- man/limit.data.Rd | 38 +- man/max_ERTs.Rd | 60 +- man/mean_FVs.Rd | 56 +- man/pairwise.test.Rd | 90 +- man/plot_general_data.Rd | 114 +- man/print.DataSet.Rd | 44 +- man/print.DataSetList.Rd | 38 +- man/read_index_file.Rd | 42 +- man/register_DSC.Rd | 58 +- man/runServer.Rd | 46 +- man/save_plotly.Rd | 56 +- man/save_table.Rd | 46 +- man/scan_index_file.Rd | 48 +- man/seq_FV.Rd | 70 +- man/seq_RT.Rd | 68 +- man/set_DSC_credentials.Rd | 40 +- man/set_color_scheme.Rd | 58 +- man/sub-.DataSetList.Rd | 48 +- man/subset.DataSet.Rd | 52 +- man/subset.DataSetList.Rd | 60 +- man/summary.DataSet.Rd | 44 +- man/summary.DataSetList.Rd | 40 +- src/align.cc | 292 +- src/read.cc | 136 +- tests/testthat.R | 8 +- tests/testthat/test_DataSetList.R | 94 +- .../test_DataSetList_named_with_strings.R | 124 +- tests/testthat/test_data_generation.R | 232 +- .../test_diagram_examples_with_string_names.R | 72 +- 228 files changed, 21693 insertions(+), 21693 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index c3b56406..3313b0a0 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,11 +1,11 @@ -^\.travis\.yml$ -^.*\.Rproj$ -^\.Rproj\.user$ -^cran-comments\.md$ -^NEWS\.md$ -^\.vscode$ -^\.vs*$ -^_config\.yml$ -^LICENSE\.md$ -^misc*$ -^\.github$ +^\.travis\.yml$ +^.*\.Rproj$ +^\.Rproj\.user$ +^cran-comments\.md$ +^NEWS\.md$ +^\.vscode$ +^\.vs*$ +^_config\.yml$ +^LICENSE\.md$ +^misc*$ +^\.github$ diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 92c9158c..c7c52dcf 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,91 +1,91 @@ -# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag. -# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions -on: - push: - branches: - - main - - master - pull_request: - branches: - - main - - master - -name: R-CMD-check - -jobs: - R-CMD-check: - runs-on: ${{ matrix.config.os }} - - name: ${{ matrix.config.os }} (${{ matrix.config.r }}) - - strategy: - fail-fast: false - matrix: - config: - - {os: windows-latest, r: 'release'} - - {os: macOS-latest, r: 'release'} - - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} - - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} - - env: - R_REMOTES_NO_ERRORS_FROM_WARNINGS: true - RSPM: ${{ matrix.config.rspm }} - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/checkout@v2 - - - uses: r-lib/actions/setup-r@v1 - with: - r-version: ${{ matrix.config.r }} - - - uses: r-lib/actions/setup-pandoc@v1 - - - name: Query dependencies - run: | - install.packages('remotes') - saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) - writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") - shell: Rscript {0} - - - name: Restore R package cache - uses: actions/cache@v2 - with: - path: ${{ env.R_LIBS_USER }} - key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} - restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- - - - name: Install system dependencies - if: runner.os == 'Linux' - run: | - while read -r cmd - do - eval sudo $cmd - done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') - - - name: Install X11 dependencies on MacOS - if: runner.os == 'macOS' - run: | - brew install xquartz --cask - - - name: Install dependencies - run: | - remotes::install_deps(dependencies = TRUE) - remotes::install_bioc('ComplexHeatmap') - remotes::install_cran("rcmdcheck") - shell: Rscript {0} - - - name: Check - env: - _R_CHECK_CRAN_INCOMING_REMOTE_: false - run: | - options(crayon.enabled = TRUE) - rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") - shell: Rscript {0} - - - name: Upload check results - if: failure() - uses: actions/upload-artifact@main - with: - name: ${{ runner.os }}-r${{ matrix.config.r }}-results - path: check +# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag. +# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + +name: R-CMD-check + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: windows-latest, r: 'release'} + - {os: macOS-latest, r: 'release'} + - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + RSPM: ${{ matrix.config.rspm }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v1 + with: + r-version: ${{ matrix.config.r }} + + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Query dependencies + run: | + install.packages('remotes') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Restore R package cache + uses: actions/cache@v2 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- + + - name: Install system dependencies + if: runner.os == 'Linux' + run: | + while read -r cmd + do + eval sudo $cmd + done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') + + - name: Install X11 dependencies on MacOS + if: runner.os == 'macOS' + run: | + brew install xquartz --cask + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_bioc('ComplexHeatmap') + remotes::install_cran("rcmdcheck") + shell: Rscript {0} + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_REMOTE_: false + run: | + options(crayon.enabled = TRUE) + rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") + shell: Rscript {0} + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@main + with: + name: ${{ runner.os }}-r${{ matrix.config.r }}-results + path: check diff --git a/.gitignore b/.gitignore index dfa0e501..466bab22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,26 @@ -*.csv -*.o -*.so -*.dll -*.log -*.RData -*.Rhistory -*.Rbuildignore -*.bk -*.pdf -*.html -.* - -!/.Rbuildignore -!/.gitignore -__pycache__ -rawdata/ -test/ -rsconnect/ -tmp/ -tmp2/ -rawdata/ -extract_data.R -.Rproj.user -*.Rproj -.Rhistory +*.csv +*.o +*.so +*.dll +*.log +*.RData +*.Rhistory +*.Rbuildignore +*.bk +*.pdf +*.html +.* + +!/.Rbuildignore +!/.gitignore +__pycache__ +rawdata/ +test/ +rsconnect/ +tmp/ +tmp2/ +rawdata/ +extract_data.R +.Rproj.user +*.Rproj +.Rhistory diff --git a/DESCRIPTION b/DESCRIPTION index 0fb8dbd0..85a4e4ae 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,59 +1,59 @@ -Package: IOHanalyzer -Type: Package -Title: Data Analysis Part of 'IOHprofiler' -Version: 0.1.6.0 -Author: Hao Wang [cre, aut], Diederick Vermetten [aut], Carola Doerr [aut], Thomas Bäck [aut] -Maintainer: Hao Wang -Authors@R: c( - person("Hao", "Wang", email = "h.wang@liacs.leidenuniv.nl", role = c("cre","aut")), - person("Diederick", "Vermetten", email="d.vermetten@gmail.com", role = "aut"), - person("Carola", "Doerr", email = "Carola.Doerr@mpi-inf.mpg.de", role = "aut"), - person("Thomas", "Bäck", email="t.h.w.baeck@liacs.leidenuniv.nl", role = "aut")) -Description: The data analysis module for the Iterative Optimization Heuristics - Profiler ('IOHprofiler'). This module provides statistical analysis methods for the - benchmark data generated by optimization heuristics, which can be visualized through a - web-based interface. The benchmark data is usually generated by the - experimentation module, called 'IOHexperimenter'. 'IOHanalyzer' also supports - the widely used 'COCO' (Comparing Continuous Optimisers) data format for benchmarking. -License: BSD_3_clause + file LICENSE -Encoding: UTF-8 -LazyData: true -URL: http://iohprofiler.liacs.nl, https://github.com/IOHprofiler/IOHAnalyzer -BugReports: https://github.com/IOHprofiler/IOHAnalyzer/issues -Imports: - magrittr, - dplyr, - data.table, - ggplot2, - plotly, - colorspace, - colorRamps, - RColorBrewer, - shiny, - markdown, - reshape2, - shinyjs, - colourpicker, - bsplus, - DT, - kableExtra, - stringi, - httr, - knitr, - methods, - rjson -LinkingTo: Rcpp -SystemRequirements: C++11 -RoxygenNote: 7.0.0 -Suggests: - Rcpp, - testthat, - withr, - shinydashboard, - ComplexHeatmap, - grid, - keyring, - PlayerRatings, - xtable, - igraph -Depends: R (>= 2.10) +Package: IOHanalyzer +Type: Package +Title: Data Analysis Part of 'IOHprofiler' +Version: 0.1.6.0 +Author: Hao Wang [cre, aut], Diederick Vermetten [aut], Carola Doerr [aut], Thomas Bäck [aut] +Maintainer: Hao Wang +Authors@R: c( + person("Hao", "Wang", email = "h.wang@liacs.leidenuniv.nl", role = c("cre","aut")), + person("Diederick", "Vermetten", email="d.vermetten@gmail.com", role = "aut"), + person("Carola", "Doerr", email = "Carola.Doerr@mpi-inf.mpg.de", role = "aut"), + person("Thomas", "Bäck", email="t.h.w.baeck@liacs.leidenuniv.nl", role = "aut")) +Description: The data analysis module for the Iterative Optimization Heuristics + Profiler ('IOHprofiler'). This module provides statistical analysis methods for the + benchmark data generated by optimization heuristics, which can be visualized through a + web-based interface. The benchmark data is usually generated by the + experimentation module, called 'IOHexperimenter'. 'IOHanalyzer' also supports + the widely used 'COCO' (Comparing Continuous Optimisers) data format for benchmarking. +License: BSD_3_clause + file LICENSE +Encoding: UTF-8 +LazyData: true +URL: http://iohprofiler.liacs.nl, https://github.com/IOHprofiler/IOHAnalyzer +BugReports: https://github.com/IOHprofiler/IOHAnalyzer/issues +Imports: + magrittr, + dplyr, + data.table, + ggplot2, + plotly, + colorspace, + colorRamps, + RColorBrewer, + shiny, + markdown, + reshape2, + shinyjs, + colourpicker, + bsplus, + DT, + kableExtra, + stringi, + httr, + knitr, + methods, + rjson +LinkingTo: Rcpp +SystemRequirements: C++11 +RoxygenNote: 7.0.0 +Suggests: + Rcpp, + testthat, + withr, + shinydashboard, + ComplexHeatmap, + grid, + keyring, + PlayerRatings, + xtable, + igraph +Depends: R (>= 2.10) diff --git a/LICENSE b/LICENSE index 772d7517..b94f835f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,3 @@ -YEAR: 2019 -COPYRIGHT HOLDER: Hao Wang +YEAR: 2019 +COPYRIGHT HOLDER: Hao Wang ORGANIZATION: Leiden University \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index d2a3b4d3..c170dbc6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,33 +1,33 @@ -# License - -This application is governed by the __BSD 3-Clause license__. - -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# License + +This application is governed by the __BSD 3-Clause license__. + +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/NAMESPACE b/NAMESPACE index 6ab7daf8..4cde42f3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,241 +1,241 @@ -# Generated by roxygen2: do not edit by hand - -S3method("==",DataSet) -S3method("[",DataSetList) -S3method(AUC,ECDF) -S3method(ECDF,DataSet) -S3method(ECDF,DataSetList) -S3method(Plot.FV.Aggregated,DataSetList) -S3method(Plot.FV.ECDF_AUC,DataSetList) -S3method(Plot.FV.ECDF_Per_Target,DataSetList) -S3method(Plot.FV.ECDF_Single_Func,DataSetList) -S3method(Plot.FV.Histogram,DataSetList) -S3method(Plot.FV.Multi_Func,DataSetList) -S3method(Plot.FV.PDF,DataSetList) -S3method(Plot.FV.Parameters,DataSetList) -S3method(Plot.FV.Single_Func,DataSetList) -S3method(Plot.RT.Aggregated,DataSetList) -S3method(Plot.RT.ECDF_AUC,DataSetList) -S3method(Plot.RT.ECDF_Multi_Func,DataSetList) -S3method(Plot.RT.ECDF_Per_Target,DataSetList) -S3method(Plot.RT.ECDF_Single_Func,DataSetList) -S3method(Plot.RT.Histogram,DataSetList) -S3method(Plot.RT.Multi_Func,DataSetList) -S3method(Plot.RT.PMF,DataSetList) -S3method(Plot.RT.Parameters,DataSetList) -S3method(Plot.RT.Single_Func,DataSetList) -S3method(Plot.Stats.Glicko2_Candlestick,DataSetList) -S3method(Plot.Stats.Significance_Graph,DataSetList) -S3method(Plot.Stats.Significance_Heatmap,DataSetList) -S3method(arrange,DataSetList) -S3method(as.character,DataSet) -S3method(c,DataSet) -S3method(c,DataSetList) -S3method(get_ERT,DataSet) -S3method(get_ERT,DataSetList) -S3method(get_FV_overview,DataSet) -S3method(get_FV_overview,DataSetList) -S3method(get_FV_sample,DataSet) -S3method(get_FV_sample,DataSetList) -S3method(get_FV_summary,DataSet) -S3method(get_FV_summary,DataSetList) -S3method(get_PAR_name,DataSet) -S3method(get_PAR_sample,DataSet) -S3method(get_PAR_sample,DataSetList) -S3method(get_PAR_summary,DataSet) -S3method(get_PAR_summary,DataSetList) -S3method(get_RT_overview,DataSet) -S3method(get_RT_overview,DataSetList) -S3method(get_RT_sample,DataSet) -S3method(get_RT_sample,DataSetList) -S3method(get_RT_summary,DataSet) -S3method(get_RT_summary,DataSetList) -S3method(get_id,DataSet) -S3method(get_id,DataSetList) -S3method(get_maxRT,DataSet) -S3method(get_maxRT,DataSetList) -S3method(get_overview,DataSet) -S3method(get_overview,DataSetList) -S3method(max_ERTs,DataSetList) -S3method(mean_FVs,DataSetList) -S3method(pairwise.test,DataSetList) -S3method(pairwise.test,list) -S3method(print,DataSet) -S3method(print,DataSetList) -S3method(subset,DataSet) -S3method(subset,DataSetList) -S3method(summary,DataSet) -S3method(summary,DataSetList) -export(AUC) -export(DataSet) -export(DataSetList) -export(ECDF) -export(IOH_plot_ly_default) -export(Plot.FV.Aggregated) -export(Plot.FV.ECDF_AUC) -export(Plot.FV.ECDF_Per_Target) -export(Plot.FV.ECDF_Single_Func) -export(Plot.FV.Histogram) -export(Plot.FV.Multi_Func) -export(Plot.FV.PDF) -export(Plot.FV.Parameters) -export(Plot.FV.Single_Func) -export(Plot.Performviz) -export(Plot.RT.Aggregated) -export(Plot.RT.ECDF_AUC) -export(Plot.RT.ECDF_Multi_Func) -export(Plot.RT.ECDF_Per_Target) -export(Plot.RT.ECDF_Single_Func) -export(Plot.RT.Histogram) -export(Plot.RT.Multi_Func) -export(Plot.RT.PMF) -export(Plot.RT.Parameters) -export(Plot.RT.Single_Func) -export(Plot.Stats.Glicko2_Candlestick) -export(Plot.Stats.Significance_Graph) -export(Plot.Stats.Significance_Heatmap) -export(SP) -export(arrange) -export(bootstrap_RT) -export(cat.DataSet) -export(change_id) -export(check_dsc_configured) -export(check_format) -export(clean_DataSetList) -export(fast_RT_samples) -export(generate_data.AUC) -export(generate_data.Aggr) -export(generate_data.ECDF) -export(generate_data.ECDF_raw) -export(generate_data.PMF) -export(generate_data.Parameters) -export(generate_data.Single_Function) -export(generate_data.hist) -export(get_ECDF_targets) -export(get_ERT) -export(get_FV_overview) -export(get_FV_sample) -export(get_FV_summary) -export(get_PAR_name) -export(get_PAR_sample) -export(get_PAR_summary) -export(get_RT_overview) -export(get_RT_sample) -export(get_RT_summary) -export(get_algId) -export(get_color_scheme) -export(get_color_scheme_dt) -export(get_default_ECDF_targets) -export(get_dim) -export(get_dsc_omnibus) -export(get_dsc_posthoc) -export(get_dsc_rank) -export(get_funcId) -export(get_funcName) -export(get_funvals) -export(get_id) -export(get_line_style) -export(get_marg_contrib_ecdf) -export(get_maxRT) -export(get_overview) -export(get_parId) -export(get_runtimes) -export(get_shapley_values) -export(get_static_attribute_values) -export(get_static_attributes) -export(get_target_dt) -export(glicko2_ranking) -export(max_ERTs) -export(mean_FVs) -export(pairwise.test) -export(plot_general_data) -export(read_index_file) -export(register_DSC) -export(runServer) -export(save_plotly) -export(save_table) -export(scan_index_file) -export(seq_FV) -export(seq_RT) -export(set_DSC_credentials) -export(set_color_scheme) -importFrom(RColorBrewer,brewer.pal) -importFrom(colorRamps,primary.colors) -importFrom(colorspace,sequential_hcl) -importFrom(data.table,":=") -importFrom(data.table,as.data.table) -importFrom(data.table,copy) -importFrom(data.table,data.table) -importFrom(data.table,frank) -importFrom(data.table,fread) -importFrom(data.table,is.data.table) -importFrom(data.table,melt) -importFrom(data.table,rbindlist) -importFrom(data.table,setnames) -importFrom(data.table,setorderv) -importFrom(data.table,transpose) -importFrom(dplyr,"%>%") -importFrom(dplyr,mutate) -importFrom(ggplot2,aes) -importFrom(ggplot2,element_text) -importFrom(ggplot2,facet_wrap) -importFrom(ggplot2,geom_jitter) -importFrom(ggplot2,geom_line) -importFrom(ggplot2,geom_ribbon) -importFrom(ggplot2,geom_violin) -importFrom(ggplot2,ggplot) -importFrom(ggplot2,guides) -importFrom(ggplot2,scale_color_manual) -importFrom(ggplot2,scale_colour_manual) -importFrom(ggplot2,scale_fill_manual) -importFrom(ggplot2,scale_x_continuous) -importFrom(ggplot2,scale_x_log10) -importFrom(ggplot2,theme) -importFrom(ggplot2,theme_grey) -importFrom(ggplot2,theme_set) -importFrom(grDevices,col2rgb) -importFrom(grDevices,colors) -importFrom(grDevices,nclass.FD) -importFrom(graphics,hist) -importFrom(httr,POST) -importFrom(httr,add_headers) -importFrom(httr,authenticate) -importFrom(httr,content) -importFrom(knitr,kable) -importFrom(magrittr,"%<>%") -importFrom(magrittr,mod) -importFrom(magrittr,set_colnames) -importFrom(magrittr,set_names) -importFrom(magrittr,set_rownames) -importFrom(methods,hasArg) -importFrom(plotly,add_annotations) -importFrom(plotly,add_trace) -importFrom(plotly,layout) -importFrom(plotly,orca) -importFrom(plotly,plot_ly) -importFrom(plotly,rename_) -importFrom(plotly,subplot) -importFrom(reshape2,acast) -importFrom(rjson,fromJSON) -importFrom(shiny,req) -importFrom(stats,dt) -importFrom(stats,ecdf) -importFrom(stats,integrate) -importFrom(stats,ks.test) -importFrom(stats,median) -importFrom(stats,p.adjust) -importFrom(stats,quantile) -importFrom(stats,rgeom) -importFrom(stats,sd) -importFrom(stringi,stri_detect_fixed) -importFrom(stringi,stri_detect_regex) -importFrom(stringi,stri_locate_all) -importFrom(stringi,stri_rand_strings) -importFrom(stringi,stri_replace) -importFrom(utils,data) -importFrom(utils,head) -importFrom(utils,read.csv) -importFrom(utils,tail) -importFrom(utils,type.convert) -importFrom(utils,write.csv) -useDynLib(IOHanalyzer) +# Generated by roxygen2: do not edit by hand + +S3method("==",DataSet) +S3method("[",DataSetList) +S3method(AUC,ECDF) +S3method(ECDF,DataSet) +S3method(ECDF,DataSetList) +S3method(Plot.FV.Aggregated,DataSetList) +S3method(Plot.FV.ECDF_AUC,DataSetList) +S3method(Plot.FV.ECDF_Per_Target,DataSetList) +S3method(Plot.FV.ECDF_Single_Func,DataSetList) +S3method(Plot.FV.Histogram,DataSetList) +S3method(Plot.FV.Multi_Func,DataSetList) +S3method(Plot.FV.PDF,DataSetList) +S3method(Plot.FV.Parameters,DataSetList) +S3method(Plot.FV.Single_Func,DataSetList) +S3method(Plot.RT.Aggregated,DataSetList) +S3method(Plot.RT.ECDF_AUC,DataSetList) +S3method(Plot.RT.ECDF_Multi_Func,DataSetList) +S3method(Plot.RT.ECDF_Per_Target,DataSetList) +S3method(Plot.RT.ECDF_Single_Func,DataSetList) +S3method(Plot.RT.Histogram,DataSetList) +S3method(Plot.RT.Multi_Func,DataSetList) +S3method(Plot.RT.PMF,DataSetList) +S3method(Plot.RT.Parameters,DataSetList) +S3method(Plot.RT.Single_Func,DataSetList) +S3method(Plot.Stats.Glicko2_Candlestick,DataSetList) +S3method(Plot.Stats.Significance_Graph,DataSetList) +S3method(Plot.Stats.Significance_Heatmap,DataSetList) +S3method(arrange,DataSetList) +S3method(as.character,DataSet) +S3method(c,DataSet) +S3method(c,DataSetList) +S3method(get_ERT,DataSet) +S3method(get_ERT,DataSetList) +S3method(get_FV_overview,DataSet) +S3method(get_FV_overview,DataSetList) +S3method(get_FV_sample,DataSet) +S3method(get_FV_sample,DataSetList) +S3method(get_FV_summary,DataSet) +S3method(get_FV_summary,DataSetList) +S3method(get_PAR_name,DataSet) +S3method(get_PAR_sample,DataSet) +S3method(get_PAR_sample,DataSetList) +S3method(get_PAR_summary,DataSet) +S3method(get_PAR_summary,DataSetList) +S3method(get_RT_overview,DataSet) +S3method(get_RT_overview,DataSetList) +S3method(get_RT_sample,DataSet) +S3method(get_RT_sample,DataSetList) +S3method(get_RT_summary,DataSet) +S3method(get_RT_summary,DataSetList) +S3method(get_id,DataSet) +S3method(get_id,DataSetList) +S3method(get_maxRT,DataSet) +S3method(get_maxRT,DataSetList) +S3method(get_overview,DataSet) +S3method(get_overview,DataSetList) +S3method(max_ERTs,DataSetList) +S3method(mean_FVs,DataSetList) +S3method(pairwise.test,DataSetList) +S3method(pairwise.test,list) +S3method(print,DataSet) +S3method(print,DataSetList) +S3method(subset,DataSet) +S3method(subset,DataSetList) +S3method(summary,DataSet) +S3method(summary,DataSetList) +export(AUC) +export(DataSet) +export(DataSetList) +export(ECDF) +export(IOH_plot_ly_default) +export(Plot.FV.Aggregated) +export(Plot.FV.ECDF_AUC) +export(Plot.FV.ECDF_Per_Target) +export(Plot.FV.ECDF_Single_Func) +export(Plot.FV.Histogram) +export(Plot.FV.Multi_Func) +export(Plot.FV.PDF) +export(Plot.FV.Parameters) +export(Plot.FV.Single_Func) +export(Plot.Performviz) +export(Plot.RT.Aggregated) +export(Plot.RT.ECDF_AUC) +export(Plot.RT.ECDF_Multi_Func) +export(Plot.RT.ECDF_Per_Target) +export(Plot.RT.ECDF_Single_Func) +export(Plot.RT.Histogram) +export(Plot.RT.Multi_Func) +export(Plot.RT.PMF) +export(Plot.RT.Parameters) +export(Plot.RT.Single_Func) +export(Plot.Stats.Glicko2_Candlestick) +export(Plot.Stats.Significance_Graph) +export(Plot.Stats.Significance_Heatmap) +export(SP) +export(arrange) +export(bootstrap_RT) +export(cat.DataSet) +export(change_id) +export(check_dsc_configured) +export(check_format) +export(clean_DataSetList) +export(fast_RT_samples) +export(generate_data.AUC) +export(generate_data.Aggr) +export(generate_data.ECDF) +export(generate_data.ECDF_raw) +export(generate_data.PMF) +export(generate_data.Parameters) +export(generate_data.Single_Function) +export(generate_data.hist) +export(get_ECDF_targets) +export(get_ERT) +export(get_FV_overview) +export(get_FV_sample) +export(get_FV_summary) +export(get_PAR_name) +export(get_PAR_sample) +export(get_PAR_summary) +export(get_RT_overview) +export(get_RT_sample) +export(get_RT_summary) +export(get_algId) +export(get_color_scheme) +export(get_color_scheme_dt) +export(get_default_ECDF_targets) +export(get_dim) +export(get_dsc_omnibus) +export(get_dsc_posthoc) +export(get_dsc_rank) +export(get_funcId) +export(get_funcName) +export(get_funvals) +export(get_id) +export(get_line_style) +export(get_marg_contrib_ecdf) +export(get_maxRT) +export(get_overview) +export(get_parId) +export(get_runtimes) +export(get_shapley_values) +export(get_static_attribute_values) +export(get_static_attributes) +export(get_target_dt) +export(glicko2_ranking) +export(max_ERTs) +export(mean_FVs) +export(pairwise.test) +export(plot_general_data) +export(read_index_file) +export(register_DSC) +export(runServer) +export(save_plotly) +export(save_table) +export(scan_index_file) +export(seq_FV) +export(seq_RT) +export(set_DSC_credentials) +export(set_color_scheme) +importFrom(RColorBrewer,brewer.pal) +importFrom(colorRamps,primary.colors) +importFrom(colorspace,sequential_hcl) +importFrom(data.table,":=") +importFrom(data.table,as.data.table) +importFrom(data.table,copy) +importFrom(data.table,data.table) +importFrom(data.table,frank) +importFrom(data.table,fread) +importFrom(data.table,is.data.table) +importFrom(data.table,melt) +importFrom(data.table,rbindlist) +importFrom(data.table,setnames) +importFrom(data.table,setorderv) +importFrom(data.table,transpose) +importFrom(dplyr,"%>%") +importFrom(dplyr,mutate) +importFrom(ggplot2,aes) +importFrom(ggplot2,element_text) +importFrom(ggplot2,facet_wrap) +importFrom(ggplot2,geom_jitter) +importFrom(ggplot2,geom_line) +importFrom(ggplot2,geom_ribbon) +importFrom(ggplot2,geom_violin) +importFrom(ggplot2,ggplot) +importFrom(ggplot2,guides) +importFrom(ggplot2,scale_color_manual) +importFrom(ggplot2,scale_colour_manual) +importFrom(ggplot2,scale_fill_manual) +importFrom(ggplot2,scale_x_continuous) +importFrom(ggplot2,scale_x_log10) +importFrom(ggplot2,theme) +importFrom(ggplot2,theme_grey) +importFrom(ggplot2,theme_set) +importFrom(grDevices,col2rgb) +importFrom(grDevices,colors) +importFrom(grDevices,nclass.FD) +importFrom(graphics,hist) +importFrom(httr,POST) +importFrom(httr,add_headers) +importFrom(httr,authenticate) +importFrom(httr,content) +importFrom(knitr,kable) +importFrom(magrittr,"%<>%") +importFrom(magrittr,mod) +importFrom(magrittr,set_colnames) +importFrom(magrittr,set_names) +importFrom(magrittr,set_rownames) +importFrom(methods,hasArg) +importFrom(plotly,add_annotations) +importFrom(plotly,add_trace) +importFrom(plotly,layout) +importFrom(plotly,orca) +importFrom(plotly,plot_ly) +importFrom(plotly,rename_) +importFrom(plotly,subplot) +importFrom(reshape2,acast) +importFrom(rjson,fromJSON) +importFrom(shiny,req) +importFrom(stats,dt) +importFrom(stats,ecdf) +importFrom(stats,integrate) +importFrom(stats,ks.test) +importFrom(stats,median) +importFrom(stats,p.adjust) +importFrom(stats,quantile) +importFrom(stats,rgeom) +importFrom(stats,sd) +importFrom(stringi,stri_detect_fixed) +importFrom(stringi,stri_detect_regex) +importFrom(stringi,stri_locate_all) +importFrom(stringi,stri_rand_strings) +importFrom(stringi,stri_replace) +importFrom(utils,data) +importFrom(utils,head) +importFrom(utils,read.csv) +importFrom(utils,tail) +importFrom(utils,type.convert) +importFrom(utils,write.csv) +useDynLib(IOHanalyzer) diff --git a/NEWS.md b/NEWS.md index bb931de7..e6c693da 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,2 +1,2 @@ -# IOHanalyzer 0.1 -* First submission to CRAN +# IOHanalyzer 0.1 +* First submission to CRAN diff --git a/R/DataSet.R b/R/DataSet.R index 9232c820..0149c098 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -1,1054 +1,1054 @@ -#' Constructor of S3 class 'DataSet' -#' -#' DataSet contains the following attributes -#' * funId -#' * DIM -#' * algId -#' * datafile -#' * instance -#' * maxEvals -#' * finalFunEvals -#' * comment -#' * Additional attributes based on the original format -#' -#' @param info A List. Contains a set of in a *.info file. -#' @param verbose Logical. -#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? -#' Set to NULL to determine automatically based on format -#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" -#' @param subsampling Logical. Whether *.cdat files are subsampled? -#' -#' @return A S3 object 'DataSet' -#' @export -#' @examples -#' path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') -#' info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) -#' DataSet(info[[1]]) -DataSet <- function(info, verbose = F, maximization = NULL, format = IOHprofiler, - subsampling = FALSE) { - if (!is.null(info)) { - path <- dirname(info$datafile) - suite <- toupper(info$suite) - - # for an unknown suite, to detect the format - if (is.null(suite) || length(suite) == 0) { - if (verbose) - warning("Suite-name not provided in .info-file, taking best guess based on - the format of data-files.") - suite <- switch(format, - IOHprofiler = "Unknown", - COCO = "BBOB", - BIOBJ_COCO = "biobj-bbob", - TWO_COL = "Unknown") - } - - if (is.null(maximization) || maximization == AUTOMATIC) { - #TODO: Better way to deal with capitalization of attributes - if (!is.null(info$maximization)) maximization <- info$maximization - else if (!is.null(info$Maximization)) maximization <- info$Maximization - else if (!is.null(suite)) { - if (verbose) - warning("maximization or minimization not specified in .info-file, - taking best guess based on the suite-name.") - if (grepl("\\w*bbob\\w*", suite, ignore.case = T) != 0) - maximization <- FALSE - else - maximization <- TRUE - } - else { - warning("Can't detect maximization based on suite-attribute, setting to - minimization by default") - maximization <- FALSE # default to minimization - } - } - - if(!(isTRUE(maximization) || isFALSE(maximization))) - warning("unclear whether we should maximize or minimize.") - - datBaseName <- sub(pattern = '(.*)\\..*$', replacement = '\\1', basename(info$datafile)) - datFile <- file.path(path, paste0(datBaseName, '.dat')) - tdatFile <- file.path(path, paste0(datBaseName, '.tdat')) - cdatFile <- file.path(path, paste0(datBaseName, '.cdat')) - - # NOTE: preference on data file for the alignment by RT: cdat > tdat > dat - if (file.exists(cdatFile)) - fvFile <- cdatFile - else if (file.exists(tdatFile)) - fvFile <- tdatFile - else if (file.exists(datFile)) - fvFile <- datFile - else - stop('No datafiles found, please verify the integrity of the chosen files') - - # NOTE: preference on data file for the alignment by FV: dat > tdat > cdat - if (file.exists(datFile)) - rtFile <- datFile - else if (file.exists(tdatFile)) - rtFile <- tdatFile - else if (file.exists(cdatFile)) - # TODO: perhaps turn on `subsampling` here as this would take quite some time - rtFile <- cdatFile - - read_raw <- switch( - format, - IOHprofiler = read_dat, - COCO = read_dat__COCO, - BIOBJ_COCO = read_dat__BIOBJ_COCO, - TWO_COL = read_dat # TODO: perhaps rename `TWO_COL` or to use a better naming - # scheme for all format names - ) - - RT_raw <- read_raw(rtFile, subsampling) - FV_raw <- read_raw(fvFile, subsampling) - - if (is.null(maximization)) { - if (verbose) - warning("Did not find maximization / minimization, auto-detecting based on - function value progression") - # TODO: idxTarget should be set depending on the data format - idxTarget <- 2 - cond <- unique(lapply(FV_raw, function(FV) FV[1, idxTarget] >= FV[nrow(FV), idxTarget])) - if (length(cond) > 1) - stop('The detected maximization differs in multiple runs') - maximization <- cond - } - - RT <- align_running_time(RT_raw, format = format, maximization = maximization) - FV <- align_function_value(FV_raw, format = format) - - PAR <- list( - 'by_FV' = RT[names(RT) != 'RT'], - 'by_RT' = FV[names(FV) != 'FV'] - ) - - RT <- RT$RT - mode(RT) <- 'integer' - FV <- FV$FV - - if (format %in% c(IOHprofiler)) { - # try to save some memory here... - FV <- tryCatch({ - .FV <- FV - mode(.FV) <- 'integer' - if (all(FV == .FV)) .FV - else FV - }, - warning = function(w) FV) # in case the type coercion gives a warning - } - - # TODO: add more data sanity checks - maxRT <- set_names(sapply(RT_raw, function(d) d[nrow(d), idxEvals]), NULL) - # Fix for old-format files which do not store used runtime in .dat-files - maxRT <- pmax(maxRT, info$maxRT) - if (any(maxRT != info$maxRT) && verbose) - warning('Inconsitent maxRT in *.info file and *.cdat file') - - # TODO: clean up these if-statements: Function to set idxTarget and n_data_column? - # `idxTarget` is a global variable? - if (format == TWO_COL) - finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget - 1]), NULL) - else - finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget]), NULL) - - if (any(finalFV != info$finalFV) && verbose) - warning('Inconsitent finalFvalue in *.info file and *.dat file') - - if (length(info$instance) != length(RT_raw)) { - if (verbose) - warning('The number of instances found in the info is inconsistent with the data!') - info$instance <- seq(length(RT_raw)) - } - - do.call( - function(...) - structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), - c(info, list(maxRT = maxRT, finalFV = finalFV, format = format, - maximization = maximization, suite = suite, ID = info$algId)) - ) - } - else - structure(list(), class = c('DataSet', 'list')) -} - -#' S3 concatenation function for DataSet -#' -#' @description Concatenation for DataSets. Combines multiple runs from separate DataSets -#' into a single DataSet object if all provided arguments have the same dimension, function ID and -#' algorithm ID, and each contains only a single run. Currently does not support parameter tracking -#' -#' @param ... The DataSets to concatenate -#' @return A new DataSet -#' @export -#' @examples -#' c(dsl[[1]], dsl[[1]]) -c.DataSet <- function(...) { - dsl <- list(...) - - if (length(dsl) == 1) - dsl <- dsl[[1]] - dsl <- dsl[sapply(dsl, length) != 0] - - if (length(dsl) == 0) - return() - if (length(dsl) == 1) - return(dsl[[1]]) - - for (ds in dsl) { - if (!any((class(ds)) == 'DataSet')) - stop("Operation only possible when all arguments are DataSets") - } - - fixed_attrs <- - c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') - info <- list() - for (attr_str in fixed_attrs) { - temp <- unique(unlist(lapply(dsl, function(x) - attr(x, attr_str)))) - if (length(temp) > 1) { - stop( - paste0( - "Attempted to add datasets with different ", - attr_str, - "-attributes! This is not supported, please keep them as separate DataSets!" - ) - ) - } - info <- c(info, temp) - } - names(info) <- fixed_attrs - - #Record number of runs to make masks of static attributes - nr_runs <- sapply(dsl, function(x) - ncol(x$FV)) - for (attr_str in names(attributes(dsl[[1]]))) { - if (attr_str %in% fixed_attrs || - attr_str %in% c("names", "class")) - next - temp <- unlist(lapply(dsl, function(x) - attr(x, attr_str))) - if (length(unique(temp)) == 1) - temp <- unique(temp) - else { - if (length(temp) == length(nr_runs)) - temp <- list(temp_name = rep(temp, nr_runs)) - else - temp <- list(temp_name = temp) - } - names(temp) <- attr_str - info <- c(info, temp) - } - - format <- info[['format']] #attr(dsl[[1]], "format") - - RT_raw <- unlist(lapply(dsl, function(ds) { - lapply(seq_len(ncol(ds$RT)), function(cnr) { - rt_temp <- as.matrix(ds$RT[, cnr]) - cbind(rt_temp, as.numeric(rownames(ds$RT))) - }) - }), recursive = F) - - RT <- - align_running_time(RT_raw, - format = "TWO_COL", - maximization = info$maximization)$RT - FV <- align_function_value(RT_raw, format = "TWO_COL")$FV - - # TODO: to deal with cases where aligned parameters are present in original DataSets - PAR <- list('by_FV' = RT[names(RT) != 'RT'], - 'by_RT' = FV[names(FV) != 'FV']) - - # Unaligned parameters - for (par_name in names(dsl[[1]]$PAR)) { - if (!par_name %in% c('by_FV', 'by_RT')) - PAR[[par_name]] <- - unlist(lapply(dsl, function(x) { - x$PAR[[par_name]] - }), recursive = F) - } - - do.call(function(...) - structure(list( - RT = RT, FV = FV, PAR = PAR - ), class = c('DataSet', 'list'), ...), - c(info)) -} - -#' S3 subset function for DataSet -#' -#' @description Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet -#' and turned into a new DataSet object. -#' -#' @param x The DataSet from which to get a subset -#' @param mask The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs -#' present in the provided dataset object x. -#' @param ... Arguments passed to underlying subset method (not yet supported) -#' -#' @return A new DataSet -#' @export -#' @examples -#' subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) -subset.DataSet <- function(x, mask, ...) { - if (length(mask) != ncol(x$FV)) - stop(paste("The input DataSet has", ncol(x$FV), "runs while the input mask array has length", length(mask))) - - info <- list() - for (attr_str in names(attributes(x))) { - if (attr_str %in% c('names', 'class')) - next - temp <- attr(x, attr_str) - if (length(unique(temp)) == 1) - temp <- unique(temp) - else { - if (length(temp) == length(mask)) - temp <- list(temp[mask]) - else{ - warning( - paste0( - "Attribute detected (", - attr_str, - ") with incorrect length for the mask-based subsetting!" - ) - ) - next - } - } - names(temp) <- attr_str - info <- c(info, temp) - } - - format <- info[['format']] - - RT <- as.matrix(x$RT[, mask]) - FV <- as.matrix(x$FV[, mask]) - - PAR <- list( - 'by_FV' = ifelse(ncol(x$PAR$by_FV) == length(mask), x$PAR$by_FV[, mask], NULL), - 'by_RT' = ifelse(ncol(x$PAR$by_RT) == length(mask), x$PAR$by_RT[, mask], NULL) - ) - - do.call(function(...) - structure(list( - RT = RT, FV = FV, PAR = PAR - ), class = c('DataSet', 'list'), ...), - c(info)) -} - -#' S3 generic print operator for DataSet -#' -#' @param x A DataSet object -#' @param ... Arguments passed to other methods -#' -#' @return A short description of the DataSet -#' @examples -#' print(dsl[[1]]) -#' @export -print.DataSet <- function(x, ...) { - cat(as.character.DataSet(x, ...)) -} - -#' S3 generic cat operator for DataSet -#' -#' @param x A DataSet object -#' -#' @return A short description of the DataSet -#' @export -#' @examples -#' cat.DataSet(dsl[[1]]) -cat.DataSet <- function(x) cat(as.character(x)) - -#' S3 generic as.character operator for DataSet -#' -#' @param x A DataSet object -#' @param verbose Verbose mode, currently not implemented -#' @param ... Arguments passed to other methods -#' -#' @return A short description of the DataSet -#' @export -#' @examples -#' as.character(dsl[[1]]) -as.character.DataSet <- function(x, verbose = F, ...) { - # TODO: implement the verbose mode - sprintf('DataSet(%s on f%s %dD)', attr(x, 'algId'), attr(x, 'funcId'), - attr(x, 'DIM')) -} - -#' S3 generic summary operator for DataSet -#' -#' @param object A DataSet object -#' @param ... Arguments passed to other methods -#' -#' @return A summary of the DataSet containing both function-value and runtime based statistics. -#' @examples -#' summary(dsl[[1]]) -#' @export -summary.DataSet <- function(object, ...) { - ds_attr <- attributes(object) - cat('DataSet Object:\n') - cat(sprintf('Source: %s\n', ds_attr$src)) - cat(sprintf('Algorithm: %s\n', ds_attr$algId)) - cat(sprintf('Function ID: %s\n', ds_attr$funcId)) - cat(sprintf('Dimension: %dD\n', ds_attr$DIM)) - - n_instance <- length(ds_attr$instance) - if (n_instance >= 15) { - inst <- paste0(paste(ds_attr$instance[1:7], collapse = ','), - ',...,', - paste(ds_attr$instance[(n_instance - 7):n_instance], collapse = ',')) - cat(sprintf('%d instance found: %s\n\n', n_instance, inst)) - } - else - cat(sprintf('%d instance found: %s\n\n', n_instance, paste(ds_attr$instance, collapse = ','))) - - cat('runtime summary:\n') - function_values <- as.numeric(rownames(object$RT)) - RT.summary <- get_RT_summary(object, function_values) - print(RT.summary) - cat('\n') - - cat('function value summary:\n') - runtimes <- as.numeric(rownames(object$FV)) - if (length(runtimes) > 100) { - runtimes <- runtimes[seq(1, length(runtimes), length.out = 100)] - } - - FV.summary <- get_FV_summary(object, runtimes) - print(FV.summary) - cat('\n') - - cat(paste('Attributes:', paste0(names(ds_attr), collapse = ', '))) -} - -#' S3 generic == operator for DataSets -#' -#' @param dsL A `DataSet` object -#' @param dsR A `DataSet` object -#' -#' -#' @return True if the DataSets contain the same function, dimension and algorithm, -#' and have the exact same attributes -#' @examples -#' dsl[[1]] == dsl[[2]] -#' @export -`==.DataSet` <- function(dsL, dsR) { - if (length(dsL) == 0 || length(dsR) == 0) - return(FALSE) - - for (attr_str in names(attributes(dsL))) { - if (any(attr(dsL, attr_str) != attr(dsR, attr_str))) return(FALSE) - } - return(TRUE) -} - -#' Get Expected RunTime -#' -#' @param ds A DataSet or DataSetList object -#' @param budget Optional; overwrites the budget found in ds for ERT-calculation -#' @param ... Arguments passed to other methods -#' @param ftarget The function target(s) for which to get the ERT -#' -#' @return A data.table containing the runtime samples for each provided target -#' function value -#' @examples -#' get_ERT(dsl, 14) -#' get_ERT(dsl[[1]], 14) -#' @export -#' -get_ERT <- function(ds, ftarget, budget, ...) UseMethod("get_ERT", ds) - -#' Get RunTime Sample -#' -#' @param ds A DataSet or DataSetList object -#' @param ftarget A Numerical vector. Function values at which runtime values are consumed -#' @param ... Arguments passed to other methods -#' -#' -#' @return A data.table containing the runtime samples for each provided target -#' function value -#' @examples -#' get_RT_sample(dsl, 14) -#' get_RT_sample(dsl[[1]], 14) -#' @export -get_RT_sample <- function(ds, ftarget, ...) UseMethod("get_RT_sample", ds) - -#' Get RunTime Summary -#' -#' @param ds A DataSet or DataSetList object -#' @param budget Optional; overwrites the budget found in ds for ERT-calculation -#' @param ... Arguments passed to other methods -#' @param ftarget The function target(s) for which to get the runtime summary -#' -#' @return A data.table containing the runtime statistics for each provided target -#' function value -#' @examples -#' get_RT_summary(dsl, 14) -#' get_RT_summary(dsl[[1]], 14) -#' @export -get_RT_summary <- function(ds, ftarget, budget, ...) UseMethod("get_RT_summary", ds) - -#' Get Funtion Value Samples -#' -#' @param ds A DataSet or DataSetList object -#' @param runtime A Numerical vector. Runtimes at which function values are reached -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the function value samples for each provided -#' target runtime -#' @examples -#' get_FV_sample(dsl, 100) -#' get_FV_sample(dsl[[1]], 100) -#' @export -get_FV_sample <- function(ds, ...) UseMethod("get_FV_sample", ds) - -#' Get Function Value Summary -#' -#' @param ds A DataSet or DataSetList object -#' @param runtime A Numerical vector. Runtimes at which function values are reached -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the function value statistics for each provided -#' target runtime value -#' @examples -#' get_FV_summary(dsl, 100) -#' get_FV_summary(dsl[[1]], 100) -#' @export -get_FV_summary <- function(ds, ...) UseMethod("get_FV_summary", ds) - -#' Get Parameter Value Samples -#' -#' @param ds A DataSet or DataSetList object -#' @param idxValue A Numerical vector. Index values at which parameter values are observed. -#' The index value can either take its value in the range of running times, or function values. -#' Such a value type is signified by `which` parameter. -#' @param ... Arguments passed to other methods -#' -#' @return A data.table object containing parameter values aligned at each given target value -#' @examples -#' get_PAR_sample(dsl, 14) -#' get_PAR_sample(dsl[[1]], 14) -#' @export -get_PAR_sample <- function(ds, idxValue, ...) UseMethod("get_PAR_sample", ds) - -#' Get Parameter Value Summary -#' -#' @param ds A DataSet or DataSetList object -#' @param idxValue A Numerical vector. Index values at which parameter values are observed. -#' The index value can either take its value in the range of running times, or function values. -#' Such a value type is signified by `which` parameter. -#' @param ... Arguments passed to other methods -#' -#' @return A data.table object containing basic statistics of parameter values aligned at each given target value -#' @examples -#' get_PAR_summary(dsl, 14) -#' get_PAR_summary(dsl[[1]], 14) -#' @export -get_PAR_summary <- function(ds, idxValue, ...) UseMethod("get_PAR_summary", ds) - -#' Get the parameter names of the algorithm -#' -#' @param ds A DataSet object -#' @param which a string takes it value in `c('by_FV', 'by_RT')`, indicating the -#' parameters aligned against the running time (RT) or function value (FV). `'by_FV'` -#' is the default value. -#' @return a character list of paramter names, if recorded in the data set -#' @examples -#' get_PAR_name(dsl[[1]]) -#' @export -get_PAR_name <- function(ds, which) UseMethod("get_PAR_name", ds) - -#' Get Function Value condensed overview -#' -#' @param ds A `DataSet` or `DataSetList` object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the algorithm ID, best, worst and mean reached function -#' values, the number of runs and available budget for the DataSet -#' @examples -#' get_FV_overview(dsl) -#' get_FV_overview(dsl[[1]]) -#' get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') -#' @export -get_FV_overview <- function(ds, ...) UseMethod("get_FV_overview", ds) - -#' Get Runtime Value condensed overview -#' -#' @param ds A DataSet or DataSetList object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the algorithm ID, minimum and maximum used evaluations, -#' number of runs and available budget for the DataSet -#' @examples -#' get_RT_overview(dsl) -#' get_RT_overview(dsl[[1]]) -#' @export -get_RT_overview <- function(ds, ...) UseMethod("get_RT_overview", ds) - -#' Get condensed overview of datasets -#' -#' @param ds A DataSet or DataSetList object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing some basic information about the provided DataSet(List) -#' @examples -#' get_overview(dsl) -#' get_overview(dsl[[1]]) -#' @export -get_overview <- function(ds, ...) UseMethod("get_overview", ds) - -#' Get condensed overview of datasets -#' -#' Get the unique identifiers for each DataSet in the provided DataSetList -#' -#' If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), -#' this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility -#' -#' @param ds The DataSetList -#' @param ... Arguments passed to other methods -#' -#' @return The list of unique identiefiers present in dsl -#' @examples -#' get_id(dsl) -#' get_id(dsl[[1]]) -#' @export -get_id <- function(ds, ...) UseMethod("get_id", ds) - -#' @rdname get_FV_overview -#' @export -get_FV_overview.DataSet <- function(ds, ...) { - data <- ds$FV - runs <- ncol(data) - last_row <- data[nrow(data), ] - budget <- max(attr(ds, 'maxRT')) - maximization <- attr(ds, 'maximization') - - op <- ifelse(maximization, max, min) - op_inv <- ifelse(maximization, min, max) - - best_fv <- op(last_row, na.rm = T) - worst_recorded_fv <- op_inv(data, na.rm = T) - worst_fv <- op_inv(last_row, na.rm = T) - mean_fv <- mean(last_row, na.rm = T) - median_fv <- median(last_row, na.rm = T) - runs_reached <- sum(last_row == best_fv) - - data.table(ID = get_id(ds), - DIM = attr(ds, 'DIM'), - funcId = attr(ds, 'funcId'), - `worst recorded` = worst_recorded_fv, - `worst reached` = worst_fv, - `best reached` = best_fv, - `mean reached` = mean_fv, - `median reached` = median_fv, - runs = runs, - `succ` = runs_reached, - budget = budget) -} - -#' @rdname get_RT_overview -#' @export -#' -get_RT_overview.DataSet <- function(ds, ...) { - - if (!is.null(attr(ds, "format")) && attr(ds, "format") == NEVERGRAD) { - data <- ds$FV - budget <- max(attr(ds, 'maxRT')) - runs <- ncol(data) - min_rt <- rownames(data) %>% as.integer %>% min - max_rt <- budget - } - - else{ - data <- ds$RT - runs <- ncol(data) - budget <- max(attr(ds, 'maxRT')) - min_rt <- min(data, na.rm = T) - max_rt <- max(data, na.rm = T) - } - - data.table(ID = get_id(ds), - DIM = attr(ds, 'DIM'), - funcId = attr(ds, 'funcId'), - `miminal runtime` = min_rt, - `maximal runtime` = max_rt, - `runs` = runs, - `Budget` = budget) -} - -#' @rdname get_overview -#' @export -#' -get_overview.DataSet <- function(ds, ...) { - data <- ds$FV - runs <- ncol(data) - - budget <- max(attr(ds, 'maxRT')) - if (!is.null(ds$RT) && length(ds$RT) > 0) { - max_rt <- max(ds$RT, na.rm = T) - budget <- max(budget, max_rt) - } - else max_rt <- budget - - last_row <- data[nrow(data), ] - maximization <- attr(ds, 'maximization') - - op <- ifelse(maximization, max, min) - op_inv <- ifelse(maximization, min, max) - - best_fv <- op(last_row, na.rm = T) - worst_recorded_fv <- op_inv(data, na.rm = T) - worst_fv <- op_inv(last_row, na.rm = T) - mean_fv <- mean(last_row, na.rm = T) - median_fv <- median(last_row, na.rm = T) - runs_reached <- sum(last_row == best_fv) - - data.table(ID = get_id(ds), - `DIM` = attr(ds, 'DIM'), - `funcId` = attr(ds, 'funcId'), - `runs` = runs, - `best reached` = best_fv, - `succ` = runs_reached, - `budget` = budget, - `max evals used` = max_rt, - `worst recorded` = worst_recorded_fv, - `worst reached` = worst_fv, - `mean reached` = mean_fv, - `median reached` = median_fv - ) - -} - -#' @rdname get_ERT -#' @export -#' -get_ERT.DataSet <- function(ds, ftarget, budget = NULL, ...) { - data <- ds$RT - if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') - else maxRT <- as.numeric(budget) - algId <- attr(ds, 'algId') - maximization <- attr(ds, 'maximization') - - ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) - FValues <- as.numeric(rownames(data)) - idx <- seq_along(FValues) - op <- ifelse(maximization, `>=`, `<=`) - - matched <- sapply(ftarget, function(f) idx[`op`(FValues, f)][1]) - - if (is.list(matched)) - return(data.table()) - - data <- data[matched, , drop = FALSE] - dt <- as.data.table(cbind(get_id(ds), ftarget, SP(data, maxRT)$ERT)) - colnames(dt) <- c('ID', 'target', 'ERT') - dt -} - -#' @rdname get_RT_summary -#' @export -#' -get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { - data <- ds$RT - if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') - else maxRT <- as.numeric(budget) - ID <- get_id(ds) - maximization <- attr(ds, 'maximization') - - ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) - FValues <- as.numeric(rownames(data)) - idx <- seq_along(FValues) - op <- ifelse(maximization, `>=`, `<=`) - - matched <- sapply( - ftarget, - function(f) { - idx[`op`(FValues, f)][1] - } - ) - - if (is.list(matched)) { - return(data.table()) - } - - data <- data[matched, , drop = FALSE] - dt_temp <- apply(data, 1, IOHanalyzer_env$D_quantile) %>% - t %>% - as.data.table %>% - cbind(as.data.table(SP(data, maxRT))) %>% - cbind(ID, ftarget, - apply(data, 1, .mean), - apply(data, 1, .median), - apply(data, 1, .sd), .) %>% - set_colnames(c('ID', 'target', 'mean', 'median', - 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), - 'ERT', 'runs', 'ps')) - dt_temp -} - -#' Get the maximal running time -#' -#' @param ds A DataSet or DataSetList object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table object containing the algorithm ID and the running time -#' when the algorithm terminates in each run -#' @examples -#' get_maxRT(dsl) -#' get_maxRT(dsl[[1]]) -#' @export -get_maxRT <- function(ds, ...) UseMethod("get_maxRT", ds) - -#' @rdname get_maxRT -#' @param output The format of the outputted table: 'wide' or 'long' -#' @export -#' -get_maxRT.DataSet <- function(ds, output = 'wide', ...) { - ID <- get_id(ds) - N <- ncol(ds$RT) - res <- t(c(ID, attr(ds, 'maxRT'))) %>% - as.data.table %>% - set_colnames(c('ID', paste0('run.', seq(N)))) - - if (output == 'long') { - res <- melt(res, id = 'ID', variable.name = 'run', value.name = 'maxRT') - res[, run := as.numeric(gsub('run.', '', run)) %>% as.integer - ][, maxRT := as.integer(maxRT) - ][order(run)] - } - res -} - -#' @rdname get_RT_sample -#' @param output A character determining the format of output data.table: 'wide' or 'long' -#' @export -get_RT_sample.DataSet <- function(ds, ftarget, output = 'wide', ...) { - data <- ds$RT - N <- ncol(data) - ID <- get_id(ds) - maximization <- attr(ds, 'maximization') - - ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) - FValues <- as.double(rownames(data)) - idx <- seq_along(FValues) - op <- ifelse(maximization, `>=`, `<=`) - - matched <- sapply( - ftarget, - function(f) { - idx[`op`(FValues, f)][1] - } - ) - - res <- cbind(ID, ftarget, as.data.table(data[matched, , drop = FALSE])) %>% - set_colnames(c('ID', 'target', paste0('run.', seq(N)))) - - if (output == 'long') { - # TODO: option to not add run etc to speed up performance of ECDF calculation? - res <- melt(res, id = c('ID', 'target'), variable.name = 'run', value.name = 'RT') - res[, run := as.integer(as.numeric(gsub('run.', '', run))) - ][, RT := as.integer(RT) - ][order(target, run)] - } - res -} - -#' Function to get just the RT samples needed, without any formatting to improve speed -#' @param RT_mat A matrix containing the RT-values of a dataset -#' @param target Which target-value to use -#' @param maximization Whether maximization is needed or not -#' @export -fast_RT_samples <- function(RT_mat, target, maximization = F) { - if (maximization) - idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) >= target] - else - idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) <= target] - if (length(idxs) > 0) { - return(RT_mat[idxs[[1]], ]) - } - return(rep(NA, 15)) -} - -#' @rdname get_FV_summary -#' @export -#' -get_FV_summary.DataSet <- function(ds, runtime, ...) { - data <- ds$FV - NC <- ncol(data) - NR <- nrow(data) - ID <- get_id(ds) - maximization <- attr(ds, 'maximization') - - runtime <- sort(as.numeric(unique(c(runtime)))) - RT <- as.numeric(rownames(data)) - idx <- seq_along(RT) - - matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) - data <- data[matched, , drop = FALSE] - - cbind(ID, runtime, NC, - apply(data, 1, .mean), - apply(data, 1, .median), - apply(data, 1, .sd), - as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile)))) %>% - set_colnames(c('ID', 'runtime', 'runs', 'mean', 'median', 'sd', - paste0(getOption("IOHanalyzer.quantiles") * 100, '%'))) -} - -#' @rdname get_FV_sample -#' @param output A String. The format of the output data: 'wide' or 'long' -#' -#' @export -#' -get_FV_sample.DataSet <- function(ds, runtime, output = 'wide', ...) { - data <- ds$FV - N <- ncol(data) - n_row <- nrow(data) - ID <- get_id(ds) - maximization <- attr(ds, 'maximization') - - runtime <- sort(as.numeric(unique(c(runtime)))) - RT <- as.numeric(rownames(data)) - idx <- seq_along(RT) - - matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) - res <- cbind(ID, runtime, as.data.table(data[matched, , drop = FALSE])) %>% - set_colnames(c('ID', 'runtime', paste0('run.', seq(N)))) - - if (output == 'long') { - res <- melt(res, id = c('ID', 'runtime'), variable.name = 'run', value.name = 'f(x)') - res[, run := as.integer(as.numeric(gsub('run.', '', run))) - ][order(runtime, run)] - } - res -} - -#' @rdname get_PAR_name -#' @export -#' -get_PAR_name.DataSet <- function(ds, which = 'by_FV') { - names(ds$PAR[[which]]) -} - -#' @rdname get_PAR_summary -#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved -#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -#' is the default value. -#' @export -get_PAR_summary.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', ...) { - if (which == 'by_FV') { - RefValues <- as.numeric(rownames(ds$RT)) - ds_par <- ds$PAR$by_FV - idx_name <- 'target' - } - else if (which == 'by_RT') { - RefValues <- as.numeric(rownames(ds$FV)) - ds_par <- ds$PAR$by_RT - idx_name <- 'runtime' - } - - idx <- seq_along(RefValues) - ID <- get_id(ds) - par_name <- get_PAR_name(ds, which = which) - - if (parId != 'all') - par_name <- intersect(par_name, parId) - if (length(par_name) == 0) - return(NULL) - - maximization <- attr(ds, 'maximization') - cond <- maximization || which == 'by_RT' - op <- ifelse(cond, `>=`, `<=`) - idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) - - matched <- sapply( - idxValue, - function(f) { - idx[`op`(RefValues, f)][1] - } - ) - - lapply(par_name, - function(par) { - data <- ds_par[[par]][matched, , drop = FALSE] - df <- cbind( - ID, par, idxValue, - apply(data, 1, function(x) length(x[!is.na(x)])), - apply(data, 1, .mean), - apply(data, 1, .median), - apply(data, 1, .sd), - as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile))) - ) - colnames(df) <- c('ID', 'parId', idx_name, 'runs', 'mean', 'median', 'sd', - paste0(getOption("IOHanalyzer.quantiles") * 100, '%')) - df - }) %>% - rbindlist -} - -#' @rdname get_PAR_sample -#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved -#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -#' is the default value. -#' @param output A character. The format of the output data: 'wide' or 'long' -#' @export -get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', - output = 'wide', ...) { - N <- length(attr(ds, 'instance')) - if (which == 'by_FV') { - RefValues <- as.numeric(rownames(ds$RT)) - ds_par <- ds$PAR$by_FV - idx_name <- 'target' - } - else if (which == 'by_RT') { - RefValues <- as.numeric(rownames(ds$FV)) - ds_par <- ds$PAR$by_RT - idx_name <- 'runtime' - } - - idx <- seq_along(RefValues) - ID <- get_id(ds) - par_name <- get_PAR_name(ds, which = which) - - if (parId != 'all') - par_name <- intersect(par_name, parId) - if (length(par_name) == 0) - return(NULL) - - maximization <- attr(ds, 'maximization') - cond <- maximization || which == 'by_RT' - op <- ifelse(cond, `>=`, `<=`) - idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) - matched <- sapply(idxValue, function(f) idx[`op`(RefValues, f)][1]) - - res <- lapply(par_name, - function(parId) { - data <- ds_par[[parId]] - data <- as.data.table(data[matched, , drop = FALSE]) - data <- cbind(ID, parId, idxValue, data) - colnames(data) <- c('ID', 'parId', idx_name, paste0('run.', seq(N))) - data - }) - res <- rbindlist(res) - - if (output == 'long') { - res <- melt(res, id = c('ID', 'parId', idx_name), variable.name = 'run', value.name = 'value') - res[, run := as.integer(as.numeric(gsub('run.', '', run)))] - if (which == 'by_FV') - res[order(target, run)] - else if (which == 'by_RT') - res[order(runtime, run)] - } - res -} - -#' @rdname get_id -#' @export -get_id.DataSet <- function(ds, ...) { - temp <- attr(ds, 'ID') - if (is.null(temp)) { - warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added - to new datasets, see the 'change_id' function.") - return(attr(ds, 'algId')) - } - return(unique(temp)) +#' Constructor of S3 class 'DataSet' +#' +#' DataSet contains the following attributes +#' * funId +#' * DIM +#' * algId +#' * datafile +#' * instance +#' * maxEvals +#' * finalFunEvals +#' * comment +#' * Additional attributes based on the original format +#' +#' @param info A List. Contains a set of in a *.info file. +#' @param verbose Logical. +#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? +#' Set to NULL to determine automatically based on format +#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" +#' @param subsampling Logical. Whether *.cdat files are subsampled? +#' +#' @return A S3 object 'DataSet' +#' @export +#' @examples +#' path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') +#' info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) +#' DataSet(info[[1]]) +DataSet <- function(info, verbose = F, maximization = NULL, format = IOHprofiler, + subsampling = FALSE) { + if (!is.null(info)) { + path <- dirname(info$datafile) + suite <- toupper(info$suite) + + # for an unknown suite, to detect the format + if (is.null(suite) || length(suite) == 0) { + if (verbose) + warning("Suite-name not provided in .info-file, taking best guess based on + the format of data-files.") + suite <- switch(format, + IOHprofiler = "Unknown", + COCO = "BBOB", + BIOBJ_COCO = "biobj-bbob", + TWO_COL = "Unknown") + } + + if (is.null(maximization) || maximization == AUTOMATIC) { + #TODO: Better way to deal with capitalization of attributes + if (!is.null(info$maximization)) maximization <- info$maximization + else if (!is.null(info$Maximization)) maximization <- info$Maximization + else if (!is.null(suite)) { + if (verbose) + warning("maximization or minimization not specified in .info-file, + taking best guess based on the suite-name.") + if (grepl("\\w*bbob\\w*", suite, ignore.case = T) != 0) + maximization <- FALSE + else + maximization <- TRUE + } + else { + warning("Can't detect maximization based on suite-attribute, setting to + minimization by default") + maximization <- FALSE # default to minimization + } + } + + if(!(isTRUE(maximization) || isFALSE(maximization))) + warning("unclear whether we should maximize or minimize.") + + datBaseName <- sub(pattern = '(.*)\\..*$', replacement = '\\1', basename(info$datafile)) + datFile <- file.path(path, paste0(datBaseName, '.dat')) + tdatFile <- file.path(path, paste0(datBaseName, '.tdat')) + cdatFile <- file.path(path, paste0(datBaseName, '.cdat')) + + # NOTE: preference on data file for the alignment by RT: cdat > tdat > dat + if (file.exists(cdatFile)) + fvFile <- cdatFile + else if (file.exists(tdatFile)) + fvFile <- tdatFile + else if (file.exists(datFile)) + fvFile <- datFile + else + stop('No datafiles found, please verify the integrity of the chosen files') + + # NOTE: preference on data file for the alignment by FV: dat > tdat > cdat + if (file.exists(datFile)) + rtFile <- datFile + else if (file.exists(tdatFile)) + rtFile <- tdatFile + else if (file.exists(cdatFile)) + # TODO: perhaps turn on `subsampling` here as this would take quite some time + rtFile <- cdatFile + + read_raw <- switch( + format, + IOHprofiler = read_dat, + COCO = read_dat__COCO, + BIOBJ_COCO = read_dat__BIOBJ_COCO, + TWO_COL = read_dat # TODO: perhaps rename `TWO_COL` or to use a better naming + # scheme for all format names + ) + + RT_raw <- read_raw(rtFile, subsampling) + FV_raw <- read_raw(fvFile, subsampling) + + if (is.null(maximization)) { + if (verbose) + warning("Did not find maximization / minimization, auto-detecting based on + function value progression") + # TODO: idxTarget should be set depending on the data format + idxTarget <- 2 + cond <- unique(lapply(FV_raw, function(FV) FV[1, idxTarget] >= FV[nrow(FV), idxTarget])) + if (length(cond) > 1) + stop('The detected maximization differs in multiple runs') + maximization <- cond + } + + RT <- align_running_time(RT_raw, format = format, maximization = maximization) + FV <- align_function_value(FV_raw, format = format) + + PAR <- list( + 'by_FV' = RT[names(RT) != 'RT'], + 'by_RT' = FV[names(FV) != 'FV'] + ) + + RT <- RT$RT + mode(RT) <- 'integer' + FV <- FV$FV + + if (format %in% c(IOHprofiler)) { + # try to save some memory here... + FV <- tryCatch({ + .FV <- FV + mode(.FV) <- 'integer' + if (all(FV == .FV)) .FV + else FV + }, + warning = function(w) FV) # in case the type coercion gives a warning + } + + # TODO: add more data sanity checks + maxRT <- set_names(sapply(RT_raw, function(d) d[nrow(d), idxEvals]), NULL) + # Fix for old-format files which do not store used runtime in .dat-files + maxRT <- pmax(maxRT, info$maxRT) + if (any(maxRT != info$maxRT) && verbose) + warning('Inconsitent maxRT in *.info file and *.cdat file') + + # TODO: clean up these if-statements: Function to set idxTarget and n_data_column? + # `idxTarget` is a global variable? + if (format == TWO_COL) + finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget - 1]), NULL) + else + finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget]), NULL) + + if (any(finalFV != info$finalFV) && verbose) + warning('Inconsitent finalFvalue in *.info file and *.dat file') + + if (length(info$instance) != length(RT_raw)) { + if (verbose) + warning('The number of instances found in the info is inconsistent with the data!') + info$instance <- seq(length(RT_raw)) + } + + do.call( + function(...) + structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), + c(info, list(maxRT = maxRT, finalFV = finalFV, format = format, + maximization = maximization, suite = suite, ID = info$algId)) + ) + } + else + structure(list(), class = c('DataSet', 'list')) +} + +#' S3 concatenation function for DataSet +#' +#' @description Concatenation for DataSets. Combines multiple runs from separate DataSets +#' into a single DataSet object if all provided arguments have the same dimension, function ID and +#' algorithm ID, and each contains only a single run. Currently does not support parameter tracking +#' +#' @param ... The DataSets to concatenate +#' @return A new DataSet +#' @export +#' @examples +#' c(dsl[[1]], dsl[[1]]) +c.DataSet <- function(...) { + dsl <- list(...) + + if (length(dsl) == 1) + dsl <- dsl[[1]] + dsl <- dsl[sapply(dsl, length) != 0] + + if (length(dsl) == 0) + return() + if (length(dsl) == 1) + return(dsl[[1]]) + + for (ds in dsl) { + if (!any((class(ds)) == 'DataSet')) + stop("Operation only possible when all arguments are DataSets") + } + + fixed_attrs <- + c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') + info <- list() + for (attr_str in fixed_attrs) { + temp <- unique(unlist(lapply(dsl, function(x) + attr(x, attr_str)))) + if (length(temp) > 1) { + stop( + paste0( + "Attempted to add datasets with different ", + attr_str, + "-attributes! This is not supported, please keep them as separate DataSets!" + ) + ) + } + info <- c(info, temp) + } + names(info) <- fixed_attrs + + #Record number of runs to make masks of static attributes + nr_runs <- sapply(dsl, function(x) + ncol(x$FV)) + for (attr_str in names(attributes(dsl[[1]]))) { + if (attr_str %in% fixed_attrs || + attr_str %in% c("names", "class")) + next + temp <- unlist(lapply(dsl, function(x) + attr(x, attr_str))) + if (length(unique(temp)) == 1) + temp <- unique(temp) + else { + if (length(temp) == length(nr_runs)) + temp <- list(temp_name = rep(temp, nr_runs)) + else + temp <- list(temp_name = temp) + } + names(temp) <- attr_str + info <- c(info, temp) + } + + format <- info[['format']] #attr(dsl[[1]], "format") + + RT_raw <- unlist(lapply(dsl, function(ds) { + lapply(seq_len(ncol(ds$RT)), function(cnr) { + rt_temp <- as.matrix(ds$RT[, cnr]) + cbind(rt_temp, as.numeric(rownames(ds$RT))) + }) + }), recursive = F) + + RT <- + align_running_time(RT_raw, + format = "TWO_COL", + maximization = info$maximization)$RT + FV <- align_function_value(RT_raw, format = "TWO_COL")$FV + + # TODO: to deal with cases where aligned parameters are present in original DataSets + PAR <- list('by_FV' = RT[names(RT) != 'RT'], + 'by_RT' = FV[names(FV) != 'FV']) + + # Unaligned parameters + for (par_name in names(dsl[[1]]$PAR)) { + if (!par_name %in% c('by_FV', 'by_RT')) + PAR[[par_name]] <- + unlist(lapply(dsl, function(x) { + x$PAR[[par_name]] + }), recursive = F) + } + + do.call(function(...) + structure(list( + RT = RT, FV = FV, PAR = PAR + ), class = c('DataSet', 'list'), ...), + c(info)) +} + +#' S3 subset function for DataSet +#' +#' @description Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet +#' and turned into a new DataSet object. +#' +#' @param x The DataSet from which to get a subset +#' @param mask The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs +#' present in the provided dataset object x. +#' @param ... Arguments passed to underlying subset method (not yet supported) +#' +#' @return A new DataSet +#' @export +#' @examples +#' subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) +subset.DataSet <- function(x, mask, ...) { + if (length(mask) != ncol(x$FV)) + stop(paste("The input DataSet has", ncol(x$FV), "runs while the input mask array has length", length(mask))) + + info <- list() + for (attr_str in names(attributes(x))) { + if (attr_str %in% c('names', 'class')) + next + temp <- attr(x, attr_str) + if (length(unique(temp)) == 1) + temp <- unique(temp) + else { + if (length(temp) == length(mask)) + temp <- list(temp[mask]) + else{ + warning( + paste0( + "Attribute detected (", + attr_str, + ") with incorrect length for the mask-based subsetting!" + ) + ) + next + } + } + names(temp) <- attr_str + info <- c(info, temp) + } + + format <- info[['format']] + + RT <- as.matrix(x$RT[, mask]) + FV <- as.matrix(x$FV[, mask]) + + PAR <- list( + 'by_FV' = ifelse(ncol(x$PAR$by_FV) == length(mask), x$PAR$by_FV[, mask], NULL), + 'by_RT' = ifelse(ncol(x$PAR$by_RT) == length(mask), x$PAR$by_RT[, mask], NULL) + ) + + do.call(function(...) + structure(list( + RT = RT, FV = FV, PAR = PAR + ), class = c('DataSet', 'list'), ...), + c(info)) +} + +#' S3 generic print operator for DataSet +#' +#' @param x A DataSet object +#' @param ... Arguments passed to other methods +#' +#' @return A short description of the DataSet +#' @examples +#' print(dsl[[1]]) +#' @export +print.DataSet <- function(x, ...) { + cat(as.character.DataSet(x, ...)) +} + +#' S3 generic cat operator for DataSet +#' +#' @param x A DataSet object +#' +#' @return A short description of the DataSet +#' @export +#' @examples +#' cat.DataSet(dsl[[1]]) +cat.DataSet <- function(x) cat(as.character(x)) + +#' S3 generic as.character operator for DataSet +#' +#' @param x A DataSet object +#' @param verbose Verbose mode, currently not implemented +#' @param ... Arguments passed to other methods +#' +#' @return A short description of the DataSet +#' @export +#' @examples +#' as.character(dsl[[1]]) +as.character.DataSet <- function(x, verbose = F, ...) { + # TODO: implement the verbose mode + sprintf('DataSet(%s on f%s %dD)', attr(x, 'algId'), attr(x, 'funcId'), + attr(x, 'DIM')) +} + +#' S3 generic summary operator for DataSet +#' +#' @param object A DataSet object +#' @param ... Arguments passed to other methods +#' +#' @return A summary of the DataSet containing both function-value and runtime based statistics. +#' @examples +#' summary(dsl[[1]]) +#' @export +summary.DataSet <- function(object, ...) { + ds_attr <- attributes(object) + cat('DataSet Object:\n') + cat(sprintf('Source: %s\n', ds_attr$src)) + cat(sprintf('Algorithm: %s\n', ds_attr$algId)) + cat(sprintf('Function ID: %s\n', ds_attr$funcId)) + cat(sprintf('Dimension: %dD\n', ds_attr$DIM)) + + n_instance <- length(ds_attr$instance) + if (n_instance >= 15) { + inst <- paste0(paste(ds_attr$instance[1:7], collapse = ','), + ',...,', + paste(ds_attr$instance[(n_instance - 7):n_instance], collapse = ',')) + cat(sprintf('%d instance found: %s\n\n', n_instance, inst)) + } + else + cat(sprintf('%d instance found: %s\n\n', n_instance, paste(ds_attr$instance, collapse = ','))) + + cat('runtime summary:\n') + function_values <- as.numeric(rownames(object$RT)) + RT.summary <- get_RT_summary(object, function_values) + print(RT.summary) + cat('\n') + + cat('function value summary:\n') + runtimes <- as.numeric(rownames(object$FV)) + if (length(runtimes) > 100) { + runtimes <- runtimes[seq(1, length(runtimes), length.out = 100)] + } + + FV.summary <- get_FV_summary(object, runtimes) + print(FV.summary) + cat('\n') + + cat(paste('Attributes:', paste0(names(ds_attr), collapse = ', '))) +} + +#' S3 generic == operator for DataSets +#' +#' @param dsL A `DataSet` object +#' @param dsR A `DataSet` object +#' +#' +#' @return True if the DataSets contain the same function, dimension and algorithm, +#' and have the exact same attributes +#' @examples +#' dsl[[1]] == dsl[[2]] +#' @export +`==.DataSet` <- function(dsL, dsR) { + if (length(dsL) == 0 || length(dsR) == 0) + return(FALSE) + + for (attr_str in names(attributes(dsL))) { + if (any(attr(dsL, attr_str) != attr(dsR, attr_str))) return(FALSE) + } + return(TRUE) +} + +#' Get Expected RunTime +#' +#' @param ds A DataSet or DataSetList object +#' @param budget Optional; overwrites the budget found in ds for ERT-calculation +#' @param ... Arguments passed to other methods +#' @param ftarget The function target(s) for which to get the ERT +#' +#' @return A data.table containing the runtime samples for each provided target +#' function value +#' @examples +#' get_ERT(dsl, 14) +#' get_ERT(dsl[[1]], 14) +#' @export +#' +get_ERT <- function(ds, ftarget, budget, ...) UseMethod("get_ERT", ds) + +#' Get RunTime Sample +#' +#' @param ds A DataSet or DataSetList object +#' @param ftarget A Numerical vector. Function values at which runtime values are consumed +#' @param ... Arguments passed to other methods +#' +#' +#' @return A data.table containing the runtime samples for each provided target +#' function value +#' @examples +#' get_RT_sample(dsl, 14) +#' get_RT_sample(dsl[[1]], 14) +#' @export +get_RT_sample <- function(ds, ftarget, ...) UseMethod("get_RT_sample", ds) + +#' Get RunTime Summary +#' +#' @param ds A DataSet or DataSetList object +#' @param budget Optional; overwrites the budget found in ds for ERT-calculation +#' @param ... Arguments passed to other methods +#' @param ftarget The function target(s) for which to get the runtime summary +#' +#' @return A data.table containing the runtime statistics for each provided target +#' function value +#' @examples +#' get_RT_summary(dsl, 14) +#' get_RT_summary(dsl[[1]], 14) +#' @export +get_RT_summary <- function(ds, ftarget, budget, ...) UseMethod("get_RT_summary", ds) + +#' Get Funtion Value Samples +#' +#' @param ds A DataSet or DataSetList object +#' @param runtime A Numerical vector. Runtimes at which function values are reached +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the function value samples for each provided +#' target runtime +#' @examples +#' get_FV_sample(dsl, 100) +#' get_FV_sample(dsl[[1]], 100) +#' @export +get_FV_sample <- function(ds, ...) UseMethod("get_FV_sample", ds) + +#' Get Function Value Summary +#' +#' @param ds A DataSet or DataSetList object +#' @param runtime A Numerical vector. Runtimes at which function values are reached +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the function value statistics for each provided +#' target runtime value +#' @examples +#' get_FV_summary(dsl, 100) +#' get_FV_summary(dsl[[1]], 100) +#' @export +get_FV_summary <- function(ds, ...) UseMethod("get_FV_summary", ds) + +#' Get Parameter Value Samples +#' +#' @param ds A DataSet or DataSetList object +#' @param idxValue A Numerical vector. Index values at which parameter values are observed. +#' The index value can either take its value in the range of running times, or function values. +#' Such a value type is signified by `which` parameter. +#' @param ... Arguments passed to other methods +#' +#' @return A data.table object containing parameter values aligned at each given target value +#' @examples +#' get_PAR_sample(dsl, 14) +#' get_PAR_sample(dsl[[1]], 14) +#' @export +get_PAR_sample <- function(ds, idxValue, ...) UseMethod("get_PAR_sample", ds) + +#' Get Parameter Value Summary +#' +#' @param ds A DataSet or DataSetList object +#' @param idxValue A Numerical vector. Index values at which parameter values are observed. +#' The index value can either take its value in the range of running times, or function values. +#' Such a value type is signified by `which` parameter. +#' @param ... Arguments passed to other methods +#' +#' @return A data.table object containing basic statistics of parameter values aligned at each given target value +#' @examples +#' get_PAR_summary(dsl, 14) +#' get_PAR_summary(dsl[[1]], 14) +#' @export +get_PAR_summary <- function(ds, idxValue, ...) UseMethod("get_PAR_summary", ds) + +#' Get the parameter names of the algorithm +#' +#' @param ds A DataSet object +#' @param which a string takes it value in `c('by_FV', 'by_RT')`, indicating the +#' parameters aligned against the running time (RT) or function value (FV). `'by_FV'` +#' is the default value. +#' @return a character list of paramter names, if recorded in the data set +#' @examples +#' get_PAR_name(dsl[[1]]) +#' @export +get_PAR_name <- function(ds, which) UseMethod("get_PAR_name", ds) + +#' Get Function Value condensed overview +#' +#' @param ds A `DataSet` or `DataSetList` object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the algorithm ID, best, worst and mean reached function +#' values, the number of runs and available budget for the DataSet +#' @examples +#' get_FV_overview(dsl) +#' get_FV_overview(dsl[[1]]) +#' get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') +#' @export +get_FV_overview <- function(ds, ...) UseMethod("get_FV_overview", ds) + +#' Get Runtime Value condensed overview +#' +#' @param ds A DataSet or DataSetList object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the algorithm ID, minimum and maximum used evaluations, +#' number of runs and available budget for the DataSet +#' @examples +#' get_RT_overview(dsl) +#' get_RT_overview(dsl[[1]]) +#' @export +get_RT_overview <- function(ds, ...) UseMethod("get_RT_overview", ds) + +#' Get condensed overview of datasets +#' +#' @param ds A DataSet or DataSetList object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing some basic information about the provided DataSet(List) +#' @examples +#' get_overview(dsl) +#' get_overview(dsl[[1]]) +#' @export +get_overview <- function(ds, ...) UseMethod("get_overview", ds) + +#' Get condensed overview of datasets +#' +#' Get the unique identifiers for each DataSet in the provided DataSetList +#' +#' If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +#' this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility +#' +#' @param ds The DataSetList +#' @param ... Arguments passed to other methods +#' +#' @return The list of unique identiefiers present in dsl +#' @examples +#' get_id(dsl) +#' get_id(dsl[[1]]) +#' @export +get_id <- function(ds, ...) UseMethod("get_id", ds) + +#' @rdname get_FV_overview +#' @export +get_FV_overview.DataSet <- function(ds, ...) { + data <- ds$FV + runs <- ncol(data) + last_row <- data[nrow(data), ] + budget <- max(attr(ds, 'maxRT')) + maximization <- attr(ds, 'maximization') + + op <- ifelse(maximization, max, min) + op_inv <- ifelse(maximization, min, max) + + best_fv <- op(last_row, na.rm = T) + worst_recorded_fv <- op_inv(data, na.rm = T) + worst_fv <- op_inv(last_row, na.rm = T) + mean_fv <- mean(last_row, na.rm = T) + median_fv <- median(last_row, na.rm = T) + runs_reached <- sum(last_row == best_fv) + + data.table(ID = get_id(ds), + DIM = attr(ds, 'DIM'), + funcId = attr(ds, 'funcId'), + `worst recorded` = worst_recorded_fv, + `worst reached` = worst_fv, + `best reached` = best_fv, + `mean reached` = mean_fv, + `median reached` = median_fv, + runs = runs, + `succ` = runs_reached, + budget = budget) +} + +#' @rdname get_RT_overview +#' @export +#' +get_RT_overview.DataSet <- function(ds, ...) { + + if (!is.null(attr(ds, "format")) && attr(ds, "format") == NEVERGRAD) { + data <- ds$FV + budget <- max(attr(ds, 'maxRT')) + runs <- ncol(data) + min_rt <- rownames(data) %>% as.integer %>% min + max_rt <- budget + } + + else{ + data <- ds$RT + runs <- ncol(data) + budget <- max(attr(ds, 'maxRT')) + min_rt <- min(data, na.rm = T) + max_rt <- max(data, na.rm = T) + } + + data.table(ID = get_id(ds), + DIM = attr(ds, 'DIM'), + funcId = attr(ds, 'funcId'), + `miminal runtime` = min_rt, + `maximal runtime` = max_rt, + `runs` = runs, + `Budget` = budget) +} + +#' @rdname get_overview +#' @export +#' +get_overview.DataSet <- function(ds, ...) { + data <- ds$FV + runs <- ncol(data) + + budget <- max(attr(ds, 'maxRT')) + if (!is.null(ds$RT) && length(ds$RT) > 0) { + max_rt <- max(ds$RT, na.rm = T) + budget <- max(budget, max_rt) + } + else max_rt <- budget + + last_row <- data[nrow(data), ] + maximization <- attr(ds, 'maximization') + + op <- ifelse(maximization, max, min) + op_inv <- ifelse(maximization, min, max) + + best_fv <- op(last_row, na.rm = T) + worst_recorded_fv <- op_inv(data, na.rm = T) + worst_fv <- op_inv(last_row, na.rm = T) + mean_fv <- mean(last_row, na.rm = T) + median_fv <- median(last_row, na.rm = T) + runs_reached <- sum(last_row == best_fv) + + data.table(ID = get_id(ds), + `DIM` = attr(ds, 'DIM'), + `funcId` = attr(ds, 'funcId'), + `runs` = runs, + `best reached` = best_fv, + `succ` = runs_reached, + `budget` = budget, + `max evals used` = max_rt, + `worst recorded` = worst_recorded_fv, + `worst reached` = worst_fv, + `mean reached` = mean_fv, + `median reached` = median_fv + ) + +} + +#' @rdname get_ERT +#' @export +#' +get_ERT.DataSet <- function(ds, ftarget, budget = NULL, ...) { + data <- ds$RT + if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') + else maxRT <- as.numeric(budget) + algId <- attr(ds, 'algId') + maximization <- attr(ds, 'maximization') + + ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) + FValues <- as.numeric(rownames(data)) + idx <- seq_along(FValues) + op <- ifelse(maximization, `>=`, `<=`) + + matched <- sapply(ftarget, function(f) idx[`op`(FValues, f)][1]) + + if (is.list(matched)) + return(data.table()) + + data <- data[matched, , drop = FALSE] + dt <- as.data.table(cbind(get_id(ds), ftarget, SP(data, maxRT)$ERT)) + colnames(dt) <- c('ID', 'target', 'ERT') + dt +} + +#' @rdname get_RT_summary +#' @export +#' +get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { + data <- ds$RT + if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') + else maxRT <- as.numeric(budget) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) + FValues <- as.numeric(rownames(data)) + idx <- seq_along(FValues) + op <- ifelse(maximization, `>=`, `<=`) + + matched <- sapply( + ftarget, + function(f) { + idx[`op`(FValues, f)][1] + } + ) + + if (is.list(matched)) { + return(data.table()) + } + + data <- data[matched, , drop = FALSE] + dt_temp <- apply(data, 1, IOHanalyzer_env$D_quantile) %>% + t %>% + as.data.table %>% + cbind(as.data.table(SP(data, maxRT))) %>% + cbind(ID, ftarget, + apply(data, 1, .mean), + apply(data, 1, .median), + apply(data, 1, .sd), .) %>% + set_colnames(c('ID', 'target', 'mean', 'median', + 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), + 'ERT', 'runs', 'ps')) + dt_temp +} + +#' Get the maximal running time +#' +#' @param ds A DataSet or DataSetList object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table object containing the algorithm ID and the running time +#' when the algorithm terminates in each run +#' @examples +#' get_maxRT(dsl) +#' get_maxRT(dsl[[1]]) +#' @export +get_maxRT <- function(ds, ...) UseMethod("get_maxRT", ds) + +#' @rdname get_maxRT +#' @param output The format of the outputted table: 'wide' or 'long' +#' @export +#' +get_maxRT.DataSet <- function(ds, output = 'wide', ...) { + ID <- get_id(ds) + N <- ncol(ds$RT) + res <- t(c(ID, attr(ds, 'maxRT'))) %>% + as.data.table %>% + set_colnames(c('ID', paste0('run.', seq(N)))) + + if (output == 'long') { + res <- melt(res, id = 'ID', variable.name = 'run', value.name = 'maxRT') + res[, run := as.numeric(gsub('run.', '', run)) %>% as.integer + ][, maxRT := as.integer(maxRT) + ][order(run)] + } + res +} + +#' @rdname get_RT_sample +#' @param output A character determining the format of output data.table: 'wide' or 'long' +#' @export +get_RT_sample.DataSet <- function(ds, ftarget, output = 'wide', ...) { + data <- ds$RT + N <- ncol(data) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) + FValues <- as.double(rownames(data)) + idx <- seq_along(FValues) + op <- ifelse(maximization, `>=`, `<=`) + + matched <- sapply( + ftarget, + function(f) { + idx[`op`(FValues, f)][1] + } + ) + + res <- cbind(ID, ftarget, as.data.table(data[matched, , drop = FALSE])) %>% + set_colnames(c('ID', 'target', paste0('run.', seq(N)))) + + if (output == 'long') { + # TODO: option to not add run etc to speed up performance of ECDF calculation? + res <- melt(res, id = c('ID', 'target'), variable.name = 'run', value.name = 'RT') + res[, run := as.integer(as.numeric(gsub('run.', '', run))) + ][, RT := as.integer(RT) + ][order(target, run)] + } + res +} + +#' Function to get just the RT samples needed, without any formatting to improve speed +#' @param RT_mat A matrix containing the RT-values of a dataset +#' @param target Which target-value to use +#' @param maximization Whether maximization is needed or not +#' @export +fast_RT_samples <- function(RT_mat, target, maximization = F) { + if (maximization) + idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) >= target] + else + idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) <= target] + if (length(idxs) > 0) { + return(RT_mat[idxs[[1]], ]) + } + return(rep(NA, 15)) +} + +#' @rdname get_FV_summary +#' @export +#' +get_FV_summary.DataSet <- function(ds, runtime, ...) { + data <- ds$FV + NC <- ncol(data) + NR <- nrow(data) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + runtime <- sort(as.numeric(unique(c(runtime)))) + RT <- as.numeric(rownames(data)) + idx <- seq_along(RT) + + matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) + data <- data[matched, , drop = FALSE] + + cbind(ID, runtime, NC, + apply(data, 1, .mean), + apply(data, 1, .median), + apply(data, 1, .sd), + as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile)))) %>% + set_colnames(c('ID', 'runtime', 'runs', 'mean', 'median', 'sd', + paste0(getOption("IOHanalyzer.quantiles") * 100, '%'))) +} + +#' @rdname get_FV_sample +#' @param output A String. The format of the output data: 'wide' or 'long' +#' +#' @export +#' +get_FV_sample.DataSet <- function(ds, runtime, output = 'wide', ...) { + data <- ds$FV + N <- ncol(data) + n_row <- nrow(data) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + runtime <- sort(as.numeric(unique(c(runtime)))) + RT <- as.numeric(rownames(data)) + idx <- seq_along(RT) + + matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) + res <- cbind(ID, runtime, as.data.table(data[matched, , drop = FALSE])) %>% + set_colnames(c('ID', 'runtime', paste0('run.', seq(N)))) + + if (output == 'long') { + res <- melt(res, id = c('ID', 'runtime'), variable.name = 'run', value.name = 'f(x)') + res[, run := as.integer(as.numeric(gsub('run.', '', run))) + ][order(runtime, run)] + } + res +} + +#' @rdname get_PAR_name +#' @export +#' +get_PAR_name.DataSet <- function(ds, which = 'by_FV') { + names(ds$PAR[[which]]) +} + +#' @rdname get_PAR_summary +#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved +#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +#' is the default value. +#' @export +get_PAR_summary.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', ...) { + if (which == 'by_FV') { + RefValues <- as.numeric(rownames(ds$RT)) + ds_par <- ds$PAR$by_FV + idx_name <- 'target' + } + else if (which == 'by_RT') { + RefValues <- as.numeric(rownames(ds$FV)) + ds_par <- ds$PAR$by_RT + idx_name <- 'runtime' + } + + idx <- seq_along(RefValues) + ID <- get_id(ds) + par_name <- get_PAR_name(ds, which = which) + + if (parId != 'all') + par_name <- intersect(par_name, parId) + if (length(par_name) == 0) + return(NULL) + + maximization <- attr(ds, 'maximization') + cond <- maximization || which == 'by_RT' + op <- ifelse(cond, `>=`, `<=`) + idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) + + matched <- sapply( + idxValue, + function(f) { + idx[`op`(RefValues, f)][1] + } + ) + + lapply(par_name, + function(par) { + data <- ds_par[[par]][matched, , drop = FALSE] + df <- cbind( + ID, par, idxValue, + apply(data, 1, function(x) length(x[!is.na(x)])), + apply(data, 1, .mean), + apply(data, 1, .median), + apply(data, 1, .sd), + as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile))) + ) + colnames(df) <- c('ID', 'parId', idx_name, 'runs', 'mean', 'median', 'sd', + paste0(getOption("IOHanalyzer.quantiles") * 100, '%')) + df + }) %>% + rbindlist +} + +#' @rdname get_PAR_sample +#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved +#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +#' is the default value. +#' @param output A character. The format of the output data: 'wide' or 'long' +#' @export +get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', + output = 'wide', ...) { + N <- length(attr(ds, 'instance')) + if (which == 'by_FV') { + RefValues <- as.numeric(rownames(ds$RT)) + ds_par <- ds$PAR$by_FV + idx_name <- 'target' + } + else if (which == 'by_RT') { + RefValues <- as.numeric(rownames(ds$FV)) + ds_par <- ds$PAR$by_RT + idx_name <- 'runtime' + } + + idx <- seq_along(RefValues) + ID <- get_id(ds) + par_name <- get_PAR_name(ds, which = which) + + if (parId != 'all') + par_name <- intersect(par_name, parId) + if (length(par_name) == 0) + return(NULL) + + maximization <- attr(ds, 'maximization') + cond <- maximization || which == 'by_RT' + op <- ifelse(cond, `>=`, `<=`) + idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) + matched <- sapply(idxValue, function(f) idx[`op`(RefValues, f)][1]) + + res <- lapply(par_name, + function(parId) { + data <- ds_par[[parId]] + data <- as.data.table(data[matched, , drop = FALSE]) + data <- cbind(ID, parId, idxValue, data) + colnames(data) <- c('ID', 'parId', idx_name, paste0('run.', seq(N))) + data + }) + res <- rbindlist(res) + + if (output == 'long') { + res <- melt(res, id = c('ID', 'parId', idx_name), variable.name = 'run', value.name = 'value') + res[, run := as.integer(as.numeric(gsub('run.', '', run)))] + if (which == 'by_FV') + res[order(target, run)] + else if (which == 'by_RT') + res[order(runtime, run)] + } + res +} + +#' @rdname get_id +#' @export +get_id.DataSet <- function(ds, ...) { + temp <- attr(ds, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'change_id' function.") + return(attr(ds, 'algId')) + } + return(unique(temp)) } \ No newline at end of file diff --git a/R/DataSetList.R b/R/DataSetList.R index 914c64e8..265605bf 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -1,1441 +1,1441 @@ -#' Read all the data file in a folfer -# -#' @param path Path to the data files. Will look for all .info-files in this directory and use -#' the corresponding datafiles to create the DataSetList -#' @param verbose Logical. -#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? -#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" -#' @param subsampling Logical. Whether *.cdat files are subsampled? -#' @param print_fun Function used to print output when in verbose mode -#' @noRd -#' @return A DataSetList object -read_dir <- function(path, verbose = T, print_fun = NULL, maximization = TRUE, - format = IOHprofiler, subsampling = FALSE) { - DataSetList( - path, - verbose, - print_fun, - maximization = maximization, - format = format, - subsampling = subsampling - ) - } - -# TODO: improve documentation of this S3 class -#' S3 constructor of the 'DataSetList' -#' -#' Attributes -#' funId -#' DIM -#' algId -# -#' @param path Path to the data files. Will look for all .info-files in this directory and use -#' the corresponding datafiles to create the DataSetList -#' @param verbose Logical. -#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? -#' @param format A character. The format of data source, options are: -#' \itemize{ -#' \item'IOHProfiler' -#' \item'COCO' -#' \item'TWO_COL' -#' \item'COCO_BIOBJ' -#' \item'NEVERGRAD' -#' \item'SOS' -#' } -#' These formats are specified in more detail in our github wiki. -#' @param subsampling Logical. Whether *.cdat files are subsampled? -#' @param print_fun Function used to print output when in verbose mode -#' @param full_aggregation If True, individual DataSets are aggregated as much as possible: all DataSets -#' with the same algorithmname, function id and dimension are combined together. This leads to information loss -#' related to static variables, so only use if that information is not required. -#' -#' @return A DataSetList object -#' @export -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -#' DataSetList(path) -DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization = NULL, - format = IOHprofiler, subsampling = FALSE, full_aggregation = TRUE) { - if (is.null(path)) - return(structure(list(), class = c('DataSetList', 'list'))) - - path <- trimws(path) - if (format == NEVERGRAD) { - if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") - return(read_nevergrad(path)) - else - indexFiles <- - file.path(path, list.files(path, pattern = '.csv', recursive = T)) - } - else if (format == SOS) { - return(read_datasetlist_SOS(path, locate_corrections_files(path))) - } - else - indexFiles <- scan_index_file(path) - - if (is.null(print_fun)) - print_fun <- cat - - object <- list() - class(object) <- c('DataSetList', class(object)) - DIM <- c() - algId <- c() - funcId <- c() - suites <- c() - maximizations <- c() - i <- 1 - - for (file in indexFiles) { - if (verbose) { - print_fun(paste('Processing', file, '...\n')) - } - - if (format == NEVERGRAD) { - dsl <- read_nevergrad(file) - object %<>% c(., dsl) - suites[i] <- NEVERGRAD - } - else { - indexInfo <- read_index_file(file) - if (verbose) - print_fun(sprintf(' algorithm %s...\n', indexInfo[[1]]$algId)) - - for (info in indexInfo) { - if (verbose) { - print_fun( - sprintf( - ' %d instances on f%s %dD...\n', - length(info$instance), - info$funcId, - info$DIM - ) - ) - } - - copy_flag <- TRUE - data <- DataSet(info, maximization = maximization, - format = format, subsampling = subsampling) - - DIM[i] <- attr(data, 'DIM') - funcId[i] <- attr(data, 'funcId') - algId[i] <- attr(data, 'algId') - - # TODO: double-check the following treatment on `instance`!!! - instance <- attr(data, 'instance') #Was instance without index? - suites[i] <- attr(data, 'suite') - maximizations[i] <- attr(data, 'maximization') - - if (length(object) != 0) { - idx <- which(sapply(object, function(obj) obj == data)) - for (k in idx) { - instance_ <- attr(object[[k]], 'instance') - if (all(instance == instance_)) { - copy_flag <- FALSE - warning('duplicated instances!') - break - } - - if (length(intersect(instance, instance_)) != 0) { - warning('duplicated instances!') - } - } - } - - if (copy_flag) { - object[[i]] <- data - i <- i + 1 - } - } - } - - if (verbose) - print_fun("\n") - } - - # TODO: sort all DataSet by multiple attributes: algId, funcId and DIM - if (format != NEVERGRAD) { - attr(object, 'DIM') <- DIM - attr(object, 'funcId') <- funcId - attr(object, 'algId') <- algId - } - - suite <- unique(suites) - maximization <- unique(maximizations) - if (length(maximization) != 1) { - warning("Multipe different optimization types detected!") - } - - attr(object, 'suite') <- suite - attr(object, 'maximization') <- maximization - attr(object, 'ID') <- attr(object, 'algId') - attr(object, 'ID_attributes') <- c('algId') - if (full_aggregation) - clean_DataSetList(object) - else - object -} - - -#' Clean DataSetList object by concatenating DataSets -#' -#' Concatenates all DataSets with the same ID, function id and dimension -#' -#' @param dsList The DataSetList object to clean -#' @export -#' @examples -#' clean_DataSetList(dsl) -clean_DataSetList <- function(dsList) { - #To ensure no uninitialized variables are present - .I <- NULL - cases <- mapply( - function(...) paste0(list(...), collapse = ','), - attr(dsList, 'funcId'), - attr(dsList, 'DIM'), - attr(dsList, 'algId'), - attr(dsList, 'ID'), - SIMPLIFY = T, - USE.NAMES = F - ) - - dt <- as.data.table(cases)[, list(list(.I)), by = cases] - idx_to_del <- c() - - for (idx in dt$V1) { - if (length(idx) > 1) { - dsList[[idx[1]]] <- c.DataSet(dsList[idx]) - idx_to_del <- c(idx_to_del, idx[-1]) - } - } - dsList[idx_to_del] <- NULL - - if (length(idx_to_del) > 0) { - attr(dsList, 'DIM') <- sapply(dsList, function(ds) attr(ds, 'DIM')) - attr(dsList, 'funcId') <- sapply(dsList, function(ds) attr(ds, 'funcId')) - attr(dsList, 'algId') <- sapply(dsList, function(ds) attr(ds, 'algId')) - attr(dsList, 'ID') <- sapply(dsList, function(ds) attr(ds, 'ID')) - } - dsList -} - -#' S3 concatenation function for DataSetList -#' -#' @param ... The DataSetLists to concatenate -#' @return A new DataSetList -#' @export -#' @examples -#' c(dsl[1], dsl[3]) -c.DataSetList <- function(...) { - # TODO: maybe remove duplicated dataset in the further - # remove the empty list first - dsl <- list(...) - dsl <- dsl[sapply(dsl, length) != 0] - - if (length(dsl) == 0) - return() - - object <- unlist(dsl, recursive = F) - if (!any((class(object)) == 'DataSetList')) - class(object) <- c('DataSetList', class(object)) - - for (attr_str in c('DIM', 'funcId', 'algId', 'ID')) { - attr(object, attr_str) <- unlist(lapply(dsl, function(x) attr(x, attr_str))) - } - - # Deal with Suites on the datasetlist level - attr(object, "suite") <- unique( - unlist( - lapply(dsl, function(x) attr(x, "suite")) - ) - ) - - # These attributes NEED to be the same across the datasetlist - for (attr_str in c('maximization', 'ID_attributes')) { - temp <- unique( - unlist(lapply(dsl, function(x) attr(x, attr_str))) - ) - - if (length(temp) > 1) { - stop(paste0("Attempted to add datasetlists with different ", attr_str, - "-attributes! This will lead to errors when processing - this data!")) - } - attr(object, attr_str) <- temp[[1]] - } - object -} - -#' S3 extraction function for DataSetList -#' -#' @param x The DataSetList to use -#' @param i The indices to extract -#' @param drop Currently unused parameter -#' @return The DataSetList of the DataSets at indices i of DataSetList x -#' @export -#' @examples -#' dsl[c(1, 3)] -`[.DataSetList` <- function(x, i, drop = FALSE) { - # remove the attributes firstly - obj <- unclass(x)[i] - class(obj) <- c('DataSetList', class(obj)) - - # also slice the attributes accordingly - attr(obj, 'DIM') <- attr(x, 'DIM')[i] - attr(obj, 'ID') <- attr(x, 'ID')[i] - attr(obj, 'funcId') <- attr(x, 'funcId')[i] - attr(obj, 'algId') <- attr(x, 'algId')[i] - attr(obj, 'suite') <- attr(x, 'suite') - attr(obj, 'maximization') <- attr(x, 'maximization') - - obj -} - -#' S3 print function for DataSetList -#' -#' @param x The DataSetList to print -#' @param ... Arguments for underlying print function? -#' @export -#' @examples -#' print(dsl) -print.DataSetList <- function(x, ...) { - cat('DataSetList:\n') - cat(paste0('Suite: ', attr(x, 'suite'), '\n')) - N <- length(x) - - # TODO: add an option for the following numbers: 15 and 5 - if (N <= 15) - idx <- seq_along(x) - else - idx <- c(1:5, '---', (N-4):N) - idx <- format(idx, justify = 'right') - - for (i in idx) { - if (trimws(i) == '---') - cat(paste0(i), '\n') - else { - .i <- as.integer(trimws(i)) - cat(sprintf('%s: %s\n', i, as.character(x[[.i]]))) - } - } -} - -# TODO: consistent use of ds, data, dsList etc. -# TODO: make decision on `DIM` or `dim`? funcId or fId or fID? algID / algid / algId? - -#' S3 summary function for DataSetList -#' -#' Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every -#' DataSet in the DataSetList -#' @param object The DataSetList to print -#' @param ... Arguments for underlying summary function? -#' @export -#' @examples -#' summary(dsl) -summary.DataSetList <- function(object, ...) { - as.data.frame( - t( - sapply( - object, - function(d) { - list( - suite = attr(d, 'suite'), - funcId = attr(d, 'funcId'), - DIM = attr(d, 'DIM'), - ID = attr(d, 'ID'), - datafile = attr(d, 'datafile'), - comment = attr(d, 'comment') - ) - } - ) - ) - ) -} - -#' S3 sort function for DataSetList -#' -#' Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). -#' Default is as ascending, can be made descending by adding a - in front of the attribute. -#' Sorting accross multiple attributes is supported, in the order they are specified. -#' -#' @param dsl The DataSetList to sort -#' @param ... attribute by which `dsl` is sorted. Multiple attributes can be specified. -#' @export -#' @examples -#' arrange(dsl, DIM, -funcId, algId) -arrange <- function(dsl, ...) UseMethod('arrange', dsl) - -#' @rdname arrange -#' @export -#' -arrange.DataSetList <- function(dsl, ...) { - cols <- substitute(list(...))[-1L] - if (identical(as.character(cols), "NULL")) - return(dsl) - - cols <- as.list(cols) - order <- as.list(rep(1L, length(cols))) - - for (i in seq_along(cols)) { - v <- as.list(cols[[i]]) - if (length(v) > 1L) { - if (v[[1L]] == '-') order[[i]] <- -1L - v <- v[[-1L]] - } - - v <- as.character(v) - if (v %in% c('DIM', 'funcId', 'ID')) - cols[[i]] <- v - else { - cols[[i]] <- NULL - order[[i]] <- NULL - } - } - - cols <- unlist(cols, use.names = F) - order <- unlist(order) - x <- c(list(1:length(dsl)), lapply(cols, function(col) attr(dsl, col))) - names(x) <- c('index', cols) - - DT <- rbindlist(list(x)) - data.table::setorderv(DT, cols, order) - idx <- DT[[1]] - dsl <- dsl[idx] - - # TODO: perhaps we do not need those attributes at all... - for (v in c('DIM', 'funcId', 'ID')) - attr(dsl, v) <- sapply(dsl, function(d) attr(d, v)) - dsl -} - -#' @rdname get_ERT -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' -get_ERT.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), attr(ds, 'funcId'), get_ERT(ds, ftarget, budget)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - })) - } - -#' @rdname get_RT_summary -#' @export -get_RT_summary.DataSetList <- function(ds, ftarget, budget = NULL, ...) { - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_RT_summary(ds, ftarget, budget)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - })) - } - -#' @rdname get_RT_sample -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' -#' @export -get_RT_sample.DataSetList <- function(ds, ftarget, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_RT_sample(ds, ftarget, ...)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - }), - fill = T) - } - -#' @rdname get_maxRT -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' -#' @export -get_maxRT.DataSetList <- function(ds, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds_) { - res <- - cbind(attr(ds_, 'DIM'), attr(ds_, 'funcId'), get_maxRT(ds_, ...)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - }), - fill = T) -} - -#' @rdname get_FV_summary -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' -get_FV_summary.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_FV_summary(ds, runtime)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - })) -} - -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' @rdname get_FV_overview -#' -get_FV_overview.DataSetList <- function(ds, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) - get_FV_overview(ds))) - -} - -#' @rdname get_RT_overview -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_RT_overview.DataSetList <- function(ds, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) - get_RT_overview(ds))) -} - -#' @rdname get_overview -#' @export -get_overview.DataSetList <- function(ds, ...) { - df <- rbindlist(lapply(ds, function(ds) - get_overview(ds))) - if (length(get_funcId(ds)) > 1 || length(get_dim(ds)) > 1) { - p2 <- df[,lapply(.SD, mean, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('mean reached')] - if (attr(ds, 'maximization')) { - p1 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'best reached')] - p3 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('worst recorded', 'worst reached')] - } - else { - p1 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('best reached')] - p3 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'worst recorded', 'worst reached')] - } - return(merge(merge(p1,p2),p3)) - } - else { - return(df) - } -} - -#' @rdname get_FV_sample -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' -get_FV_sample.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_FV_sample(ds, runtime, ...)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - }), - fill = T) - } - -#' @rdname get_PAR_summary -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_PAR_summary.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) get_PAR_summary(ds, idxValue, ...))) -} - -#' @rdname get_PAR_sample -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_PAR_sample.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) get_PAR_sample(ds, idxValue, ...)), fill = T) -} - -#' @rdname get_id -#' @export -get_id.DataSetList <- function(ds, ...) { - temp <- attr(ds, 'ID') - if (is.null(temp)) { - warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added - to new datasets, see the 'change_id' function.") - return(get_algId(ds)) - } - return(unique(temp)) -} - -#' Get all dimensions present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A sorted list of all unique dimensions which occur in the DataSetList -#' @export -#' @examples -#' get_dim(dsl) -get_dim <- function(dsList) { - sort(unique(sapply(dsList, function(d) attr(d, 'DIM')))) -} - -#' Get all function ids present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A sorted list of all unique function ids which occur in the DataSetList -#' @export -#' @examples -#' get_funcId(dsl) -get_funcId <- function(dsList) { - ll <- unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcId'))))) - - # TODO: what if the function ID is a double value? - # those are coerced to integers now - if (is.integer(ll)) return(sort(ll)) - - lli <- suppressWarnings(as.integer(ll)) - if (any(is.na(lli))) return(sort(ll)) - - if (all((lli >= 0L) & (lli <= 1000000000L))) return(ll[order(lli)]) - - # TODO: should this be even allowed? - sort(lli) -} - -#' Get all function names present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A list of all unique function names which occur in the DataSetList -#' @export -#' @examples -#' get_funcName(dsl) -get_funcName <- function(dsList) { - unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcName'))))) -} - -#' Get all algorithm ids present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A sorted list of all unique algorithm ids which occur in the DataSetList -#' @export -#' @examples -#' get_algId(dsl) -get_algId <- function(dsList) { - unique(sapply(dsList, function(d) attr(d, 'algId'))) -} - -#' Get all parameter ids present in a DataSetList -#' -#' @param dsList The DataSetList -#' @param which A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned -#' by the running time (RT) or the function value (FV). Note that parameters in each case are -#' not necessary the same. -#' -#' @return A sorted list of all unique parameter ids which occur in the DataSetList -#' @export -#' @examples -#' get_parId(dsl) -get_parId <- function(dsList, which = 'by_FV') { - unique( - unlist( - lapply(dsList, - function(d) { - if (which == 'by_FV') - names(d$PAR$by_FV) - else if (which == 'by_RT') - names(d$PAR$by_RT) - } - ) - ) - ) -} - -# TODO: let the user choose/detect whether the problem is subject to maximization -# and determine whether to sort the function values in ascending/desceding order - -#' Get all function values present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A list matrices of all function values which occur in the DataSetList -#' @export -#' @examples -#' get_funvals(dsl) -get_funvals <- function(dsList) { - if (length(dsList) == 0) - return(NULL) - if (length(dsList[[1]]$RT) == 0) { - x <- sort(unique(as.numeric(unlist( - lapply(dsList, function(x) - as.vector(x$FV)) - )))) - } - else - x <- (sort(unique(as.numeric(unlist( - lapply(dsList, function(x) - rownames(x$RT)) - ))))) - x[!is.na(x) & !is.infinite(x)] -} - -#' Get all runtime values present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A list matrices of all runtime values which occur in the DataSetList -#' @export -#' @examples -#' get_runtimes(dsl) -get_runtimes <- function(dsList) { - sort(unique(as.numeric(unlist( - lapply(dsList, function(x) - rownames(x$FV)) - )))) -} - -#' Get all attributes which can be used to subset a DataSetList -#' -#' @param dsl The DataSetList -#' -#' @return The list of available attributes -#' @export -#' @examples -#' get_static_attributes(dsl) -get_static_attributes <- function(dsl) { - full_names <- unique(unlist(lapply(dsl, function(ds) {names(attributes(ds))}))) - - reserved_attributes <- c("names", "class", "suite", "maximization", "algInfo", "comment", - "datafile", "maxRT", "finalFV", "format") - setdiff(full_names, reserved_attributes) -} - -#' Get all options for a specific attribute which can be used to subset a DataSetList -#' -#' This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. -#' Note the only attributes returned by `get_static_attributes` are supported in this funcion -#' -#' @param dsl The DataSetList -#' @param attribute the name of the attribute for which to get the available options in dsl -#' @return The list of options for the specified attribute -#' @export -#' @examples -#' get_static_attribute_values(dsl, 'funcId') -get_static_attribute_values <- function(dsl, attribute) { - unique(unlist(lapply(dsl, function(ds) {attr(ds, attribute)}))) -} - -#' Filter a DataSetList by some criteria -#' -#' @param x The DataSetList -#' @param ... The conditions to filter on. Can be any expression which assigns True or False -#' to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes -#' (funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should -#' be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a -#' string to be parsed. This allows exectution of subset commands on arbitrary variables in code. -#' -#' @return The filtered DataSetList -#' @export -#' @examples -#' subset(dsl, funcId == 1) -#' subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes -#' subset(dsl, instance == 1) -#' subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes -#' subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND -subset.DataSetList <- function(x, ...) { - enclos <- parent.frame() - if (hasArg('text')) { - text <- list(...)$text - condition_call <- parse(text = text) - } else { - condition_call <- substitute(list(...)) - condition_call <- condition_call[2:length(condition_call)] - } - - obj <- lapply(x, - function(ds){ - mask <- tryCatch(expr = { - mask <- NULL - for (idx in seq(length(condition_call))) { - mask_temp <- unlist( - eval(condition_call[[idx]], attributes(ds), enclos = enclos) - ) - if (is.null(mask)) mask <- mask_temp - else { - if (length(mask_temp) == 1 && !mask_temp) { - mask <- F - } else if (length(mask_temp) == 1) { - mask <- mask - } else if (length(mask_temp) == length(mask) || length(mask) == 1) { - mask <- mask & mask_temp - } else { - stop("Error creating mask") - } - - } - } - mask - }, error = function(e) {F}) - - if (length(mask) == 1 && mask) return(ds) - else if (length(mask) == 1 || !any(mask)) return(NULL) - return(subset(ds, mask)) - }) - - class(obj) <- c('DataSetList', class(obj)) - obj <- Filter(Negate(is.null), obj) - - # also slice the attributes accordingly - attr(obj, 'suite') <- attr(x, 'suite') - attr(obj, 'maximization') <- attr(x, 'maximization') - attr(obj, 'DIM') <- sapply(obj, function(ds) attr(ds, 'DIM')) - attr(obj, 'funcId') <- sapply(obj, function(ds) attr(ds, 'funcId')) - attr(obj, 'algId') <- sapply(obj, function(ds) attr(ds, 'algId')) - unique_ids <- unlist(sapply(obj, function(ds) attr(ds, 'unique_id'))) - if (!any(is.null(unique_ids))) { - attr(obj, 'unique_ids') <- unique_ids - } - return(obj) -} - -#' Add unique identifiers to each DataSet in the provided DataSetList based on static attributes -#' -#' Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to -#' ensure each dataset has exactly one unique identifier. -#' Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. -#' -#' @param dsl The DataSetList -#' @param attrs The list of attributes to combine into a unique identifier -#' @return A new DataSetList object where the split has been done based on the provided attributes, and the unique -#' identifier has been added. -#' @export -#' @examples -#' change_id(dsl, c('instance')) -change_id <- function(dsl, attrs) { - if (length(dsl) == 0) return(dsl) - if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") - grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(dsl, x)})) - colnames(grid) <- attrs - - dsl_new <- DataSetList() - attr_vals <- c() - for (x in transpose(grid)) { - #TODO: on Windows, UTF-8 Characters get converted into characters, which then - #don't interact correctly with the parse used in 'subset' function, leading to empty datasetlist objects. - #I'm not aware of any way to fix this, so UFT-8 characters should be avoided in ID for now. - conditions <- paste0(unlist(lapply(seq(length(attrs)), function(idx) { - paste0(attrs[[idx]], ' == "', x[[idx]], '"') - })), collapse = " & ") - dsl_temp <- subset(dsl, text = conditions) - if (length(attrs) == 1) - attr_val <- x - else - attr_val <- paste0(x, collapse = "_") - - attr_vals <- c(attr_vals, rep(attr_val, length(dsl_temp))) - dsl_new <- c(dsl_new, dsl_temp) - } - attr(dsl_new, 'ID_attributes') <- attrs - attr(dsl_new, 'ID') <- attr_vals - for (idx in seq(length(dsl_new))) { - attr(dsl_new[[idx]], 'ID') <- attr_vals[[idx]] - } - class(dsl_new) <- c("DataSetList", "list") - attr(dsl_new, 'suite') <- attr(dsl, 'suite') - attr(dsl_new, 'maximization') <- attr(dsl, 'maximization') - - attr(dsl_new, 'DIM') <- lapply(dsl_new, function(x) attr(x, 'DIM')) - attr(dsl_new, 'funcId') <- lapply(dsl_new, function(x) attr(x, 'funcId')) - attr(dsl_new, 'algId') <- lapply(dsl_new, function(x) attr(x, 'algId')) - dsl_new -} - - - - -#' Save DataTable in multiple formats -#' -#' @param df The DataTable to store -#' @param file String. The name of the figure file, with the extension of the required file-format -#' @param format Optional, string. Overwrites the extension of the `file` parameter. If not specified while -#' file does not have an extension, it defaults to csv -#' -#' @export -#' @examples -#' df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -#' save_table(df, tempfile(fileext = ".md")) -save_table <- function(df, file, format = NULL) { - if (is.null(format)) { - format <- tools::file_ext(file) - } - if (format == 'TeX' || format == 'tex') { - if (requireNamespace('xtable', quietly = T)) - print(xtable::xtable(df), file = file) - else - write(kable(df, format = 'latex'), file) - } else if (format == 'Markdown' || format == 'md') { - write(kable(df, format = 'markdown'), file) - } else if (format == 'html') { - write(kable(df, format = "html"), file) - } - else { #Default to csv - write.csv(df, file, row.names = F) - } -} - -#' Generation of default ECDF-targets -#' -#' @param dsList The DataSetList object for which to generate the targets -#' @param type The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, -#' equal for all functions / dimensions) -#' @param number_targets The amount of targets to generate -#' -#' @return A data.table with 3 columns: funcId, DIM and target -#' @export -#' @examples -#' get_ECDF_targets(dsl, 'linear', 10) -get_ECDF_targets <- function(dsList, type = "log-linear", number_targets = 10) { - funcIds <- get_funcId(dsList) - dims <- get_dim(dsList) - - dt <- rbindlist(apply(expand.grid(funcIds, dims), 1, function(x) { - if (type == 'bbob') { - fseq <- rev(seq_FV(c(100,1e-8), length.out = 51, scale = 'log')) - } - else { - dsl <- subset(dsList, funcId == x[[1]] && DIM == x[[2]]) - if (length(dsl) == 0) - return(NULL) - fall <- get_funvals(dsl) - if (length(fall) < 2) - return(NULL) - - fseq <- seq_FV(fall, length.out = number_targets, scale = ifelse(type == "log-linear", 'log', 'linear')) - } - data.table(funcId = x[[1]], DIM = x[[2]], target = fseq) - })) - dt -} - -### __________________________ Rewritten data generation functions _______________________ ### -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param start Optional start value (Runtime or target value) -#' @param stop Optional end value (Runtime or target value) -#' @param scale_log Wheterh to use logarithmic scaling or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param include_opts Whether or not to also include the best value hit by each algorithm to -#' the generated datapoints -#' @param budget Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works -#' in fixed_target mode. -#' -#' @export -#' @examples -#' generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, - scale_log = F, which = 'by_RT', include_opts = F, budget = NULL) { - - if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1 ) { - #Required because target generation is included in this function, - #which needs to be done on a per-function basis - stop("Multiple functions / dimensions are present in provided DataSetList. - Please call this function for each individual function/dimension pair instead.") - } - - by_rt <- (which == 'by_RT') - - if (by_rt) - all <- get_funvals(dsList) - else - all <- get_runtimes(dsList) - - maximization <- attr(dsList, 'maximization') - - if (is.null(maximization)) maximization <- T - if (is.null(start)) start <- min(all) - if (is.null(stop)) stop <- max(all) - - if (by_rt) { - Xseq <- seq_FV(all, start, stop, length.out = 60, - scale = ifelse(scale_log, 'log', 'linear')) - if (include_opts) { - for (uid in get_id(dsList)) { - if (maximization) - Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) - else - Xseq <- c(Xseq, min(get_funvals(subset(dsList, ID == uid)))) - } - Xseq <- unique(sort(Xseq)) - } - dt <- get_RT_summary(dsList, ftarget = Xseq, budget = budget) - } - else { - Xseq <- seq_RT(all, start, stop, length.out = 60, - scale = ifelse(scale_log, 'log', 'linear')) - if (include_opts) { - for (uid in get_id(dsList)) { - Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) - } - Xseq <- unique(sort(Xseq)) - } - dt <- get_FV_summary(dsList, Xseq) - } - - - dt[, `:=`(upper = mean + sd, lower = mean - sd)] - return(dt) -} - -#' Generate dataframe of a single function/dimension pair for creating PDF or PMF plots -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param target The target value (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -generate_data.PMF <- function(dsList, target, which = 'by_RT') { - if (which == 'by_RT') - return(get_RT_sample(dsList, target, output = 'long')) - return(get_FV_sample(dsList, target, output = 'long')) -} - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param target The target value (Runtime or target value) -#' @param use.equal.bins Whether all bins should be equal size for each algorithm or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -generate_data.hist <- function(dsList, target, use.equal.bins = F, which = 'by_RT') { - width <- NULL # Set local binding to remove warnings - - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) - stop('This function is only available a single function/dimension pair at a time.') - - if (use.equal.bins) { - if (which == 'by_RT') - res1 <- hist( - get_RT_sample(dsList, target, output = 'long')$RT, - breaks = nclass.FD, - plot = F - ) - else - res1 <- hist( - get_FV_sample(dsList, target, output = 'long')$`f(x)`, - breaks = nclass.FD, - plot = F - ) - } - - dt <- as.data.table( - rbindlist( - lapply( - dsList, - function(ds) { - ID <- get_id(ds) - - if (which == 'by_RT') - data <- get_RT_sample(ds, target, output = 'long')$RT - else if (which == 'by_FV') - data <- get_FV_sample(ds, target, output = 'long')$`f(x)` - else - stop('Invalid argument for parameter `which`.') - - if (sum(!is.na(data)) < 2) - return(NULL) - - if (use.equal.bins) - breaks <- res1$breaks - else - breaks <- nclass.FD - - res <- hist(data, breaks = breaks, plot = F) - breaks <- res$breaks - - plot_text <- paste0( - 'count: ', res$counts, '
breaks: [', - breaks[-length(breaks)], ',', breaks[-1], ']' - ) - - plot_data <- data.frame( - x = res$mids, - y = res$counts, - ID = ID, - width = breaks[2] - breaks[1], - text = plot_text - ) - } - ) %>% { - `[`(., !vapply(., is.null, logical(1))) - } - ) - ) - dt -} - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param scale_log Wheterh to use logarithmic scaling or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param use_full_range Whether or not to use the full range of the x-axis or cut it off as soon as -#' all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension -#' -#' @export -#' @examples -#' generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) -generate_data.ECDF <- function(dsList, targets, scale_log = F, which = 'by_RT', use_full_range = TRUE) { - V1 <- NULL #Set local binding to remove warnings - by_rt <- which == 'by_RT' - if (by_rt) { - RT <- get_runtimes(dsList) - if (!use_full_range) { - if (length(unique(get_funcId(dsList))) > 1 || length(unique(get_dim(dsList))) > 1) { - warning("The limiting of x-values is only supported in the case of 1 function 1 dimension!") - } - else { - maxRT <- as.integer(max(get_RT_summary(dsList, targets)[,'98%']) * 1.1) #Slight buffer for nicer plotting - RT <- c(min(RT), maxRT) - } - } - x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) - - #TODO: Some scaling by dimension? - - if (!is.data.table(targets)) { - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) - stop("Targets provided are not in data.table format, while multiple functions / dimensions - are present in provided DataSetList.") - targets <- data.table( - target = targets, - funcId = get_funcId(dsList), - DIM = get_dim(dsList) - ) - } - } - else { - FV <- get_funvals(dsList) - x <- unique(seq_FV(FV, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) - } - - dt <- as.data.table(rbindlist(lapply(dsList, function(df) { - ID <- get_id(df) - if (by_rt) { - temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] - targets_ <- temp[funcId == attr(df, 'funcId')][['target']] - } - else - targets_ <- targets - m <- lapply(targets_, function(target) { - if (by_rt) - data <- get_RT_sample(df, target, output = 'long')$RT - else - data <- get_FV_sample(df, target, output = 'long')$`f(x)` - - if (all(is.na(data))) - return(rep(0, length(x))) - fun <- ecdf(data) - if (is.function(fun)) fun(x) else NA - }) %>% - do.call(rbind, .) - - data.frame(x = x, - mean = apply(m, 2, . %>% mean(na.rm = T)), - sd = apply(m, 2, . %>% sd(na.rm = T))) %>% - mutate(upper = mean + sd, lower = mean - sd, ID = ID) - }))) - dt[, mean(mean), by = .(x, ID)][, .(mean = V1, ID = ID, x = x)] -} - - -#' Generate dataframe containing the AUC for any ECDF-curves -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param scale_log Whether to use logarithmic scaling or not -#' @param dt_ecdf A data table of the ECDF to avoid needless recomputations. Will take preference if it -#' is provided together with dsList and targets -#' @param multiple_x Boolean, whether to get only the total AUC or get stepwise AUC values -#' -#' @export -#' @examples -#' generate_data.AUC(dsl, get_ECDF_targets(dsl)) -#' generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) -generate_data.AUC <- function(dsList, targets, scale_log = F, which = 'by_RT', dt_ecdf = NULL, - multiple_x = FALSE) { - idx <- auc_contrib <- mean_pre <- mean_post <- x <- x_pre <- auc <- NULL - if (is.null(dt_ecdf)) { - if (length(dsList) == 0 || is.null(targets)) - return(NULL) - dt_ecdf <- generate_data.ECDF(dsList, targets, scale_log, which) - } - max_idx <- nrow(unique(dt_ecdf[,'x'])) - dt_ecdf[, idx := seq(max_idx), by = 'ID'] - dt3 = copy(dt_ecdf) - dt3[, idx := idx - 1] - dt_merged = merge(dt_ecdf, dt3, by = c('ID', 'idx')) - colnames(dt_merged) <- c("ID", "idx", "mean_pre", "x_pre", "mean_post", "x") - dt_merged[, auc_contrib := ((mean_pre + mean_post)/2)*(x - x_pre)] - dt_merged[, auc := cumsum(auc_contrib)/x, by = 'ID'] - #TODO: just for max x - if (multiple_x) - return(dt_merged[, c('ID','x','auc') ]) - return(dt_merged[idx == (max_idx - 1), c('ID','x','auc') ]) -} - - -# #' Generate dataframe of a single function/dimension pair -# #' -# #' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -# #' -# #' @param dsList The DataSetList object -# #' @param targets A list of the target value for which to calculate the AUC (Runtime or target value) -# #' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -# #' -# #' @export -# #' @examples -# #' generate_data.AUC(subset(dsl, funcId == 1), c(12, 16)) -# generate_data.AUC <- function(dsList, targets, which = 'by_RT') { -# if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { -# stop("This function is only available a single function/dimension pair at a time.") -# } -# by_rt <- which == 'by_RT' -# -# if (by_rt) -# RT.max <- sapply(dsList, function(ds) max(attr(ds, 'maxRT'))) %>% max -# else { -# funevals.max <- sapply(dsList, function(ds) max(attr(ds, 'finalFV'))) %>% max -# funevals.min <- sapply(dsList, function(ds) min(attr(ds, 'finalFV'))) %>% min -# } -# -# as.data.table(rbindlist(lapply(dsList, function(df) { -# algId <- attr(df, 'algId') -# if (by_rt) -# auc <- sapply(targets, function(fv) { -# ECDF(df, fv) %>% AUC(from = 1, to = RT.max) -# }) -# else { -# funs <- lapply(targets, function(r) { -# get_FV_sample(df, r, output = 'long')$'f(x)' %>% { -# if (all(is.na(.))) NULL -# else { -# f <- ecdf(.) -# attr(f, 'min') <- min(.) -# attr(f, 'max') <- max(.) -# f -# } -# } -# }) -# -# auc <- sapply(funs, -# function(fun) { -# if (is.null(fun)) 0 -# else{ -# if (attr(df, 'maximization')) -# integrate(fun, lower = attr(fun, 'min') - 1, upper = funevals.max, -# subdivisions = 1e3) %>% {'$'(., 'value') / funevals.max} -# else -# integrate(fun, lower = funevals.min, upper = attr(fun, 'max') + 1, -# subdivisions = 1e3) %>% {'$'(., 'value') / (attr(fun, 'max') + 1)} -# } -# }) -# } -# data.frame(x = targets, AUC = auc, algId = algId) -# }))) -# } - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param scale_log Wheterh to use logarithmic scaling or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.Parameters(subset(dsl, funcId == 1)) -generate_data.Parameters <- function(dsList, which = 'by_RT', scale_log = F) { - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { - stop("This function is only available a single function/dimension pair at a time.") - } - if (which == 'by_RT') { - rtall <- get_runtimes(dsList) - - rtseq <- seq_RT(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) - req(rtseq) - - dt <- get_PAR_summary(dsList, rtseq, which = 'by_RT') - } - else if (which == 'by_FV') { - rtall <- get_funvals(dsList) - - rtseq <- seq_FV(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) - req(rtseq) - - dt <- get_PAR_summary(dsList, rtseq, which = 'by_FV') - } - else stop("Invalid value for parameter `which`") - dt[, `:=`(upper = mean + sd, lower = mean - sd)] -} - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param aggr_on Which attribute to use for aggregation. Either 'funcId' or 'DIM' -#' @param targets Optional list of target values (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.Aggr(dsl) -generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which = 'by_RT') { - maximize <- attr(dsList, 'maximization') - variable <- fid <- value <- NULL #Set local binding to remove warnings - by_rt <- which == 'by_RT' - - if (is.null(targets)) { - targets <- get_target_dt(dsList, which) - } - - aggr_attr <- if (aggr_on == 'funcId') get_funcId(dsList) else get_dim(dsList) - N <- length(get_id(dsList)) - - dt <- rbindlist(lapply(aggr_attr, function(agg_val) { - if (by_rt) { - if (aggr_on == 'funcId') - dt <- get_RT_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) - else - dt <- get_RT_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) - dt[, c('ID', value = 'ERT', 'funcId', 'DIM')] - setnames(dt, 'ERT', 'value') - } - else{ - if (aggr_on == 'funcId') - dt <- get_FV_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) - else - dt <- get_FV_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) - dt[, c('ID', value = 'mean', 'funcId', 'DIM')] - setnames(dt, 'mean', 'value') - } - })) - - if (by_rt) order_sel <- 1 - else order_sel <- -1*(maximize*2 - 1) - - dt[, rank := frank(order_sel*value, na.last = T), by = .(DIM, funcId)] - return(dt) -} - - - -#' Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of -#' -#' This provides an unaggregated version of the function `generate_data.ECDF`. -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param scale_log Wheterh to use logarithmic scaling or not -#' -#' @export -#' @examples -#' generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) -generate_data.ECDF_raw <- function(dsList, targets, scale_log = F) { - V1 <- NULL #Set local binding to remove warnings - - #Get the x-coordinates at which to calculate the ECDF - RT <- get_runtimes(dsList) - x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) - #TODO: Some scaling by dimension? - - #Get targets to use - if (!is.data.table(targets)) { - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) - stop("Targets provided are not in data.table format, while multiple functions / dimensions - are present in provided DataSetList.") - targets <- data.table( - target = targets, - funcId = get_funcId(dsList), - DIM = get_dim(dsList) - ) - } - - dt <- as.data.table(rbindlist(lapply(dsList, function(df) { - ID <- get_id(df) - temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] - targets_ <- temp[funcId == attr(df, 'funcId')][['target']] - - m <- lapply(targets_, function(target) { - data <- fast_RT_samples(df$RT, target, attr(df, 'maximization')) - if (all(is.na(data))) - hit <- (rep(0, length(x))) - else { - fun <- ecdf(data) - hit <- if (is.function(fun)) fun(x) else NA - } - data.table(hit = hit, rt = x, target = target, funcId = attr(df, 'funcId'), DIM = attr(df, 'DIM'), ID = ID) - }) %>% - do.call(rbind, .) - m - }))) - dt +#' Read all the data file in a folfer +# +#' @param path Path to the data files. Will look for all .info-files in this directory and use +#' the corresponding datafiles to create the DataSetList +#' @param verbose Logical. +#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? +#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" +#' @param subsampling Logical. Whether *.cdat files are subsampled? +#' @param print_fun Function used to print output when in verbose mode +#' @noRd +#' @return A DataSetList object +read_dir <- function(path, verbose = T, print_fun = NULL, maximization = TRUE, + format = IOHprofiler, subsampling = FALSE) { + DataSetList( + path, + verbose, + print_fun, + maximization = maximization, + format = format, + subsampling = subsampling + ) + } + +# TODO: improve documentation of this S3 class +#' S3 constructor of the 'DataSetList' +#' +#' Attributes +#' funId +#' DIM +#' algId +# +#' @param path Path to the data files. Will look for all .info-files in this directory and use +#' the corresponding datafiles to create the DataSetList +#' @param verbose Logical. +#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? +#' @param format A character. The format of data source, options are: +#' \itemize{ +#' \item'IOHProfiler' +#' \item'COCO' +#' \item'TWO_COL' +#' \item'COCO_BIOBJ' +#' \item'NEVERGRAD' +#' \item'SOS' +#' } +#' These formats are specified in more detail in our github wiki. +#' @param subsampling Logical. Whether *.cdat files are subsampled? +#' @param print_fun Function used to print output when in verbose mode +#' @param full_aggregation If True, individual DataSets are aggregated as much as possible: all DataSets +#' with the same algorithmname, function id and dimension are combined together. This leads to information loss +#' related to static variables, so only use if that information is not required. +#' +#' @return A DataSetList object +#' @export +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +#' DataSetList(path) +DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization = NULL, + format = IOHprofiler, subsampling = FALSE, full_aggregation = TRUE) { + if (is.null(path)) + return(structure(list(), class = c('DataSetList', 'list'))) + + path <- trimws(path) + if (format == NEVERGRAD) { + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") + return(read_nevergrad(path)) + else + indexFiles <- + file.path(path, list.files(path, pattern = '.csv', recursive = T)) + } + else if (format == SOS) { + return(read_datasetlist_SOS(path, locate_corrections_files(path))) + } + else + indexFiles <- scan_index_file(path) + + if (is.null(print_fun)) + print_fun <- cat + + object <- list() + class(object) <- c('DataSetList', class(object)) + DIM <- c() + algId <- c() + funcId <- c() + suites <- c() + maximizations <- c() + i <- 1 + + for (file in indexFiles) { + if (verbose) { + print_fun(paste('Processing', file, '...\n')) + } + + if (format == NEVERGRAD) { + dsl <- read_nevergrad(file) + object %<>% c(., dsl) + suites[i] <- NEVERGRAD + } + else { + indexInfo <- read_index_file(file) + if (verbose) + print_fun(sprintf(' algorithm %s...\n', indexInfo[[1]]$algId)) + + for (info in indexInfo) { + if (verbose) { + print_fun( + sprintf( + ' %d instances on f%s %dD...\n', + length(info$instance), + info$funcId, + info$DIM + ) + ) + } + + copy_flag <- TRUE + data <- DataSet(info, maximization = maximization, + format = format, subsampling = subsampling) + + DIM[i] <- attr(data, 'DIM') + funcId[i] <- attr(data, 'funcId') + algId[i] <- attr(data, 'algId') + + # TODO: double-check the following treatment on `instance`!!! + instance <- attr(data, 'instance') #Was instance without index? + suites[i] <- attr(data, 'suite') + maximizations[i] <- attr(data, 'maximization') + + if (length(object) != 0) { + idx <- which(sapply(object, function(obj) obj == data)) + for (k in idx) { + instance_ <- attr(object[[k]], 'instance') + if (all(instance == instance_)) { + copy_flag <- FALSE + warning('duplicated instances!') + break + } + + if (length(intersect(instance, instance_)) != 0) { + warning('duplicated instances!') + } + } + } + + if (copy_flag) { + object[[i]] <- data + i <- i + 1 + } + } + } + + if (verbose) + print_fun("\n") + } + + # TODO: sort all DataSet by multiple attributes: algId, funcId and DIM + if (format != NEVERGRAD) { + attr(object, 'DIM') <- DIM + attr(object, 'funcId') <- funcId + attr(object, 'algId') <- algId + } + + suite <- unique(suites) + maximization <- unique(maximizations) + if (length(maximization) != 1) { + warning("Multipe different optimization types detected!") + } + + attr(object, 'suite') <- suite + attr(object, 'maximization') <- maximization + attr(object, 'ID') <- attr(object, 'algId') + attr(object, 'ID_attributes') <- c('algId') + if (full_aggregation) + clean_DataSetList(object) + else + object +} + + +#' Clean DataSetList object by concatenating DataSets +#' +#' Concatenates all DataSets with the same ID, function id and dimension +#' +#' @param dsList The DataSetList object to clean +#' @export +#' @examples +#' clean_DataSetList(dsl) +clean_DataSetList <- function(dsList) { + #To ensure no uninitialized variables are present + .I <- NULL + cases <- mapply( + function(...) paste0(list(...), collapse = ','), + attr(dsList, 'funcId'), + attr(dsList, 'DIM'), + attr(dsList, 'algId'), + attr(dsList, 'ID'), + SIMPLIFY = T, + USE.NAMES = F + ) + + dt <- as.data.table(cases)[, list(list(.I)), by = cases] + idx_to_del <- c() + + for (idx in dt$V1) { + if (length(idx) > 1) { + dsList[[idx[1]]] <- c.DataSet(dsList[idx]) + idx_to_del <- c(idx_to_del, idx[-1]) + } + } + dsList[idx_to_del] <- NULL + + if (length(idx_to_del) > 0) { + attr(dsList, 'DIM') <- sapply(dsList, function(ds) attr(ds, 'DIM')) + attr(dsList, 'funcId') <- sapply(dsList, function(ds) attr(ds, 'funcId')) + attr(dsList, 'algId') <- sapply(dsList, function(ds) attr(ds, 'algId')) + attr(dsList, 'ID') <- sapply(dsList, function(ds) attr(ds, 'ID')) + } + dsList +} + +#' S3 concatenation function for DataSetList +#' +#' @param ... The DataSetLists to concatenate +#' @return A new DataSetList +#' @export +#' @examples +#' c(dsl[1], dsl[3]) +c.DataSetList <- function(...) { + # TODO: maybe remove duplicated dataset in the further + # remove the empty list first + dsl <- list(...) + dsl <- dsl[sapply(dsl, length) != 0] + + if (length(dsl) == 0) + return() + + object <- unlist(dsl, recursive = F) + if (!any((class(object)) == 'DataSetList')) + class(object) <- c('DataSetList', class(object)) + + for (attr_str in c('DIM', 'funcId', 'algId', 'ID')) { + attr(object, attr_str) <- unlist(lapply(dsl, function(x) attr(x, attr_str))) + } + + # Deal with Suites on the datasetlist level + attr(object, "suite") <- unique( + unlist( + lapply(dsl, function(x) attr(x, "suite")) + ) + ) + + # These attributes NEED to be the same across the datasetlist + for (attr_str in c('maximization', 'ID_attributes')) { + temp <- unique( + unlist(lapply(dsl, function(x) attr(x, attr_str))) + ) + + if (length(temp) > 1) { + stop(paste0("Attempted to add datasetlists with different ", attr_str, + "-attributes! This will lead to errors when processing + this data!")) + } + attr(object, attr_str) <- temp[[1]] + } + object +} + +#' S3 extraction function for DataSetList +#' +#' @param x The DataSetList to use +#' @param i The indices to extract +#' @param drop Currently unused parameter +#' @return The DataSetList of the DataSets at indices i of DataSetList x +#' @export +#' @examples +#' dsl[c(1, 3)] +`[.DataSetList` <- function(x, i, drop = FALSE) { + # remove the attributes firstly + obj <- unclass(x)[i] + class(obj) <- c('DataSetList', class(obj)) + + # also slice the attributes accordingly + attr(obj, 'DIM') <- attr(x, 'DIM')[i] + attr(obj, 'ID') <- attr(x, 'ID')[i] + attr(obj, 'funcId') <- attr(x, 'funcId')[i] + attr(obj, 'algId') <- attr(x, 'algId')[i] + attr(obj, 'suite') <- attr(x, 'suite') + attr(obj, 'maximization') <- attr(x, 'maximization') + + obj +} + +#' S3 print function for DataSetList +#' +#' @param x The DataSetList to print +#' @param ... Arguments for underlying print function? +#' @export +#' @examples +#' print(dsl) +print.DataSetList <- function(x, ...) { + cat('DataSetList:\n') + cat(paste0('Suite: ', attr(x, 'suite'), '\n')) + N <- length(x) + + # TODO: add an option for the following numbers: 15 and 5 + if (N <= 15) + idx <- seq_along(x) + else + idx <- c(1:5, '---', (N-4):N) + idx <- format(idx, justify = 'right') + + for (i in idx) { + if (trimws(i) == '---') + cat(paste0(i), '\n') + else { + .i <- as.integer(trimws(i)) + cat(sprintf('%s: %s\n', i, as.character(x[[.i]]))) + } + } +} + +# TODO: consistent use of ds, data, dsList etc. +# TODO: make decision on `DIM` or `dim`? funcId or fId or fID? algID / algid / algId? + +#' S3 summary function for DataSetList +#' +#' Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every +#' DataSet in the DataSetList +#' @param object The DataSetList to print +#' @param ... Arguments for underlying summary function? +#' @export +#' @examples +#' summary(dsl) +summary.DataSetList <- function(object, ...) { + as.data.frame( + t( + sapply( + object, + function(d) { + list( + suite = attr(d, 'suite'), + funcId = attr(d, 'funcId'), + DIM = attr(d, 'DIM'), + ID = attr(d, 'ID'), + datafile = attr(d, 'datafile'), + comment = attr(d, 'comment') + ) + } + ) + ) + ) +} + +#' S3 sort function for DataSetList +#' +#' Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). +#' Default is as ascending, can be made descending by adding a - in front of the attribute. +#' Sorting accross multiple attributes is supported, in the order they are specified. +#' +#' @param dsl The DataSetList to sort +#' @param ... attribute by which `dsl` is sorted. Multiple attributes can be specified. +#' @export +#' @examples +#' arrange(dsl, DIM, -funcId, algId) +arrange <- function(dsl, ...) UseMethod('arrange', dsl) + +#' @rdname arrange +#' @export +#' +arrange.DataSetList <- function(dsl, ...) { + cols <- substitute(list(...))[-1L] + if (identical(as.character(cols), "NULL")) + return(dsl) + + cols <- as.list(cols) + order <- as.list(rep(1L, length(cols))) + + for (i in seq_along(cols)) { + v <- as.list(cols[[i]]) + if (length(v) > 1L) { + if (v[[1L]] == '-') order[[i]] <- -1L + v <- v[[-1L]] + } + + v <- as.character(v) + if (v %in% c('DIM', 'funcId', 'ID')) + cols[[i]] <- v + else { + cols[[i]] <- NULL + order[[i]] <- NULL + } + } + + cols <- unlist(cols, use.names = F) + order <- unlist(order) + x <- c(list(1:length(dsl)), lapply(cols, function(col) attr(dsl, col))) + names(x) <- c('index', cols) + + DT <- rbindlist(list(x)) + data.table::setorderv(DT, cols, order) + idx <- DT[[1]] + dsl <- dsl[idx] + + # TODO: perhaps we do not need those attributes at all... + for (v in c('DIM', 'funcId', 'ID')) + attr(dsl, v) <- sapply(dsl, function(d) attr(d, v)) + dsl +} + +#' @rdname get_ERT +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' +get_ERT.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), attr(ds, 'funcId'), get_ERT(ds, ftarget, budget)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + })) + } + +#' @rdname get_RT_summary +#' @export +get_RT_summary.DataSetList <- function(ds, ftarget, budget = NULL, ...) { + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_RT_summary(ds, ftarget, budget)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + })) + } + +#' @rdname get_RT_sample +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' +#' @export +get_RT_sample.DataSetList <- function(ds, ftarget, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_RT_sample(ds, ftarget, ...)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + }), + fill = T) + } + +#' @rdname get_maxRT +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' +#' @export +get_maxRT.DataSetList <- function(ds, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds_) { + res <- + cbind(attr(ds_, 'DIM'), attr(ds_, 'funcId'), get_maxRT(ds_, ...)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + }), + fill = T) +} + +#' @rdname get_FV_summary +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' +get_FV_summary.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_FV_summary(ds, runtime)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + })) +} + +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' @rdname get_FV_overview +#' +get_FV_overview.DataSetList <- function(ds, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) + get_FV_overview(ds))) + +} + +#' @rdname get_RT_overview +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +get_RT_overview.DataSetList <- function(ds, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) + get_RT_overview(ds))) +} + +#' @rdname get_overview +#' @export +get_overview.DataSetList <- function(ds, ...) { + df <- rbindlist(lapply(ds, function(ds) + get_overview(ds))) + if (length(get_funcId(ds)) > 1 || length(get_dim(ds)) > 1) { + p2 <- df[,lapply(.SD, mean, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('mean reached')] + if (attr(ds, 'maximization')) { + p1 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'best reached')] + p3 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('worst recorded', 'worst reached')] + } + else { + p1 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('best reached')] + p3 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'worst recorded', 'worst reached')] + } + return(merge(merge(p1,p2),p3)) + } + else { + return(df) + } +} + +#' @rdname get_FV_sample +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' +get_FV_sample.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_FV_sample(ds, runtime, ...)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + }), + fill = T) + } + +#' @rdname get_PAR_summary +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +get_PAR_summary.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) get_PAR_summary(ds, idxValue, ...))) +} + +#' @rdname get_PAR_sample +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +get_PAR_sample.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) get_PAR_sample(ds, idxValue, ...)), fill = T) +} + +#' @rdname get_id +#' @export +get_id.DataSetList <- function(ds, ...) { + temp <- attr(ds, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'change_id' function.") + return(get_algId(ds)) + } + return(unique(temp)) +} + +#' Get all dimensions present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A sorted list of all unique dimensions which occur in the DataSetList +#' @export +#' @examples +#' get_dim(dsl) +get_dim <- function(dsList) { + sort(unique(sapply(dsList, function(d) attr(d, 'DIM')))) +} + +#' Get all function ids present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A sorted list of all unique function ids which occur in the DataSetList +#' @export +#' @examples +#' get_funcId(dsl) +get_funcId <- function(dsList) { + ll <- unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcId'))))) + + # TODO: what if the function ID is a double value? + # those are coerced to integers now + if (is.integer(ll)) return(sort(ll)) + + lli <- suppressWarnings(as.integer(ll)) + if (any(is.na(lli))) return(sort(ll)) + + if (all((lli >= 0L) & (lli <= 1000000000L))) return(ll[order(lli)]) + + # TODO: should this be even allowed? + sort(lli) +} + +#' Get all function names present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list of all unique function names which occur in the DataSetList +#' @export +#' @examples +#' get_funcName(dsl) +get_funcName <- function(dsList) { + unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcName'))))) +} + +#' Get all algorithm ids present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A sorted list of all unique algorithm ids which occur in the DataSetList +#' @export +#' @examples +#' get_algId(dsl) +get_algId <- function(dsList) { + unique(sapply(dsList, function(d) attr(d, 'algId'))) +} + +#' Get all parameter ids present in a DataSetList +#' +#' @param dsList The DataSetList +#' @param which A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned +#' by the running time (RT) or the function value (FV). Note that parameters in each case are +#' not necessary the same. +#' +#' @return A sorted list of all unique parameter ids which occur in the DataSetList +#' @export +#' @examples +#' get_parId(dsl) +get_parId <- function(dsList, which = 'by_FV') { + unique( + unlist( + lapply(dsList, + function(d) { + if (which == 'by_FV') + names(d$PAR$by_FV) + else if (which == 'by_RT') + names(d$PAR$by_RT) + } + ) + ) + ) +} + +# TODO: let the user choose/detect whether the problem is subject to maximization +# and determine whether to sort the function values in ascending/desceding order + +#' Get all function values present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list matrices of all function values which occur in the DataSetList +#' @export +#' @examples +#' get_funvals(dsl) +get_funvals <- function(dsList) { + if (length(dsList) == 0) + return(NULL) + if (length(dsList[[1]]$RT) == 0) { + x <- sort(unique(as.numeric(unlist( + lapply(dsList, function(x) + as.vector(x$FV)) + )))) + } + else + x <- (sort(unique(as.numeric(unlist( + lapply(dsList, function(x) + rownames(x$RT)) + ))))) + x[!is.na(x) & !is.infinite(x)] +} + +#' Get all runtime values present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list matrices of all runtime values which occur in the DataSetList +#' @export +#' @examples +#' get_runtimes(dsl) +get_runtimes <- function(dsList) { + sort(unique(as.numeric(unlist( + lapply(dsList, function(x) + rownames(x$FV)) + )))) +} + +#' Get all attributes which can be used to subset a DataSetList +#' +#' @param dsl The DataSetList +#' +#' @return The list of available attributes +#' @export +#' @examples +#' get_static_attributes(dsl) +get_static_attributes <- function(dsl) { + full_names <- unique(unlist(lapply(dsl, function(ds) {names(attributes(ds))}))) + + reserved_attributes <- c("names", "class", "suite", "maximization", "algInfo", "comment", + "datafile", "maxRT", "finalFV", "format") + setdiff(full_names, reserved_attributes) +} + +#' Get all options for a specific attribute which can be used to subset a DataSetList +#' +#' This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. +#' Note the only attributes returned by `get_static_attributes` are supported in this funcion +#' +#' @param dsl The DataSetList +#' @param attribute the name of the attribute for which to get the available options in dsl +#' @return The list of options for the specified attribute +#' @export +#' @examples +#' get_static_attribute_values(dsl, 'funcId') +get_static_attribute_values <- function(dsl, attribute) { + unique(unlist(lapply(dsl, function(ds) {attr(ds, attribute)}))) +} + +#' Filter a DataSetList by some criteria +#' +#' @param x The DataSetList +#' @param ... The conditions to filter on. Can be any expression which assigns True or False +#' to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes +#' (funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should +#' be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a +#' string to be parsed. This allows exectution of subset commands on arbitrary variables in code. +#' +#' @return The filtered DataSetList +#' @export +#' @examples +#' subset(dsl, funcId == 1) +#' subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes +#' subset(dsl, instance == 1) +#' subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes +#' subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND +subset.DataSetList <- function(x, ...) { + enclos <- parent.frame() + if (hasArg('text')) { + text <- list(...)$text + condition_call <- parse(text = text) + } else { + condition_call <- substitute(list(...)) + condition_call <- condition_call[2:length(condition_call)] + } + + obj <- lapply(x, + function(ds){ + mask <- tryCatch(expr = { + mask <- NULL + for (idx in seq(length(condition_call))) { + mask_temp <- unlist( + eval(condition_call[[idx]], attributes(ds), enclos = enclos) + ) + if (is.null(mask)) mask <- mask_temp + else { + if (length(mask_temp) == 1 && !mask_temp) { + mask <- F + } else if (length(mask_temp) == 1) { + mask <- mask + } else if (length(mask_temp) == length(mask) || length(mask) == 1) { + mask <- mask & mask_temp + } else { + stop("Error creating mask") + } + + } + } + mask + }, error = function(e) {F}) + + if (length(mask) == 1 && mask) return(ds) + else if (length(mask) == 1 || !any(mask)) return(NULL) + return(subset(ds, mask)) + }) + + class(obj) <- c('DataSetList', class(obj)) + obj <- Filter(Negate(is.null), obj) + + # also slice the attributes accordingly + attr(obj, 'suite') <- attr(x, 'suite') + attr(obj, 'maximization') <- attr(x, 'maximization') + attr(obj, 'DIM') <- sapply(obj, function(ds) attr(ds, 'DIM')) + attr(obj, 'funcId') <- sapply(obj, function(ds) attr(ds, 'funcId')) + attr(obj, 'algId') <- sapply(obj, function(ds) attr(ds, 'algId')) + unique_ids <- unlist(sapply(obj, function(ds) attr(ds, 'unique_id'))) + if (!any(is.null(unique_ids))) { + attr(obj, 'unique_ids') <- unique_ids + } + return(obj) +} + +#' Add unique identifiers to each DataSet in the provided DataSetList based on static attributes +#' +#' Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to +#' ensure each dataset has exactly one unique identifier. +#' Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. +#' +#' @param dsl The DataSetList +#' @param attrs The list of attributes to combine into a unique identifier +#' @return A new DataSetList object where the split has been done based on the provided attributes, and the unique +#' identifier has been added. +#' @export +#' @examples +#' change_id(dsl, c('instance')) +change_id <- function(dsl, attrs) { + if (length(dsl) == 0) return(dsl) + if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") + grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(dsl, x)})) + colnames(grid) <- attrs + + dsl_new <- DataSetList() + attr_vals <- c() + for (x in transpose(grid)) { + #TODO: on Windows, UTF-8 Characters get converted into characters, which then + #don't interact correctly with the parse used in 'subset' function, leading to empty datasetlist objects. + #I'm not aware of any way to fix this, so UFT-8 characters should be avoided in ID for now. + conditions <- paste0(unlist(lapply(seq(length(attrs)), function(idx) { + paste0(attrs[[idx]], ' == "', x[[idx]], '"') + })), collapse = " & ") + dsl_temp <- subset(dsl, text = conditions) + if (length(attrs) == 1) + attr_val <- x + else + attr_val <- paste0(x, collapse = "_") + + attr_vals <- c(attr_vals, rep(attr_val, length(dsl_temp))) + dsl_new <- c(dsl_new, dsl_temp) + } + attr(dsl_new, 'ID_attributes') <- attrs + attr(dsl_new, 'ID') <- attr_vals + for (idx in seq(length(dsl_new))) { + attr(dsl_new[[idx]], 'ID') <- attr_vals[[idx]] + } + class(dsl_new) <- c("DataSetList", "list") + attr(dsl_new, 'suite') <- attr(dsl, 'suite') + attr(dsl_new, 'maximization') <- attr(dsl, 'maximization') + + attr(dsl_new, 'DIM') <- lapply(dsl_new, function(x) attr(x, 'DIM')) + attr(dsl_new, 'funcId') <- lapply(dsl_new, function(x) attr(x, 'funcId')) + attr(dsl_new, 'algId') <- lapply(dsl_new, function(x) attr(x, 'algId')) + dsl_new +} + + + + +#' Save DataTable in multiple formats +#' +#' @param df The DataTable to store +#' @param file String. The name of the figure file, with the extension of the required file-format +#' @param format Optional, string. Overwrites the extension of the `file` parameter. If not specified while +#' file does not have an extension, it defaults to csv +#' +#' @export +#' @examples +#' df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +#' save_table(df, tempfile(fileext = ".md")) +save_table <- function(df, file, format = NULL) { + if (is.null(format)) { + format <- tools::file_ext(file) + } + if (format == 'TeX' || format == 'tex') { + if (requireNamespace('xtable', quietly = T)) + print(xtable::xtable(df), file = file) + else + write(kable(df, format = 'latex'), file) + } else if (format == 'Markdown' || format == 'md') { + write(kable(df, format = 'markdown'), file) + } else if (format == 'html') { + write(kable(df, format = "html"), file) + } + else { #Default to csv + write.csv(df, file, row.names = F) + } +} + +#' Generation of default ECDF-targets +#' +#' @param dsList The DataSetList object for which to generate the targets +#' @param type The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, +#' equal for all functions / dimensions) +#' @param number_targets The amount of targets to generate +#' +#' @return A data.table with 3 columns: funcId, DIM and target +#' @export +#' @examples +#' get_ECDF_targets(dsl, 'linear', 10) +get_ECDF_targets <- function(dsList, type = "log-linear", number_targets = 10) { + funcIds <- get_funcId(dsList) + dims <- get_dim(dsList) + + dt <- rbindlist(apply(expand.grid(funcIds, dims), 1, function(x) { + if (type == 'bbob') { + fseq <- rev(seq_FV(c(100,1e-8), length.out = 51, scale = 'log')) + } + else { + dsl <- subset(dsList, funcId == x[[1]] && DIM == x[[2]]) + if (length(dsl) == 0) + return(NULL) + fall <- get_funvals(dsl) + if (length(fall) < 2) + return(NULL) + + fseq <- seq_FV(fall, length.out = number_targets, scale = ifelse(type == "log-linear", 'log', 'linear')) + } + data.table(funcId = x[[1]], DIM = x[[2]], target = fseq) + })) + dt +} + +### __________________________ Rewritten data generation functions _______________________ ### +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param start Optional start value (Runtime or target value) +#' @param stop Optional end value (Runtime or target value) +#' @param scale_log Wheterh to use logarithmic scaling or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param include_opts Whether or not to also include the best value hit by each algorithm to +#' the generated datapoints +#' @param budget Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works +#' in fixed_target mode. +#' +#' @export +#' @examples +#' generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, + scale_log = F, which = 'by_RT', include_opts = F, budget = NULL) { + + if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1 ) { + #Required because target generation is included in this function, + #which needs to be done on a per-function basis + stop("Multiple functions / dimensions are present in provided DataSetList. + Please call this function for each individual function/dimension pair instead.") + } + + by_rt <- (which == 'by_RT') + + if (by_rt) + all <- get_funvals(dsList) + else + all <- get_runtimes(dsList) + + maximization <- attr(dsList, 'maximization') + + if (is.null(maximization)) maximization <- T + if (is.null(start)) start <- min(all) + if (is.null(stop)) stop <- max(all) + + if (by_rt) { + Xseq <- seq_FV(all, start, stop, length.out = 60, + scale = ifelse(scale_log, 'log', 'linear')) + if (include_opts) { + for (uid in get_id(dsList)) { + if (maximization) + Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) + else + Xseq <- c(Xseq, min(get_funvals(subset(dsList, ID == uid)))) + } + Xseq <- unique(sort(Xseq)) + } + dt <- get_RT_summary(dsList, ftarget = Xseq, budget = budget) + } + else { + Xseq <- seq_RT(all, start, stop, length.out = 60, + scale = ifelse(scale_log, 'log', 'linear')) + if (include_opts) { + for (uid in get_id(dsList)) { + Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) + } + Xseq <- unique(sort(Xseq)) + } + dt <- get_FV_summary(dsList, Xseq) + } + + + dt[, `:=`(upper = mean + sd, lower = mean - sd)] + return(dt) +} + +#' Generate dataframe of a single function/dimension pair for creating PDF or PMF plots +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param target The target value (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +generate_data.PMF <- function(dsList, target, which = 'by_RT') { + if (which == 'by_RT') + return(get_RT_sample(dsList, target, output = 'long')) + return(get_FV_sample(dsList, target, output = 'long')) +} + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param target The target value (Runtime or target value) +#' @param use.equal.bins Whether all bins should be equal size for each algorithm or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +generate_data.hist <- function(dsList, target, use.equal.bins = F, which = 'by_RT') { + width <- NULL # Set local binding to remove warnings + + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) + stop('This function is only available a single function/dimension pair at a time.') + + if (use.equal.bins) { + if (which == 'by_RT') + res1 <- hist( + get_RT_sample(dsList, target, output = 'long')$RT, + breaks = nclass.FD, + plot = F + ) + else + res1 <- hist( + get_FV_sample(dsList, target, output = 'long')$`f(x)`, + breaks = nclass.FD, + plot = F + ) + } + + dt <- as.data.table( + rbindlist( + lapply( + dsList, + function(ds) { + ID <- get_id(ds) + + if (which == 'by_RT') + data <- get_RT_sample(ds, target, output = 'long')$RT + else if (which == 'by_FV') + data <- get_FV_sample(ds, target, output = 'long')$`f(x)` + else + stop('Invalid argument for parameter `which`.') + + if (sum(!is.na(data)) < 2) + return(NULL) + + if (use.equal.bins) + breaks <- res1$breaks + else + breaks <- nclass.FD + + res <- hist(data, breaks = breaks, plot = F) + breaks <- res$breaks + + plot_text <- paste0( + 'count: ', res$counts, '
breaks: [', + breaks[-length(breaks)], ',', breaks[-1], ']' + ) + + plot_data <- data.frame( + x = res$mids, + y = res$counts, + ID = ID, + width = breaks[2] - breaks[1], + text = plot_text + ) + } + ) %>% { + `[`(., !vapply(., is.null, logical(1))) + } + ) + ) + dt +} + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param scale_log Wheterh to use logarithmic scaling or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param use_full_range Whether or not to use the full range of the x-axis or cut it off as soon as +#' all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension +#' +#' @export +#' @examples +#' generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) +generate_data.ECDF <- function(dsList, targets, scale_log = F, which = 'by_RT', use_full_range = TRUE) { + V1 <- NULL #Set local binding to remove warnings + by_rt <- which == 'by_RT' + if (by_rt) { + RT <- get_runtimes(dsList) + if (!use_full_range) { + if (length(unique(get_funcId(dsList))) > 1 || length(unique(get_dim(dsList))) > 1) { + warning("The limiting of x-values is only supported in the case of 1 function 1 dimension!") + } + else { + maxRT <- as.integer(max(get_RT_summary(dsList, targets)[,'98%']) * 1.1) #Slight buffer for nicer plotting + RT <- c(min(RT), maxRT) + } + } + x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) + + #TODO: Some scaling by dimension? + + if (!is.data.table(targets)) { + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) + stop("Targets provided are not in data.table format, while multiple functions / dimensions + are present in provided DataSetList.") + targets <- data.table( + target = targets, + funcId = get_funcId(dsList), + DIM = get_dim(dsList) + ) + } + } + else { + FV <- get_funvals(dsList) + x <- unique(seq_FV(FV, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) + } + + dt <- as.data.table(rbindlist(lapply(dsList, function(df) { + ID <- get_id(df) + if (by_rt) { + temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] + targets_ <- temp[funcId == attr(df, 'funcId')][['target']] + } + else + targets_ <- targets + m <- lapply(targets_, function(target) { + if (by_rt) + data <- get_RT_sample(df, target, output = 'long')$RT + else + data <- get_FV_sample(df, target, output = 'long')$`f(x)` + + if (all(is.na(data))) + return(rep(0, length(x))) + fun <- ecdf(data) + if (is.function(fun)) fun(x) else NA + }) %>% + do.call(rbind, .) + + data.frame(x = x, + mean = apply(m, 2, . %>% mean(na.rm = T)), + sd = apply(m, 2, . %>% sd(na.rm = T))) %>% + mutate(upper = mean + sd, lower = mean - sd, ID = ID) + }))) + dt[, mean(mean), by = .(x, ID)][, .(mean = V1, ID = ID, x = x)] +} + + +#' Generate dataframe containing the AUC for any ECDF-curves +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param scale_log Whether to use logarithmic scaling or not +#' @param dt_ecdf A data table of the ECDF to avoid needless recomputations. Will take preference if it +#' is provided together with dsList and targets +#' @param multiple_x Boolean, whether to get only the total AUC or get stepwise AUC values +#' +#' @export +#' @examples +#' generate_data.AUC(dsl, get_ECDF_targets(dsl)) +#' generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) +generate_data.AUC <- function(dsList, targets, scale_log = F, which = 'by_RT', dt_ecdf = NULL, + multiple_x = FALSE) { + idx <- auc_contrib <- mean_pre <- mean_post <- x <- x_pre <- auc <- NULL + if (is.null(dt_ecdf)) { + if (length(dsList) == 0 || is.null(targets)) + return(NULL) + dt_ecdf <- generate_data.ECDF(dsList, targets, scale_log, which) + } + max_idx <- nrow(unique(dt_ecdf[,'x'])) + dt_ecdf[, idx := seq(max_idx), by = 'ID'] + dt3 = copy(dt_ecdf) + dt3[, idx := idx - 1] + dt_merged = merge(dt_ecdf, dt3, by = c('ID', 'idx')) + colnames(dt_merged) <- c("ID", "idx", "mean_pre", "x_pre", "mean_post", "x") + dt_merged[, auc_contrib := ((mean_pre + mean_post)/2)*(x - x_pre)] + dt_merged[, auc := cumsum(auc_contrib)/x, by = 'ID'] + #TODO: just for max x + if (multiple_x) + return(dt_merged[, c('ID','x','auc') ]) + return(dt_merged[idx == (max_idx - 1), c('ID','x','auc') ]) +} + + +# #' Generate dataframe of a single function/dimension pair +# #' +# #' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +# #' +# #' @param dsList The DataSetList object +# #' @param targets A list of the target value for which to calculate the AUC (Runtime or target value) +# #' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +# #' +# #' @export +# #' @examples +# #' generate_data.AUC(subset(dsl, funcId == 1), c(12, 16)) +# generate_data.AUC <- function(dsList, targets, which = 'by_RT') { +# if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { +# stop("This function is only available a single function/dimension pair at a time.") +# } +# by_rt <- which == 'by_RT' +# +# if (by_rt) +# RT.max <- sapply(dsList, function(ds) max(attr(ds, 'maxRT'))) %>% max +# else { +# funevals.max <- sapply(dsList, function(ds) max(attr(ds, 'finalFV'))) %>% max +# funevals.min <- sapply(dsList, function(ds) min(attr(ds, 'finalFV'))) %>% min +# } +# +# as.data.table(rbindlist(lapply(dsList, function(df) { +# algId <- attr(df, 'algId') +# if (by_rt) +# auc <- sapply(targets, function(fv) { +# ECDF(df, fv) %>% AUC(from = 1, to = RT.max) +# }) +# else { +# funs <- lapply(targets, function(r) { +# get_FV_sample(df, r, output = 'long')$'f(x)' %>% { +# if (all(is.na(.))) NULL +# else { +# f <- ecdf(.) +# attr(f, 'min') <- min(.) +# attr(f, 'max') <- max(.) +# f +# } +# } +# }) +# +# auc <- sapply(funs, +# function(fun) { +# if (is.null(fun)) 0 +# else{ +# if (attr(df, 'maximization')) +# integrate(fun, lower = attr(fun, 'min') - 1, upper = funevals.max, +# subdivisions = 1e3) %>% {'$'(., 'value') / funevals.max} +# else +# integrate(fun, lower = funevals.min, upper = attr(fun, 'max') + 1, +# subdivisions = 1e3) %>% {'$'(., 'value') / (attr(fun, 'max') + 1)} +# } +# }) +# } +# data.frame(x = targets, AUC = auc, algId = algId) +# }))) +# } + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param scale_log Wheterh to use logarithmic scaling or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.Parameters(subset(dsl, funcId == 1)) +generate_data.Parameters <- function(dsList, which = 'by_RT', scale_log = F) { + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { + stop("This function is only available a single function/dimension pair at a time.") + } + if (which == 'by_RT') { + rtall <- get_runtimes(dsList) + + rtseq <- seq_RT(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) + req(rtseq) + + dt <- get_PAR_summary(dsList, rtseq, which = 'by_RT') + } + else if (which == 'by_FV') { + rtall <- get_funvals(dsList) + + rtseq <- seq_FV(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) + req(rtseq) + + dt <- get_PAR_summary(dsList, rtseq, which = 'by_FV') + } + else stop("Invalid value for parameter `which`") + dt[, `:=`(upper = mean + sd, lower = mean - sd)] +} + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param aggr_on Which attribute to use for aggregation. Either 'funcId' or 'DIM' +#' @param targets Optional list of target values (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.Aggr(dsl) +generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which = 'by_RT') { + maximize <- attr(dsList, 'maximization') + variable <- fid <- value <- NULL #Set local binding to remove warnings + by_rt <- which == 'by_RT' + + if (is.null(targets)) { + targets <- get_target_dt(dsList, which) + } + + aggr_attr <- if (aggr_on == 'funcId') get_funcId(dsList) else get_dim(dsList) + N <- length(get_id(dsList)) + + dt <- rbindlist(lapply(aggr_attr, function(agg_val) { + if (by_rt) { + if (aggr_on == 'funcId') + dt <- get_RT_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) + else + dt <- get_RT_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) + dt[, c('ID', value = 'ERT', 'funcId', 'DIM')] + setnames(dt, 'ERT', 'value') + } + else{ + if (aggr_on == 'funcId') + dt <- get_FV_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) + else + dt <- get_FV_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) + dt[, c('ID', value = 'mean', 'funcId', 'DIM')] + setnames(dt, 'mean', 'value') + } + })) + + if (by_rt) order_sel <- 1 + else order_sel <- -1*(maximize*2 - 1) + + dt[, rank := frank(order_sel*value, na.last = T), by = .(DIM, funcId)] + return(dt) +} + + + +#' Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of +#' +#' This provides an unaggregated version of the function `generate_data.ECDF`. +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param scale_log Wheterh to use logarithmic scaling or not +#' +#' @export +#' @examples +#' generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) +generate_data.ECDF_raw <- function(dsList, targets, scale_log = F) { + V1 <- NULL #Set local binding to remove warnings + + #Get the x-coordinates at which to calculate the ECDF + RT <- get_runtimes(dsList) + x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) + #TODO: Some scaling by dimension? + + #Get targets to use + if (!is.data.table(targets)) { + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) + stop("Targets provided are not in data.table format, while multiple functions / dimensions + are present in provided DataSetList.") + targets <- data.table( + target = targets, + funcId = get_funcId(dsList), + DIM = get_dim(dsList) + ) + } + + dt <- as.data.table(rbindlist(lapply(dsList, function(df) { + ID <- get_id(df) + temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] + targets_ <- temp[funcId == attr(df, 'funcId')][['target']] + + m <- lapply(targets_, function(target) { + data <- fast_RT_samples(df$RT, target, attr(df, 'maximization')) + if (all(is.na(data))) + hit <- (rep(0, length(x))) + else { + fun <- ecdf(data) + hit <- if (is.function(fun)) fun(x) else NA + } + data.table(hit = hit, rt = x, target = target, funcId = attr(df, 'funcId'), DIM = attr(df, 'DIM'), ID = ID) + }) %>% + do.call(rbind, .) + m + }))) + dt } \ No newline at end of file diff --git a/R/IOHanalyzer-deprecated.R b/R/IOHanalyzer-deprecated.R index 304aaa68..6b5366d9 100644 --- a/R/IOHanalyzer-deprecated.R +++ b/R/IOHanalyzer-deprecated.R @@ -1,196 +1,196 @@ -##IOHanalyzer-deprecated.R -#' @title Deprecated function in package \pkg{IOHanalyzer} -#' @description The functions listed below are deprecated and will be defunct in -#' the near future. When possible, alternative functions with similar -#' functionality are also mentioned. -#' @name IOHanlyzer-deprecated -#' @keywords internal -NULL - -#' Get the ERT-values for all DataSets in a DataSetList at certain targets -#' -#' @param dsList The DataSetLsit -#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. -#' @param targets Predifined target function-values. Should be one for each function/dimension -#' @param maximize Whether the DataSetList is from a maximization or minimization problem -#' -#' @return A data.table containing ERT-values -#' @export -#' @examples -#' max_ERTs(dsl) -max_ERTs <- - function(dsList, - aggr_on = 'funcId', - targets = NULL, - maximize = T) - UseMethod("max_ERTs", dsList) - -#TODO: rename this function! this function needs to be rewritten -#' @rdname max_ERTs -#' @export -max_ERTs.DataSetList <- - function(dsList, - aggr_on = 'funcId', - targets = NULL, - maximize = T) { - .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. - Please use the suggested function instead.") - N <- length(get_algId(dsList)) - - aggr_attr <- - if (aggr_on == 'funcId') - get_funcId(dsList) - else - get_dim(dsList) - if (!is.null(targets) && - length(targets) != length(aggr_attr)) - targets <- NULL - - second_aggr <- - if (aggr_on == 'funcId') - get_dim(dsList) - else - get_funcId(dsList) - if (length(second_aggr) > 1) - return(NULL) - - erts <- seq(0, 0, length.out = length(get_algId(dsList))) - names(erts) <- get_algId(dsList) - - for (j in seq_along(aggr_attr)) { - dsList_filetered <- - if (aggr_on == 'funcId') - subset(dsList, funcId == aggr_attr[[j]]) - else - subset(dsList, DIM == aggr_attr[[j]]) - - if (is.null(targets)) { - Fall <- get_funvals(dsList_filetered) - Fval <- ifelse(maximize, max(Fall), min(Fall)) - } - else - Fval <- targets[[j]] - summary <- get_RT_summary(dsList_filetered, ftarget = Fval) - ert <- summary$ERT - names(ert) <- summary$algId - erts <- rbind(erts, ert[get_algId(dsList)]) - } - return(erts[-1, ]) - } - -#' Get the expected function-values for all DataSets in a DataSetList at certain runtimes -#' -#' @param dsList The DataSetLsit -#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. -#' @param runtimes Predifined target runtimes-values. Should be one for each function/dimension -#' -#' @return A data.table containing expected fucntion-values -#' @export -#' @examples -#' mean_FVs(dsl) -mean_FVs <- - function(dsList, - aggr_on = 'funcId', - runtimes = NULL) - UseMethod("mean_FVs", dsList) - -#' @rdname mean_FVs -#' @export -mean_FVs.DataSetList <- - function(dsList, - aggr_on = 'funcId', - runtimes = NULL) { - N <- length(get_algId(dsList)) - .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. - Please use the suggested function instead.") - - aggr_attr <- - if (aggr_on == 'funcId') - get_funcId(dsList) - else - get_dim(dsList) - if (!is.null(runtimes) && - length(runtimes) != length(aggr_attr)) - targets <- NULL - - second_aggr <- - if (aggr_on == 'funcId') - get_dim(dsList) - else - get_funcId(dsList) - if (length(second_aggr) > 1) - return(NULL) - - erts <- seq(0, 0, length.out = length(get_algId(dsList))) - names(erts) <- get_algId(dsList) - - for (j in seq_along(aggr_attr)) { - dsList_filetered <- - if (aggr_on == 'funcId') - subset(dsList, funcId == aggr_attr[[j]]) - else - subset(dsList, DIM == aggr_attr[[j]]) - - if (is.null(runtimes)) { - RTall <- get_runtimes(dsList_filetered) - RTval <- max(RTall) - } - else - RTval <- runtimes[[j]] - summary <- get_FV_summary(dsList_filetered, runtime = RTval) - ert <- summary$mean - names(ert) <- summary$algId - erts <- rbind(erts, ert[get_algId(dsList)]) - } - return(erts[-1, ]) - } - -# TODO: review / re-write this function -#TODO: inconsistent use of format_func gives slightly different results between -#generated and uploaded targets -#' Generate ECDF targets for a DataSetList -#' -#' @param data A DataSetList -#' @param format_func function to format the targets -#' -#' @return a vector of targets -#' @export -#' @examples -#' get_default_ECDF_targets(dsl) -get_default_ECDF_targets <- function(data, format_func = as.integer) { - .Deprecated("get_ECDF_targets", msg = "This function will be deprecated in the next release of IOHanalyzer. - Please use the suggested function instead.") - - funcIds <- get_funcId(data) - dims <- get_dim(data) - - targets <- list() - names <- list() - for (i in seq_along(funcIds)) { - Id <- funcIds[[i]] - data_sub <- subset(data, funcId == Id) - - for (j in seq_along(dims)) { - dim <- dims[[j]] - data_subsub <- subset(data_sub, DIM == dim) - if (length(data_subsub) == 0) break - fall <- get_funvals(data_subsub) - if (length(fall) < 2) break #TODO: double check why this can happen in nevergrad? - #TODO: Account for minimization / maximization - fmin <- min(fall) - fmax <- max(fall) - - fseq <- seq_FV(fall, fmin, fmax, length.out = 10) %>% - sapply(format_func) - targets <- append(targets, list(fseq)) - - if (length(funcIds) == 1) { - names <- append(names, dim) - } else if (length(dims) == 1) { - names <- append(names, Id) - } else - names <- append(names, paste0(Id, ";", dim)) - } - } - targets %>% set_names(names) +##IOHanalyzer-deprecated.R +#' @title Deprecated function in package \pkg{IOHanalyzer} +#' @description The functions listed below are deprecated and will be defunct in +#' the near future. When possible, alternative functions with similar +#' functionality are also mentioned. +#' @name IOHanlyzer-deprecated +#' @keywords internal +NULL + +#' Get the ERT-values for all DataSets in a DataSetList at certain targets +#' +#' @param dsList The DataSetLsit +#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. +#' @param targets Predifined target function-values. Should be one for each function/dimension +#' @param maximize Whether the DataSetList is from a maximization or minimization problem +#' +#' @return A data.table containing ERT-values +#' @export +#' @examples +#' max_ERTs(dsl) +max_ERTs <- + function(dsList, + aggr_on = 'funcId', + targets = NULL, + maximize = T) + UseMethod("max_ERTs", dsList) + +#TODO: rename this function! this function needs to be rewritten +#' @rdname max_ERTs +#' @export +max_ERTs.DataSetList <- + function(dsList, + aggr_on = 'funcId', + targets = NULL, + maximize = T) { + .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. + Please use the suggested function instead.") + N <- length(get_algId(dsList)) + + aggr_attr <- + if (aggr_on == 'funcId') + get_funcId(dsList) + else + get_dim(dsList) + if (!is.null(targets) && + length(targets) != length(aggr_attr)) + targets <- NULL + + second_aggr <- + if (aggr_on == 'funcId') + get_dim(dsList) + else + get_funcId(dsList) + if (length(second_aggr) > 1) + return(NULL) + + erts <- seq(0, 0, length.out = length(get_algId(dsList))) + names(erts) <- get_algId(dsList) + + for (j in seq_along(aggr_attr)) { + dsList_filetered <- + if (aggr_on == 'funcId') + subset(dsList, funcId == aggr_attr[[j]]) + else + subset(dsList, DIM == aggr_attr[[j]]) + + if (is.null(targets)) { + Fall <- get_funvals(dsList_filetered) + Fval <- ifelse(maximize, max(Fall), min(Fall)) + } + else + Fval <- targets[[j]] + summary <- get_RT_summary(dsList_filetered, ftarget = Fval) + ert <- summary$ERT + names(ert) <- summary$algId + erts <- rbind(erts, ert[get_algId(dsList)]) + } + return(erts[-1, ]) + } + +#' Get the expected function-values for all DataSets in a DataSetList at certain runtimes +#' +#' @param dsList The DataSetLsit +#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. +#' @param runtimes Predifined target runtimes-values. Should be one for each function/dimension +#' +#' @return A data.table containing expected fucntion-values +#' @export +#' @examples +#' mean_FVs(dsl) +mean_FVs <- + function(dsList, + aggr_on = 'funcId', + runtimes = NULL) + UseMethod("mean_FVs", dsList) + +#' @rdname mean_FVs +#' @export +mean_FVs.DataSetList <- + function(dsList, + aggr_on = 'funcId', + runtimes = NULL) { + N <- length(get_algId(dsList)) + .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. + Please use the suggested function instead.") + + aggr_attr <- + if (aggr_on == 'funcId') + get_funcId(dsList) + else + get_dim(dsList) + if (!is.null(runtimes) && + length(runtimes) != length(aggr_attr)) + targets <- NULL + + second_aggr <- + if (aggr_on == 'funcId') + get_dim(dsList) + else + get_funcId(dsList) + if (length(second_aggr) > 1) + return(NULL) + + erts <- seq(0, 0, length.out = length(get_algId(dsList))) + names(erts) <- get_algId(dsList) + + for (j in seq_along(aggr_attr)) { + dsList_filetered <- + if (aggr_on == 'funcId') + subset(dsList, funcId == aggr_attr[[j]]) + else + subset(dsList, DIM == aggr_attr[[j]]) + + if (is.null(runtimes)) { + RTall <- get_runtimes(dsList_filetered) + RTval <- max(RTall) + } + else + RTval <- runtimes[[j]] + summary <- get_FV_summary(dsList_filetered, runtime = RTval) + ert <- summary$mean + names(ert) <- summary$algId + erts <- rbind(erts, ert[get_algId(dsList)]) + } + return(erts[-1, ]) + } + +# TODO: review / re-write this function +#TODO: inconsistent use of format_func gives slightly different results between +#generated and uploaded targets +#' Generate ECDF targets for a DataSetList +#' +#' @param data A DataSetList +#' @param format_func function to format the targets +#' +#' @return a vector of targets +#' @export +#' @examples +#' get_default_ECDF_targets(dsl) +get_default_ECDF_targets <- function(data, format_func = as.integer) { + .Deprecated("get_ECDF_targets", msg = "This function will be deprecated in the next release of IOHanalyzer. + Please use the suggested function instead.") + + funcIds <- get_funcId(data) + dims <- get_dim(data) + + targets <- list() + names <- list() + for (i in seq_along(funcIds)) { + Id <- funcIds[[i]] + data_sub <- subset(data, funcId == Id) + + for (j in seq_along(dims)) { + dim <- dims[[j]] + data_subsub <- subset(data_sub, DIM == dim) + if (length(data_subsub) == 0) break + fall <- get_funvals(data_subsub) + if (length(fall) < 2) break #TODO: double check why this can happen in nevergrad? + #TODO: Account for minimization / maximization + fmin <- min(fall) + fmax <- max(fall) + + fseq <- seq_FV(fall, fmin, fmax, length.out = 10) %>% + sapply(format_func) + targets <- append(targets, list(fseq)) + + if (length(funcIds) == 1) { + names <- append(names, dim) + } else if (length(dims) == 1) { + names <- append(names, Id) + } else + names <- append(names, paste0(Id, ";", dim)) + } + } + targets %>% set_names(names) } \ No newline at end of file diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index e2b54eb0..4f6bd829 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -1,129 +1,129 @@ -#' @importFrom stats dt ecdf integrate median quantile sd rgeom ks.test p.adjust -#' @importFrom grDevices col2rgb colors nclass.FD -#' @importFrom graphics hist -#' @importFrom utils data head read.csv tail type.convert write.csv -#' @importFrom dplyr %>% mutate -#' @importFrom magrittr set_names set_rownames set_colnames %<>% mod -#' @importFrom colorspace sequential_hcl -#' @importFrom RColorBrewer brewer.pal -#' @importFrom colorRamps primary.colors -#' @importFrom data.table as.data.table rbindlist data.table fread := melt is.data.table -#' @importFrom data.table setorderv frank setnames rbindlist copy transpose -#' @importFrom plotly add_annotations add_trace orca plot_ly rename_ subplot layout -#' @importFrom ggplot2 aes geom_jitter geom_line geom_ribbon geom_violin ggplot element_text -#' @importFrom ggplot2 guides scale_color_manual scale_colour_manual scale_fill_manual -#' @importFrom ggplot2 scale_x_continuous scale_x_log10 facet_wrap theme_set theme_grey theme -#' @importFrom shiny req -#' @importFrom stringi stri_detect_regex stri_detect_fixed stri_locate_all stri_replace -#' @importFrom stringi stri_rand_strings -#' @importFrom httr POST add_headers content authenticate -#' @importFrom reshape2 acast -#' @importFrom knitr kable -#' @importFrom methods hasArg -#' @importFrom rjson fromJSON -#' @useDynLib IOHanalyzer -NULL -# Ugly hack, but appears to be required to appease CRAN -utils::globalVariables(c(".", "algId", "run", "ERT", "RT", "group", - "DIM", "Fvalue", "lower", "upper", "target", "format", - "runtime", "parId", "instance", "input", "funcId", - "budget", "dimension", "loss", "name", "optimizer_name", - "rescale", "maxRT", "algnames", ".SD", "function_class", "ID", "ids")) - -options(shiny.port = 4242) - -.onLoad <- function(libname, pkgname) { - op <- options() - op.IOHanalyzer <- list( - IOHanalyzer.ID_vars = c("algId"), - IOHanalyzer.quantiles = c(2, 5, 10, 25, 50, 75, 90, 95, 98) / 100., - IOHanalyzer.max_samples = 100, - IOHanalyzer.backend = 'plotly', - IOHanalyzer.bgcolor = 'rgb(230,230,230)', - IOHanalyzer.gridcolor = 'rgb(255,255,255)', - IOHanalyzer.tickcolor = 'rgb(51,51,51)', - IOHanalyzer.figure_width = 1000, - IOHanalyzer.figure_height = 1000, - IOHanalyzer.legend_location = 'below', - IOHanalyzer.legend_fontsize = 13, - IOHanalyzer.custom_legend_x = 0.5, - IOHanalyzer.custom_legend_y = -0.2, - IOHanalyzer.label_fontsize = 16, - IOHanalyzer.title_fontsize = 16, - IOHanalyzer.tick_fontsize = 12, - IOHanalyzer.linewidth = 2, - IOHanalyzer.markersize = 4, - IOHanalyzer.max_colors = 2 #Set to 2 since colorbrewer only works with >= 3 colors - ) - toset <- !(names(op.IOHanalyzer) %in% names(op)) - if (any(toset)) options(op.IOHanalyzer[toset]) - - invisible() -} - -IOHanalyzer_env <- new.env(parent = emptyenv()) - -.mean <- function(x) mean(x, na.rm = T) -.median <- function(x) median(x, na.rm = T) -.sd <- function(x) sd(x, na.rm = T) -.sum <- function(x) sum(x, na.rm = T) - -# Quantile function for discrete values -IOHanalyzer_env$D_quantile <- function(x, pct = NULL) { - if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") - tryCatch( - quantile(x, pct, names = F, type = 3, na.rm = T), - error = function(e) rep(NA, length(pct)), - warning = function(w) rep(NA, length(pct)) - ) -} - -# Quantile function for real values -IOHanalyzer_env$C_quantile <- function(x, pct = NULL) { - if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") - tryCatch( - quantile(x, pct, names = F, na.rm = T), - error = function(e) rep(NA, length(pct)), - warning = function(w) rep(NA, length(pct)) - ) -} - -IOHprofiler <- 'IOHprofiler' -COCO <- 'COCO' -BIBOJ_COCO <- 'BIBOJ_COCO' -TWO_COL <- 'TWO_COL' -AUTOMATIC <- 'AUTOMATIC' -NEVERGRAD <- 'NEVERGRAD' -SOS <- 'SOS' - -#' IOHanalyzer: Data Analysis Part of IOHprofiler -#' -#' The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). -#' This module provides statistical analysis methods for the benchmark data generated by -#' optimization heuristics, which can be visualized through a -#' web-based interface. The benchmark data is usually generated by the -#' experimentation module, called IOHexperimenter. IOHanalyzer also supports -#' the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. -#' -#' @section Functions: -#' The IOHanalyzer consists of 3 main functionalities: -#' \itemize{ -#' \item Reading and alligning data from different heuristics, such as IOHExperimenter. -#' This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions -#' \item Processing and summarizing this data -#' \item Creating various plots -#' } -#' -#' @docType package -#' @name IOHanalyzer -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -#' dsList <- DataSetList(path) -#' summary(dsList) -#' Plot.RT.Single_Func(dsList[1]) -#' -#' @examples -#' \dontrun{ -#' runServer() -#' } +#' @importFrom stats dt ecdf integrate median quantile sd rgeom ks.test p.adjust +#' @importFrom grDevices col2rgb colors nclass.FD +#' @importFrom graphics hist +#' @importFrom utils data head read.csv tail type.convert write.csv +#' @importFrom dplyr %>% mutate +#' @importFrom magrittr set_names set_rownames set_colnames %<>% mod +#' @importFrom colorspace sequential_hcl +#' @importFrom RColorBrewer brewer.pal +#' @importFrom colorRamps primary.colors +#' @importFrom data.table as.data.table rbindlist data.table fread := melt is.data.table +#' @importFrom data.table setorderv frank setnames rbindlist copy transpose +#' @importFrom plotly add_annotations add_trace orca plot_ly rename_ subplot layout +#' @importFrom ggplot2 aes geom_jitter geom_line geom_ribbon geom_violin ggplot element_text +#' @importFrom ggplot2 guides scale_color_manual scale_colour_manual scale_fill_manual +#' @importFrom ggplot2 scale_x_continuous scale_x_log10 facet_wrap theme_set theme_grey theme +#' @importFrom shiny req +#' @importFrom stringi stri_detect_regex stri_detect_fixed stri_locate_all stri_replace +#' @importFrom stringi stri_rand_strings +#' @importFrom httr POST add_headers content authenticate +#' @importFrom reshape2 acast +#' @importFrom knitr kable +#' @importFrom methods hasArg +#' @importFrom rjson fromJSON +#' @useDynLib IOHanalyzer +NULL +# Ugly hack, but appears to be required to appease CRAN +utils::globalVariables(c(".", "algId", "run", "ERT", "RT", "group", + "DIM", "Fvalue", "lower", "upper", "target", "format", + "runtime", "parId", "instance", "input", "funcId", + "budget", "dimension", "loss", "name", "optimizer_name", + "rescale", "maxRT", "algnames", ".SD", "function_class", "ID", "ids")) + +options(shiny.port = 4242) + +.onLoad <- function(libname, pkgname) { + op <- options() + op.IOHanalyzer <- list( + IOHanalyzer.ID_vars = c("algId"), + IOHanalyzer.quantiles = c(2, 5, 10, 25, 50, 75, 90, 95, 98) / 100., + IOHanalyzer.max_samples = 100, + IOHanalyzer.backend = 'plotly', + IOHanalyzer.bgcolor = 'rgb(230,230,230)', + IOHanalyzer.gridcolor = 'rgb(255,255,255)', + IOHanalyzer.tickcolor = 'rgb(51,51,51)', + IOHanalyzer.figure_width = 1000, + IOHanalyzer.figure_height = 1000, + IOHanalyzer.legend_location = 'below', + IOHanalyzer.legend_fontsize = 13, + IOHanalyzer.custom_legend_x = 0.5, + IOHanalyzer.custom_legend_y = -0.2, + IOHanalyzer.label_fontsize = 16, + IOHanalyzer.title_fontsize = 16, + IOHanalyzer.tick_fontsize = 12, + IOHanalyzer.linewidth = 2, + IOHanalyzer.markersize = 4, + IOHanalyzer.max_colors = 2 #Set to 2 since colorbrewer only works with >= 3 colors + ) + toset <- !(names(op.IOHanalyzer) %in% names(op)) + if (any(toset)) options(op.IOHanalyzer[toset]) + + invisible() +} + +IOHanalyzer_env <- new.env(parent = emptyenv()) + +.mean <- function(x) mean(x, na.rm = T) +.median <- function(x) median(x, na.rm = T) +.sd <- function(x) sd(x, na.rm = T) +.sum <- function(x) sum(x, na.rm = T) + +# Quantile function for discrete values +IOHanalyzer_env$D_quantile <- function(x, pct = NULL) { + if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") + tryCatch( + quantile(x, pct, names = F, type = 3, na.rm = T), + error = function(e) rep(NA, length(pct)), + warning = function(w) rep(NA, length(pct)) + ) +} + +# Quantile function for real values +IOHanalyzer_env$C_quantile <- function(x, pct = NULL) { + if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") + tryCatch( + quantile(x, pct, names = F, na.rm = T), + error = function(e) rep(NA, length(pct)), + warning = function(w) rep(NA, length(pct)) + ) +} + +IOHprofiler <- 'IOHprofiler' +COCO <- 'COCO' +BIBOJ_COCO <- 'BIBOJ_COCO' +TWO_COL <- 'TWO_COL' +AUTOMATIC <- 'AUTOMATIC' +NEVERGRAD <- 'NEVERGRAD' +SOS <- 'SOS' + +#' IOHanalyzer: Data Analysis Part of IOHprofiler +#' +#' The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). +#' This module provides statistical analysis methods for the benchmark data generated by +#' optimization heuristics, which can be visualized through a +#' web-based interface. The benchmark data is usually generated by the +#' experimentation module, called IOHexperimenter. IOHanalyzer also supports +#' the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. +#' +#' @section Functions: +#' The IOHanalyzer consists of 3 main functionalities: +#' \itemize{ +#' \item Reading and alligning data from different heuristics, such as IOHExperimenter. +#' This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions +#' \item Processing and summarizing this data +#' \item Creating various plots +#' } +#' +#' @docType package +#' @name IOHanalyzer +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +#' dsList <- DataSetList(path) +#' summary(dsList) +#' Plot.RT.Single_Func(dsList[1]) +#' +#' @examples +#' \dontrun{ +#' runServer() +#' } NULL \ No newline at end of file diff --git a/R/RcppExports.R b/R/RcppExports.R index 81825ee7..0b8ddd7f 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -1,31 +1,31 @@ -# Generated by using Rcpp::compileAttributes() -> do not edit by hand -# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 - -align_by_target_inner_loop <- function(t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) { - .Call('_IOHanalyzer_align_by_target_inner_loop', PACKAGE = 'IOHanalyzer', t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) -} - -c_impute <- function(x, y, rowname) { - .Call('_IOHanalyzer_c_impute', PACKAGE = 'IOHanalyzer', x, y, rowname) -} - -c_impute_running_time <- function(index, value, FV, maximization) { - .Call('_IOHanalyzer_c_impute_running_time', PACKAGE = 'IOHanalyzer', index, value, FV, maximization) -} - -#' Align a list of data set by function values -#' -#' @param data the data -#' @param FV Function values -#' @param idxValue index of the function values -#' @param maximization Boolean -#' @param idxTarget index of the target -#' @noRd -c_align_running_time <- function(data, FV, idxValue, maximization, idxTarget) { - .Call('_IOHanalyzer_c_align_running_time', PACKAGE = 'IOHanalyzer', data, FV, idxValue, maximization, idxTarget) -} - -c_read_dat <- function(dat, NC, leading) { - .Call('_IOHanalyzer_c_read_dat', PACKAGE = 'IOHanalyzer', dat, NC, leading) -} - +# Generated by using Rcpp::compileAttributes() -> do not edit by hand +# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 + +align_by_target_inner_loop <- function(t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) { + .Call('_IOHanalyzer_align_by_target_inner_loop', PACKAGE = 'IOHanalyzer', t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) +} + +c_impute <- function(x, y, rowname) { + .Call('_IOHanalyzer_c_impute', PACKAGE = 'IOHanalyzer', x, y, rowname) +} + +c_impute_running_time <- function(index, value, FV, maximization) { + .Call('_IOHanalyzer_c_impute_running_time', PACKAGE = 'IOHanalyzer', index, value, FV, maximization) +} + +#' Align a list of data set by function values +#' +#' @param data the data +#' @param FV Function values +#' @param idxValue index of the function values +#' @param maximization Boolean +#' @param idxTarget index of the target +#' @noRd +c_align_running_time <- function(data, FV, idxValue, maximization, idxTarget) { + .Call('_IOHanalyzer_c_align_running_time', PACKAGE = 'IOHanalyzer', data, FV, idxValue, maximization, idxTarget) +} + +c_read_dat <- function(dat, NC, leading) { + .Call('_IOHanalyzer_c_read_dat', PACKAGE = 'IOHanalyzer', dat, NC, leading) +} + diff --git a/R/data.R b/R/data.R index da9552b8..aa917beb 100644 --- a/R/data.R +++ b/R/data.R @@ -1,17 +1,17 @@ -#' Example DataSetList used in tests / examples -#' -#' A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D -#' -#' @format DataSetList -#' @examples -#' summary(dsl) -"dsl" - -#' Larger example DataSetList used in tests / examples -#' -#' A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D -#' -#' @format DataSetList -#' @examples -#' summary(dsl_large) +#' Example DataSetList used in tests / examples +#' +#' A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D +#' +#' @format DataSetList +#' @examples +#' summary(dsl) +"dsl" + +#' Larger example DataSetList used in tests / examples +#' +#' A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D +#' +#' @format DataSetList +#' @examples +#' summary(dsl_large) "dsl_large" \ No newline at end of file diff --git a/R/plot.R b/R/plot.R index c6f96e81..6ac88d88 100644 --- a/R/plot.R +++ b/R/plot.R @@ -1,369 +1,369 @@ -# font No. 1... -f1 <- list( - family = 'Old Standard TT, serif', - size = 11, - color = 'black' -) - -# font No. 2... -f2 <- list( - family = 'Old Standard TT, serif', - size = 13, - color = 'black' -) - -# font No. 3... -f3 <- function() { - list( - family = 'Old Standard TT, serif', - size = getOption("IOHanalyzer.tick_fontsize", default = 12), - color = 'black' - ) -} - -legend_right <- function() { - list(x = 1.01, y = 1, orientation = 'v', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -legend_inside <- function() { - list(x = .01, y = 1, orientation = 'v', - bgcolor = 'rgba(255, 255, 255, 0)', - bordercolor = 'rgba(255, 255, 255, 0)', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -legend_inside2 <- function() { - list(x = 0.7, y = 0.1, orientation = 'v', - bgcolor = 'rgba(255, 255, 255, 0.5)', - bordercolor = 'rgba(255, 255, 255, 0.8)', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -#TODO: Make the y-value configurable -legend_below <- function() { - list(y = -0.2, orientation = 'h', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -legend_custom <- function() { - list(x = getOption("IOHanalyzer.custom_legend_x", default = 0.5), - y = getOption("IOHanalyzer.custom_legend_y", default = -0.2), orientation = 'h', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - - -legend_location <- function(){ - opt <- getOption('IOHanalyzer.legend_location', default = 'below') - if (opt == 'outside_right') return(legend_right()) - else if (opt == 'inside_left') return(legend_inside()) - else if (opt == 'inside_right') return(legend_inside2()) - else if (opt == 'below') return(legend_below()) - else if (opt == 'custom') return(legend_custom()) - # else if (opt == 'below2') return(legend_below2()) - else warning(paste0("The selected legend option (", opt, ") is not implemented")) -} - -# TODO: create font object as above for title, axis... - -#' Template for creating plots in the IOHanalyzer-style -#' -#' @param title Title for the plot -#' @param x.title X-axis label -#' @param y.title Y-axis label -#' -#' @export -#' @examples -#' IOH_plot_ly_default("Example plot","x-axis","y-axis") -IOH_plot_ly_default <- function(title = NULL, x.title = NULL, y.title = NULL) { - plot_ly() %>% - layout(title = list(text = title, - font = list(size = getOption("IOHanalyzer.title_fontsize", default = 16), - family = 'Old Standard TT, serif')), - autosize = T, hovermode = 'compare', - legend = legend_location(), - paper_bgcolor = 'rgb(255,255,255)', - plot_bgcolor = getOption('IOHanalyzer.bgcolor'), - font = list(size = getOption("IOHanalyzer.label_fontsize", default = 16), - family = 'Old Standard TT, serif'), - autosize = T, - showlegend = T, - xaxis = list( - # title = list(text = x.title, font = f3), - title = x.title, - gridcolor = getOption('IOHanalyzer.gridcolor'), - showgrid = TRUE, - showline = FALSE, - showticklabels = TRUE, - tickcolor = getOption('IOHanalyzer.tickcolor'), - ticks = 'outside', - ticklen = 9, - tickfont = f3(), - exponentformat = 'e', - zeroline = F), - yaxis = list( - # title = list(text = y.title, font = f3), - title = y.title, - gridcolor = getOption('IOHanalyzer.gridcolor'), - showgrid = TRUE, - showline = FALSE, - showticklabels = TRUE, - tickcolor = getOption('IOHanalyzer.tickcolor'), - ticks = 'outside', - ticklen = 9, - tickfont = f3(), - exponentformat = 'e', - zeroline = F)) -} - -t <- theme_grey() + - theme(text = element_text(size = 15), - plot.title = element_text(hjust = 0.5) - # legend.position = c(0.15, 0.7), # legend position - # legend.key = element_blank(), # no small box around symbol - # legend.key.size = unit(1.3, "line"), # bigger symbols - # legend.background = element_rect(color = alpha("black", 0.5), - # fill = alpha('blue', 0.0), - # size = 1, - # linetype = "solid") - ) -theme_set(t) - -gg_beanplot <- function(mapping, data, p = NULL, width = 3, fill = 'grey', - colour = 'grey', alpha = 1, kernel = 'gaussian', bw = 'SJ', - draw_quantiles = NULL, trim = TRUE, na.rm = FALSE, - show.legend = NA, point.shape = 20, show.sample = T, - show.violin = T, linetype = 'solid') { - - set.seed(42) - x <- as.character(mapping$x) - y <- as.character(mapping$y) - df <- data[, c(x, y)] %>% rename_(.dots = c('x' = x, 'y' = y)) - - if (!is.numeric(df$x)) - df$x <- tryCatch(as.numeric(df$x), # in case x is a factor... - warning = function(w) return(match(x, as.factor(x)))) - - if (is.null(p)) - p <- ggplot() - - if (show.violin) - p <- p + geom_violin(data = data, mapping = mapping, trim = trim, - draw_quantiles = draw_quantiles, bw = bw, - kernel = kernel, scale = 'width', - width = width, alpha = alpha) - if (show.sample) - p <- p + geom_jitter(data = df, aes(x, y), height = 0, width = width / 2, - alpha = 0.45, shape = point.shape, size = 3.5) - # geom_segment(aes(x = x - width / 2.2, xend = x + width / 2.2, y = y, yend = y), - # df, col = 'black', size = 0.2, alpha = 0.3, linetype = linetype) - p -} - - -Set1 <- function(n) colorspace::sequential_hcl(n, h = c(360, 40), c. = c(100, NA, 90), l = c(28, 90), - power = c(1, 1.1), gamma = NULL, fixup = TRUE, - alpha = 1)#, palette = NULL, rev = FALSE) - -Set2 <- function(n) colorspace::sequential_hcl(n, c(261, 26), c. = c(50, NA, 70), l = c(54, 77), - power = c(0.5, NA), gamma = NULL, - fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) - -Set3 <- function(n) colorspace::sequential_hcl(n, c(-88, 59), c. = c(60, 75, 55), l = c(40, 90), - power = c(0.1, 1.2), gamma = NULL, - fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) - -IOHanalyzer_env$used_colorscheme <- Set2 -IOHanalyzer_env$id_colors <- NULL - -#' Set the colorScheme of the IOHanalyzer plots -#' -#' @param schemename Three default colorschemes are implemented: -#' \itemize{ -#' \item Default -#' \item Variant 1 -#' \item Variant 2 -#' \item Variant 3 -#' } -#' And it is also possible to select "Custom", which allows uploading of a custom set of colors -#' @param ids The names of the algorithms (or custom ids, see `change_id`) for which to set the colors -#' @param path The path to the file containing the colors to use. Only used if -#' schemename is "Custom" -#' -#' @export -#' -#' @examples -#' set_color_scheme("Default", get_algId(dsl)) -set_color_scheme <- function(schemename, ids, path = NULL){ - if (schemename == "Custom" && !is.null(path)) { - dt <- fread(path, header = T) - if (any(colnames(dt) != c("ids", "colors", "linestyles"))) { - warning("Incorrect file-format has been uploaded.") - } - else - IOHanalyzer_env$id_colors <- dt - return() - } - else { - if (schemename == "Default") { - options(IOHanalyzer.max_colors = 2) - } - else { - if (schemename == "Variant 1") IOHanalyzer_env$used_colorscheme <- Set1 - else if (schemename == "Variant 2") IOHanalyzer_env$used_colorscheme <- Set2 - else if (schemename == "Variant 3") IOHanalyzer_env$used_colorscheme <- Set3 - options(IOHanalyzer.max_colors = length(ids)) - } - create_color_scheme(ids) - } -} - -#' Get datatable of current color (and linestyle) scheme to file -#' -#' @return data.table object with 3 columns: ids, colors, linestyles -#' @export -#' @examples -#' get_color_scheme_dt() -get_color_scheme_dt <- function(){ - return(IOHanalyzer_env$id_colors) -} - -#' Helper function to create default color scheme -#' -#' @noRd -create_color_scheme <- function(ids) { - if (length(ids) == 0) { - return(NULL) - } - colors <- color_palettes(length(ids)) - linestyles <- rep(c("solid", "dash", "dot"), ceiling(length(colors)/3))[1:length(colors)] - IOHanalyzer_env$id_colors <- data.table(ids, colors, linestyles) -} - -#' Get colors according to the current colorScheme of the IOHanalyzer -#' -#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get colors -#' -#' @export -#' -#' @examples -#' get_color_scheme(get_algId(dsl)) -get_color_scheme <- function(ids_in){ - if (is.null(IOHanalyzer_env$id_colors)) - create_color_scheme(ids_in) - cdt <- IOHanalyzer_env$id_colors - colors <- subset(cdt, ids %in% ids_in)[['colors']] - if (is.null(colors) || length(colors) != length(ids_in)) { - return(color_palettes(length(ids_in))) - } - names(colors) <- ids_in - return(colors) -} - - -#' Get line styles according to the current styleScheme of the IOHanalyzer -#' -#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get linestyles -#' -#' @export -#' -#' @examples -#' get_line_style(get_algId(dsl)) -get_line_style <- function(ids_in){ - if (is.null(IOHanalyzer_env$id_colors)) - create_color_scheme(ids_in) - cdt <- IOHanalyzer_env$id_colors - linestyles <- subset(cdt, ids %in% ids_in)[['linestyles']] - if (is.null(linestyles) || length(linestyles) != length(ids_in)) { - return(rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), - ceiling(length(ids_in)/3))[1:length(ids_in)]) - } - return(linestyles) -} - -# TODO: incoporate more colors -color_palettes <- function(ncolor) { - # TODO: FIX IT! - max_colors <- getOption("IOHanalyzer.max_colors", 2) - if (ncolor <= max_colors) return(IOHanalyzer_env$used_colorscheme(ncolor)) - - brewer <- function(n) { - colors <- RColorBrewer::brewer.pal(n, 'Spectral') - colors[colors == "#FFFFBF"] <- "#B2B285" - colors[colors == "#E6F598"] <- "#86FF33" - colors[colors == '#FEE08B'] <- "#FFFF33" - colors - } - - color_fcts <- c(colorRamps::primary.colors, IOHanalyzer_env$used_colorscheme) - - n <- min(11, ncolor) - colors <- brewer(n) - ncolor <- ncolor - n - - i <- 1 - while (ncolor > 0) { - n <- min(8, ncolor) - if (i > length(color_fcts)) { - colors <- c(colors, colorRamps::primary.colors(ncolor)) - break - } else { - colors <- c(colors, color_fcts[[i]](n)) - ncolor <- ncolor - n - } - i <- i + 1 - } - colors -} - -#' Save plotly figure in multiple format -#' -#' NOTE: This function requires orca to be installed -#' -#' @param p plotly object. The plot to be saved -#' @param file String. The name of the figure file, with the extension of the required file-format -#' @param width Optional. Width of the figure -#' @param height Optional. Height of the figure -#' @param ... Additional arguments for orca -#' @export -#' @examples -#' \dontrun{ -#' p <- Plot.RT.Single_Func(dsl[1]) -#' save_plotly(p, 'example_file.png') -#' } -save_plotly <- function(p, file, width = NULL, height = NULL, ...) { - - if (!requireNamespace("withr", quietly = TRUE)) { - stop("Package \"withr\" needed for this function to work. Please install it.", - call. = FALSE) - } - des <- dirname(file) - file <- basename(file) - format <- tools::file_ext(file) - - pwd <- tempdir() - if (is.null(width)) width <- getOption("IOHanalyzer.figure_width", default = NULL) - if (is.null(height)) height <- getOption("IOHanalyzer.figure_height", default = NULL) - - if (format %in% c('svg', 'png', 'jpeg', 'webp', 'pdf', 'eps')) - withr::with_dir(pwd, orca(p, file, format = format, width = width, height = height, ...)) - else { - file_svg <- paste0(file, '.svg') - withr::with_dir(pwd, orca(p, file_svg, format = 'svg', width = width, height = height, ...)) - invisible( - system( - paste( - 'inkscape', file.path(pwd,file_svg), - paste0('--export-', format, ' ', file.path(pwd, file)) - ), - intern = T - ) - ) - } - file.rename(file.path(pwd, file), file.path(des, file)) -} +# font No. 1... +f1 <- list( + family = 'Old Standard TT, serif', + size = 11, + color = 'black' +) + +# font No. 2... +f2 <- list( + family = 'Old Standard TT, serif', + size = 13, + color = 'black' +) + +# font No. 3... +f3 <- function() { + list( + family = 'Old Standard TT, serif', + size = getOption("IOHanalyzer.tick_fontsize", default = 12), + color = 'black' + ) +} + +legend_right <- function() { + list(x = 1.01, y = 1, orientation = 'v', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +legend_inside <- function() { + list(x = .01, y = 1, orientation = 'v', + bgcolor = 'rgba(255, 255, 255, 0)', + bordercolor = 'rgba(255, 255, 255, 0)', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +legend_inside2 <- function() { + list(x = 0.7, y = 0.1, orientation = 'v', + bgcolor = 'rgba(255, 255, 255, 0.5)', + bordercolor = 'rgba(255, 255, 255, 0.8)', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +#TODO: Make the y-value configurable +legend_below <- function() { + list(y = -0.2, orientation = 'h', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +legend_custom <- function() { + list(x = getOption("IOHanalyzer.custom_legend_x", default = 0.5), + y = getOption("IOHanalyzer.custom_legend_y", default = -0.2), orientation = 'h', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + + +legend_location <- function(){ + opt <- getOption('IOHanalyzer.legend_location', default = 'below') + if (opt == 'outside_right') return(legend_right()) + else if (opt == 'inside_left') return(legend_inside()) + else if (opt == 'inside_right') return(legend_inside2()) + else if (opt == 'below') return(legend_below()) + else if (opt == 'custom') return(legend_custom()) + # else if (opt == 'below2') return(legend_below2()) + else warning(paste0("The selected legend option (", opt, ") is not implemented")) +} + +# TODO: create font object as above for title, axis... + +#' Template for creating plots in the IOHanalyzer-style +#' +#' @param title Title for the plot +#' @param x.title X-axis label +#' @param y.title Y-axis label +#' +#' @export +#' @examples +#' IOH_plot_ly_default("Example plot","x-axis","y-axis") +IOH_plot_ly_default <- function(title = NULL, x.title = NULL, y.title = NULL) { + plot_ly() %>% + layout(title = list(text = title, + font = list(size = getOption("IOHanalyzer.title_fontsize", default = 16), + family = 'Old Standard TT, serif')), + autosize = T, hovermode = 'compare', + legend = legend_location(), + paper_bgcolor = 'rgb(255,255,255)', + plot_bgcolor = getOption('IOHanalyzer.bgcolor'), + font = list(size = getOption("IOHanalyzer.label_fontsize", default = 16), + family = 'Old Standard TT, serif'), + autosize = T, + showlegend = T, + xaxis = list( + # title = list(text = x.title, font = f3), + title = x.title, + gridcolor = getOption('IOHanalyzer.gridcolor'), + showgrid = TRUE, + showline = FALSE, + showticklabels = TRUE, + tickcolor = getOption('IOHanalyzer.tickcolor'), + ticks = 'outside', + ticklen = 9, + tickfont = f3(), + exponentformat = 'e', + zeroline = F), + yaxis = list( + # title = list(text = y.title, font = f3), + title = y.title, + gridcolor = getOption('IOHanalyzer.gridcolor'), + showgrid = TRUE, + showline = FALSE, + showticklabels = TRUE, + tickcolor = getOption('IOHanalyzer.tickcolor'), + ticks = 'outside', + ticklen = 9, + tickfont = f3(), + exponentformat = 'e', + zeroline = F)) +} + +t <- theme_grey() + + theme(text = element_text(size = 15), + plot.title = element_text(hjust = 0.5) + # legend.position = c(0.15, 0.7), # legend position + # legend.key = element_blank(), # no small box around symbol + # legend.key.size = unit(1.3, "line"), # bigger symbols + # legend.background = element_rect(color = alpha("black", 0.5), + # fill = alpha('blue', 0.0), + # size = 1, + # linetype = "solid") + ) +theme_set(t) + +gg_beanplot <- function(mapping, data, p = NULL, width = 3, fill = 'grey', + colour = 'grey', alpha = 1, kernel = 'gaussian', bw = 'SJ', + draw_quantiles = NULL, trim = TRUE, na.rm = FALSE, + show.legend = NA, point.shape = 20, show.sample = T, + show.violin = T, linetype = 'solid') { + + set.seed(42) + x <- as.character(mapping$x) + y <- as.character(mapping$y) + df <- data[, c(x, y)] %>% rename_(.dots = c('x' = x, 'y' = y)) + + if (!is.numeric(df$x)) + df$x <- tryCatch(as.numeric(df$x), # in case x is a factor... + warning = function(w) return(match(x, as.factor(x)))) + + if (is.null(p)) + p <- ggplot() + + if (show.violin) + p <- p + geom_violin(data = data, mapping = mapping, trim = trim, + draw_quantiles = draw_quantiles, bw = bw, + kernel = kernel, scale = 'width', + width = width, alpha = alpha) + if (show.sample) + p <- p + geom_jitter(data = df, aes(x, y), height = 0, width = width / 2, + alpha = 0.45, shape = point.shape, size = 3.5) + # geom_segment(aes(x = x - width / 2.2, xend = x + width / 2.2, y = y, yend = y), + # df, col = 'black', size = 0.2, alpha = 0.3, linetype = linetype) + p +} + + +Set1 <- function(n) colorspace::sequential_hcl(n, h = c(360, 40), c. = c(100, NA, 90), l = c(28, 90), + power = c(1, 1.1), gamma = NULL, fixup = TRUE, + alpha = 1)#, palette = NULL, rev = FALSE) + +Set2 <- function(n) colorspace::sequential_hcl(n, c(261, 26), c. = c(50, NA, 70), l = c(54, 77), + power = c(0.5, NA), gamma = NULL, + fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) + +Set3 <- function(n) colorspace::sequential_hcl(n, c(-88, 59), c. = c(60, 75, 55), l = c(40, 90), + power = c(0.1, 1.2), gamma = NULL, + fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) + +IOHanalyzer_env$used_colorscheme <- Set2 +IOHanalyzer_env$id_colors <- NULL + +#' Set the colorScheme of the IOHanalyzer plots +#' +#' @param schemename Three default colorschemes are implemented: +#' \itemize{ +#' \item Default +#' \item Variant 1 +#' \item Variant 2 +#' \item Variant 3 +#' } +#' And it is also possible to select "Custom", which allows uploading of a custom set of colors +#' @param ids The names of the algorithms (or custom ids, see `change_id`) for which to set the colors +#' @param path The path to the file containing the colors to use. Only used if +#' schemename is "Custom" +#' +#' @export +#' +#' @examples +#' set_color_scheme("Default", get_algId(dsl)) +set_color_scheme <- function(schemename, ids, path = NULL){ + if (schemename == "Custom" && !is.null(path)) { + dt <- fread(path, header = T) + if (any(colnames(dt) != c("ids", "colors", "linestyles"))) { + warning("Incorrect file-format has been uploaded.") + } + else + IOHanalyzer_env$id_colors <- dt + return() + } + else { + if (schemename == "Default") { + options(IOHanalyzer.max_colors = 2) + } + else { + if (schemename == "Variant 1") IOHanalyzer_env$used_colorscheme <- Set1 + else if (schemename == "Variant 2") IOHanalyzer_env$used_colorscheme <- Set2 + else if (schemename == "Variant 3") IOHanalyzer_env$used_colorscheme <- Set3 + options(IOHanalyzer.max_colors = length(ids)) + } + create_color_scheme(ids) + } +} + +#' Get datatable of current color (and linestyle) scheme to file +#' +#' @return data.table object with 3 columns: ids, colors, linestyles +#' @export +#' @examples +#' get_color_scheme_dt() +get_color_scheme_dt <- function(){ + return(IOHanalyzer_env$id_colors) +} + +#' Helper function to create default color scheme +#' +#' @noRd +create_color_scheme <- function(ids) { + if (length(ids) == 0) { + return(NULL) + } + colors <- color_palettes(length(ids)) + linestyles <- rep(c("solid", "dash", "dot"), ceiling(length(colors)/3))[1:length(colors)] + IOHanalyzer_env$id_colors <- data.table(ids, colors, linestyles) +} + +#' Get colors according to the current colorScheme of the IOHanalyzer +#' +#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get colors +#' +#' @export +#' +#' @examples +#' get_color_scheme(get_algId(dsl)) +get_color_scheme <- function(ids_in){ + if (is.null(IOHanalyzer_env$id_colors)) + create_color_scheme(ids_in) + cdt <- IOHanalyzer_env$id_colors + colors <- subset(cdt, ids %in% ids_in)[['colors']] + if (is.null(colors) || length(colors) != length(ids_in)) { + return(color_palettes(length(ids_in))) + } + names(colors) <- ids_in + return(colors) +} + + +#' Get line styles according to the current styleScheme of the IOHanalyzer +#' +#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get linestyles +#' +#' @export +#' +#' @examples +#' get_line_style(get_algId(dsl)) +get_line_style <- function(ids_in){ + if (is.null(IOHanalyzer_env$id_colors)) + create_color_scheme(ids_in) + cdt <- IOHanalyzer_env$id_colors + linestyles <- subset(cdt, ids %in% ids_in)[['linestyles']] + if (is.null(linestyles) || length(linestyles) != length(ids_in)) { + return(rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), + ceiling(length(ids_in)/3))[1:length(ids_in)]) + } + return(linestyles) +} + +# TODO: incoporate more colors +color_palettes <- function(ncolor) { + # TODO: FIX IT! + max_colors <- getOption("IOHanalyzer.max_colors", 2) + if (ncolor <= max_colors) return(IOHanalyzer_env$used_colorscheme(ncolor)) + + brewer <- function(n) { + colors <- RColorBrewer::brewer.pal(n, 'Spectral') + colors[colors == "#FFFFBF"] <- "#B2B285" + colors[colors == "#E6F598"] <- "#86FF33" + colors[colors == '#FEE08B'] <- "#FFFF33" + colors + } + + color_fcts <- c(colorRamps::primary.colors, IOHanalyzer_env$used_colorscheme) + + n <- min(11, ncolor) + colors <- brewer(n) + ncolor <- ncolor - n + + i <- 1 + while (ncolor > 0) { + n <- min(8, ncolor) + if (i > length(color_fcts)) { + colors <- c(colors, colorRamps::primary.colors(ncolor)) + break + } else { + colors <- c(colors, color_fcts[[i]](n)) + ncolor <- ncolor - n + } + i <- i + 1 + } + colors +} + +#' Save plotly figure in multiple format +#' +#' NOTE: This function requires orca to be installed +#' +#' @param p plotly object. The plot to be saved +#' @param file String. The name of the figure file, with the extension of the required file-format +#' @param width Optional. Width of the figure +#' @param height Optional. Height of the figure +#' @param ... Additional arguments for orca +#' @export +#' @examples +#' \dontrun{ +#' p <- Plot.RT.Single_Func(dsl[1]) +#' save_plotly(p, 'example_file.png') +#' } +save_plotly <- function(p, file, width = NULL, height = NULL, ...) { + + if (!requireNamespace("withr", quietly = TRUE)) { + stop("Package \"withr\" needed for this function to work. Please install it.", + call. = FALSE) + } + des <- dirname(file) + file <- basename(file) + format <- tools::file_ext(file) + + pwd <- tempdir() + if (is.null(width)) width <- getOption("IOHanalyzer.figure_width", default = NULL) + if (is.null(height)) height <- getOption("IOHanalyzer.figure_height", default = NULL) + + if (format %in% c('svg', 'png', 'jpeg', 'webp', 'pdf', 'eps')) + withr::with_dir(pwd, orca(p, file, format = format, width = width, height = height, ...)) + else { + file_svg <- paste0(file, '.svg') + withr::with_dir(pwd, orca(p, file_svg, format = 'svg', width = width, height = height, ...)) + invisible( + system( + paste( + 'inkscape', file.path(pwd,file_svg), + paste0('--export-', format, ' ', file.path(pwd, file)) + ), + intern = T + ) + ) + } + file.rename(file.path(pwd, file), file.path(des, file)) +} diff --git a/R/plotDataSetList.R b/R/plotDataSetList.R index 53b50005..0360fa6e 100644 --- a/R/plotDataSetList.R +++ b/R/plotDataSetList.R @@ -1,1417 +1,1417 @@ -symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", - "triangle-up-open", "triangle-down-open") - -get_legends <- function(dsList) { - N <- length(dsList) - legends <- sapply(dsList, function(d) get_id(d)) - - if (length(unique(legends)) < N) { - funcId <- sapply(dsList, function(d) attr(d, 'funcId')) - if (length(unique(funcId)) > 1) - legends <- paste0(legends, '-F', funcId) - } - - if (length(unique(legends)) < N) { - DIM <- sapply(dsList, function(d) attr(d, 'DIM')) - if (length(unique(DIM)) > 1) - legends <- paste0(legends, '-', DIM, 'D') - } - legends -} - -insert_best_parts <- function(from_data, to_data, best_is_min) { - if (all(is.na(from_data))) - to_data - else - if (best_is_min) - pmin(from_data, to_data, na.rm = T) - else - pmax(from_data, to_data, na.rm = T) -} - -generate_rbga <- function(color,a){ - paste0('rgba(', paste0(color, collapse = ','), ',', a,')') -} - -grad_functions <- c( - scaled_edges = function(count, amount, intensity){ - scale <- (intensity + 1)/2 - color_end <- floor(scale*amount*2) - if (count < color_end) - 1/color_end - else - 0 - } - , - fixed_edges = function(count, amount, intensity) { - scale <- (intensity + 1) / 2 - color_center <- floor(scale * amount) + 1 - if (count <= color_center) - 1 / (2 * color_center) - else - 1 / (2 * (amount - color_center)) - } -) - -#S3 generics -# TODO: decide which parameters need to be in the generics - -#' Plot lineplot of the ERTs of a DataSetList -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param Fstart The starting function value. -#' @param Fstop The final function value. -#' @param show.ERT Whether or not to show the ERT-values -#' @param show.CI Whether or not to show the standard deviations -#' @param show.mean Whether or not to show the mean hitting times -#' @param show.median Whether or not to show the median hitting times -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) -#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' -#' @param includeOpts Whether or not to include all best points reached by each algorithm -#' @param p Existing plot to which to add the current data -#' @return A plot of ERT-values of the DataSetList -#' @export -#' @examples -#' Plot.RT.Single_Func(subset(dsl, funcId == 1)) -Plot.RT.Single_Func <- function(dsList, Fstart = NULL, Fstop = NULL, - show.ERT = T, show.CI = F, show.mean = F, - show.median = F, backend = NULL, - scale.xlog = F, scale.ylog = F, scale.reverse = F, - includeOpts = F, p = NULL) - UseMethod("Plot.RT.Single_Func", dsList) -#' Plot lineplot of the expected function values of a DataSetList -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param RTstart The starting runtime value. -#' @param RTstop The final runtime value. -#' @param show.CI Whether or not to show the standard deviations -#' @param show.mean Whether or not to show the mean runtimes -#' @param show.median Whether or not to show the median runtimes -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) -#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' -#' -#' @return A plot of ERT-values of the DataSetList -#' @export -#' @examples -#' Plot.FV.Single_Func(subset(dsl, funcId == 1)) -Plot.FV.Single_Func <- function(dsList, RTstart = NULL, RTstop = NULL, show.CI = F, show.mean = T, - show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F) UseMethod("Plot.FV.Single_Func", dsList) -#' Plot probablity mass function of the runtimes of a DataSetList at a certain target function value -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value. -#' @param show.sample Whether or not to show the individual runtime samples -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' -#' -#' @return A plot of the probablity mass function of the runtimes at a the -#' target function value of the DataSetList -#' @export -#' @examples -#' Plot.RT.PMF(subset(dsl, funcId == 1), 14) -Plot.RT.PMF <- function(dsList, ftarget, show.sample = F, scale.ylog = F, backend = NULL) - UseMethod("Plot.RT.PMF", dsList) -#' Plot histograms of the runtimes of a DataSetList at a certain target function value -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value. -#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either -#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. -#' @param use.equal.bins Whether to determine one bin size for all plots or have individual -#' bin sizes for each algorithm -#' -#' @return A plot of the histograms of the runtimes at a the -#' target function value of the DataSetList -#' @export -#' @examples -#' Plot.RT.Histogram(subset(dsl, funcId == 1), 14) -Plot.RT.Histogram <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F) - UseMethod("Plot.RT.Histogram", dsList) -#' Plot the empirical cumulative distriburtion as a function of the running times of -#' a DataSetList at certain target function values -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftargets The target function values -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the running times of the DataSetList at the target function values -#' @export -#' @examples -#' Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) -Plot.RT.ECDF_Per_Target <- function(dsList, ftargets, scale.xlog = F) - UseMethod("Plot.RT.ECDF_Per_Target", dsList) -#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of -#' a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param fstart The starting function value -#' @param fstop The final function value -#' @param fstep The spacing between starting and final function values -#' @param show.per_target Whether or not to show the individual ECDF-curves for each target -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the running times of the DataSetList -#' @export -#' @examples -#' Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) -Plot.RT.ECDF_Single_Func <- function(dsList, fstart = NULL, fstop = NULL, - fstep = NULL, show.per_target = F, - scale.xlog = F) UseMethod("Plot.RT.ECDF_Single_Func", dsList) -#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param fstart The starting function value -#' @param fstop The final function value -#' @param fstep The spacing between starting and final function values -#' @param fval_formatter Function to format the function-value labels -#' -#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList -#' @export -#' @examples -#' Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) -Plot.RT.ECDF_AUC <- function(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, - fval_formatter = as.integer) UseMethod("Plot.RT.ECDF_AUC", dsList) -#' Plot probablity density function of the function values of a DataSetList at -#' a certain target runtime -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param runtime The target runtime -#' @param show.sample Whether or not to show the individual function value samples -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' -#' @return A plot of the probablity density function of the runtimes at a the -#' target function value of the DataSetList -#' @export -#' @examples -#' Plot.FV.PDF(subset(dsl, funcId == 1), 100) -Plot.FV.PDF <- function(dsList, runtime, show.sample = F, scale.ylog = F) - UseMethod("Plot.FV.PDF", dsList) -#' Plot histograms of the function values of a DataSetList at a certain target runtime -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param runtime The target runtime -#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either -#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. -#' @param use.equal.bins Whether to determine one bin size for all plots or have individual -#' bin sizes for each algorithm -#' -#' @return A plot of the histograms of the function values at a the -#' target runtime of the DataSetList -#' @export -#' @examples -#' Plot.FV.Histogram(subset(dsl, funcId == 1), 100) -Plot.FV.Histogram <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F) - UseMethod("Plot.FV.Histogram", dsList) -#' Plot the empirical cumulative distriburtion as a function of the target values of -#' a DataSetList at certain target runtimes -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param runtimes The target runtimes -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the fucntion values of the DataSetList at the target runtimes -#' @export -#' @examples -#' Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) -Plot.FV.ECDF_Per_Target <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F) - UseMethod("Plot.FV.ECDF_Per_Target", dsList) -#' Plot the aggregated empirical cumulative distriburtion as a function of the function values of -#' a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param rt_min The starting runtime -#' @param rt_max The final runtime -#' @param rt_step The spacing between starting and final runtimes -#' @param show.per_target Whether or not to show the individual ECDF-curves for each runtime -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the function values of the DataSetList -#' @export -#' @examples -#' Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) -Plot.FV.ECDF_Single_Func <- function(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL, scale.xlog = F, - show.per_target = F, scale.reverse = F) - UseMethod("Plot.FV.ECDF_Single_Func", dsList) -#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param rt_min The starting runtime -#' @param rt_max The final runtime -#' @param rt_step The spacing between starting and final runtimes -#' -#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList -#' @export -#' @examples -#' Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) -Plot.FV.ECDF_AUC <- function(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL) UseMethod("Plot.FV.ECDF_AUC", dsList) -#' Plot the parameter values recorded in a DataSetList (aligned by funcion value) -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param f_min The starting function value. -#' @param f_max The final function value. -#' @param show.mean Whether or not to show the mean parameter values -#' @param show.median Whether or not to show the median parameter values -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param algids Which algorithms from dsList to use -#' @param par_name Which parameters to create plots for; set to NULL to use all -#' parameters found in dsList. -#' @param show.CI Whether or not to show the standard deviation -#' -#' @return A plot of for every recorded parameter in the DataSetList -#' @export -#' @examples -#' Plot.RT.Parameters(subset(dsl, funcId == 1)) -Plot.RT.Parameters <- function(dsList, f_min = NULL, f_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) UseMethod("Plot.RT.Parameters", dsList) -#' Plot the parameter values recorded in a DataSetList (aligned by budget) -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param rt_min The starting budget value. -#' @param rt_max The final budget value. -#' @param show.mean Whether or not to show the mean parameter values -#' @param show.median Whether or not to show the median parameter values -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param algids Which algorithms from dsList to use -#' @param par_name Which parameters to create plots for; set to NULL to use all -#' parameters found in dsList. -#' @param show.CI Whether or not to show the standard deviation -#' -#' @return A plot of for every recorded parameter in the DataSetList -#' @export -#' @examples -#' Plot.FV.Parameters(subset(dsl, funcId == 1)) -Plot.FV.Parameters <- function(dsList, rt_min = NULL, rt_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) UseMethod("Plot.FV.Parameters", dsList) -#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of -#' a DataSetList. Aggregated over multiple functions or dimensions. -#' -#' @param dsList A DataSetList. -#' @param targets The target function values. Specified in a data.frame, as can be generated -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' by the function 'get_ECDF_targets' -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the running times of the DataSetList -#' @export -#' @examples -#' Plot.RT.ECDF_Multi_Func(dsl) -Plot.RT.ECDF_Multi_Func <- function(dsList, targets = NULL, scale.xlog = F) - UseMethod("Plot.RT.ECDF_Multi_Func", dsList) -#' Plot ERT-plots for multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) -#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. -#' -#' @return A plot of ERT-values of the DataSetList -#' @export -#' @examples -#' Plot.RT.Multi_Func(dsl) -Plot.RT.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, scale.reverse = F, - backend = NULL) - UseMethod("Plot.RT.Multi_Func", dsList) -#' Plot ERT-based comparison over multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' -#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param maximize Wheter or not to the data is of a maximization problem -#' @param targets Custom list of function-value targets, one for each function or dimension. -#' @param use_rank Wheter to use a ranking system. If False, the actual ERT-values will be used. -#' @param erts Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function -#' of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating -#' this plot. -#' @param inf.action How to handle infinite ERTs ('overlap' or 'jitter') -#' @return A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList -#' @export -#' @examples -#' Plot.RT.Aggregated(dsl) -Plot.RT.Aggregated <- function(dsList, aggr_on = 'funcId', targets = NULL, - plot_mode = 'radar', use_rank = F, scale.ylog = T, maximize = T, - erts = NULL, inf.action = 'overlap') - UseMethod("Plot.RT.Aggregated", dsList) -#' Plot expected function value-based comparison over multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' -#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param runtimes Custom list of function-value targets, one for each function or dimension. -#' @param use_rank Wheter to use a ranking system. If False, the actual expected function- -#' values will be used. -#' @param fvs Pre-calculated expected function-values for the provided runtimes Created by the -#' max_ERTs function of DataSetList. Can be provided to prevent needless computation -#' in recalculating ERTs when recreating this plot. -#' -#' @return A plot of expected function value-based comparison on the provided functions -#' or dimensions of the DataSetList -#' @export -#' @examples -#' Plot.FV.Aggregated(dsl) -Plot.FV.Aggregated <- function(dsList, aggr_on = 'funcId', runtimes = NULL, plot_mode = 'radar', - use_rank = F, scale.ylog = T, fvs = NULL) - UseMethod("Plot.FV.Aggregated", dsList) - -#' Plot FV-plots for multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. -#' -#' @return A plot of Function-values of the DataSetList -#' @export -#' @examples -#' Plot.FV.Multi_Func(dsl) -Plot.FV.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, backend = NULL) - UseMethod("Plot.FV.Multi_Func", dsList) - -#' Plot a heatmap showing the statistically different algorithms -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value to use -#' @param alpha The cutoff for statistical significance -#' @param bootstrap.size The amound of bootstrapped samples used -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' -#' @return A heatmap showing the statistical significance between algorithms -#' @export -#' @examples -#' Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) -Plot.Stats.Significance_Heatmap <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, - which = 'by_FV') - UseMethod("Plot.Stats.Significance_Heatmap", dsList) - -#' Plot a network graph showing the statistically different algorithms -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value to use -#' @param alpha The cutoff for statistical significance -#' @param bootstrap.size The amound of bootstrapped samples used -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' -#' @return A graph showing the statistical significance between algorithms -#' @export -#' @examples -#' Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) -Plot.Stats.Significance_Graph <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, - which = 'by_FV') - UseMethod("Plot.Stats.Significance_Graph", dsList) - -#' Create a candlestick plot of Glicko2-rankings -#' -#' @param dsList A DataSetList -#' @param nr_rounds The number of rounds in the tournament -#' @param glicko2_rank_df Optional. Dataframe containing the glicko2 rating to avoid needless recalculation. -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' @param target_dt Optional: data table containing the targets for each function and dimension -#' -#' @export -#' @examples -#' Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) -Plot.Stats.Glicko2_Candlestick <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, - which = 'by_FV', target_dt = NULL) - UseMethod("Plot.Stats.Glicko2_Candlestick", dsList) - - -##Implementations - -#' @rdname Plot.RT.Single_Func -#' @export -Plot.RT.Single_Func.DataSetList <- function(dsList, Fstart = NULL, Fstop = NULL, - show.ERT = T, show.CI = T, show.mean = F, - show.median = F, backend = NULL, - scale.xlog = F, scale.ylog = F, - scale.reverse = F, includeOpts = F, p = NULL) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - if (backend == 'plotly') { - data <- generate_data.Single_Function(dsList, Fstart, Fstop, scale.xlog, 'by_RT', includeOpts) - - y_attrs <- c() - if (show.ERT) y_attrs <- c(y_attrs, 'ERT') - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'ID', show.legend = show_legend, - scale.ylog = scale.ylog, p = p, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - show_legend <- F - } - if (show.CI) { - p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - } - } - # } else if (backend == 'ggplot2') { - # dt[, 'group' := paste(algId, funcId, DIM, sep = '-')] - # p <- ggplot(data = dt, aes(group = `group`, colour = `group`)) - # - # if (show.CI) p <- p + geom_ribbon(aes(target, ymin = lower, ymax = upper, fill = `group`), - # alpha = 0.2, colour = NA) - # if (show.ERT) p <- p + geom_line(aes(target, ERT), size = 1.2) - # if (show.mean) p <- p + geom_line(aes(target, mean), linetype = 'dashed') - # if (show.median) p <- p + geom_line(aes(target, median), linetype = 'dotted') - # - # p <- p + - # scale_color_manual(values = colors) + - # scale_fill_manual(values = colors) - # } - return(p) -} - -#' @rdname Plot.FV.Single_Func -#' @export -Plot.FV.Single_Func.DataSetList <- function(dsList, RTstart = NULL, RTstop = NULL, - show.CI = F, - show.mean = T, show.median = F, - backend = NULL, - scale.xlog = F, scale.ylog = F, - scale.reverse = F) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - if (backend == 'plotly') { - data <- generate_data.Single_Function(dsList, RTstart, RTstop, scale.xlog, 'by_FV') - - y_attrs <- c() - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'ID', show.legend = show_legend, - scale.ylog = scale.ylog, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - show_legend <- F - } - else - p <- NULL - if (show.CI) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - } - - } - # } else if (backend == 'ggplot2') { - # fce[, 'group' := paste(algId, funcId, DIM, sep = '-')] - # p <- ggplot(data = fce, aes(group = `group`, colour = `group`)) - # - # if (show.mean) p <- p + geom_line(aes(runtime, mean), linetype = 'dashed') - # if (show.median) p <- p + geom_line(aes(runtime, median), linetype = 'dotted') - # - # p <- p + - # scale_color_manual(values = colors) + - # scale_fill_manual(values = colors) - # - # #TODO: add individual run etc - # } - return(p) -} - -#' @rdname Plot.RT.PMF -#' @export -Plot.RT.PMF.DataSetList <- function(dsList, ftarget, show.sample = F, - scale.ylog = F, backend = NULL){ - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - data <- generate_data.PMF(dsList, ftarget, 'by_RT') - - plot_general_data(data, 'ID', 'RT', scale.ylog = scale.ylog, - x_title = "Algorithm", y_title = "Function Evaluations") -} - -#' @rdname Plot.RT.Histogram -#' @export -Plot.RT.Histogram.DataSetList <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F){ - if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { - warning("Invalid dataset uploaded. Please ensure the datasetlist contains data - from only one function and only one dimension.") - return(NULL) - } - data <- generate_data.hist(dsList, ftarget, use.equal.bins, 'by_RT') - - subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL - plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Function Evaluations", - y_title = "Runs") -} - -#' @rdname Plot.RT.ECDF_Per_Target -#' @export -Plot.RT.ECDF_Per_Target.DataSetList <- function(dsList, ftargets, scale.xlog = F){ - req(length(ftargets) != 0) - data <- generate_data.ECDF(dsList, ftargets, scale.xlog) - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of runs", scale.xlog = scale.xlog, show.legend = T) -} - -#' @rdname Plot.RT.ECDF_Single_Func -#' @export -Plot.RT.ECDF_Single_Func.DataSetList <- function(dsList, fstart = NULL, fstop = NULL, - fstep = NULL, show.per_target = F, - scale.xlog = F) { - - targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep) - req(targets) - - data <- generate_data.ECDF(dsList, targets, scale.xlog) - - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of (run, target) pairs", - scale.xlog = scale.xlog, show.legend = T) -} - -#' @rdname Plot.RT.ECDF_AUC -#' @export -Plot.RT.ECDF_AUC.DataSetList <- function(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, - fval_formatter = as.integer) { - - targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep, length.out = 10) - req(targets) - - data <- generate_data.AUC(dsList, targets, multiple_x = TRUE) - - plot_general_data(data, 'x', 'auc', 'radar') -} - -#' @rdname Plot.FV.PDF -#' @export -Plot.FV.PDF.DataSetList <- function(dsList, runtime, show.sample = F, scale.ylog = F){ - - data <- generate_data.PMF(dsList, runtime, 'by_FV') - - plot_general_data(data, 'ID', 'f(x)', scale.ylog = scale.ylog, - x_title = "Algorithm", y_title = "Target Value") -} - -#' @rdname Plot.FV.Histogram -#' @export -Plot.FV.Histogram.DataSetList <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F){ - if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { - warning("Invalid dataset uploaded. Please ensure the datasetlist contains data - from only one function and only one dimension.") - return(NULL) - } - data <- generate_data.hist(dsList, runtime, use.equal.bins, 'by_FV') - - subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL - plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Target Values", - y_title = "Runs") -} - -#' @rdname Plot.FV.ECDF_Per_Target -#' @export -Plot.FV.ECDF_Per_Target.DataSetList <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F){ - #TODO: Fvals in legend need to be formatted properly - runtimes <- runtimes[!is.na(runtimes)] - req(length(runtimes) != 0) - - data <- generate_data.ECDF(dsList, runtimes, scale.xlog, which = 'by_FV') - - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of runs", scale.xlog = scale.xlog, - show.legend = T, - scale.reverse = scale.reverse) -} - -#' @rdname Plot.FV.ECDF_Single_Func -#' @export -Plot.FV.ECDF_Single_Func.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL, scale.xlog = F, - show.per_target = F, scale.reverse = F){ - - targets <- seq_RT(get_funvals(dsList), rt_min, rt_max, rt_step) - req(targets) - data <- generate_data.ECDF(dsList, targets, scale.xlog, which = 'by_FV') - - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of (run, target) pairs", - scale.xlog = scale.xlog, - scale.reverse = scale.reverse, show.legend = T) -} - -#' @rdname Plot.FV.ECDF_AUC -#' @export -Plot.FV.ECDF_AUC.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, rt_step = NULL) { - targets <- seq_RT(get_runtimes(dsList), rt_min, rt_max, rt_step, length.out = 10) - req(targets) - data <- generate_data.AUC(dsList, targets, which = 'by_FV', multiple_x = TRUE) - - plot_general_data(data, 'x', 'auc', 'radar') -} - -#' @rdname Plot.RT.Parameters -#' @export -Plot.RT.Parameters.DataSetList <- function(dsList, f_min = NULL, f_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) { - data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_FV') - - y_attrs <- c() - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'ID', show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - show_legend <- F - } - else - p <- NULL - if (show.CI) { - p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - } - p -} - - -#' @rdname Plot.FV.Parameters -#' @export -Plot.FV.Parameters.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) { - data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_RT') - - y_attrs <- c() - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'ID', show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - show_legend <- F - } - else - p <- NULL - if (show.CI) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - } - p -} - -#' @rdname Plot.RT.ECDF_Multi_Func -#' @export -Plot.RT.ECDF_Multi_Func.DataSetList <- function(dsList, targets = NULL, - scale.xlog = F) { - if (is.null(targets) || !is.data.table(targets)) { - targets <- get_ECDF_targets(dsList) - } - - data <- generate_data.ECDF(dsList, targets, scale.xlog) - - plot_general_data(data, 'x', 'mean', 'line', - scale.xlog = scale.xlog, - x_title = "Function Evaluations", - y_title = "Proportion of (run, target, ...) pairs", - show.legend = T) -} - -#' @rdname Plot.RT.Multi_Func -#' @export -Plot.RT.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, - scale.ylog = F, - scale.reverse = F, - backend = NULL) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - data <- rbindlist(lapply(get_funcId(dsList), function(fid) { - generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, - which = 'by_RT') - })) - - plot_general_data(data, x_attr = 'target', y_attr = 'ERT', - subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, - scale.ylog = scale.ylog, x_title = 'Best-so-far f(x)', - y_title = 'ERT', show.legend = T, - scale.reverse = scale.reverse) -} - -#' @rdname Plot.FV.Multi_Func -#' @export -Plot.FV.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, - scale.ylog = F, - backend = NULL) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - data <- rbindlist(lapply(get_funcId(dsList), function(fid) { - generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, - which = 'by_FV') - })) - - plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, - scale.ylog = scale.ylog, x_title = 'Runtime', - y_title = 'Best-so-far f(x)', show.legend = T) -} - -#' @rdname Plot.RT.Aggregated -#' @export -Plot.RT.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', targets = NULL, - plot_mode = 'radar', use_rank = F, - scale.ylog = T, maximize = T, erts = NULL, - inf.action = 'overlap') { - targets <- get_target_dt(dsList) - data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) - y_attr <- if (use_rank) 'rank' else 'value' - y_title <- if (use_rank) 'Rank' else 'ERT' - plot_general_data(data, type = plot_mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = scale.ylog, - inf.action = inf.action) -} - -#' @rdname Plot.FV.Aggregated -#' @export -Plot.FV.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', runtimes = NULL, - plot_mode = 'radar', use_rank = F, - scale.ylog = T, fvs = NULL){ - targets <- get_target_dt(dsList, which = 'by_FV') - data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, which = 'by_FV') - y_attr <- if (use_rank) 'rank' else 'value' - y_title <- if (use_rank) 'Rank' else 'Best-so-far f(x)' - plot_general_data(data, type = plot_mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = scale.ylog) -} - -#' @rdname Plot.Stats.Significance_Heatmap -#' @export -Plot.Stats.Significance_Heatmap.DataSetList <- function(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = 'by_FV'){ - if (length(get_dim(dsList)) != 1 || - length(get_funcId(dsList)) != 1 || - length(get_id(dsList)) < 2) - return(NULL) - - p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) - y <- p_matrix <= alpha - colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), - col = c('blue', 'blue', 'white', 'white', 'red', 'red') - ) - heatmap <- y - t(y) - - p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', - xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) - p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), - xaxis = list(tickangle = 45)) - p -} - -#' Helper function for Plot.Stats.Significance_Graph -#' -#' @param x x -#' @param start default is 0 -#' @param direction default is 1 -#' -#' @noRd -radian.rescale <- function(x, start=0, direction=1) { - c.rotate <- function(x) (x + start) %% (2 * pi) * direction - c.rotate((2 * pi * (x - min(x)) / (max(x) - min(x)))) -} - -#' @rdname Plot.Stats.Significance_Graph -#' @export -Plot.Stats.Significance_Graph.DataSetList <- function(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = 'by_FV'){ - if (!requireNamespace("igraph", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || length(get_id(dsList)) < 2) { - return(NULL) - } - p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) - g <- igraph::graph_from_adjacency_matrix(p_matrix <= alpha, mode = 'directed', diag = F) - lab.locs <- radian.rescale(x = 1:nrow(p_matrix), direction = -1, start = 0) - - igraph::plot.igraph(g, layout = igraph::layout.circle(g), vertex.size = 10, edge.arrow.size = 1, - vertex.label.color = 'black', - vertex.label.dist = 2, - vertex.label.cex = 1, - vertex.label.degree = lab.locs) -} - -#' @rdname Plot.Stats.Glicko2_Candlestick -#' @export -Plot.Stats.Glicko2_Candlestick.DataSetList <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, - which = 'by_FV', target_dt = NULL) { - df <- glicko2_rank_df - - if (is.null(df)) { - df <- glicko2_ranking(dsList, nr_rounds, which, target_dt = target_dt)$ratings - Ids <- df$Player$ID - } - else{ - Ids <- df$ID - } - p <- IOH_plot_ly_default(title = "Glicko2-rating", - x.title = "ID", - y.title = "Rating") - df$Rating %<>% as.numeric - df$Deviation %<>% as.numeric - high <- df$Rating + 3*df$Deviation - low <- df$Rating - 3*df$Deviation - open <- df$Rating + df$Deviation - close <- df$Rating - df$Deviation - - N <- length(df$Rating) - colors <- get_color_scheme(Ids) - if (length(colors != N)) { - colors <- get_color_scheme(get_id(dsList)) - } - - for (i in seq(N)) { - # rgba_str <- paste0('rgba(', paste0(col2rgb(colors[i]), collapse = ','), ',0.52)') - color <- list(line = list(color = colors[[i]])) - p %<>% add_trace(type = "candlestick", x = Ids[[i]], open = open[[i]], close = close[[i]], - high = high[[i]], low = low[[i]], legendgroup = Ids[[i]], - name = Ids[[i]], increasing = color, decreasing = color, - hovertext = paste0(format(df$Rating[[i]], digits = 3), '+-', - format(df$Deviation[[i]], digits = 3)), - hoverinfo = "text") - } - p %<>% layout(xaxis = list(rangeslider = list(visible = F))) - p -} - -### _______________________ Rewritten plotting function ____________________ ### - -#' Add transparancy to named list of colors -#' -#' @param colors Named list of colors (in hex-notation) -#' @param percentage The percentage of opacity. 0 is fully transparant, 1 is fully opaque -#' -#' @noRd -add_transparancy <- function(colors, percentage){ - hex_val <- format(as.hexmode(as.integer(255 * percentage)), upper.case = T, width = 2) - sapply(colors, function(col) { col <- paste0('#', substr(col, 2, 7), hex_val) }) -} - -#' General function for plotting within IOHanalyzer -#' -#' @param df The dataframe containing the data to plot. It should contain at least two columns: -#' 'x_attr' and 'y_attr' -#' @param x_attr The column to specify the x_axis. Default is 'algId' -#' @param legend_attr Default is 'algId' This is also used for the selection of colorschemes -#' @param y_attr The column to specify the y_axis -#' @param type The type of plot to use. Currently available: 'violin', 'line', 'radar', -#' 'bar', hist' and 'ribbon' -#' @param upper_attr When using ribbon-plot, this can be used to create a shaded area. -#' Only works in combination with`lower_attr` and `type` == 'ribbon' -#' @param lower_attr When using ribbon-plot, this can be used to create a shaded area. -#' Only works in combination with`upper_attr` and `type` == 'ribbon' -#' @param subplot_attr Which attribute of the dataframe to use for creating subplots -#' @param scale.xlog Logarithmic scaling of x-axis -#' @param scale.ylog Logarithmic scaling of y-axis -#' @param scale.reverse Decreasing or increasing x-axis -#' @param x_title Title of x-axis. Defaults to x_attr -#' @param y_title Title of x-axis. Defaults to x_attr -#' @param plot_title Title of x-axis. Defaults to no title -#' @param p A previously existing plot on which to add traces. If NULL, a new canvas is created -#' @param show.legend Whether or not to include a legend -#' @param inf.action How to deal with infinite values. Can be 'none', 'overlap' or 'jitter' -#' @param ... Additional parameters for the add_trace function -#' -#' @export -plot_general_data <- function(df, x_attr = 'ID', y_attr = 'vals', type = 'violin', - legend_attr = 'ID', scale.xlog = F, scale.ylog = F, - scale.reverse = F, p = NULL, x_title = NULL, - y_title = NULL, plot_title = NULL, upper_attr = NULL, - lower_attr = NULL, subplot_attr = NULL, show.legend = F, - inf.action = 'none', ...) { - - l <- x <- isinf <- y <- text <- l_orig <- NULL #Set local binding to remove warnings - - #Only allow valid plot types - if (!(type %in% c('violin', 'line', 'radar', 'hist', 'ribbon', 'line+ribbon', 'bar'))) { - stop(paste0("Provided plot type ('", type, "') is not supported")) - } - - #And valid number of y-attributes - if (length(y_attr) == 0) { - stop("At least one y-attribute is needed to plot") - } - - #Deal with subplots - if (!is.null(subplot_attr)) { - if (!subplot_attr %in% colnames(df)) { - stop("Provided subplot-attribut is not a colname of the selected data.table.") - } - colnames(df)[colnames(df) == subplot_attr] <- "subplot_attr" - attrs <- unique(df[, subplot_attr]) - if (length(attrs) == 0) stop("Attempting to create subplots with fewer than 2 unique values of - `subplot_attrs`-column") - if (length(attrs) == 1) return(plot_general_data(df, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, - scale.reverse, p, x_title, y_title, attrs, upper_attr, lower_attr, - show.legend = show.legend, subplot_attr = NULL, ...)) - if (subplot_attr == legend_attr) { - df[, l := subplot_attr] - } - - #Only need one legend for the whole plot - legends_show <- rep(F, length(attrs)) - legends_show[[1]] <- show.legend - names(legends_show) <- attrs - - #Get some number of rows and columns - n_cols <- 1 + ceiling(length(attrs)/10) - n_rows <- ceiling(length(attrs) / n_cols) - - p <- lapply(seq(length(attrs)), function(idx) { - attr_val <- attrs[[idx]] - df_sub <- df[subplot_attr == attr_val] - disp_y <- idx %% n_cols == 1 - disp_x <- idx > (length(attrs) - n_cols) - x.title = if (disp_x) x_title else "" - y.title = if (disp_y) y_title else "" - - #Generate title for the subplots - if (stri_detect_regex(subplot_attr, "(?i)fun")) - sub_title <- paste0('F', attr_val) - else if (stri_detect_regex(subplot_attr, "(?i)dim")) - sub_title <- paste0('D', attr_val) - else - sub_title <- paste0(attr_val) - p <- NULL - if (stri_detect_fixed(type, '+')) { - type1 <- substr(type, 0, stri_locate_all(type, fixed = '+')[[1]][[1]] - 1) - p <- plot_general_data(df_sub, x_attr, y_attr, type1, legend_attr, scale.xlog, scale.ylog, - scale.reverse, NULL, x.title, y.title, plot_title, upper_attr, lower_attr, - show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) - type <- substr(type, stri_locate_all(type, fixed = '+')[[1]][[1]] + 1, nchar(type)) - } - plot_general_data(df_sub, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, - scale.reverse, p, x.title, y.title, plot_title, upper_attr, lower_attr, - show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) %>% - layout( - annotations = list( - text = sub_title, - font = f2, - xref = "paper", yref = "paper", align = "center", - yanchor = "bottom", - xanchor = "center", textangle = 0, - x = 0.5, y = 1, showarrow = FALSE - ) - ) - }) - - p <- subplot( - p, nrows = n_rows, titleX = T, titleY = T, - margin = c(0.02, 0.02, 0.06, 0.06) - ) %>% - layout(title = plot_title) - return(p) - } - - # Replace colnames to have easier matching - if (!x_attr %in% colnames(df) || !all(y_attr %in% colnames(df))) { - stop("Not all provided attributes are colnames of the selected data.table.") - } - colnames(df)[colnames(df) == x_attr] <- "x" - - - if (length(y_attr) == 1 && type != 'line') - colnames(df)[colnames(df) == y_attr] <- "y" - else if (type != 'line') stop("Multiple y-attrs is currently only supported for line-plots") - - if ( !is.null(upper_attr) && !is.null(lower_attr)) { - if (!upper_attr %in% colnames(df) || !lower_attr %in% colnames(df)) { - stop("Provided upper and lower attributes are not colnames of the selected data.table.") - } - colnames(df)[colnames(df) == upper_attr] <- "upper" - colnames(df)[colnames(df) == lower_attr] <- "lower" - } - - if ( x_attr != legend_attr) { - colnames(df)[colnames(df) == legend_attr] <- "l" - xs <- unique(df[['l']]) - } - else{ - xs <- unique(df[['x']]) - } - - #Get color and based on legend-attribute - colors <- get_color_scheme(xs) - if (is.null(names(colors)) || !all(names(colors) %in% xs) ) names(colors) <- xs - - xscale <- if (scale.xlog) 'log' else 'linear' - yscale <- if (scale.ylog) 'log' else 'linear' - - #If new plot is needed, create one. Store in bool to decide if axis scaling is needed. - is_new_plot <- F - if (is.null(p)) { - p <- IOH_plot_ly_default(x.title = ifelse(is.null(x_title), x_attr, x_title), - y.title = ifelse(is.null(y_title), y_attr, y_title), - title = plot_title) - is_new_plot <- T - } - - switch(type, - 'violin' = { - if (legend_attr != x_attr) { - warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") - } - #Update names to aviod numerical legend - if (is.numeric(df[['x']])) { - if (stri_detect_regex(x_attr, "(?i)fun")) - df <- df[, x := paste0('F', sprintf("%02d", x))] - else if (stri_detect_regex(x_attr, "(?i)dim")) - df <- df[, x := paste0('D', as.character(x))] - else - df <- df[, x := as.character(x)] - } - #Update color names as well, since the value changed - names(colors) <- unique(df[['x']]) - - p %<>% - add_trace(data = df, - x = ~x, y = ~y, type = 'violin', - hoveron = "points+kde", - points = F, - pointpos = 1.5, - jitter = 0, - scalemode = 'count', - meanline = list(visible = F), - name = ~x, - colors = colors, - color = ~x, - split = ~x, - line = list(color = 'black', width = 1.1), - box = list(visible = T), - spanmode = 'hard', - showlegend = show.legend, - ... - ) - if (is_new_plot) { - p %<>% layout(yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - }, - 'line' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - - # Force legend to be categorical - df[, l_orig := l] - if (is.numeric(df[['l']])) { - df[, l := paste0('A', l)] - names(colors) <- paste0('A', names(colors)) - } - - #Use linestyles to differentiate traces if only one attribute is selected to be plotted - #TODO: Combine these two options more elegantly - if (length(y_attr) == 1) { - dashes <- get_line_style(xs) - names(dashes) <- xs - colnames(df)[colnames(df) == y_attr] <- "y" - - df[, isinf := is.infinite(y)] - df[, text := as.character(round(y, getOption("IOHanalyzer.precision", 2)))] - - if (inf.action == 'overlap') { - maxval <- max(df[isinf == F, 'y']) - df[['y']][df[['isinf']]] <- 10**(ceiling(log10(maxval)) + 1) - } - else if (inf.action == 'jitter') { - #TODO: Faster way to compute this - maxval <- max(df[isinf == F, 'y']) - for (xval in unique(df[['x']])) { - tempval <- 10**(ceiling(log10(maxval)) + 1) - for (lval in unique(df[['l']])) { - temp <- df[l == lval][x == xval] - if (nrow(temp) > 0 && df[l == lval][x == xval][['isinf']]) { - df[l == lval][x == xval][['y']] <- tempval - tempval <- 1.2 * tempval - } - } - } - } - - suppressWarnings( - p %<>% - add_trace( - data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, - type = 'scatter', mode = 'lines+markers', - linetype = ~l_orig, marker = list(size = getOption('IOHanalyzer.markersize', 4)), - linetypes = dashes, - colors = colors, showlegend = show.legend, - text = ~text, line = list(width = getOption('IOHanalyzer.linewidth', 2)), - hovertemplate = '%{text}', - ... - ) - ) - if (inf.action != 'none') { - p %<>% add_trace(data = df[isinf == T], x = ~x, y = ~y, legendgroup = ~l_orig, name = ~l_orig, - type = 'scatter', mode = 'markers', color = ~l, - marker = list(symbol = 'circle-open', size = 8 + getOption('IOHanalyzer.markersize', 4)), - colors = colors, showlegend = F, text = 'Inf', hoverinfo = 'none', - ... - ) - } - - } - else { - if (inf.action != 'none') { - warning("inf.action is not yet supported for multiple y-attributes") - } - - dashes_full <- rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), - ceiling(length(y_attr)/3))[1:length(y_attr)] - names(dashes_full) <- y_attr - - for (y_atr in y_attr) { - colnames(df)[colnames(df) == y_atr] <- "y" - - #TODO: Figure out how to supress warning about 6 linetypes - dashstyle <- dashes_full[[y_atr]] - suppressWarnings( - p %<>% - add_trace( - data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, - type = 'scatter', mode = 'lines+markers', - marker = list(size = getOption('IOHanalyzer.markersize', 4)), linetype = dashstyle, - colors = colors, showlegend = show.legend, name = ~l, - text = y_atr, line = list(width = getOption('IOHanalyzer.linewidth', 2)), - ... - ) - ) - colnames(df)[colnames(df) == "y"] <- y_atr - show.legend <- F - } - } - if (is_new_plot) { - if (is.numeric(df[['x']])) - p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - else - p %<>% layout(xaxis = list(type = 'category', tickfont = f3(), ticklen = 3), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - - } - }, - 'ribbon' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - if (is.null(upper_attr) || is.null(lower_attr)) { - stop("No upper or lower attribute provided for ribbon-plot") - } - - for (name in xs) { - df_small <- df[l == name] - legend_name <- as.character(name) - rgba_str <- paste0('rgba(', paste0(col2rgb(colors[[name]]), collapse = ','), ',0.2)') - p %<>% - add_trace(data = df_small, x = ~x, y = ~upper, type = 'scatter', mode = 'lines', - line = list(color = rgba_str, width = 0), legendgroup = legend_name, - showlegend = F, name = 'upper', ...) %>% - add_trace(x = ~x, y = ~lower, type = 'scatter', mode = 'lines', - fill = 'tonexty', line = list(color = 'transparent'), legendgroup = legend_name, - fillcolor = rgba_str, showlegend = F, name = 'lower', ...) - } - - - - if (is_new_plot) { - p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - }, - 'radar' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - if (is.numeric(df[['x']])) { - if (stri_detect_regex(x_attr, "(?i)fun")) - df <- df[, x := paste0('F', sprintf("%02d", x))] - else if (stri_detect_regex(x_attr, "(?i)dim")) - df <- df[, x := paste0('D', as.character(x))] - else - df <- df[, x := as.character(x)] - } - df <- df[, col := add_transparancy(colors, 0.4)[l]] - p %<>% - add_trace(data = df, type = 'scatterpolar', r = ~y, - theta = ~x, mode = 'markers', #marker = list(color = 'lightgrey', size=0), - fill = 'toself', connectgaps = T, fillcolor = ~col, color = ~l, colors = colors, - name = ~l, legendgroup = ~l, ...) - if (is_new_plot) { - p %<>% layout(polar = list(radialaxis = list(type = yscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)))) - } - }, - 'hist' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - if (!'width' %in% colnames(df)) { - stop("No 'width'-column included in the provided dataframe. This is required for a histogram-plot") - } - p %<>% - add_trace(data = df, x = ~x, y = ~y, width = ~width, type = 'bar', - name = ~l, text = ~text, hoverinfo = 'text', - colors = add_transparancy(colors, 0.6), color = ~l, - marker = list(line = list(color = 'rgb(8,48,107)')), - ...) - - if (is_new_plot) { - p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - }, - 'bar' = { - if (legend_attr != x_attr) { - warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") - } - colors = add_transparancy(colors, 0.6) - for (xv in xs) { - p %<>% - add_trace(x = xv, y = df[x == xv, y], type = 'bar', - name = xv, - color = colors[xv], - marker = list(line = list(color = 'rgb(8,48,107)')), - ...) - } - - if (is_new_plot) { - p %<>% layout(xaxis = list(tickfont = f3(), ticklen = 3), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - } - ) - return(p) -} - - -#' Create the PerformViz plot -#' -#' From the paper: -#' -#' @param DSC_rank_result The result from a call to DSCtool rank service (`get_dsc_rank`) -#' -#' @return A performviz plot -#' @export -#' @examples -#' \dontrun{ -#' Plot.Performviz(get_dsc_rank(dsl)) -#' } -Plot.Performviz <- function(DSC_rank_result) { - if (!requireNamespace("ComplexHeatmap", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - if (!requireNamespace("reshape2", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - if (!requireNamespace("grid", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - mlist <- DSC_rank_result$ranked_matrix - - problem <- NULL #Assign variable to remove warnings - # df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - # function(x) { - # list(algorithm = x$algorithm, rank = x$rank) - # })) - # df_temp[, problem := mlist[[problem_idx]]$problem] - - df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { - df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - function(x) { - list(algorithm = x$algorithm, rank = x$rank) - })) - df_temp[, problem := mlist[[problem_idx]]$problem] - })) - - rank_matrix <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') - df <- rank_matrix - # colnames(df)<-index - # rownames(df)<-vector - # Define some graphics to display the distribution of columns - # library(ComplexHeatmap) - .hist = ComplexHeatmap::anno_histogram(df, gp = grid::gpar(fill = "lightblue")) - .density = ComplexHeatmap::anno_density(df, type = "line", gp = grid::gpar(col = "blue")) - ha_mix_top = ComplexHeatmap::HeatmapAnnotation(hist = .hist, density = .density) - # Define some graphics to display the distribution of rows - .violin = ComplexHeatmap::anno_density(df, type = "violin", - gp = grid::gpar(fill = "lightblue"), which = "row") - .boxplot = ComplexHeatmap::anno_boxplot(df, which = "row") - ha_mix_right = ComplexHeatmap::HeatmapAnnotation(violin = .violin, bxplt = .boxplot, - which = "row", width = grid::unit(4, "cm")) - # Combine annotation with heatmap - heatmap_main <- ComplexHeatmap::Heatmap(df, name = "Ranking", - column_names_gp = grid::gpar(fontsize = 8), - top_annotation = ha_mix_top, - top_annotation_height = grid::unit(3.8, "cm")) - return(ComplexHeatmap::draw( - ComplexHeatmap::`+.AdditiveUnit`(heatmap_main, ha_mix_right)) - ) -} +symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", + "triangle-up-open", "triangle-down-open") + +get_legends <- function(dsList) { + N <- length(dsList) + legends <- sapply(dsList, function(d) get_id(d)) + + if (length(unique(legends)) < N) { + funcId <- sapply(dsList, function(d) attr(d, 'funcId')) + if (length(unique(funcId)) > 1) + legends <- paste0(legends, '-F', funcId) + } + + if (length(unique(legends)) < N) { + DIM <- sapply(dsList, function(d) attr(d, 'DIM')) + if (length(unique(DIM)) > 1) + legends <- paste0(legends, '-', DIM, 'D') + } + legends +} + +insert_best_parts <- function(from_data, to_data, best_is_min) { + if (all(is.na(from_data))) + to_data + else + if (best_is_min) + pmin(from_data, to_data, na.rm = T) + else + pmax(from_data, to_data, na.rm = T) +} + +generate_rbga <- function(color,a){ + paste0('rgba(', paste0(color, collapse = ','), ',', a,')') +} + +grad_functions <- c( + scaled_edges = function(count, amount, intensity){ + scale <- (intensity + 1)/2 + color_end <- floor(scale*amount*2) + if (count < color_end) + 1/color_end + else + 0 + } + , + fixed_edges = function(count, amount, intensity) { + scale <- (intensity + 1) / 2 + color_center <- floor(scale * amount) + 1 + if (count <= color_center) + 1 / (2 * color_center) + else + 1 / (2 * (amount - color_center)) + } +) + +#S3 generics +# TODO: decide which parameters need to be in the generics + +#' Plot lineplot of the ERTs of a DataSetList +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param Fstart The starting function value. +#' @param Fstop The final function value. +#' @param show.ERT Whether or not to show the ERT-values +#' @param show.CI Whether or not to show the standard deviations +#' @param show.mean Whether or not to show the mean hitting times +#' @param show.median Whether or not to show the median hitting times +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) +#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' +#' @param includeOpts Whether or not to include all best points reached by each algorithm +#' @param p Existing plot to which to add the current data +#' @return A plot of ERT-values of the DataSetList +#' @export +#' @examples +#' Plot.RT.Single_Func(subset(dsl, funcId == 1)) +Plot.RT.Single_Func <- function(dsList, Fstart = NULL, Fstop = NULL, + show.ERT = T, show.CI = F, show.mean = F, + show.median = F, backend = NULL, + scale.xlog = F, scale.ylog = F, scale.reverse = F, + includeOpts = F, p = NULL) + UseMethod("Plot.RT.Single_Func", dsList) +#' Plot lineplot of the expected function values of a DataSetList +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param RTstart The starting runtime value. +#' @param RTstop The final runtime value. +#' @param show.CI Whether or not to show the standard deviations +#' @param show.mean Whether or not to show the mean runtimes +#' @param show.median Whether or not to show the median runtimes +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) +#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' +#' +#' @return A plot of ERT-values of the DataSetList +#' @export +#' @examples +#' Plot.FV.Single_Func(subset(dsl, funcId == 1)) +Plot.FV.Single_Func <- function(dsList, RTstart = NULL, RTstop = NULL, show.CI = F, show.mean = T, + show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F) UseMethod("Plot.FV.Single_Func", dsList) +#' Plot probablity mass function of the runtimes of a DataSetList at a certain target function value +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value. +#' @param show.sample Whether or not to show the individual runtime samples +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' +#' +#' @return A plot of the probablity mass function of the runtimes at a the +#' target function value of the DataSetList +#' @export +#' @examples +#' Plot.RT.PMF(subset(dsl, funcId == 1), 14) +Plot.RT.PMF <- function(dsList, ftarget, show.sample = F, scale.ylog = F, backend = NULL) + UseMethod("Plot.RT.PMF", dsList) +#' Plot histograms of the runtimes of a DataSetList at a certain target function value +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value. +#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either +#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. +#' @param use.equal.bins Whether to determine one bin size for all plots or have individual +#' bin sizes for each algorithm +#' +#' @return A plot of the histograms of the runtimes at a the +#' target function value of the DataSetList +#' @export +#' @examples +#' Plot.RT.Histogram(subset(dsl, funcId == 1), 14) +Plot.RT.Histogram <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F) + UseMethod("Plot.RT.Histogram", dsList) +#' Plot the empirical cumulative distriburtion as a function of the running times of +#' a DataSetList at certain target function values +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftargets The target function values +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the running times of the DataSetList at the target function values +#' @export +#' @examples +#' Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) +Plot.RT.ECDF_Per_Target <- function(dsList, ftargets, scale.xlog = F) + UseMethod("Plot.RT.ECDF_Per_Target", dsList) +#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of +#' a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param fstart The starting function value +#' @param fstop The final function value +#' @param fstep The spacing between starting and final function values +#' @param show.per_target Whether or not to show the individual ECDF-curves for each target +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the running times of the DataSetList +#' @export +#' @examples +#' Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) +Plot.RT.ECDF_Single_Func <- function(dsList, fstart = NULL, fstop = NULL, + fstep = NULL, show.per_target = F, + scale.xlog = F) UseMethod("Plot.RT.ECDF_Single_Func", dsList) +#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param fstart The starting function value +#' @param fstop The final function value +#' @param fstep The spacing between starting and final function values +#' @param fval_formatter Function to format the function-value labels +#' +#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList +#' @export +#' @examples +#' Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) +Plot.RT.ECDF_AUC <- function(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, + fval_formatter = as.integer) UseMethod("Plot.RT.ECDF_AUC", dsList) +#' Plot probablity density function of the function values of a DataSetList at +#' a certain target runtime +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param runtime The target runtime +#' @param show.sample Whether or not to show the individual function value samples +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' +#' @return A plot of the probablity density function of the runtimes at a the +#' target function value of the DataSetList +#' @export +#' @examples +#' Plot.FV.PDF(subset(dsl, funcId == 1), 100) +Plot.FV.PDF <- function(dsList, runtime, show.sample = F, scale.ylog = F) + UseMethod("Plot.FV.PDF", dsList) +#' Plot histograms of the function values of a DataSetList at a certain target runtime +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param runtime The target runtime +#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either +#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. +#' @param use.equal.bins Whether to determine one bin size for all plots or have individual +#' bin sizes for each algorithm +#' +#' @return A plot of the histograms of the function values at a the +#' target runtime of the DataSetList +#' @export +#' @examples +#' Plot.FV.Histogram(subset(dsl, funcId == 1), 100) +Plot.FV.Histogram <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F) + UseMethod("Plot.FV.Histogram", dsList) +#' Plot the empirical cumulative distriburtion as a function of the target values of +#' a DataSetList at certain target runtimes +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param runtimes The target runtimes +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the fucntion values of the DataSetList at the target runtimes +#' @export +#' @examples +#' Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) +Plot.FV.ECDF_Per_Target <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F) + UseMethod("Plot.FV.ECDF_Per_Target", dsList) +#' Plot the aggregated empirical cumulative distriburtion as a function of the function values of +#' a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param rt_min The starting runtime +#' @param rt_max The final runtime +#' @param rt_step The spacing between starting and final runtimes +#' @param show.per_target Whether or not to show the individual ECDF-curves for each runtime +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the function values of the DataSetList +#' @export +#' @examples +#' Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) +Plot.FV.ECDF_Single_Func <- function(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL, scale.xlog = F, + show.per_target = F, scale.reverse = F) + UseMethod("Plot.FV.ECDF_Single_Func", dsList) +#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param rt_min The starting runtime +#' @param rt_max The final runtime +#' @param rt_step The spacing between starting and final runtimes +#' +#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList +#' @export +#' @examples +#' Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) +Plot.FV.ECDF_AUC <- function(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL) UseMethod("Plot.FV.ECDF_AUC", dsList) +#' Plot the parameter values recorded in a DataSetList (aligned by funcion value) +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param f_min The starting function value. +#' @param f_max The final function value. +#' @param show.mean Whether or not to show the mean parameter values +#' @param show.median Whether or not to show the median parameter values +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param algids Which algorithms from dsList to use +#' @param par_name Which parameters to create plots for; set to NULL to use all +#' parameters found in dsList. +#' @param show.CI Whether or not to show the standard deviation +#' +#' @return A plot of for every recorded parameter in the DataSetList +#' @export +#' @examples +#' Plot.RT.Parameters(subset(dsl, funcId == 1)) +Plot.RT.Parameters <- function(dsList, f_min = NULL, f_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) UseMethod("Plot.RT.Parameters", dsList) +#' Plot the parameter values recorded in a DataSetList (aligned by budget) +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param rt_min The starting budget value. +#' @param rt_max The final budget value. +#' @param show.mean Whether or not to show the mean parameter values +#' @param show.median Whether or not to show the median parameter values +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param algids Which algorithms from dsList to use +#' @param par_name Which parameters to create plots for; set to NULL to use all +#' parameters found in dsList. +#' @param show.CI Whether or not to show the standard deviation +#' +#' @return A plot of for every recorded parameter in the DataSetList +#' @export +#' @examples +#' Plot.FV.Parameters(subset(dsl, funcId == 1)) +Plot.FV.Parameters <- function(dsList, rt_min = NULL, rt_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) UseMethod("Plot.FV.Parameters", dsList) +#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of +#' a DataSetList. Aggregated over multiple functions or dimensions. +#' +#' @param dsList A DataSetList. +#' @param targets The target function values. Specified in a data.frame, as can be generated +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' by the function 'get_ECDF_targets' +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the running times of the DataSetList +#' @export +#' @examples +#' Plot.RT.ECDF_Multi_Func(dsl) +Plot.RT.ECDF_Multi_Func <- function(dsList, targets = NULL, scale.xlog = F) + UseMethod("Plot.RT.ECDF_Multi_Func", dsList) +#' Plot ERT-plots for multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) +#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. +#' +#' @return A plot of ERT-values of the DataSetList +#' @export +#' @examples +#' Plot.RT.Multi_Func(dsl) +Plot.RT.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, scale.reverse = F, + backend = NULL) + UseMethod("Plot.RT.Multi_Func", dsList) +#' Plot ERT-based comparison over multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' +#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param maximize Wheter or not to the data is of a maximization problem +#' @param targets Custom list of function-value targets, one for each function or dimension. +#' @param use_rank Wheter to use a ranking system. If False, the actual ERT-values will be used. +#' @param erts Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function +#' of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating +#' this plot. +#' @param inf.action How to handle infinite ERTs ('overlap' or 'jitter') +#' @return A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList +#' @export +#' @examples +#' Plot.RT.Aggregated(dsl) +Plot.RT.Aggregated <- function(dsList, aggr_on = 'funcId', targets = NULL, + plot_mode = 'radar', use_rank = F, scale.ylog = T, maximize = T, + erts = NULL, inf.action = 'overlap') + UseMethod("Plot.RT.Aggregated", dsList) +#' Plot expected function value-based comparison over multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' +#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param runtimes Custom list of function-value targets, one for each function or dimension. +#' @param use_rank Wheter to use a ranking system. If False, the actual expected function- +#' values will be used. +#' @param fvs Pre-calculated expected function-values for the provided runtimes Created by the +#' max_ERTs function of DataSetList. Can be provided to prevent needless computation +#' in recalculating ERTs when recreating this plot. +#' +#' @return A plot of expected function value-based comparison on the provided functions +#' or dimensions of the DataSetList +#' @export +#' @examples +#' Plot.FV.Aggregated(dsl) +Plot.FV.Aggregated <- function(dsList, aggr_on = 'funcId', runtimes = NULL, plot_mode = 'radar', + use_rank = F, scale.ylog = T, fvs = NULL) + UseMethod("Plot.FV.Aggregated", dsList) + +#' Plot FV-plots for multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. +#' +#' @return A plot of Function-values of the DataSetList +#' @export +#' @examples +#' Plot.FV.Multi_Func(dsl) +Plot.FV.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, backend = NULL) + UseMethod("Plot.FV.Multi_Func", dsList) + +#' Plot a heatmap showing the statistically different algorithms +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value to use +#' @param alpha The cutoff for statistical significance +#' @param bootstrap.size The amound of bootstrapped samples used +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' +#' @return A heatmap showing the statistical significance between algorithms +#' @export +#' @examples +#' Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) +Plot.Stats.Significance_Heatmap <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, + which = 'by_FV') + UseMethod("Plot.Stats.Significance_Heatmap", dsList) + +#' Plot a network graph showing the statistically different algorithms +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value to use +#' @param alpha The cutoff for statistical significance +#' @param bootstrap.size The amound of bootstrapped samples used +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' +#' @return A graph showing the statistical significance between algorithms +#' @export +#' @examples +#' Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) +Plot.Stats.Significance_Graph <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, + which = 'by_FV') + UseMethod("Plot.Stats.Significance_Graph", dsList) + +#' Create a candlestick plot of Glicko2-rankings +#' +#' @param dsList A DataSetList +#' @param nr_rounds The number of rounds in the tournament +#' @param glicko2_rank_df Optional. Dataframe containing the glicko2 rating to avoid needless recalculation. +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' @param target_dt Optional: data table containing the targets for each function and dimension +#' +#' @export +#' @examples +#' Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) +Plot.Stats.Glicko2_Candlestick <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, + which = 'by_FV', target_dt = NULL) + UseMethod("Plot.Stats.Glicko2_Candlestick", dsList) + + +##Implementations + +#' @rdname Plot.RT.Single_Func +#' @export +Plot.RT.Single_Func.DataSetList <- function(dsList, Fstart = NULL, Fstop = NULL, + show.ERT = T, show.CI = T, show.mean = F, + show.median = F, backend = NULL, + scale.xlog = F, scale.ylog = F, + scale.reverse = F, includeOpts = F, p = NULL) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + if (backend == 'plotly') { + data <- generate_data.Single_Function(dsList, Fstart, Fstop, scale.xlog, 'by_RT', includeOpts) + + y_attrs <- c() + if (show.ERT) y_attrs <- c(y_attrs, 'ERT') + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, p = p, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + show_legend <- F + } + if (show.CI) { + p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + } + } + # } else if (backend == 'ggplot2') { + # dt[, 'group' := paste(algId, funcId, DIM, sep = '-')] + # p <- ggplot(data = dt, aes(group = `group`, colour = `group`)) + # + # if (show.CI) p <- p + geom_ribbon(aes(target, ymin = lower, ymax = upper, fill = `group`), + # alpha = 0.2, colour = NA) + # if (show.ERT) p <- p + geom_line(aes(target, ERT), size = 1.2) + # if (show.mean) p <- p + geom_line(aes(target, mean), linetype = 'dashed') + # if (show.median) p <- p + geom_line(aes(target, median), linetype = 'dotted') + # + # p <- p + + # scale_color_manual(values = colors) + + # scale_fill_manual(values = colors) + # } + return(p) +} + +#' @rdname Plot.FV.Single_Func +#' @export +Plot.FV.Single_Func.DataSetList <- function(dsList, RTstart = NULL, RTstop = NULL, + show.CI = F, + show.mean = T, show.median = F, + backend = NULL, + scale.xlog = F, scale.ylog = F, + scale.reverse = F) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + if (backend == 'plotly') { + data <- generate_data.Single_Function(dsList, RTstart, RTstop, scale.xlog, 'by_FV') + + y_attrs <- c() + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + show_legend <- F + } + else + p <- NULL + if (show.CI) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + } + + } + # } else if (backend == 'ggplot2') { + # fce[, 'group' := paste(algId, funcId, DIM, sep = '-')] + # p <- ggplot(data = fce, aes(group = `group`, colour = `group`)) + # + # if (show.mean) p <- p + geom_line(aes(runtime, mean), linetype = 'dashed') + # if (show.median) p <- p + geom_line(aes(runtime, median), linetype = 'dotted') + # + # p <- p + + # scale_color_manual(values = colors) + + # scale_fill_manual(values = colors) + # + # #TODO: add individual run etc + # } + return(p) +} + +#' @rdname Plot.RT.PMF +#' @export +Plot.RT.PMF.DataSetList <- function(dsList, ftarget, show.sample = F, + scale.ylog = F, backend = NULL){ + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + data <- generate_data.PMF(dsList, ftarget, 'by_RT') + + plot_general_data(data, 'ID', 'RT', scale.ylog = scale.ylog, + x_title = "Algorithm", y_title = "Function Evaluations") +} + +#' @rdname Plot.RT.Histogram +#' @export +Plot.RT.Histogram.DataSetList <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F){ + if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { + warning("Invalid dataset uploaded. Please ensure the datasetlist contains data + from only one function and only one dimension.") + return(NULL) + } + data <- generate_data.hist(dsList, ftarget, use.equal.bins, 'by_RT') + + subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL + plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Function Evaluations", + y_title = "Runs") +} + +#' @rdname Plot.RT.ECDF_Per_Target +#' @export +Plot.RT.ECDF_Per_Target.DataSetList <- function(dsList, ftargets, scale.xlog = F){ + req(length(ftargets) != 0) + data <- generate_data.ECDF(dsList, ftargets, scale.xlog) + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of runs", scale.xlog = scale.xlog, show.legend = T) +} + +#' @rdname Plot.RT.ECDF_Single_Func +#' @export +Plot.RT.ECDF_Single_Func.DataSetList <- function(dsList, fstart = NULL, fstop = NULL, + fstep = NULL, show.per_target = F, + scale.xlog = F) { + + targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep) + req(targets) + + data <- generate_data.ECDF(dsList, targets, scale.xlog) + + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of (run, target) pairs", + scale.xlog = scale.xlog, show.legend = T) +} + +#' @rdname Plot.RT.ECDF_AUC +#' @export +Plot.RT.ECDF_AUC.DataSetList <- function(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, + fval_formatter = as.integer) { + + targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep, length.out = 10) + req(targets) + + data <- generate_data.AUC(dsList, targets, multiple_x = TRUE) + + plot_general_data(data, 'x', 'auc', 'radar') +} + +#' @rdname Plot.FV.PDF +#' @export +Plot.FV.PDF.DataSetList <- function(dsList, runtime, show.sample = F, scale.ylog = F){ + + data <- generate_data.PMF(dsList, runtime, 'by_FV') + + plot_general_data(data, 'ID', 'f(x)', scale.ylog = scale.ylog, + x_title = "Algorithm", y_title = "Target Value") +} + +#' @rdname Plot.FV.Histogram +#' @export +Plot.FV.Histogram.DataSetList <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F){ + if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { + warning("Invalid dataset uploaded. Please ensure the datasetlist contains data + from only one function and only one dimension.") + return(NULL) + } + data <- generate_data.hist(dsList, runtime, use.equal.bins, 'by_FV') + + subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL + plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Target Values", + y_title = "Runs") +} + +#' @rdname Plot.FV.ECDF_Per_Target +#' @export +Plot.FV.ECDF_Per_Target.DataSetList <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F){ + #TODO: Fvals in legend need to be formatted properly + runtimes <- runtimes[!is.na(runtimes)] + req(length(runtimes) != 0) + + data <- generate_data.ECDF(dsList, runtimes, scale.xlog, which = 'by_FV') + + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of runs", scale.xlog = scale.xlog, + show.legend = T, + scale.reverse = scale.reverse) +} + +#' @rdname Plot.FV.ECDF_Single_Func +#' @export +Plot.FV.ECDF_Single_Func.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL, scale.xlog = F, + show.per_target = F, scale.reverse = F){ + + targets <- seq_RT(get_funvals(dsList), rt_min, rt_max, rt_step) + req(targets) + data <- generate_data.ECDF(dsList, targets, scale.xlog, which = 'by_FV') + + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of (run, target) pairs", + scale.xlog = scale.xlog, + scale.reverse = scale.reverse, show.legend = T) +} + +#' @rdname Plot.FV.ECDF_AUC +#' @export +Plot.FV.ECDF_AUC.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, rt_step = NULL) { + targets <- seq_RT(get_runtimes(dsList), rt_min, rt_max, rt_step, length.out = 10) + req(targets) + data <- generate_data.AUC(dsList, targets, which = 'by_FV', multiple_x = TRUE) + + plot_general_data(data, 'x', 'auc', 'radar') +} + +#' @rdname Plot.RT.Parameters +#' @export +Plot.RT.Parameters.DataSetList <- function(dsList, f_min = NULL, f_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) { + data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_FV') + + y_attrs <- c() + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + show_legend <- F + } + else + p <- NULL + if (show.CI) { + p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + } + p +} + + +#' @rdname Plot.FV.Parameters +#' @export +Plot.FV.Parameters.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) { + data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_RT') + + y_attrs <- c() + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + show_legend <- F + } + else + p <- NULL + if (show.CI) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + } + p +} + +#' @rdname Plot.RT.ECDF_Multi_Func +#' @export +Plot.RT.ECDF_Multi_Func.DataSetList <- function(dsList, targets = NULL, + scale.xlog = F) { + if (is.null(targets) || !is.data.table(targets)) { + targets <- get_ECDF_targets(dsList) + } + + data <- generate_data.ECDF(dsList, targets, scale.xlog) + + plot_general_data(data, 'x', 'mean', 'line', + scale.xlog = scale.xlog, + x_title = "Function Evaluations", + y_title = "Proportion of (run, target, ...) pairs", + show.legend = T) +} + +#' @rdname Plot.RT.Multi_Func +#' @export +Plot.RT.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, + scale.ylog = F, + scale.reverse = F, + backend = NULL) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + data <- rbindlist(lapply(get_funcId(dsList), function(fid) { + generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, + which = 'by_RT') + })) + + plot_general_data(data, x_attr = 'target', y_attr = 'ERT', + subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, + scale.ylog = scale.ylog, x_title = 'Best-so-far f(x)', + y_title = 'ERT', show.legend = T, + scale.reverse = scale.reverse) +} + +#' @rdname Plot.FV.Multi_Func +#' @export +Plot.FV.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, + scale.ylog = F, + backend = NULL) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + data <- rbindlist(lapply(get_funcId(dsList), function(fid) { + generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, + which = 'by_FV') + })) + + plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', + subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, + scale.ylog = scale.ylog, x_title = 'Runtime', + y_title = 'Best-so-far f(x)', show.legend = T) +} + +#' @rdname Plot.RT.Aggregated +#' @export +Plot.RT.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', targets = NULL, + plot_mode = 'radar', use_rank = F, + scale.ylog = T, maximize = T, erts = NULL, + inf.action = 'overlap') { + targets <- get_target_dt(dsList) + data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) + y_attr <- if (use_rank) 'rank' else 'value' + y_title <- if (use_rank) 'Rank' else 'ERT' + plot_general_data(data, type = plot_mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = scale.ylog, + inf.action = inf.action) +} + +#' @rdname Plot.FV.Aggregated +#' @export +Plot.FV.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', runtimes = NULL, + plot_mode = 'radar', use_rank = F, + scale.ylog = T, fvs = NULL){ + targets <- get_target_dt(dsList, which = 'by_FV') + data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, which = 'by_FV') + y_attr <- if (use_rank) 'rank' else 'value' + y_title <- if (use_rank) 'Rank' else 'Best-so-far f(x)' + plot_general_data(data, type = plot_mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = scale.ylog) +} + +#' @rdname Plot.Stats.Significance_Heatmap +#' @export +Plot.Stats.Significance_Heatmap.DataSetList <- function(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = 'by_FV'){ + if (length(get_dim(dsList)) != 1 || + length(get_funcId(dsList)) != 1 || + length(get_id(dsList)) < 2) + return(NULL) + + p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) + y <- p_matrix <= alpha + colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), + col = c('blue', 'blue', 'white', 'white', 'red', 'red') + ) + heatmap <- y - t(y) + + p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', + xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) + p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), + xaxis = list(tickangle = 45)) + p +} + +#' Helper function for Plot.Stats.Significance_Graph +#' +#' @param x x +#' @param start default is 0 +#' @param direction default is 1 +#' +#' @noRd +radian.rescale <- function(x, start=0, direction=1) { + c.rotate <- function(x) (x + start) %% (2 * pi) * direction + c.rotate((2 * pi * (x - min(x)) / (max(x) - min(x)))) +} + +#' @rdname Plot.Stats.Significance_Graph +#' @export +Plot.Stats.Significance_Graph.DataSetList <- function(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = 'by_FV'){ + if (!requireNamespace("igraph", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || length(get_id(dsList)) < 2) { + return(NULL) + } + p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) + g <- igraph::graph_from_adjacency_matrix(p_matrix <= alpha, mode = 'directed', diag = F) + lab.locs <- radian.rescale(x = 1:nrow(p_matrix), direction = -1, start = 0) + + igraph::plot.igraph(g, layout = igraph::layout.circle(g), vertex.size = 10, edge.arrow.size = 1, + vertex.label.color = 'black', + vertex.label.dist = 2, + vertex.label.cex = 1, + vertex.label.degree = lab.locs) +} + +#' @rdname Plot.Stats.Glicko2_Candlestick +#' @export +Plot.Stats.Glicko2_Candlestick.DataSetList <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, + which = 'by_FV', target_dt = NULL) { + df <- glicko2_rank_df + + if (is.null(df)) { + df <- glicko2_ranking(dsList, nr_rounds, which, target_dt = target_dt)$ratings + Ids <- df$Player$ID + } + else{ + Ids <- df$ID + } + p <- IOH_plot_ly_default(title = "Glicko2-rating", + x.title = "ID", + y.title = "Rating") + df$Rating %<>% as.numeric + df$Deviation %<>% as.numeric + high <- df$Rating + 3*df$Deviation + low <- df$Rating - 3*df$Deviation + open <- df$Rating + df$Deviation + close <- df$Rating - df$Deviation + + N <- length(df$Rating) + colors <- get_color_scheme(Ids) + if (length(colors != N)) { + colors <- get_color_scheme(get_id(dsList)) + } + + for (i in seq(N)) { + # rgba_str <- paste0('rgba(', paste0(col2rgb(colors[i]), collapse = ','), ',0.52)') + color <- list(line = list(color = colors[[i]])) + p %<>% add_trace(type = "candlestick", x = Ids[[i]], open = open[[i]], close = close[[i]], + high = high[[i]], low = low[[i]], legendgroup = Ids[[i]], + name = Ids[[i]], increasing = color, decreasing = color, + hovertext = paste0(format(df$Rating[[i]], digits = 3), '+-', + format(df$Deviation[[i]], digits = 3)), + hoverinfo = "text") + } + p %<>% layout(xaxis = list(rangeslider = list(visible = F))) + p +} + +### _______________________ Rewritten plotting function ____________________ ### + +#' Add transparancy to named list of colors +#' +#' @param colors Named list of colors (in hex-notation) +#' @param percentage The percentage of opacity. 0 is fully transparant, 1 is fully opaque +#' +#' @noRd +add_transparancy <- function(colors, percentage){ + hex_val <- format(as.hexmode(as.integer(255 * percentage)), upper.case = T, width = 2) + sapply(colors, function(col) { col <- paste0('#', substr(col, 2, 7), hex_val) }) +} + +#' General function for plotting within IOHanalyzer +#' +#' @param df The dataframe containing the data to plot. It should contain at least two columns: +#' 'x_attr' and 'y_attr' +#' @param x_attr The column to specify the x_axis. Default is 'algId' +#' @param legend_attr Default is 'algId' This is also used for the selection of colorschemes +#' @param y_attr The column to specify the y_axis +#' @param type The type of plot to use. Currently available: 'violin', 'line', 'radar', +#' 'bar', hist' and 'ribbon' +#' @param upper_attr When using ribbon-plot, this can be used to create a shaded area. +#' Only works in combination with`lower_attr` and `type` == 'ribbon' +#' @param lower_attr When using ribbon-plot, this can be used to create a shaded area. +#' Only works in combination with`upper_attr` and `type` == 'ribbon' +#' @param subplot_attr Which attribute of the dataframe to use for creating subplots +#' @param scale.xlog Logarithmic scaling of x-axis +#' @param scale.ylog Logarithmic scaling of y-axis +#' @param scale.reverse Decreasing or increasing x-axis +#' @param x_title Title of x-axis. Defaults to x_attr +#' @param y_title Title of x-axis. Defaults to x_attr +#' @param plot_title Title of x-axis. Defaults to no title +#' @param p A previously existing plot on which to add traces. If NULL, a new canvas is created +#' @param show.legend Whether or not to include a legend +#' @param inf.action How to deal with infinite values. Can be 'none', 'overlap' or 'jitter' +#' @param ... Additional parameters for the add_trace function +#' +#' @export +plot_general_data <- function(df, x_attr = 'ID', y_attr = 'vals', type = 'violin', + legend_attr = 'ID', scale.xlog = F, scale.ylog = F, + scale.reverse = F, p = NULL, x_title = NULL, + y_title = NULL, plot_title = NULL, upper_attr = NULL, + lower_attr = NULL, subplot_attr = NULL, show.legend = F, + inf.action = 'none', ...) { + + l <- x <- isinf <- y <- text <- l_orig <- NULL #Set local binding to remove warnings + + #Only allow valid plot types + if (!(type %in% c('violin', 'line', 'radar', 'hist', 'ribbon', 'line+ribbon', 'bar'))) { + stop(paste0("Provided plot type ('", type, "') is not supported")) + } + + #And valid number of y-attributes + if (length(y_attr) == 0) { + stop("At least one y-attribute is needed to plot") + } + + #Deal with subplots + if (!is.null(subplot_attr)) { + if (!subplot_attr %in% colnames(df)) { + stop("Provided subplot-attribut is not a colname of the selected data.table.") + } + colnames(df)[colnames(df) == subplot_attr] <- "subplot_attr" + attrs <- unique(df[, subplot_attr]) + if (length(attrs) == 0) stop("Attempting to create subplots with fewer than 2 unique values of + `subplot_attrs`-column") + if (length(attrs) == 1) return(plot_general_data(df, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, + scale.reverse, p, x_title, y_title, attrs, upper_attr, lower_attr, + show.legend = show.legend, subplot_attr = NULL, ...)) + if (subplot_attr == legend_attr) { + df[, l := subplot_attr] + } + + #Only need one legend for the whole plot + legends_show <- rep(F, length(attrs)) + legends_show[[1]] <- show.legend + names(legends_show) <- attrs + + #Get some number of rows and columns + n_cols <- 1 + ceiling(length(attrs)/10) + n_rows <- ceiling(length(attrs) / n_cols) + + p <- lapply(seq(length(attrs)), function(idx) { + attr_val <- attrs[[idx]] + df_sub <- df[subplot_attr == attr_val] + disp_y <- idx %% n_cols == 1 + disp_x <- idx > (length(attrs) - n_cols) + x.title = if (disp_x) x_title else "" + y.title = if (disp_y) y_title else "" + + #Generate title for the subplots + if (stri_detect_regex(subplot_attr, "(?i)fun")) + sub_title <- paste0('F', attr_val) + else if (stri_detect_regex(subplot_attr, "(?i)dim")) + sub_title <- paste0('D', attr_val) + else + sub_title <- paste0(attr_val) + p <- NULL + if (stri_detect_fixed(type, '+')) { + type1 <- substr(type, 0, stri_locate_all(type, fixed = '+')[[1]][[1]] - 1) + p <- plot_general_data(df_sub, x_attr, y_attr, type1, legend_attr, scale.xlog, scale.ylog, + scale.reverse, NULL, x.title, y.title, plot_title, upper_attr, lower_attr, + show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) + type <- substr(type, stri_locate_all(type, fixed = '+')[[1]][[1]] + 1, nchar(type)) + } + plot_general_data(df_sub, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, + scale.reverse, p, x.title, y.title, plot_title, upper_attr, lower_attr, + show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) %>% + layout( + annotations = list( + text = sub_title, + font = f2, + xref = "paper", yref = "paper", align = "center", + yanchor = "bottom", + xanchor = "center", textangle = 0, + x = 0.5, y = 1, showarrow = FALSE + ) + ) + }) + + p <- subplot( + p, nrows = n_rows, titleX = T, titleY = T, + margin = c(0.02, 0.02, 0.06, 0.06) + ) %>% + layout(title = plot_title) + return(p) + } + + # Replace colnames to have easier matching + if (!x_attr %in% colnames(df) || !all(y_attr %in% colnames(df))) { + stop("Not all provided attributes are colnames of the selected data.table.") + } + colnames(df)[colnames(df) == x_attr] <- "x" + + + if (length(y_attr) == 1 && type != 'line') + colnames(df)[colnames(df) == y_attr] <- "y" + else if (type != 'line') stop("Multiple y-attrs is currently only supported for line-plots") + + if ( !is.null(upper_attr) && !is.null(lower_attr)) { + if (!upper_attr %in% colnames(df) || !lower_attr %in% colnames(df)) { + stop("Provided upper and lower attributes are not colnames of the selected data.table.") + } + colnames(df)[colnames(df) == upper_attr] <- "upper" + colnames(df)[colnames(df) == lower_attr] <- "lower" + } + + if ( x_attr != legend_attr) { + colnames(df)[colnames(df) == legend_attr] <- "l" + xs <- unique(df[['l']]) + } + else{ + xs <- unique(df[['x']]) + } + + #Get color and based on legend-attribute + colors <- get_color_scheme(xs) + if (is.null(names(colors)) || !all(names(colors) %in% xs) ) names(colors) <- xs + + xscale <- if (scale.xlog) 'log' else 'linear' + yscale <- if (scale.ylog) 'log' else 'linear' + + #If new plot is needed, create one. Store in bool to decide if axis scaling is needed. + is_new_plot <- F + if (is.null(p)) { + p <- IOH_plot_ly_default(x.title = ifelse(is.null(x_title), x_attr, x_title), + y.title = ifelse(is.null(y_title), y_attr, y_title), + title = plot_title) + is_new_plot <- T + } + + switch(type, + 'violin' = { + if (legend_attr != x_attr) { + warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") + } + #Update names to aviod numerical legend + if (is.numeric(df[['x']])) { + if (stri_detect_regex(x_attr, "(?i)fun")) + df <- df[, x := paste0('F', sprintf("%02d", x))] + else if (stri_detect_regex(x_attr, "(?i)dim")) + df <- df[, x := paste0('D', as.character(x))] + else + df <- df[, x := as.character(x)] + } + #Update color names as well, since the value changed + names(colors) <- unique(df[['x']]) + + p %<>% + add_trace(data = df, + x = ~x, y = ~y, type = 'violin', + hoveron = "points+kde", + points = F, + pointpos = 1.5, + jitter = 0, + scalemode = 'count', + meanline = list(visible = F), + name = ~x, + colors = colors, + color = ~x, + split = ~x, + line = list(color = 'black', width = 1.1), + box = list(visible = T), + spanmode = 'hard', + showlegend = show.legend, + ... + ) + if (is_new_plot) { + p %<>% layout(yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + }, + 'line' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + + # Force legend to be categorical + df[, l_orig := l] + if (is.numeric(df[['l']])) { + df[, l := paste0('A', l)] + names(colors) <- paste0('A', names(colors)) + } + + #Use linestyles to differentiate traces if only one attribute is selected to be plotted + #TODO: Combine these two options more elegantly + if (length(y_attr) == 1) { + dashes <- get_line_style(xs) + names(dashes) <- xs + colnames(df)[colnames(df) == y_attr] <- "y" + + df[, isinf := is.infinite(y)] + df[, text := as.character(round(y, getOption("IOHanalyzer.precision", 2)))] + + if (inf.action == 'overlap') { + maxval <- max(df[isinf == F, 'y']) + df[['y']][df[['isinf']]] <- 10**(ceiling(log10(maxval)) + 1) + } + else if (inf.action == 'jitter') { + #TODO: Faster way to compute this + maxval <- max(df[isinf == F, 'y']) + for (xval in unique(df[['x']])) { + tempval <- 10**(ceiling(log10(maxval)) + 1) + for (lval in unique(df[['l']])) { + temp <- df[l == lval][x == xval] + if (nrow(temp) > 0 && df[l == lval][x == xval][['isinf']]) { + df[l == lval][x == xval][['y']] <- tempval + tempval <- 1.2 * tempval + } + } + } + } + + suppressWarnings( + p %<>% + add_trace( + data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, + type = 'scatter', mode = 'lines+markers', + linetype = ~l_orig, marker = list(size = getOption('IOHanalyzer.markersize', 4)), + linetypes = dashes, + colors = colors, showlegend = show.legend, + text = ~text, line = list(width = getOption('IOHanalyzer.linewidth', 2)), + hovertemplate = '%{text}', + ... + ) + ) + if (inf.action != 'none') { + p %<>% add_trace(data = df[isinf == T], x = ~x, y = ~y, legendgroup = ~l_orig, name = ~l_orig, + type = 'scatter', mode = 'markers', color = ~l, + marker = list(symbol = 'circle-open', size = 8 + getOption('IOHanalyzer.markersize', 4)), + colors = colors, showlegend = F, text = 'Inf', hoverinfo = 'none', + ... + ) + } + + } + else { + if (inf.action != 'none') { + warning("inf.action is not yet supported for multiple y-attributes") + } + + dashes_full <- rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), + ceiling(length(y_attr)/3))[1:length(y_attr)] + names(dashes_full) <- y_attr + + for (y_atr in y_attr) { + colnames(df)[colnames(df) == y_atr] <- "y" + + #TODO: Figure out how to supress warning about 6 linetypes + dashstyle <- dashes_full[[y_atr]] + suppressWarnings( + p %<>% + add_trace( + data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, + type = 'scatter', mode = 'lines+markers', + marker = list(size = getOption('IOHanalyzer.markersize', 4)), linetype = dashstyle, + colors = colors, showlegend = show.legend, name = ~l, + text = y_atr, line = list(width = getOption('IOHanalyzer.linewidth', 2)), + ... + ) + ) + colnames(df)[colnames(df) == "y"] <- y_atr + show.legend <- F + } + } + if (is_new_plot) { + if (is.numeric(df[['x']])) + p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + else + p %<>% layout(xaxis = list(type = 'category', tickfont = f3(), ticklen = 3), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + + } + }, + 'ribbon' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + if (is.null(upper_attr) || is.null(lower_attr)) { + stop("No upper or lower attribute provided for ribbon-plot") + } + + for (name in xs) { + df_small <- df[l == name] + legend_name <- as.character(name) + rgba_str <- paste0('rgba(', paste0(col2rgb(colors[[name]]), collapse = ','), ',0.2)') + p %<>% + add_trace(data = df_small, x = ~x, y = ~upper, type = 'scatter', mode = 'lines', + line = list(color = rgba_str, width = 0), legendgroup = legend_name, + showlegend = F, name = 'upper', ...) %>% + add_trace(x = ~x, y = ~lower, type = 'scatter', mode = 'lines', + fill = 'tonexty', line = list(color = 'transparent'), legendgroup = legend_name, + fillcolor = rgba_str, showlegend = F, name = 'lower', ...) + } + + + + if (is_new_plot) { + p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + }, + 'radar' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + if (is.numeric(df[['x']])) { + if (stri_detect_regex(x_attr, "(?i)fun")) + df <- df[, x := paste0('F', sprintf("%02d", x))] + else if (stri_detect_regex(x_attr, "(?i)dim")) + df <- df[, x := paste0('D', as.character(x))] + else + df <- df[, x := as.character(x)] + } + df <- df[, col := add_transparancy(colors, 0.4)[l]] + p %<>% + add_trace(data = df, type = 'scatterpolar', r = ~y, + theta = ~x, mode = 'markers', #marker = list(color = 'lightgrey', size=0), + fill = 'toself', connectgaps = T, fillcolor = ~col, color = ~l, colors = colors, + name = ~l, legendgroup = ~l, ...) + if (is_new_plot) { + p %<>% layout(polar = list(radialaxis = list(type = yscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)))) + } + }, + 'hist' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + if (!'width' %in% colnames(df)) { + stop("No 'width'-column included in the provided dataframe. This is required for a histogram-plot") + } + p %<>% + add_trace(data = df, x = ~x, y = ~y, width = ~width, type = 'bar', + name = ~l, text = ~text, hoverinfo = 'text', + colors = add_transparancy(colors, 0.6), color = ~l, + marker = list(line = list(color = 'rgb(8,48,107)')), + ...) + + if (is_new_plot) { + p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + }, + 'bar' = { + if (legend_attr != x_attr) { + warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") + } + colors = add_transparancy(colors, 0.6) + for (xv in xs) { + p %<>% + add_trace(x = xv, y = df[x == xv, y], type = 'bar', + name = xv, + color = colors[xv], + marker = list(line = list(color = 'rgb(8,48,107)')), + ...) + } + + if (is_new_plot) { + p %<>% layout(xaxis = list(tickfont = f3(), ticklen = 3), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + } + ) + return(p) +} + + +#' Create the PerformViz plot +#' +#' From the paper: +#' +#' @param DSC_rank_result The result from a call to DSCtool rank service (`get_dsc_rank`) +#' +#' @return A performviz plot +#' @export +#' @examples +#' \dontrun{ +#' Plot.Performviz(get_dsc_rank(dsl)) +#' } +Plot.Performviz <- function(DSC_rank_result) { + if (!requireNamespace("ComplexHeatmap", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + if (!requireNamespace("reshape2", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + if (!requireNamespace("grid", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + mlist <- DSC_rank_result$ranked_matrix + + problem <- NULL #Assign variable to remove warnings + # df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + # function(x) { + # list(algorithm = x$algorithm, rank = x$rank) + # })) + # df_temp[, problem := mlist[[problem_idx]]$problem] + + df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { + df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + function(x) { + list(algorithm = x$algorithm, rank = x$rank) + })) + df_temp[, problem := mlist[[problem_idx]]$problem] + })) + + rank_matrix <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') + df <- rank_matrix + # colnames(df)<-index + # rownames(df)<-vector + # Define some graphics to display the distribution of columns + # library(ComplexHeatmap) + .hist = ComplexHeatmap::anno_histogram(df, gp = grid::gpar(fill = "lightblue")) + .density = ComplexHeatmap::anno_density(df, type = "line", gp = grid::gpar(col = "blue")) + ha_mix_top = ComplexHeatmap::HeatmapAnnotation(hist = .hist, density = .density) + # Define some graphics to display the distribution of rows + .violin = ComplexHeatmap::anno_density(df, type = "violin", + gp = grid::gpar(fill = "lightblue"), which = "row") + .boxplot = ComplexHeatmap::anno_boxplot(df, which = "row") + ha_mix_right = ComplexHeatmap::HeatmapAnnotation(violin = .violin, bxplt = .boxplot, + which = "row", width = grid::unit(4, "cm")) + # Combine annotation with heatmap + heatmap_main <- ComplexHeatmap::Heatmap(df, name = "Ranking", + column_names_gp = grid::gpar(fontsize = 8), + top_annotation = ha_mix_top, + top_annotation_height = grid::unit(3.8, "cm")) + return(ComplexHeatmap::draw( + ComplexHeatmap::`+.AdditiveUnit`(heatmap_main, ha_mix_right)) + ) +} diff --git a/R/readFiles.R b/R/readFiles.R index 07f9d45a..550d1138 100644 --- a/R/readFiles.R +++ b/R/readFiles.R @@ -1,956 +1,956 @@ -# sourceCpp('src/align.cc') -# sourceCpp('src/read.cc') - -#' Reduce the size of the data set by evenly subsampling the records -#' -#' @param df The data to subsample -#' @param n The amount of samples -#' @return A smaller data.frame -limit.data <- function(df, n) { - N <- nrow(df) - if (N > n) { - idx <- unique(c(1, seq(1, N, length.out = n), N)) - df[idx, ] - } else - df -} - -#' Scan *.info files for IOHProfiler or COCO -#' -#' @param folder The folder containing the .info and .json files -#' @return The paths to all found .info and .json-files -#' @export -#' @note This automatically filetrs our files of size 0 -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -#' scan_index_file(path) -scan_index_file <- function(folder) { - folder <- trimws(folder) - files <- list.files(folder, pattern = '.(info|json)$', recursive = T, full.names = T) - files[file.size(files) > 0] -} - -#' Read .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @export -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -#' info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) -read_index_file <- function(fname) { - format <- tools::file_ext(file) - if (format == 'json') - read_index_file__json(fname) - else { - tryCatch( - read_index_file__IOH(fname), - warning = function(e) read_index_file__COCO(fname), - error = function(e) read_index_file__COCO(fname), - finally = function(e) stop(paste0('Error in reading .info files ', e)) - ) - } -} - -#' Read IOHprofiler-based .json files and extract information -#' -#' @param fname The path to the json info-file -#' @return The data contained in the json info-file -#' @noRd -read_index_file__json <- function(fname) { - - json_data <- fromJSON(file = fname) - attribute_names <- attributes(json_data) - - exp_attrs <- json_data$experiment_attributes - - data <- list() - - tryCatch({ - fid <- json_data$function_id - fname <- json_data$function_name - suite <- json_data$suite - maximization <- json_data$maximization - algid <- json_data$algorithm$name - }, error = function(e) {return(NULL)}) - - data <- lapply(json_data$dimensions, function(scenario) { - run_attrs <- list() - - for (run_attr in json_data$run_attributes) { - attr(run_attrs, run_attr) <- sapply(scenario$runs, function(x) x$run_attr) - } - - temp <- c(list( - funcId = fid, - funcName = fname, - suite = suite, - maximization = maximization, - algId = algid, - DIM = scenario$dimension, - datafile = scenario$path, - instance = sapply(scenario$runs, function(x) x$instance), - maxRT = sapply(scenario$runs, function(x) x$evals), - finalFV = sapply(scenario$runs, function(x) x$best$y), - final_pos = sapply(scenario$runs, function(x) x$best$x) - ), exp_attrs, - run_attrs) - - }) - data -} - -#' Read IOHprofiler-based .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @noRd -read_index_file__IOH <- function(fname) { - f <- file(fname, 'r') - path <- dirname(fname) - data <- list() - i <- 1 - - while (TRUE) { - # TODO: remove suppressWarnings later - lines <- suppressWarnings(readLines(f, n = 3)) - if (length(lines) == 0) - break - - # TODO: make this quote symbol ' or " as the configurable parameter - # TODO: Fix this - name_value <- read.csv(text = lines[1], header = F, quote = c("\"","'")) %>% - as.list %>% - unlist %>% - as.vector - - header <- name_value %>% - trimws %>% { - regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' - } %>% - unlist %>% - trimws %>% - matrix(nrow = 2) %>% { - ans <- as.list(.[2, ]) - names(ans) <- .[1, ] - for (name in .[1, ]) { - value <- ans[[name]] - ans[[name]] <- gsub("'", '', value) - - if (name == 'maximization') - value <- as.logical(value) - else - value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - - if (!is.na(value)) ans[[name]] <- value - } - ans - } - - record <- trimws(strsplit(lines[3], ',')[[1]]) - - # TODO: this must also be removed... - if (record[2] == "") { - warning(sprintf('File %s is incomplete!', fname)) - finalFVs <- NULL - instances <- NULL - maxRTs <- NULL - } else { - res <- matrix(unlist(strsplit(record[-1], ':')), nrow = 2) - info <- matrix(unlist(strsplit(res[2, ], '\\|')), nrow = 2) - #Check for incorrect usages of reset_problem and remove them - maxRTs <- as.numeric(info[1,]) - idx_correct <- which(maxRTs > 0) - finalFVs <- as.numeric(info[2,])[idx_correct] - instances <- as.numeric(res[1,])[idx_correct] - maxRTs <- maxRTs[idx_correct] - } - - record[1] <- gsub("\\\\", "/", record[1]) - datafile <- file.path(path, record[1]) - - # TODO: check the name of the attributes and fix them! - data[[i]] <- c( - header, - list( - comment = lines[2], - datafile = datafile, - instance = instances, - maxRT = maxRTs, - finalFV = finalFVs - ) - ) - i <- i + 1 - } - close(f) - data -} - -#' Read single-objective COCO-based .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @noRd -read_index_file__COCO <- function(fname) { - f <- file(fname, 'r') - path <- dirname(fname) - data <- list() - i <- 1 - while (TRUE) { - - lines <- suppressWarnings(readLines(f, n = 3)) # read header and comments - if (length(lines) < 3) { - break - } - comment <- lines[2] - name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) - - header <- trimws(name_value) %>% { - regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' - } %>% - unlist %>% - trimws %>% - matrix(nrow = 2) %>% { - ans <- as.list(.[2, ]) - names(ans) <- .[1, ] - for (name in .[1, ]) { - value <- ans[[name]] - ans[[name]] <- gsub("'", '', value) - value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - if (!is.na(value)) - ans[[name]] <- value - } - ans - } - - names(header) <- gsub('algorithm', 'algId', names(header)) - - record <- strsplit(lines[3], ',')[[1]] %>% trimws - - if (length(record) < 2) { - warning(sprintf('File %s is incomplete!', fname)) - res <- NULL - info <- NULL - } else { - res <- matrix(unlist(strsplit(record[-c(1)], ':')), nrow = 2) - info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) - } - - record[1] <- gsub("\\\\", "/", record[1]) - if ('folder' %in% names(header)) - datafile <- file.path(path, header$folder, record[1]) - else - datafile <- file.path(path, record[1]) - - - # TODO: check the name of the attributes and fix them! - data[[i]] <- c( - header, - list( - comment = comment, - datafile = datafile, - instance = as.numeric(res[1, ]), - maxRT = info[1, ], - finalFV = info[2, ] - ) - ) - i <- i + 1 - } - close(f) - data -} - -#' Read bi-objective COCO-based .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @noRd -read_index_file__BIOBJ_COCO <- function(fname) { - f <- file(fname, 'r') - path <- dirname(fname) - data <- list() - i <- 1 - - lines <- suppressWarnings(readLines(f, n = 2)) # read header and comments - comment <- lines[2] - name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) - - header <- trimws(name_value) %>% { - regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' - } %>% - unlist %>% - trimws %>% - matrix(nrow = 2) %>% { - ans <- as.list(.[2, ]) - names(ans) <- .[1, ] - for (name in .[1, ]) { - value <- ans[[name]] - ans[[name]] <- gsub("'", '', value) - value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - if (!is.na(value)) - ans[[name]] <- value - } - ans - } - - names(header) <- gsub('algorithm', 'algId', names(header)) - while (TRUE) { - # TODO: remove suppressWarnings later - lines <- suppressWarnings(readLines(f, n = 1)) - if (length(lines) == 0) - break - - record <- strsplit(lines[1], ',')[[1]] %>% trimws - - # TODO: this must also be removed... - if (record[4] == "") { - warning(sprintf('File %s is incomplete!', fname)) - res <- NULL - info <- NULL - } else { - res <- matrix(unlist(strsplit(record[-c(1, 2, 3)], ':')), nrow = 2) - info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) - } - - record[3] <- gsub("\\\\", "/", record[3]) - if ('folder' %in% names(header)) - datafile <- file.path(path, header$folder, record[3]) - else - datafile <- file.path(path, record[3]) - - funcId <- trimws(strsplit(record[1], '=')[[1]][2]) - funcId.int <- suppressWarnings(as.integer(funcId.int)) - if(!any(is.na(funcId.int))) { - if(all((funcId.int >= 0L) & (funcId.int <= 1000000000L))) { - funcId <- funcId.int - } - } - - DIM <- as.numeric(trimws(strsplit(record[2], '=')[[1]][2])) - - # TODO: check the name of the attributes and fix them! - data[[i]] <- c( - header, - list( - comment = comment, - funcId = funcId, - DIM = DIM, - datafile = datafile, - instance = as.numeric(res[1, ]), - maxRT = info[1, ], - finalFV = info[2, ] - ) - ) - i <- i + 1 - } - close(f) - data -} - -#' Check the format of data -#' -#' Throws a warning when multiple formats are found in the same folder. -#' -#' @param path The path to the folder to check -#' @return The format of the data in the given folder. Either 'COCO', 'IOHprofiler', -#' 'NEVERGRAD' or 'SOS'. -#' @export -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -#' check_format(path) -check_format <- function(path) { - if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") - return(NEVERGRAD) - - if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "rds") - return("RDS") - - index_files <- scan_index_file(path) - if (length(index_files) == 0) - return(SOS) - - info <- unlist(lapply(index_files, read_index_file), recursive = F) - datafile <- sapply(info, function(item) item$datafile) - - format <- lapply(datafile, function(file) { - tryCatch({ - if (!file.exists(file)) { - cdatfile <- stri_replace(file, ".cdat", fixed = ".dat") - tdatfile <- stri_replace(file, ".tdat", fixed = ".dat") - if (file.exists(cdatfile)) file <- cdatfile - else file <- tdatfile - } - first_line <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) - }, error = function(e) { - stop("Error detecting data files specified in .info, please verify the - integrity of the provided files.") - }) - - if (startsWith(first_line, '% function') || startsWith(first_line, '% f evaluations')) - COCO - else if (startsWith(first_line, '\"function')) { - n_col <- ncol(fread(file, header = FALSE, sep = ' ', - colClasses = 'character', fill = T, nrows = 1)) - if (n_col == 2) - TWO_COL - else - IOHprofiler - } - else if (first_line == '%') # Bi-objective COCO format... - BIBOJ_COCO - else { - stop("Error detecting file format of file ", file, "; Please verify - the integrity of this file.") - } - }) %>% - unlist %>% - unique - - csv_files <- file.path(path, list.files(path, pattern = '.csv', recursive = T)) - if (length(csv_files) > 0) - format <- c(format, NEVERGRAD) - - txt_files <- file.path(path, list.files(path, pattern = '.txt', recursive = T)) - if (length(txt_files) > 0) - format <- c(format, SOS) - - if (length(format) > 1) { - stop( - paste( - path, - 'contains multiple data formats. This is not allowed for data processing. - Please check the returned dataframe for more information.' - ) - ) - } else - format -} - -#' Read IOHProfiler *.dat files -#' -#' @param fname The path to the .dat file -#' @param subsampling Whether to subsample the data or not -#' @noRd -#' @return A list of data.frames -read_dat <- function(fname, subsampling = FALSE) { - # TODO: use the same data loading method as in read_dat__COCO - df <- fread(fname, header = FALSE, sep = ' ', colClasses = 'character', fill = T) - colnames(df) <- as.character(df[1, ]) - idx <- which(!grepl('\\d+', df[[1]], perl = T)) - - # check for data consistence - header_len <- min(apply(df[idx, ] != "", 1, sum)) - idx <- c(idx, nrow(df) + 1) - df <- df[, 1:header_len] - - # turn off the warnings of the data coersion below - options(warn = -1) - # TOOD: this opeartor is the bottelneck - df <- sapply(df, function(c) {class(c) <- 'numeric'; c}) - options(warn = 0) - - res <- lapply(seq(length(idx) - 1), function(i) { - i1 <- idx[i] + 1 - i2 <- idx[i + 1] - 1 - ans <- df[i1:i2, ] - if (i1 == i2) - ans <- as.matrix(t(ans)) - - # TODO: determine the number of record in the 'efficient mode' - if (subsampling) - ans <- limit.data(ans, n = 500) - else - ans - }) - res -} - -# TODO: this method is deprecated. Remove it later -# TODO: maybe not subsampling for COCO data -#' read COCO '.dat'-like file -#' -#' @param fname The path to the .dat file -#' @param subsampling Whether to subsample the data or not -#' @noRd -#' @return A list of data.frames -read_dat__COCO_ <- function(fname, subsampling = FALSE) { - c_read_dat(path.expand(fname), 7, '%') -} - -#' read COCO '.dat'-like file directly in R -#' -#' @param fname The path to the .dat file -#' @param subsampling Whether to subsample the data or not -#' @noRd -#' @return A list of data.frames -read_dat__COCO <- function(fname, subsampling = FALSE) { - select <- seq(5) - # read the file as a character vector (one string per row) - X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] - idx <- which(startsWith(X, '%')) - X <- gsub('\\s+|\\t', ' ', X, perl = T) - - df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) - idx <- c((idx + 1) - seq_along(idx), nrow(df)) - - lapply(seq(length(idx) - 1), - function(i) { - i1 <- idx[i] - i2 <- idx[i + 1] - 1 - as.matrix(df[i1:i2, ]) - }) -} - -read_dat__BIOBJ_COCO <- function(fname, subsampling = FALSE) { - if (endsWith(fname, '.dat')) - select <- seq(3) - else if (endsWith(fname, '.tdat')) - select <- seq(2) - - # read the file as a character vector (one string per row) - X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] - idx <- which(startsWith(X, '%')) - X <- gsub('\\s+|\\t', ' ', X, perl = T) - - df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) - idx <- which(startsWith(X, '% function')) - idx <- c((idx + 1) - seq_along(idx) * 4, nrow(df)) - - lapply(seq(length(idx) - 1), function(i) { - i1 <- idx[i] - i2 <- idx[i + 1] - 1 - as.matrix(df[i1:i2, ]) - }) -} - -# global variables for the alignment functions -idxEvals <- 1 -idxTarget <- 3 -n_data_column <- 5 - -# TODO: add docs to the following three functions -check_contiguous <- function(data) { - sapply(data, - function(d) { - v <- d[, idxEvals] - N <- length(v) - v[1] == 1 && v[N] == N - }) %>% - all -} - -align_contiguous <- function(data, idx, rownames) { - N <- length(data) - nrow <- length(rownames) - lapply(data, - function(d) { - v <- d[, idx] - r <- nrow - length(v) - if (r > 0) { - v <- c(v, rep(v[length(v)], r)) - } - v - }) %>% - unlist %>% - matrix(nrow = nrow, ncol = N) %>% - set_rownames(rownames) -} - -align_non_contiguous <- function(data, idx, rownames) { - N <- length(data) - nrow <- length(rownames) - lapply(data, - function(d) { - c_impute(d[, idx], d[, idxEvals], rownames) - }) %>% - unlist %>% - matrix(nrow = nrow, ncol = N) %>% - set_rownames(rownames) -} - -#' Align data by runtimes -#' @param data The data to align -#' @param format Whether the data is form IOHprofiler or COCO -#' @param include_param Whether to include the recorded parameters in the alignment -#' @noRd -#' @return Data aligned by the running time -align_running_time <- function(data, format = IOHprofiler, include_param = TRUE, - maximization = TRUE) { - if (format == IOHprofiler) - idxTarget <- 3 - else if (format == COCO) - idxTarget <- 3 - else if (format == BIBOJ_COCO) { - n_data_column <- 3 - idxTarget <- 2 - } - else if (format == TWO_COL) { - n_data_column <- 2 - idxTarget <- 2 - } - - FV <- sort(unique(unlist(lapply(data, function(x) x[, idxTarget]))), - decreasing = !maximization) - n_column <- unique(sapply(data, ncol)) - - if (format == COCO) { - n_param <- 0 - idxValue <- idxEvals - param_names <- NULL - } - else if (format == IOHprofiler) { - n_param <- n_column - n_data_column - if (include_param && n_param > 0) { - param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] - idxValue <- c(idxEvals, (n_data_column + 1):n_column) - } - else { - param_names <- NULL - idxValue <- idxEvals - } - } - else { - param_names <- NULL - idxValue <- idxEvals - } - - res <- c_align_running_time(data, FV, idxValue - 1, maximization, idxTarget - 1) - names(res) <- c('RT', param_names) - res -} - -#' Align data by function values -#' @param data The data to align -#' @param format Whether the data is form IOHprofiler or COCO. -#' @param include_param Whether to include the recorded parameters in the alignment -#' @noRd -#' @return Data aligned by the function value -align_function_value <- function(data, include_param = TRUE, format = IOHprofiler) { - n_column <- unique(sapply(data, ncol)) - stopifnot(length(n_column) == 1) - - if (format == COCO) { - maximization <- FALSE - idxTarget <- 3 - n_param <- 0 - } - else if (format == IOHprofiler) { - maximization <- TRUE - idxTarget <- 3 - n_param <- n_column - n_data_column - } - else if (format == BIBOJ_COCO) { # bi-objective COCO format - maximization <- FALSE - idxTarget <- 2 - n_data_column <- 2 - n_param <- 0 # no parameter is allowed in this case - } - else if (format == TWO_COL) { - maximization <- TRUE - idxTarget <- 2 - n_param <- 0 - } - - if (check_contiguous(data)) { - nrow <- sapply(data, nrow) %>% max - runtime <- seq(nrow) - align_func <- align_contiguous - } else { - runtime <- sort(unique(unlist(lapply(data, function(x) x[, idxEvals])))) - nrow <- length(runtime) - align_func <- align_non_contiguous - } - - FV <- align_func(data, idxTarget, runtime) - include_param <- include_param && (n_param > 0) - - if (include_param) { - param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] - param <- list() - for (i in seq(n_param)) { - name <- param_names[i] - param[[name]] <- align_func(data, i + n_data_column, runtime) - } - } - - if (include_param) { - c(list(FV = FV), param) - } else { - list(FV = FV) - } -} - - -#' Read Nevergrad data -#' -#' Read .csv files in nevergrad format and extract information as a DataSetList -#' -#' @param fname The path to the .csv file -#' @return The DataSetList extracted from the .csv file provided -#' @noRd -read_nevergrad <- function(path){ - dt <- fread(path) - - if (!'name' %in% colnames(dt)) { - dt[, name := function_class] - } - - triplets <- unique(dt[, .(optimizer_name, dimension, name)]) - algIds <- unique(triplets$optimizer_name) - DIMs <- unique(triplets$dimension) - funcIds <- unique(triplets$name) - - res <- list() - - idx <- 1 - - for (i in seq(nrow(triplets))) { - algId <- triplets$optimizer_name[i] - DIM <- triplets$dimension[i] - funcId <- triplets$name[i] - - rescale_name <- 'rescale' - if ( !('rescale' %in% colnames(dt))) { - if ( 'transform' %in% colnames(dt)) - colnames(dt)[colnames(dt) == "transform"] <- "rescale" - else{ - dt$rescale <- NA - } - } - - data <- dt[optimizer_name == algId & dimension == DIM & name == funcId, - .(budget, loss, rescale)] - - for (scaled in unique(data$rescale)) { - if (!is.na(scaled)) { - data_reduced <- data[rescale == scaled, .(budget, loss)] - } - else { - data_reduced <- data[is.na(rescale), .(budget, loss)] - } - - if (!is.na(scaled) && scaled) { - funcId_name <- paste0(funcId, '_rescaled') - } - else { - funcId_name <- funcId - } - - rows <- unique(data_reduced$budget) %>% sort - FV <- lapply(rows, - function(b) { - data_reduced[budget == b, loss] - } - ) %>% - do.call(rbind, .) %>% - set_rownames(rows) - - RT <- list() - - ds <- structure(list(RT = RT, FV = FV), - class = c('DataSet', 'list'), - maxRT = max(rows), - finalFV = min(FV), - format = 'NEVERGRAD', - maximization = FALSE, - algId = algId, - funcId = funcId_name, - DIM = DIM) - res[[idx]] <- ds - idx <- idx + 1 - } - } - class(res) %<>% c('DataSetList') - attr(res, 'DIM') <- DIMs - attr(res, 'funcId') <- funcIds - attr(res, 'algId') <- algIds - attr(res, 'suite') <- 'NEVERGRAD' - attr(res, 'maximization') <- F - res - -} - -#' Read single DataSet of SOS-based data -#' -#' Read single .txt files in SOS format and extract information as a DataSet -#' -#' @param file The path to the .txt file -#' @return The DataSet extracted from the .txt file provided -#' @noRd -read_single_file_SOS <- function(file) { - V1 <- NULL #Local binding to remove CRAN warnings - - algId <- substr(basename(file), 1, stringi::stri_locate_last(basename(file), fixed = 'D')[[1]] - 1) - - dt <- fread(file, header = F) - header <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) - splitted <- header %>% trimws %>% strsplit("\\s+") %>% .[[1]] %>% .[2:length(.)] - info <- list(algId = algId) - for (i in seq_len(length(splitted) / 2)) { - temp <- splitted[[2*i]] - name <- splitted[[2*i - 1]] - if (name == 'function') name <- 'funcId' - if (name == 'dim') name <- 'DIM' - names(temp) <- name - info <- c(info, temp) - } - - dim <- as.numeric(info$DIM) - #Hardcoded fix for SB-related data - if (is.null(dim) || length(dim) == 0) { - warning("Dimension not explicitly defined, setting as 30 by default") - dim <- 30 - info$DIM <- dim - } - - RT_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] - names(RT_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] - RT <- as.matrix(RT_raw) - mode(RT) <- 'integer' - - FV_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] - names(FV_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] - FV <- as.matrix(FV_raw) - - - pos <- dt[, (ncol(dt) - dim + 1):ncol(dt)] - colnames(pos) <- as.character(seq_len(dim)) - - maxRT <- max(RT) - finalFV <- min(FV) - - idxs_avail <- dt[['V1']] - idxs_replaced <- dt[['V6']] - - idxs_final <- setdiff(idxs_avail, idxs_replaced) - - idx_final_best <- idxs_final[[which.min(FV[idxs_final])]] - final_pos <- as.numeric(pos[idx_final_best, ]) - # if (sum(FV == finalFV) > 1) { - # #Reconstruct population to determine which best solution is final position - # ids_min <- dt[FV_raw == finalFV, V1] - # replaced_idxs <- dt[[colnames(dt)[[ncol(dt) - dim]]]] - # #If none, take the last one added - # pos_idx <- max(ids_min) - # for (i in ids_min) { - # if (all(replaced_idxs != i)) { - # #If multiple, take the first one added - # pos_idx <- i - # break - # } - # } - # final_pos <- as.numeric(pos[pos_idx, ]) - # } - # else { - # final_pos <- as.numeric(pos[which.min(FV), ]) - # } - - PAR <- list( - # 'position' = list(pos), - 'final_position' = list(final_pos), - 'by_FV' = NULL, - 'by_RT' = NULL - ) - - - - object <- list() - class(object) <- c('DataSet', class(object)) - object$RT <- RT - object$FV <- FV - object$PAR <- PAR - attr(object, 'maxRT') <- maxRT - attr(object, 'finalFV') <- finalFV - attr(object, 'format') <- "SOS" - attr(object, 'maximization') <- F - attr(object, 'suite') <- "SOS" - for (i in seq_along(info)) { - attr(object, names(info)[[i]]) <- type.convert(info[[i]], as.is = T) - } - attr(object, 'ID') <- attr(object, 'algId') - object -} - - -#' Read DataSetList of SOS-based data -#' -#' Read directory containing .txt files in SOS format and extract information as a DataSetList -#' -#' @param dir The path to the directory file -#' @param corrections_file A file containing boundary-correction ratios for the files in `dir` -#' @return The DataSetList extracted from the directory provided -#' @noRd -read_datasetlist_SOS <- function(dir, corrections_files = NULL) { - V1 <- V3 <- V4 <- NULL #Local binding to remove CRAN warnings - res <- list() - dims <- list() - funcIds <- list() - algIds <- list() - suites <- list() - maximizations <- list() - - idx <- 1 - - corrs <- as.data.table(rbindlist(lapply(corrections_files, fread))) - - for (f in list.files(dir, recursive = T, pattern = "*.txt", full.names = T)) { - if (f %in% corrections_files) next - ds <- read_single_file_SOS(f) - - dims[[idx]] <- attr(ds, 'DIM') - funcIds[[idx]] <- attr(ds, 'funcId') - algIds[[idx]] <- attr(ds, 'algId') - suites[[idx]] <- attr(ds, 'suite') - maximizations[[idx]] <- attr(ds, 'maximization') - - if (nrow(corrs) > 0) { - fn <- substr(basename(f), 1, nchar(basename(f)) - 4) - corr_opts <- corrs[V1 == fn, ] - if (stri_detect_fixed(fn, "DE")) { - corr <- corr_opts[V3 == attr(ds, 'F'), ][V4 == attr(ds, 'CR'), 'V2'][['V2']] - } - else if (stri_detect_fixed(fn, "RIS")) { - corr <- corr_opts[['V2']] - } - else { - warning("Unknown algorithm, so skipping lookup of boundary corrections ratio") - corr <- NULL - } - if (length(corr) == 1) - ds$PAR$'corrections' <- corr[[1]] - else - warning(paste0("No boundary corrections ratio found for ", fn)) - } - - res[[idx]] <- ds - idx <- idx + 1 - } - class(res) %<>% c('DataSetList') - attr(res, 'DIM') <- dims - attr(res, 'funcId') <- funcIds - attr(res, 'algId') <- algIds - attr(res, 'ID_attributes') <- c('algId') - - suite <- unique(suites) - maximization <- unique(maximizations) - if (length(suite) != 1 || length(maximization) != 1) { - warning("Multipe different suites detected!") - } - - attr(res, 'suite') <- suite - attr(res, 'maximization') <- maximization - res - clean_DataSetList(res) -} - -#' Find corrections-files in SOS-based folder -#' -#' Read directory containing .txt files in SOS format and extract the corrections-files -#' -#' @param path The path to the directory file -#' @return The relative paths to the corection files -#' @noRd -locate_corrections_files <- function(path) { - files <- list.files(path, recursive = T, pattern = "*.txt", full.names = T) - files[stri_detect_fixed(files, 'corrections')] -} +# sourceCpp('src/align.cc') +# sourceCpp('src/read.cc') + +#' Reduce the size of the data set by evenly subsampling the records +#' +#' @param df The data to subsample +#' @param n The amount of samples +#' @return A smaller data.frame +limit.data <- function(df, n) { + N <- nrow(df) + if (N > n) { + idx <- unique(c(1, seq(1, N, length.out = n), N)) + df[idx, ] + } else + df +} + +#' Scan *.info files for IOHProfiler or COCO +#' +#' @param folder The folder containing the .info and .json files +#' @return The paths to all found .info and .json-files +#' @export +#' @note This automatically filetrs our files of size 0 +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +#' scan_index_file(path) +scan_index_file <- function(folder) { + folder <- trimws(folder) + files <- list.files(folder, pattern = '.(info|json)$', recursive = T, full.names = T) + files[file.size(files) > 0] +} + +#' Read .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @export +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +#' info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) +read_index_file <- function(fname) { + format <- tools::file_ext(fname) + if (format == 'json') + read_index_file__json(fname) + else { + tryCatch( + read_index_file__IOH(fname), + warning = function(e) read_index_file__COCO(fname), + error = function(e) read_index_file__COCO(fname), + finally = function(e) stop(paste0('Error in reading .info files ', e)) + ) + } +} + +#' Read IOHprofiler-based .json files and extract information +#' +#' @param fname The path to the json info-file +#' @return The data contained in the json info-file +#' @noRd +read_index_file__json <- function(fname) { + + json_data <- fromJSON(file = fname) + attribute_names <- attributes(json_data) + + exp_attrs <- json_data$experiment_attributes + + data <- list() + + tryCatch({ + fid <- json_data$function_id + fname <- json_data$function_name + suite <- json_data$suite + maximization <- json_data$maximization + algid <- json_data$algorithm$name + }, error = function(e) {return(NULL)}) + + data <- lapply(json_data$dimensions, function(scenario) { + run_attrs <- list() + + for (run_attr in json_data$run_attributes) { + attr(run_attrs, run_attr) <- sapply(scenario$runs, function(x) x$run_attr) + } + + temp <- c(list( + funcId = fid, + funcName = fname, + suite = suite, + maximization = maximization, + algId = algid, + DIM = scenario$dimension, + datafile = scenario$path, + instance = sapply(scenario$runs, function(x) x$instance), + maxRT = sapply(scenario$runs, function(x) x$evals), + finalFV = sapply(scenario$runs, function(x) x$best$y), + final_pos = sapply(scenario$runs, function(x) x$best$x) + ), exp_attrs, + run_attrs) + + }) + data +} + +#' Read IOHprofiler-based .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @noRd +read_index_file__IOH <- function(fname) { + f <- file(fname, 'r') + path <- dirname(fname) + data <- list() + i <- 1 + + while (TRUE) { + # TODO: remove suppressWarnings later + lines <- suppressWarnings(readLines(f, n = 3)) + if (length(lines) == 0) + break + + # TODO: make this quote symbol ' or " as the configurable parameter + # TODO: Fix this + name_value <- read.csv(text = lines[1], header = F, quote = c("\"","'")) %>% + as.list %>% + unlist %>% + as.vector + + header <- name_value %>% + trimws %>% { + regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' + } %>% + unlist %>% + trimws %>% + matrix(nrow = 2) %>% { + ans <- as.list(.[2, ]) + names(ans) <- .[1, ] + for (name in .[1, ]) { + value <- ans[[name]] + ans[[name]] <- gsub("'", '', value) + + if (name == 'maximization') + value <- as.logical(value) + else + value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric + + if (!is.na(value)) ans[[name]] <- value + } + ans + } + + record <- trimws(strsplit(lines[3], ',')[[1]]) + + # TODO: this must also be removed... + if (record[2] == "") { + warning(sprintf('File %s is incomplete!', fname)) + finalFVs <- NULL + instances <- NULL + maxRTs <- NULL + } else { + res <- matrix(unlist(strsplit(record[-1], ':')), nrow = 2) + info <- matrix(unlist(strsplit(res[2, ], '\\|')), nrow = 2) + #Check for incorrect usages of reset_problem and remove them + maxRTs <- as.numeric(info[1,]) + idx_correct <- which(maxRTs > 0) + finalFVs <- as.numeric(info[2,])[idx_correct] + instances <- as.numeric(res[1,])[idx_correct] + maxRTs <- maxRTs[idx_correct] + } + + record[1] <- gsub("\\\\", "/", record[1]) + datafile <- file.path(path, record[1]) + + # TODO: check the name of the attributes and fix them! + data[[i]] <- c( + header, + list( + comment = lines[2], + datafile = datafile, + instance = instances, + maxRT = maxRTs, + finalFV = finalFVs + ) + ) + i <- i + 1 + } + close(f) + data +} + +#' Read single-objective COCO-based .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @noRd +read_index_file__COCO <- function(fname) { + f <- file(fname, 'r') + path <- dirname(fname) + data <- list() + i <- 1 + while (TRUE) { + + lines <- suppressWarnings(readLines(f, n = 3)) # read header and comments + if (length(lines) < 3) { + break + } + comment <- lines[2] + name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) + + header <- trimws(name_value) %>% { + regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' + } %>% + unlist %>% + trimws %>% + matrix(nrow = 2) %>% { + ans <- as.list(.[2, ]) + names(ans) <- .[1, ] + for (name in .[1, ]) { + value <- ans[[name]] + ans[[name]] <- gsub("'", '', value) + value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric + if (!is.na(value)) + ans[[name]] <- value + } + ans + } + + names(header) <- gsub('algorithm', 'algId', names(header)) + + record <- strsplit(lines[3], ',')[[1]] %>% trimws + + if (length(record) < 2) { + warning(sprintf('File %s is incomplete!', fname)) + res <- NULL + info <- NULL + } else { + res <- matrix(unlist(strsplit(record[-c(1)], ':')), nrow = 2) + info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) + } + + record[1] <- gsub("\\\\", "/", record[1]) + if ('folder' %in% names(header)) + datafile <- file.path(path, header$folder, record[1]) + else + datafile <- file.path(path, record[1]) + + + # TODO: check the name of the attributes and fix them! + data[[i]] <- c( + header, + list( + comment = comment, + datafile = datafile, + instance = as.numeric(res[1, ]), + maxRT = info[1, ], + finalFV = info[2, ] + ) + ) + i <- i + 1 + } + close(f) + data +} + +#' Read bi-objective COCO-based .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @noRd +read_index_file__BIOBJ_COCO <- function(fname) { + f <- file(fname, 'r') + path <- dirname(fname) + data <- list() + i <- 1 + + lines <- suppressWarnings(readLines(f, n = 2)) # read header and comments + comment <- lines[2] + name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) + + header <- trimws(name_value) %>% { + regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' + } %>% + unlist %>% + trimws %>% + matrix(nrow = 2) %>% { + ans <- as.list(.[2, ]) + names(ans) <- .[1, ] + for (name in .[1, ]) { + value <- ans[[name]] + ans[[name]] <- gsub("'", '', value) + value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric + if (!is.na(value)) + ans[[name]] <- value + } + ans + } + + names(header) <- gsub('algorithm', 'algId', names(header)) + while (TRUE) { + # TODO: remove suppressWarnings later + lines <- suppressWarnings(readLines(f, n = 1)) + if (length(lines) == 0) + break + + record <- strsplit(lines[1], ',')[[1]] %>% trimws + + # TODO: this must also be removed... + if (record[4] == "") { + warning(sprintf('File %s is incomplete!', fname)) + res <- NULL + info <- NULL + } else { + res <- matrix(unlist(strsplit(record[-c(1, 2, 3)], ':')), nrow = 2) + info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) + } + + record[3] <- gsub("\\\\", "/", record[3]) + if ('folder' %in% names(header)) + datafile <- file.path(path, header$folder, record[3]) + else + datafile <- file.path(path, record[3]) + + funcId <- trimws(strsplit(record[1], '=')[[1]][2]) + funcId.int <- suppressWarnings(as.integer(funcId.int)) + if(!any(is.na(funcId.int))) { + if(all((funcId.int >= 0L) & (funcId.int <= 1000000000L))) { + funcId <- funcId.int + } + } + + DIM <- as.numeric(trimws(strsplit(record[2], '=')[[1]][2])) + + # TODO: check the name of the attributes and fix them! + data[[i]] <- c( + header, + list( + comment = comment, + funcId = funcId, + DIM = DIM, + datafile = datafile, + instance = as.numeric(res[1, ]), + maxRT = info[1, ], + finalFV = info[2, ] + ) + ) + i <- i + 1 + } + close(f) + data +} + +#' Check the format of data +#' +#' Throws a warning when multiple formats are found in the same folder. +#' +#' @param path The path to the folder to check +#' @return The format of the data in the given folder. Either 'COCO', 'IOHprofiler', +#' 'NEVERGRAD' or 'SOS'. +#' @export +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +#' check_format(path) +check_format <- function(path) { + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") + return(NEVERGRAD) + + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "rds") + return("RDS") + + index_files <- scan_index_file(path) + if (length(index_files) == 0) + return(SOS) + + info <- unlist(lapply(index_files, read_index_file), recursive = F) + datafile <- sapply(info, function(item) item$datafile) + + format <- lapply(datafile, function(file) { + tryCatch({ + if (!file.exists(file)) { + cdatfile <- stri_replace(file, ".cdat", fixed = ".dat") + tdatfile <- stri_replace(file, ".tdat", fixed = ".dat") + if (file.exists(cdatfile)) file <- cdatfile + else file <- tdatfile + } + first_line <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) + }, error = function(e) { + stop("Error detecting data files specified in .info, please verify the + integrity of the provided files.") + }) + + if (startsWith(first_line, '% function') || startsWith(first_line, '% f evaluations')) + COCO + else if (startsWith(first_line, '\"function')) { + n_col <- ncol(fread(file, header = FALSE, sep = ' ', + colClasses = 'character', fill = T, nrows = 1)) + if (n_col == 2) + TWO_COL + else + IOHprofiler + } + else if (first_line == '%') # Bi-objective COCO format... + BIBOJ_COCO + else { + stop("Error detecting file format of file ", file, "; Please verify + the integrity of this file.") + } + }) %>% + unlist %>% + unique + + csv_files <- file.path(path, list.files(path, pattern = '.csv', recursive = T)) + if (length(csv_files) > 0) + format <- c(format, NEVERGRAD) + + txt_files <- file.path(path, list.files(path, pattern = '.txt', recursive = T)) + if (length(txt_files) > 0) + format <- c(format, SOS) + + if (length(format) > 1) { + stop( + paste( + path, + 'contains multiple data formats. This is not allowed for data processing. + Please check the returned dataframe for more information.' + ) + ) + } else + format +} + +#' Read IOHProfiler *.dat files +#' +#' @param fname The path to the .dat file +#' @param subsampling Whether to subsample the data or not +#' @noRd +#' @return A list of data.frames +read_dat <- function(fname, subsampling = FALSE) { + # TODO: use the same data loading method as in read_dat__COCO + df <- fread(fname, header = FALSE, sep = ' ', colClasses = 'character', fill = T) + colnames(df) <- as.character(df[1, ]) + idx <- which(!grepl('\\d+', df[[1]], perl = T)) + + # check for data consistence + header_len <- min(apply(df[idx, ] != "", 1, sum)) + idx <- c(idx, nrow(df) + 1) + df <- df[, 1:header_len] + + # turn off the warnings of the data coersion below + options(warn = -1) + # TOOD: this opeartor is the bottelneck + df <- sapply(df, function(c) {class(c) <- 'numeric'; c}) + options(warn = 0) + + res <- lapply(seq(length(idx) - 1), function(i) { + i1 <- idx[i] + 1 + i2 <- idx[i + 1] - 1 + ans <- df[i1:i2, ] + if (i1 == i2) + ans <- as.matrix(t(ans)) + + # TODO: determine the number of record in the 'efficient mode' + if (subsampling) + ans <- limit.data(ans, n = 500) + else + ans + }) + res +} + +# TODO: this method is deprecated. Remove it later +# TODO: maybe not subsampling for COCO data +#' read COCO '.dat'-like file +#' +#' @param fname The path to the .dat file +#' @param subsampling Whether to subsample the data or not +#' @noRd +#' @return A list of data.frames +read_dat__COCO_ <- function(fname, subsampling = FALSE) { + c_read_dat(path.expand(fname), 7, '%') +} + +#' read COCO '.dat'-like file directly in R +#' +#' @param fname The path to the .dat file +#' @param subsampling Whether to subsample the data or not +#' @noRd +#' @return A list of data.frames +read_dat__COCO <- function(fname, subsampling = FALSE) { + select <- seq(5) + # read the file as a character vector (one string per row) + X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] + idx <- which(startsWith(X, '%')) + X <- gsub('\\s+|\\t', ' ', X, perl = T) + + df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) + idx <- c((idx + 1) - seq_along(idx), nrow(df)) + + lapply(seq(length(idx) - 1), + function(i) { + i1 <- idx[i] + i2 <- idx[i + 1] - 1 + as.matrix(df[i1:i2, ]) + }) +} + +read_dat__BIOBJ_COCO <- function(fname, subsampling = FALSE) { + if (endsWith(fname, '.dat')) + select <- seq(3) + else if (endsWith(fname, '.tdat')) + select <- seq(2) + + # read the file as a character vector (one string per row) + X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] + idx <- which(startsWith(X, '%')) + X <- gsub('\\s+|\\t', ' ', X, perl = T) + + df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) + idx <- which(startsWith(X, '% function')) + idx <- c((idx + 1) - seq_along(idx) * 4, nrow(df)) + + lapply(seq(length(idx) - 1), function(i) { + i1 <- idx[i] + i2 <- idx[i + 1] - 1 + as.matrix(df[i1:i2, ]) + }) +} + +# global variables for the alignment functions +idxEvals <- 1 +idxTarget <- 3 +n_data_column <- 5 + +# TODO: add docs to the following three functions +check_contiguous <- function(data) { + sapply(data, + function(d) { + v <- d[, idxEvals] + N <- length(v) + v[1] == 1 && v[N] == N + }) %>% + all +} + +align_contiguous <- function(data, idx, rownames) { + N <- length(data) + nrow <- length(rownames) + lapply(data, + function(d) { + v <- d[, idx] + r <- nrow - length(v) + if (r > 0) { + v <- c(v, rep(v[length(v)], r)) + } + v + }) %>% + unlist %>% + matrix(nrow = nrow, ncol = N) %>% + set_rownames(rownames) +} + +align_non_contiguous <- function(data, idx, rownames) { + N <- length(data) + nrow <- length(rownames) + lapply(data, + function(d) { + c_impute(d[, idx], d[, idxEvals], rownames) + }) %>% + unlist %>% + matrix(nrow = nrow, ncol = N) %>% + set_rownames(rownames) +} + +#' Align data by runtimes +#' @param data The data to align +#' @param format Whether the data is form IOHprofiler or COCO +#' @param include_param Whether to include the recorded parameters in the alignment +#' @noRd +#' @return Data aligned by the running time +align_running_time <- function(data, format = IOHprofiler, include_param = TRUE, + maximization = TRUE) { + if (format == IOHprofiler) + idxTarget <- 3 + else if (format == COCO) + idxTarget <- 3 + else if (format == BIBOJ_COCO) { + n_data_column <- 3 + idxTarget <- 2 + } + else if (format == TWO_COL) { + n_data_column <- 2 + idxTarget <- 2 + } + + FV <- sort(unique(unlist(lapply(data, function(x) x[, idxTarget]))), + decreasing = !maximization) + n_column <- unique(sapply(data, ncol)) + + if (format == COCO) { + n_param <- 0 + idxValue <- idxEvals + param_names <- NULL + } + else if (format == IOHprofiler) { + n_param <- n_column - n_data_column + if (include_param && n_param > 0) { + param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] + idxValue <- c(idxEvals, (n_data_column + 1):n_column) + } + else { + param_names <- NULL + idxValue <- idxEvals + } + } + else { + param_names <- NULL + idxValue <- idxEvals + } + + res <- c_align_running_time(data, FV, idxValue - 1, maximization, idxTarget - 1) + names(res) <- c('RT', param_names) + res +} + +#' Align data by function values +#' @param data The data to align +#' @param format Whether the data is form IOHprofiler or COCO. +#' @param include_param Whether to include the recorded parameters in the alignment +#' @noRd +#' @return Data aligned by the function value +align_function_value <- function(data, include_param = TRUE, format = IOHprofiler) { + n_column <- unique(sapply(data, ncol)) + stopifnot(length(n_column) == 1) + + if (format == COCO) { + maximization <- FALSE + idxTarget <- 3 + n_param <- 0 + } + else if (format == IOHprofiler) { + maximization <- TRUE + idxTarget <- 3 + n_param <- n_column - n_data_column + } + else if (format == BIBOJ_COCO) { # bi-objective COCO format + maximization <- FALSE + idxTarget <- 2 + n_data_column <- 2 + n_param <- 0 # no parameter is allowed in this case + } + else if (format == TWO_COL) { + maximization <- TRUE + idxTarget <- 2 + n_param <- 0 + } + + if (check_contiguous(data)) { + nrow <- sapply(data, nrow) %>% max + runtime <- seq(nrow) + align_func <- align_contiguous + } else { + runtime <- sort(unique(unlist(lapply(data, function(x) x[, idxEvals])))) + nrow <- length(runtime) + align_func <- align_non_contiguous + } + + FV <- align_func(data, idxTarget, runtime) + include_param <- include_param && (n_param > 0) + + if (include_param) { + param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] + param <- list() + for (i in seq(n_param)) { + name <- param_names[i] + param[[name]] <- align_func(data, i + n_data_column, runtime) + } + } + + if (include_param) { + c(list(FV = FV), param) + } else { + list(FV = FV) + } +} + + +#' Read Nevergrad data +#' +#' Read .csv files in nevergrad format and extract information as a DataSetList +#' +#' @param fname The path to the .csv file +#' @return The DataSetList extracted from the .csv file provided +#' @noRd +read_nevergrad <- function(path){ + dt <- fread(path) + + if (!'name' %in% colnames(dt)) { + dt[, name := function_class] + } + + triplets <- unique(dt[, .(optimizer_name, dimension, name)]) + algIds <- unique(triplets$optimizer_name) + DIMs <- unique(triplets$dimension) + funcIds <- unique(triplets$name) + + res <- list() + + idx <- 1 + + for (i in seq(nrow(triplets))) { + algId <- triplets$optimizer_name[i] + DIM <- triplets$dimension[i] + funcId <- triplets$name[i] + + rescale_name <- 'rescale' + if ( !('rescale' %in% colnames(dt))) { + if ( 'transform' %in% colnames(dt)) + colnames(dt)[colnames(dt) == "transform"] <- "rescale" + else{ + dt$rescale <- NA + } + } + + data <- dt[optimizer_name == algId & dimension == DIM & name == funcId, + .(budget, loss, rescale)] + + for (scaled in unique(data$rescale)) { + if (!is.na(scaled)) { + data_reduced <- data[rescale == scaled, .(budget, loss)] + } + else { + data_reduced <- data[is.na(rescale), .(budget, loss)] + } + + if (!is.na(scaled) && scaled) { + funcId_name <- paste0(funcId, '_rescaled') + } + else { + funcId_name <- funcId + } + + rows <- unique(data_reduced$budget) %>% sort + FV <- lapply(rows, + function(b) { + data_reduced[budget == b, loss] + } + ) %>% + do.call(rbind, .) %>% + set_rownames(rows) + + RT <- list() + + ds <- structure(list(RT = RT, FV = FV), + class = c('DataSet', 'list'), + maxRT = max(rows), + finalFV = min(FV), + format = 'NEVERGRAD', + maximization = FALSE, + algId = algId, + funcId = funcId_name, + DIM = DIM) + res[[idx]] <- ds + idx <- idx + 1 + } + } + class(res) %<>% c('DataSetList') + attr(res, 'DIM') <- DIMs + attr(res, 'funcId') <- funcIds + attr(res, 'algId') <- algIds + attr(res, 'suite') <- 'NEVERGRAD' + attr(res, 'maximization') <- F + res + +} + +#' Read single DataSet of SOS-based data +#' +#' Read single .txt files in SOS format and extract information as a DataSet +#' +#' @param file The path to the .txt file +#' @return The DataSet extracted from the .txt file provided +#' @noRd +read_single_file_SOS <- function(file) { + V1 <- NULL #Local binding to remove CRAN warnings + + algId <- substr(basename(file), 1, stringi::stri_locate_last(basename(file), fixed = 'D')[[1]] - 1) + + dt <- fread(file, header = F) + header <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) + splitted <- header %>% trimws %>% strsplit("\\s+") %>% .[[1]] %>% .[2:length(.)] + info <- list(algId = algId) + for (i in seq_len(length(splitted) / 2)) { + temp <- splitted[[2*i]] + name <- splitted[[2*i - 1]] + if (name == 'function') name <- 'funcId' + if (name == 'dim') name <- 'DIM' + names(temp) <- name + info <- c(info, temp) + } + + dim <- as.numeric(info$DIM) + #Hardcoded fix for SB-related data + if (is.null(dim) || length(dim) == 0) { + warning("Dimension not explicitly defined, setting as 30 by default") + dim <- 30 + info$DIM <- dim + } + + RT_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] + names(RT_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] + RT <- as.matrix(RT_raw) + mode(RT) <- 'integer' + + FV_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] + names(FV_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] + FV <- as.matrix(FV_raw) + + + pos <- dt[, (ncol(dt) - dim + 1):ncol(dt)] + colnames(pos) <- as.character(seq_len(dim)) + + maxRT <- max(RT) + finalFV <- min(FV) + + idxs_avail <- dt[['V1']] + idxs_replaced <- dt[['V6']] + + idxs_final <- setdiff(idxs_avail, idxs_replaced) + + idx_final_best <- idxs_final[[which.min(FV[idxs_final])]] + final_pos <- as.numeric(pos[idx_final_best, ]) + # if (sum(FV == finalFV) > 1) { + # #Reconstruct population to determine which best solution is final position + # ids_min <- dt[FV_raw == finalFV, V1] + # replaced_idxs <- dt[[colnames(dt)[[ncol(dt) - dim]]]] + # #If none, take the last one added + # pos_idx <- max(ids_min) + # for (i in ids_min) { + # if (all(replaced_idxs != i)) { + # #If multiple, take the first one added + # pos_idx <- i + # break + # } + # } + # final_pos <- as.numeric(pos[pos_idx, ]) + # } + # else { + # final_pos <- as.numeric(pos[which.min(FV), ]) + # } + + PAR <- list( + # 'position' = list(pos), + 'final_position' = list(final_pos), + 'by_FV' = NULL, + 'by_RT' = NULL + ) + + + + object <- list() + class(object) <- c('DataSet', class(object)) + object$RT <- RT + object$FV <- FV + object$PAR <- PAR + attr(object, 'maxRT') <- maxRT + attr(object, 'finalFV') <- finalFV + attr(object, 'format') <- "SOS" + attr(object, 'maximization') <- F + attr(object, 'suite') <- "SOS" + for (i in seq_along(info)) { + attr(object, names(info)[[i]]) <- type.convert(info[[i]], as.is = T) + } + attr(object, 'ID') <- attr(object, 'algId') + object +} + + +#' Read DataSetList of SOS-based data +#' +#' Read directory containing .txt files in SOS format and extract information as a DataSetList +#' +#' @param dir The path to the directory file +#' @param corrections_file A file containing boundary-correction ratios for the files in `dir` +#' @return The DataSetList extracted from the directory provided +#' @noRd +read_datasetlist_SOS <- function(dir, corrections_files = NULL) { + V1 <- V3 <- V4 <- NULL #Local binding to remove CRAN warnings + res <- list() + dims <- list() + funcIds <- list() + algIds <- list() + suites <- list() + maximizations <- list() + + idx <- 1 + + corrs <- as.data.table(rbindlist(lapply(corrections_files, fread))) + + for (f in list.files(dir, recursive = T, pattern = "*.txt", full.names = T)) { + if (f %in% corrections_files) next + ds <- read_single_file_SOS(f) + + dims[[idx]] <- attr(ds, 'DIM') + funcIds[[idx]] <- attr(ds, 'funcId') + algIds[[idx]] <- attr(ds, 'algId') + suites[[idx]] <- attr(ds, 'suite') + maximizations[[idx]] <- attr(ds, 'maximization') + + if (nrow(corrs) > 0) { + fn <- substr(basename(f), 1, nchar(basename(f)) - 4) + corr_opts <- corrs[V1 == fn, ] + if (stri_detect_fixed(fn, "DE")) { + corr <- corr_opts[V3 == attr(ds, 'F'), ][V4 == attr(ds, 'CR'), 'V2'][['V2']] + } + else if (stri_detect_fixed(fn, "RIS")) { + corr <- corr_opts[['V2']] + } + else { + warning("Unknown algorithm, so skipping lookup of boundary corrections ratio") + corr <- NULL + } + if (length(corr) == 1) + ds$PAR$'corrections' <- corr[[1]] + else + warning(paste0("No boundary corrections ratio found for ", fn)) + } + + res[[idx]] <- ds + idx <- idx + 1 + } + class(res) %<>% c('DataSetList') + attr(res, 'DIM') <- dims + attr(res, 'funcId') <- funcIds + attr(res, 'algId') <- algIds + attr(res, 'ID_attributes') <- c('algId') + + suite <- unique(suites) + maximization <- unique(maximizations) + if (length(suite) != 1 || length(maximization) != 1) { + warning("Multipe different suites detected!") + } + + attr(res, 'suite') <- suite + attr(res, 'maximization') <- maximization + res + clean_DataSetList(res) +} + +#' Find corrections-files in SOS-based folder +#' +#' Read directory containing .txt files in SOS format and extract the corrections-files +#' +#' @param path The path to the directory file +#' @return The relative paths to the corection files +#' @noRd +locate_corrections_files <- function(path) { + files <- list.files(path, recursive = T, pattern = "*.txt", full.names = T) + files[stri_detect_fixed(files, 'corrections')] +} diff --git a/R/runServer.R b/R/runServer.R index dd7741e7..00a51868 100644 --- a/R/runServer.R +++ b/R/runServer.R @@ -1,18 +1,18 @@ -#' Create a shiny-server GUI to interactively use the IOHanalyzer -#' @param port Optional; which port the server should be opened at. Defaults -#' to the option set for 'shiny.port' -#' @param open_browser Whether or not to open a browser tab with the -#' IOHanalyzer GUI. Defaults to TRUE. -#' @export -#' @examples -#' \dontrun{ -#' runServer(6563, TRUE) -#' } -runServer <- function(port = getOption('shiny.port'), open_browser = TRUE) { - appDir <- system.file("shiny-server", package = "IOHanalyzer") - if (appDir == "") { - stop("Could not find example directory. Try re-installing `IOHanalyzer`.", call. = FALSE) - } - - shiny::runApp(appDir, port = port, launch.browser = open_browser, display.mode = "normal") -} +#' Create a shiny-server GUI to interactively use the IOHanalyzer +#' @param port Optional; which port the server should be opened at. Defaults +#' to the option set for 'shiny.port' +#' @param open_browser Whether or not to open a browser tab with the +#' IOHanalyzer GUI. Defaults to TRUE. +#' @export +#' @examples +#' \dontrun{ +#' runServer(6563, TRUE) +#' } +runServer <- function(port = getOption('shiny.port'), open_browser = TRUE) { + appDir <- system.file("shiny-server", package = "IOHanalyzer") + if (appDir == "") { + stop("Could not find example directory. Try re-installing `IOHanalyzer`.", call. = FALSE) + } + + shiny::runApp(appDir, port = port, launch.browser = open_browser, display.mode = "normal") +} diff --git a/R/stats.R b/R/stats.R index d46092c3..3d57c137 100644 --- a/R/stats.R +++ b/R/stats.R @@ -1,1024 +1,1024 @@ -#' Estimator 'SP' for the Expected Running Time (ERT) -#' -#' @param data A dataframe or matrix. Each row stores the runtime sample points from -#' several runs -#' @param max_runtime The budget to use for calculating ERT. If this is a vector, the largest value is taken. -#' Using this as a vector is being deprecated, and will be removed in a future update -#' -#' @return A list containing ERTs, number of succesfull runs and the succes rate -#' @export -#' @examples -#' SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) -SP <- function(data, max_runtime) { - N <- ncol(data) - if (length(max_runtime) > 1) { - max_runtime <- max(max_runtime) - } - idx <- is.na(data) | data > max_runtime - succ <- N - rowSums(idx) - succ_rate <- succ / N - data[idx] <- max_runtime - - list(ERT = rowSums(data) / succ, runs = succ, succ_rate = succ_rate) -} - -#' Bootstrapping for running time samples -#' -#' @param x A numeric vector. A sample of the running time. -#' @param max_eval A numeric vector, containing the maximal running time in -#' each run. It should have the same size as x -#' @param bootstrap.size integer, the size of the bootstrapped sample -#' -#' @return A numeric vector of the bootstrapped running time sample -#' @export -#' @examples -#' ds <- dsl[[1]] -#' x <- get_RT_sample(ds, ftarget = 16, output = 'long') -#' max_eval <- get_maxRT(dsl, output = 'long') -#' bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) -bootstrap_RT <- function(x, max_eval, bootstrap.size) { - x_succ <- x[!is.na(x)] - x_unsucc <- max_eval[is.na(x)] - n_succ <- length(x_succ) - n_unsucc <- length(x_unsucc) - - p <- n_succ / length(x) - N <- rgeom(bootstrap.size, p) - - if (n_succ == 0){ - return (rep(Inf, bootstrap.size)) - } - - sapply(N, - function(size) { - if (size > 0) - x <- sum(sample(x_unsucc, size, replace = T)) - else - x <- 0 - x <- x + sample(x_succ, 1, replace = T) - }) -} - -# TODO: remove the bootstrapping part as it does not make much sense here... -#' Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times -#' among a data set -#' -#' @description This function performs a Kolmogorov-Smirnov test on each pair of -#' algorithms in the input x to determine which algorithm gives a significantly -#' smaller running time. The resulting p-values are arranged in a matrix, where -#' each cell (i, j) contains a p-value from the test with alternative hypothesis: -#' the running time of algorithm i is smaller (thus better) than that of j. -#' -#' @param x either a list that contains running time sample for each algorithm as -#' sub-lists, or a DataSetList object -#' @param bootstrap.size integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping -#' @param ... all other options -#' @return A matrix containing p-values of the test -#' @export -#' @examples -#' pairwise.test(subset(dsl, funcId == 1), 16) -pairwise.test <- function(x, ...) UseMethod('pairwise.test', x) - -#' @param max_eval list that contains the maximal running time for each algorithm -#' as sub-lists -#' @export -#' @rdname pairwise.test -pairwise.test.list <- function(x, max_eval, bootstrap.size = 30, ...) { - if ("DataSetList" %in% class(x)) { - class(x) <- rev(class(x)) - pairwise.test.DataSetList(x) - } - - N <- length(x) - p.value <- matrix(NA, N, N) - - for (i in seq(1, N - 1)) { - for (j in seq(i + 1, N)) { - if (bootstrap.size == 0) { - x1 <- x[[i]] - x2 <- x[[j]] - } else { - x1 <- bootstrap_RT(x[[i]], max_eval[[i]], bootstrap.size) - x2 <- bootstrap_RT(x[[j]], max_eval[[j]], bootstrap.size) - } - if (all(is.na(x1))) { - if (all(is.na(x2))) { - next - } - else { - p.value[i, j] <- 1 - p.value[j, i] <- 0 - } - } - else if (all(is.na(x2))) { - p.value[i, j] <- 0 - p.value[j, i] <- 1 - } - else { - options(warn = -1) - p.value[i, j] <- ks.test(x1, x2, alternative = 'greater', exact = F)$p.value - p.value[j, i] <- ks.test(x1, x2, alternative = 'less', exact = F)$p.value - options(warn = 0) - } - } - } - - p.value.adjust <- p.adjust(as.vector(p.value), method = 'holm') - p.value <- matrix(p.value.adjust, N, N) - colnames(p.value) <- rownames(p.value) <- names(x) - p.value -} - -#' @param ftarget float, the target value used to determine the running / hitting -#' @param which wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison -#' time -#' @export -#' @rdname pairwise.test -pairwise.test.DataSetList <- function(x, ftarget, bootstrap.size = 0, which = 'by_FV', ...) { - if (which == 'by_FV') { - dt <- get_RT_sample(x, ftarget, output = 'long') - maxRT <- get_maxRT(x, output = 'long') - maxRT <- split(maxRT$maxRT, maxRT$ID) - s <- split(dt$RT, dt$ID) - } - else if (which == 'by_RT') { - dt <- get_FV_sample(x, ftarget, output = 'long') - maxRT <- NULL - if (bootstrap.size > 0) warning("Bootstrapping is currently not supported for - fixed-budget statistics.") - bootstrap.size = 0 - s <- split(dt$`f(x)`, dt$ID) - } - else stop("Unsupported argument 'which'. Available options are 'by_FV' and 'by_RT'") - - return(pairwise.test.list(s, maxRT, bootstrap.size)) -} - -# TODO: move those two functions to a separate file -# TODO: review / re-write this function -#' Function for generating sequences of function values -#' -#' @param FV A list of function values -#' @param from Starting function value. Will be replaced by min(FV) if it is NULL or too small -#' @param to Stopping function value. Will be replaced by max(FV) if it is NULL or too large -#' @param by Stepsize of the sequence. Will be replaced if it is too small -#' @param length.out Number of values in the sequence. -#' 'by' takes preference if both it and length.out are provided. -#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a -#' linear or log-linear spacing respectively. If NULL, the scale will be predicted -#' based on FV -#' -#' @return A sequence of function values -#' @export -#' @examples -#' FVall <- get_runtimes(dsl) -#' seq_FV(FVall, 10, 16, 1, scale='linear') -seq_FV <- function(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, scale = NULL) { - from <- max(from, min(FV)) - to <- min(to, max(FV)) - - rev_trans <- function(x) x - - # Auto detect scaling - # TODO: Improve this detection (based on FV?). Currently very arbitrary - if (is.null(scale)) { - if (to < 0 || from < 0) - scale <- 'linear' - else if (abs(log10(mean(FV)) - log10(median(FV))) > 1) - scale <- 'log' - else - scale <- 'linear' - } - - if (scale == 'log') { - trans <- log10 - rev_trans <- function(x) 10 ^ x - # TODO: Better way to deal with negative values - # set lowest possible target globally instead of arbitrary 1e-12 - from <- max(1e-12, from) - to <- max(1e-12 ,to) - from <- trans(from) - to <- trans(to) - } - - #Avoid generating too many samples - if (!is.null(by)) { - nr_samples_generated <- (to - from) / by - if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { - by <- NULL - if (is.null(length.out)) - length.out <- getOption("IOHanalyzer.max_samples", default = 100) - } - } - - if (is.null(by) || by > to - from) { - if (is.null(length.out)) { - length.out <- 10 - args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) - } else - args <- list(from = from, to = to, length.out = length.out) - } else - args <- list(from = from, to = to, by = by) - - # tryCatch({ - do.call(seq, args) %>% - c(from, ., to) %>% # always include the starting / ending value - unique %>% - rev_trans - # }, error = function(e) { - # c() - # }) -} - -# TODO: review / re-write this function -#' Function for generating sequences of runtime values -#' -#' @param RT A list of runtime values -#' @param from Starting runtime value. Will be replaced by min(RT) if it is NULL or too small -#' @param to Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large -#' @param by Stepsize of the sequence. Will be replaced if it is too small -#' @param length.out Number of values in the sequence. -#' 'by' takes preference if both it and length.out are provided. -#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a -#' linear or log-linear spacing respectively. -#' -#' @return A sequence of runtime values -#' @export -#' @examples -#' RTall <- get_runtimes(dsl) -#' seq_RT(RTall, 0, 500, length.out=10, scale='log') -seq_RT <- function(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, - scale = 'linear') { - rev_trans <- function(x) x - - # Do this first to avoid the log-values being overwritten. - from <- max(from, min(RT)) - to <- min(to, max(RT)) - - if (scale == 'log') { - RT <- log10(RT) - rev_trans <- function(x) 10 ^ x - if (!is.null(from)) - from <- log10(from) - if (!is.null(to)) - to <- log10(to) - if (!is.null(by)) - by <- log10(by) - } - - #Avoid generating too many samples - if (!is.null(by)) { - nr_samples_generated <- (to - from) / by - if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { - by <- NULL - if (is.null(length.out)) - length.out <- getOption("IOHanalyzer.max_samples", default = 100) - } - } - - # Also reset by if it is too large - if (is.null(by) || by > to - from) { - if (is.null(length.out)) { - length.out <- 10 - args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) - } else - args <- list(from = from, to = to, length.out = length.out) - } else - args <- list(from = from, to = to, by = by) - - do.call(seq, args) %>% - c(from, ., to) %>% # always include the starting / ending value - unique %>% - rev_trans %>% - round -} - -# TODO: implement the empirical p.m.f. for runtime -EPMF <- function() { - -} - -#' Empirical Cumulative Dsitribution Function of Runtime of a single data set -#' -#' @param ds A DataSet or DataSetList object. -#' @param ftarget A Numerical vector. Function values at which runtime values are consumed -#' @param ... Arguments passed to other methods -#' -#' @return a object of type 'ECDF' -#' @export -#' @examples -#' ECDF(dsl,c(12,14)) -#' ECDF(dsl[[1]],c(12,14)) -ECDF <- function(ds, ftarget, ...) UseMethod("ECDF", ds) - -# TODO: also implement the ecdf functions for function values and parameters -#' @rdname ECDF -#' @export -ECDF.DataSet <- function(ds, ftarget, ...) { - runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT - runtime <- runtime[!is.na(runtime)] - if (length(runtime) == 0) - fun <- ecdf(Inf) - else - fun <- ecdf(runtime) - - class(fun)[1] <- 'ECDF' - attr(fun, 'min') <- min(runtime) - attr(fun, 'samples') <- length(runtime) - attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) - fun -} - -# TODO: review / re-write this function -#' @rdname ECDF -#' @export -ECDF.DataSetList <- function(ds, ftarget, ...) { - if (length(ds) == 0) return(NULL) - - dims <- unique(get_dim(ds)) - funcs <- unique(get_funcId(ds)) - - if (is.data.table(ftarget)) { - runtime <- sapply(seq(nrow(ftarget)), function(i) { - if (length(dims) > 1 && length(funcs) > 1) { - names_temp <- ftarget[i][[1]] %>% - strsplit(., ';') - FuncId <- names_temp[[1]][[1]] - Dim <- names_temp[[1]][[2]] - } - else if (length(dims) > 1) { - FuncId <- funcs[[1]] - Dim <- ftarget[i][[1]] - } - else if (length(funcs) > 1) { - FuncId <- ftarget[i][[1]] - Dim <- dims[[1]] - } - data <- subset(ds, funcId == FuncId, DIM == Dim) - if (length(data) == 0) return(NA) - temp_targets <- ftarget[i] %>% - unlist %>% - as.numeric - names(temp_targets) <- NULL - res <- get_RT_sample(data, temp_targets[2:11], output = 'long')$RT - res[is.na(res)] <- Inf - res - }) %>% - unlist - } else if (is.list(ftarget)) { - runtime <- sapply(seq_along(ftarget), function(i) { - if(length(dims) > 1 && length(funcs) >1){ - names_temp <- names(ftarget[i])[[1]] %>% - strsplit(., ';') - FuncId <- names_temp[[1]][[1]] - Dim <- names_temp[[1]][[2]] - } - else if(length(dims) > 1){ - FuncId <- funcs[[1]] - Dim <- names(ftarget[i])[[1]] - } - else if(length(funcs) > 1){ - FuncId <- names(ftarget[i])[[1]] - Dim <- dims[[1]] - } else { - FuncId <- funcs[[1]] - Dim <- dims[[1]] - } - data <- subset(ds, funcId == FuncId, DIM == Dim) - if (length(data) == 0) return(NA) - res <- get_RT_sample(data, ftarget[[i]], output = 'long')$RT - res[is.na(res)] <- Inf - res - }) %>% - unlist - } else { - runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT - runtime[is.na(runtime)] <- Inf - } - - - if (length(runtime) == 0) return(NULL) - - fun <- ecdf(runtime) - class(fun)[1] <- 'ECDF' - attr(fun, 'min') <- min(runtime) - attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) - fun -} - -#' Area Under Curve (Empirical Cumulative Dsitribution Function) -#' -#' @param fun A ECDF object. -#' @param from double. Starting point of the area on x-axis -#' @param to double. Ending point of the area on x-axis -#' -#' @return a object of type 'ECDF' -#' @export -#' @examples -#' ecdf <- ECDF(dsl,c(12,14)) -#' AUC(ecdf, 0, 100) -AUC <- function(fun, from = NULL, to = NULL) UseMethod('AUC', fun) - -#' @rdname AUC -#' @export -AUC.ECDF <- function(fun, from = NULL, to = NULL) { - if (is.null(from)) - from <- attr(fun, 'min') - if (is.null(to)) - to <- attr(fun, 'max') - - if (is.null(fun)) - 0 - else - integrate(fun, lower = from, upper = to, subdivisions = 1e3L)$value / (to - from) -} - -#' Generate datatables of runtime or function value targets for a DataSetList -#' -#' Only one target is generated per (function, dimension)-pair, as opposed to the -#' function `get_default_ECDF_targets`, which generates multiple targets. -#' -#' @param dsList A DataSetList -#' @param which Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets -#' -#' @return a data.table of targets -#' @export -#' @examples -#' get_target_dt(dsl) -get_target_dt <- function(dsList, which = 'by_RT') { - vals <- c() - funcs <- get_funcId(dsList) - dims <- get_dim(dsList) - dt <- as.data.table(expand.grid(funcs, dims)) - colnames(dt) <- c("funcId", "DIM") - if (which == 'by_FV') { - target_func <- get_target_FV - } - else if (which == 'by_RT') { - target_func <- get_target_RT - } - else stop("Invalid argument for `which`; can only be `by_FV` or `by_RT`.") - targets <- apply(dt, 1, - function(x) { - dsl_sub <- subset(dsList, funcId == x[[1]], DIM == x[[2]]) - if (length(dsl_sub) > 0) - target_func(dsl_sub) - else - NULL - }) - dt[, target := targets] - - return(dt[!sapply(dt[['target']], is.null)]) -} - -#' Helper function for `get_target_dt` -#' @noRd -get_target_FV <- function(dsList){ - vals <- get_FV_summary(dsList, Inf)[[paste0(100*getOption("IOHanalyzer.quantiles", 0.02)[[1]], '%')]] - if (is.null(attr(dsList, 'maximization')) || attr(dsList, 'maximization')) { - return(max(vals)) - } - else { - return(min(vals)) - } -} - -#' Helper function for `get_target_dt` -#' @noRd -get_target_RT <- function(dsList){ - return(max(get_runtimes(dsList))) -} - -#' Glicko2 raning of algorithms -#' -#' This procedure ranks algorithms based on a glicko2-procedure. -#' Every round (total nr_rounds), for every function and dimension of the datasetlist, -#' each pair of algorithms competes. This competition samples a random runtime for the -#' provided target (defaults to best achieved target). Whichever algorithm has the lower -#' runtime wins the game. Then, from these games, the glicko2-rating is determined. -#' -#' @param dsl The DataSetList, can contain multiple functions and dimensions, but should have the -#' same algorithms for all of them -#' @param nr_rounds The number of rounds to run. More rounds leads to a more accurate ranking. -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' @param target_dt Custom data.table target value to use. When NULL, this is selected automatically. -#' @return A dataframe containing the glicko2-ratings and some additional info -#' -#' @export -#' @examples -#' glicko2_ranking(dsl, nr_round = 25) -#' glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') -glicko2_ranking <- function(dsl, nr_rounds = 100, which = 'by_FV', target_dt = NULL){ - if (!requireNamespace("PlayerRatings", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - req(length(get_id(dsl)) > 1) - - if (!is.null(target_dt) && !('data.table' %in% class(target_dt))) { - warning("Provided `target_dt` argument is not a data.table") - target_dt <- NULL - } - - if (is.null(target_dt)) - target_dt <- get_target_dt(dsl, which) - - if (!(which %in% c('by_FV', 'by_RT'))) - stop("Invalid argument: 'which' can only be 'by_FV' or 'by_RT'") - p1 <- NULL - p2 <- NULL - scores <- NULL - weeks <- NULL - get_dims <- function(){ - dims <- get_dim(dsl) - if (length(dims) > 1) { - dims <- sample(dims) - } - dims - } - get_funcs <- function(){ - funcs <- get_funcId(dsl) - if (length(funcs) > 1) { - funcs <- sample(funcs) - } - funcs - } - n_algs = length(get_id(dsl)) - alg_names <- NULL - for (k in seq(1,nr_rounds)) { - for (dim in get_dims()) { - targets_temp <- target_dt[target_dt$DIM == dim] - for (fid in get_funcs()) { - dsl_s <- subset(dsl, funcId == fid && DIM == dim) - if (length(dsl_s) == 0) - next - if (which == 'by_FV') { - target <- targets_temp[targets_temp$funcId == fid]$target - x_arr <- get_RT_sample(dsl_s, target) - win_operator <- `<` - } - else { - target <- targets_temp[targets_temp$funcId == fid]$target - x_arr <- get_FV_sample(dsl_s, target) - win_operator <- ifelse(attr(dsl, 'maximization'), `>`, `<`) - } - vals = array(dim = c(n_algs,ncol(x_arr) - 4)) - for (i in seq(1,n_algs)) { - z <- x_arr[i] - y <- as.numeric(z[,5:ncol(x_arr)]) - vals[i,] = y - } - if (is.null(alg_names)) alg_names <- x_arr[,3] - - for (i in seq(1,n_algs)) { - for (j in seq(i,n_algs)) { - if (i == j) next - weeks <- c(weeks, k) - s1 <- sample(vals[i,], 1) - s2 <- sample(vals[j,], 1) - if (is.na(s1)) { - if (is.na(s2)) { - won <- 0.5 - } - else{ - won <- 0 - } - } - else{ - if (is.na(s2)) { - won <- 1 - } - else if (s1 == s2) { - won <- 0.5 #Tie - } - else { - won <- win_operator(s1, s2) - } - } - p1 <- c(p1, i) - p2 <- c(p2, j) - scores <- c(scores, won) - - } - } - } - } - } - # weeks <- seq(1,1,length.out = length(p1)) - games <- data.frame(Weeks = weeks, Player1 = p1, Player2 = p2, Scores = as.numeric(scores)) - lout <- PlayerRatings::glicko2(games, init = c(1500,350,0.06)) - lout$ratings$Player <- alg_names[lout$ratings$Player] - lout -} - - -#' Verify that the credentials for DSCtool have been set -#' -#' This uses the keyring package to store and load credentials. -#' If the keyring package does not exists, it will default to look for -#' a config-file in the 'repository'-folder, under your home directory. -#' This can be changed by setting the option IOHprofiler.config_dir -#' If you already have an account, please call `set_DSC_credentials` -#' with the corresponding username and password. -#' If you don't have an account, you can register for one using `register_DSC` -#' -#' -#' -#' @export -#' @examples -#' check_dsc_configured() -check_dsc_configured <- function() { - if (!requireNamespace("keyring", quietly = TRUE)) { - warning("It is recommended to have the 'keyring' package installed to store - DSCtool settings. Since this package is not found, we default - to a local settings-file instead.") - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - if (!file.exists(paste0(repo_dir, "/config.rds"))) { - return(FALSE) - } - data <- readRDS(paste0(repo_dir, "/config.rds")) - if (is.null(data$DSCusername) || is.null(data$DSCpassword)) { - return(FALSE) - } - return(TRUE) - } - tryCatch({ - username <- keyring::key_get("DSCtool_name") - pwd <- keyring::key_get("DSCtool") - return(TRUE) - }, - error = function(e) { - print("DSCtool connection not yet set. Please use the function `register_DSC` to - create an account to use the DSC tool locally (or use 'add_DSC_credentials' - to use an existing account)") - return(FALSE) - } - ) -} - -#' Register an account to the DSCtool API -#' -#' This uses the keyring package to store and load credentials. -#' If you already have an account, please call `set_DSC_credentials` instead -#' -#' @param name Your name -#' @param username A usename to be identified with. Will be stored on keyring under 'DSCtool_name' -#' @param affiliation Your affiliation (university / company) -#' @param email Your email adress -#' @param password The password to use. If NULL, this will be generated at random. -#' Will be stored on keyring under 'DSCtool' -#' -#' @export -#' @examples -#' \dontrun{ -#' register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") -#' } -register_DSC <- function(name, username, affiliation, email, password = NULL) { - if (is.null(password)) password <- stri_rand_strings(1, 10) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/manage/user" - result_json <- POST(url, add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = list(name = name, affiliation = affiliation, email = email, - username = username, password = password), - encode = "json") - if (result_json$status_code == 200) { - if (!requireNamespace("keyring", quietly = TRUE)) { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - saveRDS(list(DSCusername = username, DSCpassword = password), - paste0(repo_dir, "/config.rds")) - return(TRUE) - } - else { - keyring::key_set_with_value("DSCtool", password = password) - keyring::key_set_with_value("DSCtool_name", password = username) - return(TRUE) - } - } - else { - stop("Something went wrong in registering for DSCtool") - } -} - -#' Register an account to the DSCtool API -#' -#' This uses the keyring package to store and load credentials. -#' If you already have an account, please call `add_DSC_credentials` instead -#' -#' @param username The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name' -#' @param password The password you use on DSCtool. Will be stored on keyring under 'DSCtool' -#' -#' @export -#' @examples -#' \dontrun{set_DSC_credentials('jdoe', 'monkey123')} -set_DSC_credentials <- function(username, password) { - if (!requireNamespace("keyring", quietly = TRUE)) { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - saveRDS(list(DSCusername = username, DSCpassword = password), - paste0(repo_dir, "/config.rds")) - } - else { - keyring::key_set_with_value("DSCtool", password = password) - keyring::key_set_with_value("DSCtool_name", password = username) - } -} - -#' Load stored credentials for DSCtool -#' -#' -#' @noRd -#' @examples -#' \dontrun{ -#' get_DSC_credentials() -#' } -get_DSC_credentials <- function() { - if (!requireNamespace("keyring", quietly = TRUE)) { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - data <- readRDS(paste0(repo_dir, "/config.rds")) - return(list(name = data$DSCusername, pwd = data$DSCpassword)) - } - else { - return(list(name = keyring::key_get("DSCtool_name"), - pwd = keyring::key_get("DSCtool"))) - } -} - -#' Convert a DataSetList to the format needed for the DSCtool -#' -#' -#' @param dsList The DataSetList object -#' @param targets Optional list of target values (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. -#' -#' @noRd -#' @examples -#' convert_to_dsc_compliant(dsl) -convert_to_dsc_compliant <- function(dsList, targets = NULL, which = 'by_RT', - na.correction = NULL) { - maximize <- attr(dsList, 'maximization') - variable <- fid <- value <- NULL #Set local binding to remove warnings - by_rt <- which == 'by_RT' - - if (is.null(targets)) { - targets <- get_target_dt(dsList, which) - } - - final_list <- lapply(get_id(dsList), function(id) { - dsl_sub <- subset(dsList, ID == id) - problems <- lapply(dsl_sub, function(ds) { - target <- targets[funcId == attr(ds, 'funcId') & DIM == attr(ds, 'DIM'), 'target'] - if (!is.numeric(target)) { #TODO: more robust solution - target <- target[[1]][[1]] - } - if (by_rt) { - data <- get_RT_sample(ds, target, output = 'long')$RT - if (any(is.na(data)) & !is.null(na.correction)) { - if (na.correction == 'PAR-1') { - budgets <- attr(ds, 'maxRT') - data[is.na(data)] <- budgets[is.na(data)] - } - else if (na.correction == 'PAR-10') { - budgets <- attr(ds, 'maxRT') - data[is.na(data)] <- 10 * budgets[is.na(data)] - } - else if (na.correction == 'ERT') { - ert <- as.numeric(get_ERT(ds, target)$ERT) - if (is.infinite(ert)) { - ert <- sum(attr(ds, 'maxRT')) - } - data[is.na(data)] <- ert - } - else if (na.correction == 'Remove-na') { - data <- data[!is.na(data)] - } - else { - stop('This value for `na.correction` is not supported') - } - } - } - else { - data <- get_FV_sample(ds, target, output = 'long')$`f(x)` - if (attr(ds, 'maximization')) data <- -1 * data - } - if (any(is.na(data))) { - warning("NA-values detected in data preparation for DSCtool. This will likely result in an - error. Please verify the provided DataSetList.") - } - return(list(name = paste0("F", attr(ds, 'funcId'), "_", attr(ds, 'DIM'),"D"), - data = data)) - }) - return(list(ID = id, problems = problems)) - }) - - - return(final_list) -} - -#' Get the matrix of rankings using the DSCtool api for a DataSetList -#' -#' -#' @param dsList The DataSetList object -#' @param targets Optional list of target values (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param test_type Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests -#' @param alpha Threshold value for statistical significance -#' @param epsilon Minimum threshold to have practical difference between algorithms (eDSC) -#' @param monte_carlo_iterations How many monte-carlo-simulations to perform -#' (set to 0 to use regular DSC) -#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. -#' Options are: -#' - 'NULL': No correction is done. This will likely result in an error, as the DSCtool -#' does not allow for na values -#' - 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) -#' - 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) -#' - 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this -#' reverts to nr_runs * budget -#' - 'Remove-na' Removes all NA values -#' -#' @return A named list containing a ranked-matrix which has the rankin of each algorithm -#' on each problem, as well as a list of which omnibus tests can be used to further process -#' this data. This can be further analyzed using `get_dsc_omnibus` -#' -#' @export -#' @examples -#' get_dsc_rank(dsl) -get_dsc_rank <- function(dsList, targets = NULL, which = 'by_RT', test_type = "AD", alpha = 0.05, - epsilon = 0, monte_carlo_iterations = 0, na.correction = NULL) { - if (!check_dsc_configured()) return(NULL) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/rank" - dsc_list <- convert_to_dsc_compliant(dsList, targets, which, na.correction = na.correction) - json_list <- list(method = list(name = test_type, alpha = alpha), epsilon = epsilon, - data = dsc_list, monte_carlo_iterations = monte_carlo_iterations) - - result_json <- POST(url, - authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), - add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = json_list, encode = "json") - - return(content(result_json)$result) -} - - -#' Perform omnibus statistical tests on the matrix of rankings from the DSCtool api -#' -#' -#' @param res The result of a call to the `get_dsc_rank` -#' @param method Which method to use to do the tests. -#' Has be be one of the allowed ones in `res$valid_methods`. -#' When NULL, the first valid option is chosen by default -#' @param alpha Threshold value for statistical significance -#' -#' @return A named list containing the algorithm means -#' -#' @export -#' @examples -#' get_dsc_omnibus(get_dsc_rank(dsl)) -get_dsc_omnibus <- function(res, method = NULL, alpha = 0.05) { - if (!check_dsc_configured()) return(NULL) - if (is.null(method)) method <- res$valid_methods[[1]] - else req(method %in% res$valid_methods) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/omnibus" - new_json <- list(method = list(name = method, alpha = alpha), - ranked_matrix = res$ranked_matrix, - number_algorithms = res$number_algorithms, - parametric_tests = res$parametric_tests) - result_json <- POST(url, - authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), - add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = new_json, encode = "json") - - return(content(result_json)$result) -} - -#' Perform post-hoc processing on data from DSCtool -#' -#' -#' @param omni_res The result from a call to `get_dsc_omnibus` -#' @param nr_algs The number of algorithms present in `omni_res` -#' @param nr_problems The number of problems present in `omni_res` -#' @param base_algorithm The base algorithm to which the other are compared. -#' This has to be present in `omni_res$algorithm_means` as an `algorithm` property -#' @param method Either 'friedman' or 'friedman-aligned-rank' -#' @param alpha Threshold value for statistical significance -#' -#' @return A named list containing 4 types of analyses: -#' * Zvalue -#' * UnadjustedPValue -#' * Holm -#' * Hochberg -#' -#' @export -#' @examples -#' get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) -get_dsc_posthoc <- function(omni_res, nr_algs, nr_problems, base_algorithm = NULL, - method = "friedman", alpha = 0.05) { - if (!check_dsc_configured()) return(NULL) - if (is.null(base_algorithm)) base_algorithm <- omni_res$algorithm_means[[1]]$algorithm - else req(base_algorithm %in% unlist(omni_res$algorithm_means)) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/posthoc" - new_json <- list(method = list(name = method, alpha = alpha), - algorithm_means = omni_res$algorithm_means, - n = nr_problems, - k = nr_algs, - base_algorithm = base_algorithm) - result_json <- POST(url, - authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), - add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = new_json, encode = "json") - - return(content(result_json)$result) -} - - -#' Get the marginal contribution of an algorithm to a portfolio -#' -#' Based on the contribution to the ECDF-curve of the VBS of the portfolio -#' -#' @param id The id for which to get the contribution -#' @param perm The permutation of algorithms to which is being contributed -#' @param j At which point in the permutation the contribution should be measured -#' @param dt The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`) -#' -#' @export -#' @examples -#' dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) -#' get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) -get_marg_contrib_ecdf <- function(id, perm, j, dt) { - hit <- NULL #Bind to avoid notes - ids <- perm[0:j] - v_pre <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - ids <- perm[0:(j - 1)] - if (j == 1) { - v_post <- 0 - } - else - v_post <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - v_pre - v_post -} - - -#' Get the shapley-values of a portfolio of algorithms -#' -#' Based on the contribution to the ECDF-curve of the VBS of the portfolio -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param scale.log Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not -#' @param group_size How many permutation groups will be considered -#' @param max_perm_size The maximum limit for permutations to be considered -#' @param normalize Whether or not to ensure the resulting values will be in [0,1] -#' -#' @export -#' @examples -#' get_shapley_values(dsl, get_ECDF_targets(dsl)) -get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, max_perm_size = 10, normalize = T){ - hit <- NULL #Bind to avoid notes - ids_full <- get_id(dsList) - - dt <- generate_data.ECDF_raw(dsList, targets, scale_log = scale.log) - - nr_players <- length(ids_full) - - perms <- lapply(seq_len(nr_players * group_size), function(i) {sample(ids_full)}) - - max_val <- sum(dt[, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - ### For each algorithm, calculate shapley value - shapleys <- lapply(ids_full, function(id) { - ### For each group of permutations - temp <- mean(unlist( - lapply(seq_len(max_perm_size), function(j) { - perm_sub <- perms[((group_size * (j - 1)) + 1):(group_size * (j))] - ### - mean(unlist( - lapply(perm_sub, function(perm) { - perm <- replace(perm, c(j, which(perm == id)), c(id,perm[[j]])) - get_marg_contrib_ecdf(id, perm, j, dt) - }) - )) - - }) - )) - if (normalize) - temp <- temp / max_val #Scale so Shapleys always fall between 0 and 1 - temp - }) - - data.table(ID = ids_full, shapley = shapleys) -} - +#' Estimator 'SP' for the Expected Running Time (ERT) +#' +#' @param data A dataframe or matrix. Each row stores the runtime sample points from +#' several runs +#' @param max_runtime The budget to use for calculating ERT. If this is a vector, the largest value is taken. +#' Using this as a vector is being deprecated, and will be removed in a future update +#' +#' @return A list containing ERTs, number of succesfull runs and the succes rate +#' @export +#' @examples +#' SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) +SP <- function(data, max_runtime) { + N <- ncol(data) + if (length(max_runtime) > 1) { + max_runtime <- max(max_runtime) + } + idx <- is.na(data) | data > max_runtime + succ <- N - rowSums(idx) + succ_rate <- succ / N + data[idx] <- max_runtime + + list(ERT = rowSums(data) / succ, runs = succ, succ_rate = succ_rate) +} + +#' Bootstrapping for running time samples +#' +#' @param x A numeric vector. A sample of the running time. +#' @param max_eval A numeric vector, containing the maximal running time in +#' each run. It should have the same size as x +#' @param bootstrap.size integer, the size of the bootstrapped sample +#' +#' @return A numeric vector of the bootstrapped running time sample +#' @export +#' @examples +#' ds <- dsl[[1]] +#' x <- get_RT_sample(ds, ftarget = 16, output = 'long') +#' max_eval <- get_maxRT(dsl, output = 'long') +#' bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) +bootstrap_RT <- function(x, max_eval, bootstrap.size) { + x_succ <- x[!is.na(x)] + x_unsucc <- max_eval[is.na(x)] + n_succ <- length(x_succ) + n_unsucc <- length(x_unsucc) + + p <- n_succ / length(x) + N <- rgeom(bootstrap.size, p) + + if (n_succ == 0){ + return (rep(Inf, bootstrap.size)) + } + + sapply(N, + function(size) { + if (size > 0) + x <- sum(sample(x_unsucc, size, replace = T)) + else + x <- 0 + x <- x + sample(x_succ, 1, replace = T) + }) +} + +# TODO: remove the bootstrapping part as it does not make much sense here... +#' Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times +#' among a data set +#' +#' @description This function performs a Kolmogorov-Smirnov test on each pair of +#' algorithms in the input x to determine which algorithm gives a significantly +#' smaller running time. The resulting p-values are arranged in a matrix, where +#' each cell (i, j) contains a p-value from the test with alternative hypothesis: +#' the running time of algorithm i is smaller (thus better) than that of j. +#' +#' @param x either a list that contains running time sample for each algorithm as +#' sub-lists, or a DataSetList object +#' @param bootstrap.size integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping +#' @param ... all other options +#' @return A matrix containing p-values of the test +#' @export +#' @examples +#' pairwise.test(subset(dsl, funcId == 1), 16) +pairwise.test <- function(x, ...) UseMethod('pairwise.test', x) + +#' @param max_eval list that contains the maximal running time for each algorithm +#' as sub-lists +#' @export +#' @rdname pairwise.test +pairwise.test.list <- function(x, max_eval, bootstrap.size = 30, ...) { + if ("DataSetList" %in% class(x)) { + class(x) <- rev(class(x)) + pairwise.test.DataSetList(x) + } + + N <- length(x) + p.value <- matrix(NA, N, N) + + for (i in seq(1, N - 1)) { + for (j in seq(i + 1, N)) { + if (bootstrap.size == 0) { + x1 <- x[[i]] + x2 <- x[[j]] + } else { + x1 <- bootstrap_RT(x[[i]], max_eval[[i]], bootstrap.size) + x2 <- bootstrap_RT(x[[j]], max_eval[[j]], bootstrap.size) + } + if (all(is.na(x1))) { + if (all(is.na(x2))) { + next + } + else { + p.value[i, j] <- 1 + p.value[j, i] <- 0 + } + } + else if (all(is.na(x2))) { + p.value[i, j] <- 0 + p.value[j, i] <- 1 + } + else { + options(warn = -1) + p.value[i, j] <- ks.test(x1, x2, alternative = 'greater', exact = F)$p.value + p.value[j, i] <- ks.test(x1, x2, alternative = 'less', exact = F)$p.value + options(warn = 0) + } + } + } + + p.value.adjust <- p.adjust(as.vector(p.value), method = 'holm') + p.value <- matrix(p.value.adjust, N, N) + colnames(p.value) <- rownames(p.value) <- names(x) + p.value +} + +#' @param ftarget float, the target value used to determine the running / hitting +#' @param which wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison +#' time +#' @export +#' @rdname pairwise.test +pairwise.test.DataSetList <- function(x, ftarget, bootstrap.size = 0, which = 'by_FV', ...) { + if (which == 'by_FV') { + dt <- get_RT_sample(x, ftarget, output = 'long') + maxRT <- get_maxRT(x, output = 'long') + maxRT <- split(maxRT$maxRT, maxRT$ID) + s <- split(dt$RT, dt$ID) + } + else if (which == 'by_RT') { + dt <- get_FV_sample(x, ftarget, output = 'long') + maxRT <- NULL + if (bootstrap.size > 0) warning("Bootstrapping is currently not supported for + fixed-budget statistics.") + bootstrap.size = 0 + s <- split(dt$`f(x)`, dt$ID) + } + else stop("Unsupported argument 'which'. Available options are 'by_FV' and 'by_RT'") + + return(pairwise.test.list(s, maxRT, bootstrap.size)) +} + +# TODO: move those two functions to a separate file +# TODO: review / re-write this function +#' Function for generating sequences of function values +#' +#' @param FV A list of function values +#' @param from Starting function value. Will be replaced by min(FV) if it is NULL or too small +#' @param to Stopping function value. Will be replaced by max(FV) if it is NULL or too large +#' @param by Stepsize of the sequence. Will be replaced if it is too small +#' @param length.out Number of values in the sequence. +#' 'by' takes preference if both it and length.out are provided. +#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a +#' linear or log-linear spacing respectively. If NULL, the scale will be predicted +#' based on FV +#' +#' @return A sequence of function values +#' @export +#' @examples +#' FVall <- get_runtimes(dsl) +#' seq_FV(FVall, 10, 16, 1, scale='linear') +seq_FV <- function(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, scale = NULL) { + from <- max(from, min(FV)) + to <- min(to, max(FV)) + + rev_trans <- function(x) x + + # Auto detect scaling + # TODO: Improve this detection (based on FV?). Currently very arbitrary + if (is.null(scale)) { + if (to < 0 || from < 0) + scale <- 'linear' + else if (abs(log10(mean(FV)) - log10(median(FV))) > 1) + scale <- 'log' + else + scale <- 'linear' + } + + if (scale == 'log') { + trans <- log10 + rev_trans <- function(x) 10 ^ x + # TODO: Better way to deal with negative values + # set lowest possible target globally instead of arbitrary 1e-12 + from <- max(1e-12, from) + to <- max(1e-12 ,to) + from <- trans(from) + to <- trans(to) + } + + #Avoid generating too many samples + if (!is.null(by)) { + nr_samples_generated <- (to - from) / by + if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { + by <- NULL + if (is.null(length.out)) + length.out <- getOption("IOHanalyzer.max_samples", default = 100) + } + } + + if (is.null(by) || by > to - from) { + if (is.null(length.out)) { + length.out <- 10 + args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) + } else + args <- list(from = from, to = to, length.out = length.out) + } else + args <- list(from = from, to = to, by = by) + + # tryCatch({ + do.call(seq, args) %>% + c(from, ., to) %>% # always include the starting / ending value + unique %>% + rev_trans + # }, error = function(e) { + # c() + # }) +} + +# TODO: review / re-write this function +#' Function for generating sequences of runtime values +#' +#' @param RT A list of runtime values +#' @param from Starting runtime value. Will be replaced by min(RT) if it is NULL or too small +#' @param to Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large +#' @param by Stepsize of the sequence. Will be replaced if it is too small +#' @param length.out Number of values in the sequence. +#' 'by' takes preference if both it and length.out are provided. +#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a +#' linear or log-linear spacing respectively. +#' +#' @return A sequence of runtime values +#' @export +#' @examples +#' RTall <- get_runtimes(dsl) +#' seq_RT(RTall, 0, 500, length.out=10, scale='log') +seq_RT <- function(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, + scale = 'linear') { + rev_trans <- function(x) x + + # Do this first to avoid the log-values being overwritten. + from <- max(from, min(RT)) + to <- min(to, max(RT)) + + if (scale == 'log') { + RT <- log10(RT) + rev_trans <- function(x) 10 ^ x + if (!is.null(from)) + from <- log10(from) + if (!is.null(to)) + to <- log10(to) + if (!is.null(by)) + by <- log10(by) + } + + #Avoid generating too many samples + if (!is.null(by)) { + nr_samples_generated <- (to - from) / by + if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { + by <- NULL + if (is.null(length.out)) + length.out <- getOption("IOHanalyzer.max_samples", default = 100) + } + } + + # Also reset by if it is too large + if (is.null(by) || by > to - from) { + if (is.null(length.out)) { + length.out <- 10 + args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) + } else + args <- list(from = from, to = to, length.out = length.out) + } else + args <- list(from = from, to = to, by = by) + + do.call(seq, args) %>% + c(from, ., to) %>% # always include the starting / ending value + unique %>% + rev_trans %>% + round +} + +# TODO: implement the empirical p.m.f. for runtime +EPMF <- function() { + +} + +#' Empirical Cumulative Dsitribution Function of Runtime of a single data set +#' +#' @param ds A DataSet or DataSetList object. +#' @param ftarget A Numerical vector. Function values at which runtime values are consumed +#' @param ... Arguments passed to other methods +#' +#' @return a object of type 'ECDF' +#' @export +#' @examples +#' ECDF(dsl,c(12,14)) +#' ECDF(dsl[[1]],c(12,14)) +ECDF <- function(ds, ftarget, ...) UseMethod("ECDF", ds) + +# TODO: also implement the ecdf functions for function values and parameters +#' @rdname ECDF +#' @export +ECDF.DataSet <- function(ds, ftarget, ...) { + runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT + runtime <- runtime[!is.na(runtime)] + if (length(runtime) == 0) + fun <- ecdf(Inf) + else + fun <- ecdf(runtime) + + class(fun)[1] <- 'ECDF' + attr(fun, 'min') <- min(runtime) + attr(fun, 'samples') <- length(runtime) + attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) + fun +} + +# TODO: review / re-write this function +#' @rdname ECDF +#' @export +ECDF.DataSetList <- function(ds, ftarget, ...) { + if (length(ds) == 0) return(NULL) + + dims <- unique(get_dim(ds)) + funcs <- unique(get_funcId(ds)) + + if (is.data.table(ftarget)) { + runtime <- sapply(seq(nrow(ftarget)), function(i) { + if (length(dims) > 1 && length(funcs) > 1) { + names_temp <- ftarget[i][[1]] %>% + strsplit(., ';') + FuncId <- names_temp[[1]][[1]] + Dim <- names_temp[[1]][[2]] + } + else if (length(dims) > 1) { + FuncId <- funcs[[1]] + Dim <- ftarget[i][[1]] + } + else if (length(funcs) > 1) { + FuncId <- ftarget[i][[1]] + Dim <- dims[[1]] + } + data <- subset(ds, funcId == FuncId, DIM == Dim) + if (length(data) == 0) return(NA) + temp_targets <- ftarget[i] %>% + unlist %>% + as.numeric + names(temp_targets) <- NULL + res <- get_RT_sample(data, temp_targets[2:11], output = 'long')$RT + res[is.na(res)] <- Inf + res + }) %>% + unlist + } else if (is.list(ftarget)) { + runtime <- sapply(seq_along(ftarget), function(i) { + if(length(dims) > 1 && length(funcs) >1){ + names_temp <- names(ftarget[i])[[1]] %>% + strsplit(., ';') + FuncId <- names_temp[[1]][[1]] + Dim <- names_temp[[1]][[2]] + } + else if(length(dims) > 1){ + FuncId <- funcs[[1]] + Dim <- names(ftarget[i])[[1]] + } + else if(length(funcs) > 1){ + FuncId <- names(ftarget[i])[[1]] + Dim <- dims[[1]] + } else { + FuncId <- funcs[[1]] + Dim <- dims[[1]] + } + data <- subset(ds, funcId == FuncId, DIM == Dim) + if (length(data) == 0) return(NA) + res <- get_RT_sample(data, ftarget[[i]], output = 'long')$RT + res[is.na(res)] <- Inf + res + }) %>% + unlist + } else { + runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT + runtime[is.na(runtime)] <- Inf + } + + + if (length(runtime) == 0) return(NULL) + + fun <- ecdf(runtime) + class(fun)[1] <- 'ECDF' + attr(fun, 'min') <- min(runtime) + attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) + fun +} + +#' Area Under Curve (Empirical Cumulative Dsitribution Function) +#' +#' @param fun A ECDF object. +#' @param from double. Starting point of the area on x-axis +#' @param to double. Ending point of the area on x-axis +#' +#' @return a object of type 'ECDF' +#' @export +#' @examples +#' ecdf <- ECDF(dsl,c(12,14)) +#' AUC(ecdf, 0, 100) +AUC <- function(fun, from = NULL, to = NULL) UseMethod('AUC', fun) + +#' @rdname AUC +#' @export +AUC.ECDF <- function(fun, from = NULL, to = NULL) { + if (is.null(from)) + from <- attr(fun, 'min') + if (is.null(to)) + to <- attr(fun, 'max') + + if (is.null(fun)) + 0 + else + integrate(fun, lower = from, upper = to, subdivisions = 1e3L)$value / (to - from) +} + +#' Generate datatables of runtime or function value targets for a DataSetList +#' +#' Only one target is generated per (function, dimension)-pair, as opposed to the +#' function `get_default_ECDF_targets`, which generates multiple targets. +#' +#' @param dsList A DataSetList +#' @param which Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets +#' +#' @return a data.table of targets +#' @export +#' @examples +#' get_target_dt(dsl) +get_target_dt <- function(dsList, which = 'by_RT') { + vals <- c() + funcs <- get_funcId(dsList) + dims <- get_dim(dsList) + dt <- as.data.table(expand.grid(funcs, dims)) + colnames(dt) <- c("funcId", "DIM") + if (which == 'by_FV') { + target_func <- get_target_FV + } + else if (which == 'by_RT') { + target_func <- get_target_RT + } + else stop("Invalid argument for `which`; can only be `by_FV` or `by_RT`.") + targets <- apply(dt, 1, + function(x) { + dsl_sub <- subset(dsList, funcId == x[[1]], DIM == x[[2]]) + if (length(dsl_sub) > 0) + target_func(dsl_sub) + else + NULL + }) + dt[, target := targets] + + return(dt[!sapply(dt[['target']], is.null)]) +} + +#' Helper function for `get_target_dt` +#' @noRd +get_target_FV <- function(dsList){ + vals <- get_FV_summary(dsList, Inf)[[paste0(100*getOption("IOHanalyzer.quantiles", 0.02)[[1]], '%')]] + if (is.null(attr(dsList, 'maximization')) || attr(dsList, 'maximization')) { + return(max(vals)) + } + else { + return(min(vals)) + } +} + +#' Helper function for `get_target_dt` +#' @noRd +get_target_RT <- function(dsList){ + return(max(get_runtimes(dsList))) +} + +#' Glicko2 raning of algorithms +#' +#' This procedure ranks algorithms based on a glicko2-procedure. +#' Every round (total nr_rounds), for every function and dimension of the datasetlist, +#' each pair of algorithms competes. This competition samples a random runtime for the +#' provided target (defaults to best achieved target). Whichever algorithm has the lower +#' runtime wins the game. Then, from these games, the glicko2-rating is determined. +#' +#' @param dsl The DataSetList, can contain multiple functions and dimensions, but should have the +#' same algorithms for all of them +#' @param nr_rounds The number of rounds to run. More rounds leads to a more accurate ranking. +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' @param target_dt Custom data.table target value to use. When NULL, this is selected automatically. +#' @return A dataframe containing the glicko2-ratings and some additional info +#' +#' @export +#' @examples +#' glicko2_ranking(dsl, nr_round = 25) +#' glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') +glicko2_ranking <- function(dsl, nr_rounds = 100, which = 'by_FV', target_dt = NULL){ + if (!requireNamespace("PlayerRatings", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + req(length(get_id(dsl)) > 1) + + if (!is.null(target_dt) && !('data.table' %in% class(target_dt))) { + warning("Provided `target_dt` argument is not a data.table") + target_dt <- NULL + } + + if (is.null(target_dt)) + target_dt <- get_target_dt(dsl, which) + + if (!(which %in% c('by_FV', 'by_RT'))) + stop("Invalid argument: 'which' can only be 'by_FV' or 'by_RT'") + p1 <- NULL + p2 <- NULL + scores <- NULL + weeks <- NULL + get_dims <- function(){ + dims <- get_dim(dsl) + if (length(dims) > 1) { + dims <- sample(dims) + } + dims + } + get_funcs <- function(){ + funcs <- get_funcId(dsl) + if (length(funcs) > 1) { + funcs <- sample(funcs) + } + funcs + } + n_algs = length(get_id(dsl)) + alg_names <- NULL + for (k in seq(1,nr_rounds)) { + for (dim in get_dims()) { + targets_temp <- target_dt[target_dt$DIM == dim] + for (fid in get_funcs()) { + dsl_s <- subset(dsl, funcId == fid && DIM == dim) + if (length(dsl_s) == 0) + next + if (which == 'by_FV') { + target <- targets_temp[targets_temp$funcId == fid]$target + x_arr <- get_RT_sample(dsl_s, target) + win_operator <- `<` + } + else { + target <- targets_temp[targets_temp$funcId == fid]$target + x_arr <- get_FV_sample(dsl_s, target) + win_operator <- ifelse(attr(dsl, 'maximization'), `>`, `<`) + } + vals = array(dim = c(n_algs,ncol(x_arr) - 4)) + for (i in seq(1,n_algs)) { + z <- x_arr[i] + y <- as.numeric(z[,5:ncol(x_arr)]) + vals[i,] = y + } + if (is.null(alg_names)) alg_names <- x_arr[,3] + + for (i in seq(1,n_algs)) { + for (j in seq(i,n_algs)) { + if (i == j) next + weeks <- c(weeks, k) + s1 <- sample(vals[i,], 1) + s2 <- sample(vals[j,], 1) + if (is.na(s1)) { + if (is.na(s2)) { + won <- 0.5 + } + else{ + won <- 0 + } + } + else{ + if (is.na(s2)) { + won <- 1 + } + else if (s1 == s2) { + won <- 0.5 #Tie + } + else { + won <- win_operator(s1, s2) + } + } + p1 <- c(p1, i) + p2 <- c(p2, j) + scores <- c(scores, won) + + } + } + } + } + } + # weeks <- seq(1,1,length.out = length(p1)) + games <- data.frame(Weeks = weeks, Player1 = p1, Player2 = p2, Scores = as.numeric(scores)) + lout <- PlayerRatings::glicko2(games, init = c(1500,350,0.06)) + lout$ratings$Player <- alg_names[lout$ratings$Player] + lout +} + + +#' Verify that the credentials for DSCtool have been set +#' +#' This uses the keyring package to store and load credentials. +#' If the keyring package does not exists, it will default to look for +#' a config-file in the 'repository'-folder, under your home directory. +#' This can be changed by setting the option IOHprofiler.config_dir +#' If you already have an account, please call `set_DSC_credentials` +#' with the corresponding username and password. +#' If you don't have an account, you can register for one using `register_DSC` +#' +#' +#' +#' @export +#' @examples +#' check_dsc_configured() +check_dsc_configured <- function() { + if (!requireNamespace("keyring", quietly = TRUE)) { + warning("It is recommended to have the 'keyring' package installed to store + DSCtool settings. Since this package is not found, we default + to a local settings-file instead.") + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + if (!file.exists(paste0(repo_dir, "/config.rds"))) { + return(FALSE) + } + data <- readRDS(paste0(repo_dir, "/config.rds")) + if (is.null(data$DSCusername) || is.null(data$DSCpassword)) { + return(FALSE) + } + return(TRUE) + } + tryCatch({ + username <- keyring::key_get("DSCtool_name") + pwd <- keyring::key_get("DSCtool") + return(TRUE) + }, + error = function(e) { + print("DSCtool connection not yet set. Please use the function `register_DSC` to + create an account to use the DSC tool locally (or use 'add_DSC_credentials' + to use an existing account)") + return(FALSE) + } + ) +} + +#' Register an account to the DSCtool API +#' +#' This uses the keyring package to store and load credentials. +#' If you already have an account, please call `set_DSC_credentials` instead +#' +#' @param name Your name +#' @param username A usename to be identified with. Will be stored on keyring under 'DSCtool_name' +#' @param affiliation Your affiliation (university / company) +#' @param email Your email adress +#' @param password The password to use. If NULL, this will be generated at random. +#' Will be stored on keyring under 'DSCtool' +#' +#' @export +#' @examples +#' \dontrun{ +#' register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") +#' } +register_DSC <- function(name, username, affiliation, email, password = NULL) { + if (is.null(password)) password <- stri_rand_strings(1, 10) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/manage/user" + result_json <- POST(url, add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = list(name = name, affiliation = affiliation, email = email, + username = username, password = password), + encode = "json") + if (result_json$status_code == 200) { + if (!requireNamespace("keyring", quietly = TRUE)) { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + saveRDS(list(DSCusername = username, DSCpassword = password), + paste0(repo_dir, "/config.rds")) + return(TRUE) + } + else { + keyring::key_set_with_value("DSCtool", password = password) + keyring::key_set_with_value("DSCtool_name", password = username) + return(TRUE) + } + } + else { + stop("Something went wrong in registering for DSCtool") + } +} + +#' Register an account to the DSCtool API +#' +#' This uses the keyring package to store and load credentials. +#' If you already have an account, please call `add_DSC_credentials` instead +#' +#' @param username The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name' +#' @param password The password you use on DSCtool. Will be stored on keyring under 'DSCtool' +#' +#' @export +#' @examples +#' \dontrun{set_DSC_credentials('jdoe', 'monkey123')} +set_DSC_credentials <- function(username, password) { + if (!requireNamespace("keyring", quietly = TRUE)) { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + saveRDS(list(DSCusername = username, DSCpassword = password), + paste0(repo_dir, "/config.rds")) + } + else { + keyring::key_set_with_value("DSCtool", password = password) + keyring::key_set_with_value("DSCtool_name", password = username) + } +} + +#' Load stored credentials for DSCtool +#' +#' +#' @noRd +#' @examples +#' \dontrun{ +#' get_DSC_credentials() +#' } +get_DSC_credentials <- function() { + if (!requireNamespace("keyring", quietly = TRUE)) { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + data <- readRDS(paste0(repo_dir, "/config.rds")) + return(list(name = data$DSCusername, pwd = data$DSCpassword)) + } + else { + return(list(name = keyring::key_get("DSCtool_name"), + pwd = keyring::key_get("DSCtool"))) + } +} + +#' Convert a DataSetList to the format needed for the DSCtool +#' +#' +#' @param dsList The DataSetList object +#' @param targets Optional list of target values (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. +#' +#' @noRd +#' @examples +#' convert_to_dsc_compliant(dsl) +convert_to_dsc_compliant <- function(dsList, targets = NULL, which = 'by_RT', + na.correction = NULL) { + maximize <- attr(dsList, 'maximization') + variable <- fid <- value <- NULL #Set local binding to remove warnings + by_rt <- which == 'by_RT' + + if (is.null(targets)) { + targets <- get_target_dt(dsList, which) + } + + final_list <- lapply(get_id(dsList), function(id) { + dsl_sub <- subset(dsList, ID == id) + problems <- lapply(dsl_sub, function(ds) { + target <- targets[funcId == attr(ds, 'funcId') & DIM == attr(ds, 'DIM'), 'target'] + if (!is.numeric(target)) { #TODO: more robust solution + target <- target[[1]][[1]] + } + if (by_rt) { + data <- get_RT_sample(ds, target, output = 'long')$RT + if (any(is.na(data)) & !is.null(na.correction)) { + if (na.correction == 'PAR-1') { + budgets <- attr(ds, 'maxRT') + data[is.na(data)] <- budgets[is.na(data)] + } + else if (na.correction == 'PAR-10') { + budgets <- attr(ds, 'maxRT') + data[is.na(data)] <- 10 * budgets[is.na(data)] + } + else if (na.correction == 'ERT') { + ert <- as.numeric(get_ERT(ds, target)$ERT) + if (is.infinite(ert)) { + ert <- sum(attr(ds, 'maxRT')) + } + data[is.na(data)] <- ert + } + else if (na.correction == 'Remove-na') { + data <- data[!is.na(data)] + } + else { + stop('This value for `na.correction` is not supported') + } + } + } + else { + data <- get_FV_sample(ds, target, output = 'long')$`f(x)` + if (attr(ds, 'maximization')) data <- -1 * data + } + if (any(is.na(data))) { + warning("NA-values detected in data preparation for DSCtool. This will likely result in an + error. Please verify the provided DataSetList.") + } + return(list(name = paste0("F", attr(ds, 'funcId'), "_", attr(ds, 'DIM'),"D"), + data = data)) + }) + return(list(ID = id, problems = problems)) + }) + + + return(final_list) +} + +#' Get the matrix of rankings using the DSCtool api for a DataSetList +#' +#' +#' @param dsList The DataSetList object +#' @param targets Optional list of target values (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param test_type Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests +#' @param alpha Threshold value for statistical significance +#' @param epsilon Minimum threshold to have practical difference between algorithms (eDSC) +#' @param monte_carlo_iterations How many monte-carlo-simulations to perform +#' (set to 0 to use regular DSC) +#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. +#' Options are: +#' - 'NULL': No correction is done. This will likely result in an error, as the DSCtool +#' does not allow for na values +#' - 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) +#' - 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) +#' - 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this +#' reverts to nr_runs * budget +#' - 'Remove-na' Removes all NA values +#' +#' @return A named list containing a ranked-matrix which has the rankin of each algorithm +#' on each problem, as well as a list of which omnibus tests can be used to further process +#' this data. This can be further analyzed using `get_dsc_omnibus` +#' +#' @export +#' @examples +#' get_dsc_rank(dsl) +get_dsc_rank <- function(dsList, targets = NULL, which = 'by_RT', test_type = "AD", alpha = 0.05, + epsilon = 0, monte_carlo_iterations = 0, na.correction = NULL) { + if (!check_dsc_configured()) return(NULL) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/rank" + dsc_list <- convert_to_dsc_compliant(dsList, targets, which, na.correction = na.correction) + json_list <- list(method = list(name = test_type, alpha = alpha), epsilon = epsilon, + data = dsc_list, monte_carlo_iterations = monte_carlo_iterations) + + result_json <- POST(url, + authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), + add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = json_list, encode = "json") + + return(content(result_json)$result) +} + + +#' Perform omnibus statistical tests on the matrix of rankings from the DSCtool api +#' +#' +#' @param res The result of a call to the `get_dsc_rank` +#' @param method Which method to use to do the tests. +#' Has be be one of the allowed ones in `res$valid_methods`. +#' When NULL, the first valid option is chosen by default +#' @param alpha Threshold value for statistical significance +#' +#' @return A named list containing the algorithm means +#' +#' @export +#' @examples +#' get_dsc_omnibus(get_dsc_rank(dsl)) +get_dsc_omnibus <- function(res, method = NULL, alpha = 0.05) { + if (!check_dsc_configured()) return(NULL) + if (is.null(method)) method <- res$valid_methods[[1]] + else req(method %in% res$valid_methods) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/omnibus" + new_json <- list(method = list(name = method, alpha = alpha), + ranked_matrix = res$ranked_matrix, + number_algorithms = res$number_algorithms, + parametric_tests = res$parametric_tests) + result_json <- POST(url, + authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), + add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = new_json, encode = "json") + + return(content(result_json)$result) +} + +#' Perform post-hoc processing on data from DSCtool +#' +#' +#' @param omni_res The result from a call to `get_dsc_omnibus` +#' @param nr_algs The number of algorithms present in `omni_res` +#' @param nr_problems The number of problems present in `omni_res` +#' @param base_algorithm The base algorithm to which the other are compared. +#' This has to be present in `omni_res$algorithm_means` as an `algorithm` property +#' @param method Either 'friedman' or 'friedman-aligned-rank' +#' @param alpha Threshold value for statistical significance +#' +#' @return A named list containing 4 types of analyses: +#' * Zvalue +#' * UnadjustedPValue +#' * Holm +#' * Hochberg +#' +#' @export +#' @examples +#' get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) +get_dsc_posthoc <- function(omni_res, nr_algs, nr_problems, base_algorithm = NULL, + method = "friedman", alpha = 0.05) { + if (!check_dsc_configured()) return(NULL) + if (is.null(base_algorithm)) base_algorithm <- omni_res$algorithm_means[[1]]$algorithm + else req(base_algorithm %in% unlist(omni_res$algorithm_means)) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/posthoc" + new_json <- list(method = list(name = method, alpha = alpha), + algorithm_means = omni_res$algorithm_means, + n = nr_problems, + k = nr_algs, + base_algorithm = base_algorithm) + result_json <- POST(url, + authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), + add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = new_json, encode = "json") + + return(content(result_json)$result) +} + + +#' Get the marginal contribution of an algorithm to a portfolio +#' +#' Based on the contribution to the ECDF-curve of the VBS of the portfolio +#' +#' @param id The id for which to get the contribution +#' @param perm The permutation of algorithms to which is being contributed +#' @param j At which point in the permutation the contribution should be measured +#' @param dt The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`) +#' +#' @export +#' @examples +#' dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) +#' get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) +get_marg_contrib_ecdf <- function(id, perm, j, dt) { + hit <- NULL #Bind to avoid notes + ids <- perm[0:j] + v_pre <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + ids <- perm[0:(j - 1)] + if (j == 1) { + v_post <- 0 + } + else + v_post <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + v_pre - v_post +} + + +#' Get the shapley-values of a portfolio of algorithms +#' +#' Based on the contribution to the ECDF-curve of the VBS of the portfolio +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param scale.log Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not +#' @param group_size How many permutation groups will be considered +#' @param max_perm_size The maximum limit for permutations to be considered +#' @param normalize Whether or not to ensure the resulting values will be in [0,1] +#' +#' @export +#' @examples +#' get_shapley_values(dsl, get_ECDF_targets(dsl)) +get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, max_perm_size = 10, normalize = T){ + hit <- NULL #Bind to avoid notes + ids_full <- get_id(dsList) + + dt <- generate_data.ECDF_raw(dsList, targets, scale_log = scale.log) + + nr_players <- length(ids_full) + + perms <- lapply(seq_len(nr_players * group_size), function(i) {sample(ids_full)}) + + max_val <- sum(dt[, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + ### For each algorithm, calculate shapley value + shapleys <- lapply(ids_full, function(id) { + ### For each group of permutations + temp <- mean(unlist( + lapply(seq_len(max_perm_size), function(j) { + perm_sub <- perms[((group_size * (j - 1)) + 1):(group_size * (j))] + ### + mean(unlist( + lapply(perm_sub, function(perm) { + perm <- replace(perm, c(j, which(perm == id)), c(id,perm[[j]])) + get_marg_contrib_ecdf(id, perm, j, dt) + }) + )) + + }) + )) + if (normalize) + temp <- temp / max_val #Scale so Shapleys always fall between 0 and 1 + temp + }) + + data.table(ID = ids_full, shapley = shapleys) +} + diff --git a/README.md b/README.md index 2772079d..461ea083 100644 --- a/README.md +++ b/README.md @@ -1,168 +1,168 @@ -# IOHanalyzer - - -[![metacran downloads](https://cranlogs.r-pkg.org/badges/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) -[![CRAN_Status_Badge_version_last_release](https://www.r-pkg.org/badges/version-last-release/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) -[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) -[![R-CMD-check](https://github.com/IOHprofiler/IOHanalyzer/workflows/R-CMD-check/badge.svg)](https://github.com/IOHprofiler/IOHanalyzer/actions) - - -The __performance analyzer__ for **I**terative **O**ptimization **H**euristics (IOHs). - -* __Documentation__: [https://arxiv.org/abs/2007.03953](https://arxiv.org/abs/2007.03953) -* __Wiki page__: [https://iohprofiler.github.io/IOHanalyzer/](https://iohprofiler.github.io/IOHanalyzer/) -* __Bug reports__: [https://github.com/IOHprofiler/IOHAnalyzer/issues](https://github.com/IOHprofiler/IOHAnalyzer/issues) -* __Online service__: [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) -* __General Contact__: [mailto:iohprofiler@liacs.leidenuniv.nl](mailto:iohprofiler@liacs.leidenuniv.nl) -* __Mailing List__: [https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler](https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler) - -![](./misc/demo.gif) - -It _provides_: - -* a web-based graphical user interface (GUI) to analyze and visualize the empirical performance of IOHs, -* interactive plotting, -* statistical evaluation, -* report generation, and -* a command-line interface (CLI) for the `R` console allowing for fine-grained controls. - -It is _built mainly on_: - -* `R` packages [Shiny](https://shiny.rstudio.com/), [Plotly](https://plotly.com/) and [Rcpp](http://www.rcpp.org/). - - -It is _available through_: - -* a free [online service](#server) that you can use right away, or -* the local [installation](#install) of the package. - -## Online Service - -A free server [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) running the stable version of __IOHanalyzer__ is hosted in [Leiden Institute of Advanced Computer Science](https://liacs.leidenuniv.nl/). You're welcome to check it out! - -## Installation - -### Software dependency - -* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). -* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. - -### Stable version - -Please start up an `R` console and install the stable version as: - -```r -R> install.packages('IOHanalyzer') -``` - -which is maintained on [CRAN](https://CRAN.R-project.org/package=IOHanalyzer) (Comprehensive R Archive Network). - -### Lastest version - -The lastest development is always hosted on Github. In case you'd like to try out this version, the `R` package **devtool** is needed: - -```r -R> install.packages('devtools') -R> devtools::install_github('IOHprofiler/IOHanalyzer') -``` - -### Development version - -If you want to run the version on which you develop: - -```r -R> devtools::install_git("/path/to/your/IOHanalyzer/git/repo") -``` - -## Runinng the Web Interface locally - -The IOHanalyzer package can be loaded using the following commands: - -```r -R> library('IOHanalyzer') -R> runServer() -``` - -It should open a browser on the `localhost` server, using a random port number. Of course, you -could also specify the port number directly: - -```r -R> runServer(port = 1234) -``` - -Have fun! For the complete reference on usage, please check out our [Wiki page](https://iohprofiler.github.io/). - -## Host it online - -We provide a docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. - -## Supported Data Format - -Specific formats are required to load your benchmark data to **IOHanalyzer**. If your data sets are generated in the format of - -* **COCO/BBOB** data format as regulated in [https://hal.inria.fr/inria-00362649](https://hal.inria.fr/inria-00362649), -* **Nevergrad** data format (explained in [https://github.com/facebookresearch/nevergrad](https://github.com/facebookresearch/nevergrad)), or -* **IOHprofiler** data format, which is motivated and modified from **COCO** data format, - -then you just need to compress the data folder obtained from the experiment into a __zip__ file and uploaded it. However, **you are encouraged to convert your own benchmark data to the format regulated here!**. The supported data format is specified in [this page](https://iohprofiler.github.io/IOHanalyzer/data/). Please follow the instruction there to convert your data sets. - -## Our Team - -* [Hao Wang](https://www.lip6.fr/actualite/personnes-fiche.php?ident=D2381), Sorbonne Université, LIP6, France. -* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science, The Netherlands. -* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science, The Netherlands. -* [Ofer M. Shir](https://ofersh.github.io/telhai/), Tel-Hai College, Israel. -* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), Sorbonne Université, CNRS, LIP6, France. -* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science, The Netherlands. - -When using IOHprofiler and parts thereof, please kindly cite this work as - -Hao Wang, Diederick Vermettern, Furong Ye, Carola Doerr and Thomas Bäck: _IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic_, arXiv e-prints:2007.03953, 2020. - -```bibtex -@ARTICLE{IOHprofiler, - author = {Hao Wang and Diederick Vermettern and Furong Ye and Carola Doerr and Thomas B{\"a}ck}, - title = {{IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic}}, - journal = {arXiv e-prints:2007.03953}, - archivePrefix = "arXiv", - eprint = {2007.03953}, - year = 2020, - month = July, - keywords = {Computer Science - Neural and Evolutionary Computing}, - url = {https://arxiv.org/abs/2007.03953} -} -``` - -## License - -This application is governed by the __BSD 3-Clause license__. - -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# IOHanalyzer + + +[![metacran downloads](https://cranlogs.r-pkg.org/badges/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) +[![CRAN_Status_Badge_version_last_release](https://www.r-pkg.org/badges/version-last-release/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) +[![R-CMD-check](https://github.com/IOHprofiler/IOHanalyzer/workflows/R-CMD-check/badge.svg)](https://github.com/IOHprofiler/IOHanalyzer/actions) + + +The __performance analyzer__ for **I**terative **O**ptimization **H**euristics (IOHs). + +* __Documentation__: [https://arxiv.org/abs/2007.03953](https://arxiv.org/abs/2007.03953) +* __Wiki page__: [https://iohprofiler.github.io/IOHanalyzer/](https://iohprofiler.github.io/IOHanalyzer/) +* __Bug reports__: [https://github.com/IOHprofiler/IOHAnalyzer/issues](https://github.com/IOHprofiler/IOHAnalyzer/issues) +* __Online service__: [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) +* __General Contact__: [mailto:iohprofiler@liacs.leidenuniv.nl](mailto:iohprofiler@liacs.leidenuniv.nl) +* __Mailing List__: [https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler](https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler) + +![](./misc/demo.gif) + +It _provides_: + +* a web-based graphical user interface (GUI) to analyze and visualize the empirical performance of IOHs, +* interactive plotting, +* statistical evaluation, +* report generation, and +* a command-line interface (CLI) for the `R` console allowing for fine-grained controls. + +It is _built mainly on_: + +* `R` packages [Shiny](https://shiny.rstudio.com/), [Plotly](https://plotly.com/) and [Rcpp](http://www.rcpp.org/). + + +It is _available through_: + +* a free [online service](#server) that you can use right away, or +* the local [installation](#install) of the package. + +## Online Service + +A free server [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) running the stable version of __IOHanalyzer__ is hosted in [Leiden Institute of Advanced Computer Science](https://liacs.leidenuniv.nl/). You're welcome to check it out! + +## Installation + +### Software dependency + +* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). +* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. + +### Stable version + +Please start up an `R` console and install the stable version as: + +```r +R> install.packages('IOHanalyzer') +``` + +which is maintained on [CRAN](https://CRAN.R-project.org/package=IOHanalyzer) (Comprehensive R Archive Network). + +### Lastest version + +The lastest development is always hosted on Github. In case you'd like to try out this version, the `R` package **devtool** is needed: + +```r +R> install.packages('devtools') +R> devtools::install_github('IOHprofiler/IOHanalyzer') +``` + +### Development version + +If you want to run the version on which you develop: + +```r +R> devtools::install_git("/path/to/your/IOHanalyzer/git/repo") +``` + +## Runinng the Web Interface locally + +The IOHanalyzer package can be loaded using the following commands: + +```r +R> library('IOHanalyzer') +R> runServer() +``` + +It should open a browser on the `localhost` server, using a random port number. Of course, you +could also specify the port number directly: + +```r +R> runServer(port = 1234) +``` + +Have fun! For the complete reference on usage, please check out our [Wiki page](https://iohprofiler.github.io/). + +## Host it online + +We provide a docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. + +## Supported Data Format + +Specific formats are required to load your benchmark data to **IOHanalyzer**. If your data sets are generated in the format of + +* **COCO/BBOB** data format as regulated in [https://hal.inria.fr/inria-00362649](https://hal.inria.fr/inria-00362649), +* **Nevergrad** data format (explained in [https://github.com/facebookresearch/nevergrad](https://github.com/facebookresearch/nevergrad)), or +* **IOHprofiler** data format, which is motivated and modified from **COCO** data format, + +then you just need to compress the data folder obtained from the experiment into a __zip__ file and uploaded it. However, **you are encouraged to convert your own benchmark data to the format regulated here!**. The supported data format is specified in [this page](https://iohprofiler.github.io/IOHanalyzer/data/). Please follow the instruction there to convert your data sets. + +## Our Team + +* [Hao Wang](https://www.lip6.fr/actualite/personnes-fiche.php?ident=D2381), Sorbonne Université, LIP6, France. +* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science, The Netherlands. +* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science, The Netherlands. +* [Ofer M. Shir](https://ofersh.github.io/telhai/), Tel-Hai College, Israel. +* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), Sorbonne Université, CNRS, LIP6, France. +* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science, The Netherlands. + +When using IOHprofiler and parts thereof, please kindly cite this work as + +Hao Wang, Diederick Vermettern, Furong Ye, Carola Doerr and Thomas Bäck: _IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic_, arXiv e-prints:2007.03953, 2020. + +```bibtex +@ARTICLE{IOHprofiler, + author = {Hao Wang and Diederick Vermettern and Furong Ye and Carola Doerr and Thomas B{\"a}ck}, + title = {{IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic}}, + journal = {arXiv e-prints:2007.03953}, + archivePrefix = "arXiv", + eprint = {2007.03953}, + year = 2020, + month = July, + keywords = {Computer Science - Neural and Evolutionary Computing}, + url = {https://arxiv.org/abs/2007.03953} +} +``` + +## License + +This application is governed by the __BSD 3-Clause license__. + +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cran-comments.md b/cran-comments.md index e3f92550..ff5d7d15 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,16 +1,16 @@ -## This is a resubmission after a previous rejection -All exported functions needed to have examples documented. This has now been implemented. - -## Test environments -* local windows install, R 3.5.1 and 3.5.3 -* local ubuntu install, R 3.4.4 -* local macOS install, R 3.5.2 - -## R CMD check results -There were no ERRORs or WARNINGs. - -There was 1 NOTE: -* New submission, since this is the first submission of the IOHanalyzer package to CRAN - -## Downstream dependencies -Since this is the first submission of the IOHanalyzer package, there are no downstream dependencies +## This is a resubmission after a previous rejection +All exported functions needed to have examples documented. This has now been implemented. + +## Test environments +* local windows install, R 3.5.1 and 3.5.3 +* local ubuntu install, R 3.4.4 +* local macOS install, R 3.5.2 + +## R CMD check results +There were no ERRORs or WARNINGs. + +There was 1 NOTE: +* New submission, since this is the first submission of the IOHanalyzer package to CRAN + +## Downstream dependencies +Since this is the first submission of the IOHanalyzer package, there are no downstream dependencies diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info b/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info index 15c8c102..aabfcb20 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info +++ b/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info @@ -1,3 +1,3 @@ -suite = 'PBO', funcId = 1, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' -% +suite = 'PBO', funcId = 1, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' +% data_f1/IOHprofiler_f1_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 \ No newline at end of file diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat b/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat index 26105167..de8cbf7b 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat +++ b/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat @@ -1,472 +1,472 @@ -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 -2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 -3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 -4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 -5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 -6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 -7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 -9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 -13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 -23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 -94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 -111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 -2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 -19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 -57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 -72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 -75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 -80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 -3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 -6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 -7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 -20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 -40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 -134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 -7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 -28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 -33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 -113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 -190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 -458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 -47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 -3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 -50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 -4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 -19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 -60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 -184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 -186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 -4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 -115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 -118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 -129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 +2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 +3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 +4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 +5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 +6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 +7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 +9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 +13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 +23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 +94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 +111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 +2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 +19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 +57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 +72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 +75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 +80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 +3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 +6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 +7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 +20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 +40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 +134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 +7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 +28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 +33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 +113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 +190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 +458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 +47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 +3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 +50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 +4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 +19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 +60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 +184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 +186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 +4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 +115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 +118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 +129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info index 129b40c6..f021512e 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info @@ -1,3 +1,3 @@ -funcId = bla, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' -% -data_fbla/IOHprofiler_fbla_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 +funcId = bla, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' +% +data_fbla/IOHprofiler_fbla_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info index 213c2096..021ca405 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info @@ -1,3 +1,3 @@ -funcId = blubb, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' -% -data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 +funcId = blubb, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' +% +data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat index 26105167..de8cbf7b 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat @@ -1,472 +1,472 @@ -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 -2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 -3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 -4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 -5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 -6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 -7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 -9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 -13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 -23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 -94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 -111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 -2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 -19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 -57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 -72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 -75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 -80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 -3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 -6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 -7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 -20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 -40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 -134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 -7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 -28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 -33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 -113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 -190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 -458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 -47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 -3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 -50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 -4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 -19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 -60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 -184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 -186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 -4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 -115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 -118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 -129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 +2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 +3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 +4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 +5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 +6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 +7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 +9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 +13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 +23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 +94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 +111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 +2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 +19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 +57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 +72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 +75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 +80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 +3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 +6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 +7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 +20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 +40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 +134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 +7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 +28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 +33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 +113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 +190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 +458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 +47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 +3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 +50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 +4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 +19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 +60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 +184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 +186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 +4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 +115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 +118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 +129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat index 26105167..de8cbf7b 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat @@ -1,472 +1,472 @@ -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 -2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 -3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 -4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 -5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 -6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 -7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 -9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 -13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 -23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 -94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 -111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 -2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 -19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 -57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 -72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 -75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 -80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 -3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 -6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 -7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 -20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 -40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 -134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 -7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 -28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 -33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 -113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 -190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 -458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 -47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 -3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 -50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 -4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 -19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 -60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 -184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 -186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 -4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 -115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 -118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 -129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 +2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 +3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 +4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 +5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 +6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 +7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 +9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 +13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 +23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 +94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 +111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 +2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 +19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 +57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 +72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 +75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 +80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 +3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 +6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 +7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 +20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 +40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 +134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 +7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 +28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 +33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 +113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 +190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 +458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 +47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 +3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 +50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 +4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 +19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 +60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 +184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 +186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 +4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 +115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 +118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 +129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 diff --git a/inst/shiny-server/LICENSE.txt b/inst/shiny-server/LICENSE.txt index d0795340..1b5b71d5 100644 --- a/inst/shiny-server/LICENSE.txt +++ b/inst/shiny-server/LICENSE.txt @@ -1,29 +1,29 @@ -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/inst/shiny-server/global.R b/inst/shiny-server/global.R index ef645e7b..1bf460a1 100644 --- a/inst/shiny-server/global.R +++ b/inst/shiny-server/global.R @@ -1,262 +1,262 @@ -suppressMessages(library(IOHanalyzer)) -suppressMessages(library(shiny)) -suppressMessages(library(shinyjs)) -suppressMessages(library(reshape2)) -suppressMessages(library(magrittr)) -suppressMessages(library(data.table)) -suppressMessages(library(dplyr)) -suppressMessages(library(plotly)) -suppressMessages(library(shinydashboard)) -suppressMessages(library(xtable)) -suppressMessages(library(colourpicker)) -suppressMessages(library(bsplus)) -suppressMessages(library(DT)) -suppressMessages(library(knitr)) -suppressMessages(library(kableExtra)) - -# global options -options(datatable.print.nrows = 20) -options(width = 80) -options(shiny.maxRequestSize = 500 * 1024 ^ 2) # maximal upload file size - -# for customized 'plotlyOutput' function ----- -widget_html <- function(name, package, id, style, class, inline = FALSE, ...) { - # attempt to lookup custom html function for widget - fn <- tryCatch(get(paste0(name, "_html"), - asNamespace(package), - inherits = FALSE), - error = function(e) NULL) - - # call the custom function if we have one, otherwise create a div - if (is.function(fn)) { - fn(id = id, style = style, class = class, ...) - } else if (inline) { - tags$span(id = id, style = style, class = class) - } else { - tags$div(id = id, style = style, class = class) - } -} - -checkShinyVersion <- function(error = TRUE) { - x <- utils::packageDescription('htmlwidgets', fields = 'Enhances') - r <- '^.*?shiny \\(>= ([0-9.]+)\\).*$' - if (is.na(x) || length(grep(r, x)) == 0 || system.file(package = 'shiny') == '') - return() - v <- gsub(r, '\\1', x) - f <- if (error) stop else packageStartupMessage - if (utils::packageVersion('shiny') < v) - f("Please upgrade the 'shiny' package to (at least) version ", v) -} - -widget_dependencies <- function(name, package){ - htmlwidgets::getDependency(name, package) -} - -plotlyOutput.IOHanalyzer <- function(outputId, width = '100%', aspect_ratio = 16/10) { - padding_bottom <- paste0(100 / aspect_ratio, '%') - reportSize <- TRUE - inline <- FALSE - - checkShinyVersion() - html <- htmltools::tagList( - widget_html('plotly', 'plotly', id = outputId, - class = paste0('plotly', " html-widget html-widget-output", - if (reportSize) - " shiny-report-size"), - style = sprintf("width:%s; height: 0; padding-bottom:%s; %s", - htmltools::validateCssUnit(width), - htmltools::validateCssUnit(padding_bottom), - if (inline) - "display: inline-block;" - else ""), - width = width, height = 0) - ) - dependencies <- widget_dependencies('plotly', 'plotly') - htmltools::attachDependencies(html, dependencies) -} - -# markers for plotly -symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", - "triangle-up-open", "triangle-down-open") - -# ploting settings for UI --------------------- -plotly_height <- "auto" -plotly_width <- "auto" -plotly_height2 <- "auto" -plotly_width2 <- "auto" - -IOHprofiler <- 'IOHprofiler' -COCO <- 'COCO' -TWO_COL <- 'TWO_COL' -AUTOMATIC <- 'AUTOMATIC' -BIBOJ_COCO <- 'BIBOJ_COCO' -NEVERGRAD <- 'NEVERGRAD' - -# directory where rds-data is stored -get_repo_location <- function() { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - if (file.exists(repo_dir)) repo_dir else '' -} - -print_html <- function(s, widget_id = 'process_data_promt') - shinyjs::html(widget_id, s, add = TRUE) - -# download file names: csv, image --------------------- -AUC_ECDF_aggr_name <- parse(text = "paste0('AUC_ECDF_MULTI.', input$RTECDF.AUC.Table.Format)") -overview_single_name <- parse(text = "paste0('Overview-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$Overview.Single.Format)") -overview_all_name <- parse(text = "paste0('Overview-All-', '.', input$Overview.All.Format)") -RT_csv_name <- parse(text = "paste0('RTStats-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RTSummary.Statistics.Format)") -RT_overview_name <- parse(text = "paste0('RTOverview-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RTSummary.Overview.Format)") -RTSample_csv_name <- parse(text = "paste0('RTSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RTSummary.Sample.Format)") -FV_csv_name <- parse(text = "paste0('FVStats-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FCESummary.Statistics.Format)") -FV_overview_name <- parse(text = "paste0('FVOverview-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FCESummary.Overview.Format)") -FVSample_csv_name <- parse(text = "paste0('FVSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FCESummary.Sample.FileFormat)") -FV_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Summary.Format)") -FV_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Sample.FileFormat)") -RT_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Summary.Format)") -RT_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Sample.FileFormat)") -ERT_multi_func_name <- parse(text = "paste0('MultiERT-', paste0(input$Overall.Dim, 'D'), - '.', input$ERTPlot.Aggr.TableFormat)") -ERT_multi_dim_name <- parse(text = "paste0('MultiERT-', paste0('F', input$Overall.Funcid), - '.', input$ERTPlot.Aggr_Dim.TableFormat)") -FCE_multi_func_name <- parse(text = "paste0('MultiFCE-', paste0(input$Overall.Dim, 'D'), - '.', input$FCEPlot.Aggr.TableFormat)") -RT_Glicko2_table_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.TableFormat)") -RT_Glicko2_figure_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.Format)") - -RT_DSC_table_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.TableFormat)") -RT_DSC_figure_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.Format)") -RT_DSC_figure_name_rank <- parse(text = "paste0('RT_DSC_PerformViz', '.', input$RT_Stats.DSC.Format_rank)") -RT_DSC_table_name_rank <- parse(text = "paste0('RT_DSC_Rank', '.', input$RT_Stats.DSC.TableFormat_rank)") - -FV_DSC_table_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.TableFormat)") -FV_DSC_figure_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.Format)") -FV_DSC_figure_name_rank <- parse(text = "paste0('FV_DSC_PerformViz', '.', input$FV_Stats.DSC.Format_rank)") -FV_DSC_table_name_rank <- parse(text = "paste0('FV_DSC_Rank', '.', input$FV_Stats.DSC.TableFormat_rank)") - -RT_Stats_table_name <- parse(text = "paste0('RT_Stat_Comp-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.TableFormat)") -RT_Stats_heatmap_name <- parse(text = "paste0('RT_Stat_Heatmap-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") -RT_Stats_network_name <- parse(text = "paste0('RT_Stat_Network-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") -RT_multifunc_ERT <- parse(text = "paste0('ERT_Table_Multi', '.', input$RT.MultiERT.Format)") -RT_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi', '.', input$RT.MultiSample.Format)") -FV_multifunc_FV <- parse(text = "paste0('FV_Table_Multi', '.', input$FV.MultiFV.Format)") -FV_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi_FV', '.', input$FV.MultiSample.Format)") -# max_samples <- 100 - -FIG_NAME_ERT_PER_FUN <- parse(text = "paste0('ERT-', Sys.Date(), '.', input$ERTPlot.Format)") -FIG_NAME_ERT_PER_FUN_MULTI <- parse(text = "paste0('ERT_Mult-', Sys.Date(), '.', input$ERTPlot.Multi.Format)") -FIG_NAME_ERT_AGGR <- parse(text = "paste0('ERT_Aggr-', Sys.Date(), '.', input$ERTPlot.Aggr.Format)") -FIG_NAME_ERT_AGGR_DIM <- parse(text = "paste0('ERT_Aggr_Dim-', Sys.Date(), '.', input$ERTPlot.Aggr_Dim.Format)") -FIG_NAME_RT_PMF <- parse(text = "paste0('RT_PMF-', Sys.Date(), '.', input$RTPMF.Bar.Format)") -FIG_NAME_RT_HIST <- parse(text = "paste0('RT_HIST-', Sys.Date(), '.', input$RTPMF.Hist.Format)") -FIG_NAME_RT_ECDF_AGGR <- parse(text = "paste0('RT_ECDF_AGGR-', Sys.Date(), '.', input$RTECDF.Multi.Format)") -FIG_NAME_RT_ECDF_MULT <- parse(text = "paste0('RT_ECDF_MULT-', Sys.Date(), '.', input$RTECDF.Aggr.Format)") -FIG_NAME_RT_AUC <- parse(text = "paste0('RT_AUC-', Sys.Date(), '.', input$RTECDF.AUC.Format)") - -FIG_NAME_FV_PER_FUN <- parse(text = "paste0('FV-', Sys.Date(), '.', input$FCEPlot.Format)") -FIG_NAME_FV_PER_FUN_MULTI <- parse(text = "paste0('FCE_Mult-', Sys.Date(), '.', input$FCEPlot.Multi.Format)") -FIG_NAME_FV_AGGR <- parse(text = "paste0('FCE_Aggr-', Sys.Date(), '.', input$FCEPlot.Aggr.Format)") -FIG_NAME_FV_PDF <- parse(text = "paste0('FV_PMF-', Sys.Date(), '.', input$FCEPDF.Bar.Format)") -FIG_NAME_FV_HIST <- parse(text = "paste0('FV_HIST-', Sys.Date(), '.', input$FCEPDF.Hist.Format)") -FIG_NAME_FV_ECDF_AGGR <- parse(text = "paste0('FV_ECDF_AGGR-', Sys.Date(), '.', input$FCEECDF.Mult.Format)") -FIG_NAME_FV_AUC <- parse(text = "paste0('FV_AUC-', Sys.Date(), '.', input$FCEECDF.AUC.Format)") - -FIG_NAME_RT_PAR_PER_FUN <- parse(text = "paste0('RT_PAR-', Sys.Date(), '.', input$RT_PAR.Plot.Format)") -FIG_NAME_FV_PAR_PER_FUN <- parse(text = "paste0('FV_PAR-', Sys.Date(), '.', input$FV_PAR.Plot.Format)") - -FIG_NAME_RT_SHAPLEY <- parse(text = "paste0('RT-Shapley-', Sys.Date(), '.', input$RTportfolio.Shapley.Format)") - -# ID of the control widget, whose current value should de always recorded and restored ---- -# those control widget are switched on and off -widget_id <- c('RTSummary.Statistics.Min', - 'RTSummary.Statistics.Max', - 'RTSummary.Statistics.Step', - 'RTSummary.Sample.Min', - 'RTSummary.Sample.Max', - 'RTSummary.Sample.Step', - 'RTECDF.Multi.Min', - 'RTECDF.Multi.Max', - 'RTECDF.Multi.Step', - 'RTECDF.Single.Target', - 'RTPMF.Bar.Target', - 'RTPMF.Hist.Target', - 'ERTPlot.Min', - 'ERTPlot.Max', - 'ERTPlot.Aggr.Targets', - 'RTECDF.AUC.Min', - 'RTECDF.AUC.Max', - 'RTECDF.AUC.Step', - 'FV_PAR.Plot.Min', - 'FV_PAR.Plot.Max', - 'FV_PAR.Summary.Min', - 'FV_PAR.Summary.Max', - 'FV_PAR.Summary.Step', - 'FV_PAR.Sample.Min', - 'FV_PAR.Sample.Max', - 'FV_PAR.Sample.Step', - 'RT_PAR.Plot.Min', - 'RT_PAR.Plot.Max', - 'RT_PAR.Summary.Min', - 'RT_PAR.Summary.Max', - 'RT_PAR.Summary.Step', - 'RT_PAR.Sample.Min', - 'RT_PAR.Sample.Max', - 'RT_PAR.Sample.Step', - 'FCESummary.Statistics.Min', - 'FCESummary.Statistics.Max', - 'FCESummary.Statistics.Step', - 'FCESummary.Sample.Min', - 'FCESummary.Sample.Max', - 'FCESummary.Sample.Step', - 'FCEPDF.Hist.Runtime', - 'FCEPDF.Bar.Runtime', - 'FCEPlot.Min', - 'FCEPlot.Max', - 'FCEECDF.Mult.Min', - 'FCEECDF.Mult.Max', - 'FCEECDF.Mult.Step', - 'FCEECDF.AUC.Min', - 'FCEECDF.AUC.Max', - 'FCEECDF.AUC.Step', - 'FCEECDF.Single.Target') - -eventExpr <- parse(text = paste0('{', paste(paste0('input$', widget_id), collapse = "\n"), '}')) - -# Supported formats table -supported_table_format <- c("csv", "tex", "md", "html") - -# token needed for mapbox, which is again needed for ocra... ------ -supported_fig_format <- c('pdf', 'png', 'eps', 'svg') -Sys.setenv('MAPBOX_TOKEN' = 'pk.eyJ1Ijoid2FuZ3JvbmluIiwiYSI6ImNqcmIzemhvMDBudnYzeWxoejh5c2Y5cXkifQ.9XGMWTDOsgi3-b5qG594kQ') - -sanity_check_id <- function(input) { - for (id in widget_id) { - tryCatch(eval(parse(text = paste0('input$', id))), - error = function(e) { - cat(paste('widget', id, 'does not exist!\n')) - }) - } -} - - - - - - +suppressMessages(library(IOHanalyzer)) +suppressMessages(library(shiny)) +suppressMessages(library(shinyjs)) +suppressMessages(library(reshape2)) +suppressMessages(library(magrittr)) +suppressMessages(library(data.table)) +suppressMessages(library(dplyr)) +suppressMessages(library(plotly)) +suppressMessages(library(shinydashboard)) +suppressMessages(library(xtable)) +suppressMessages(library(colourpicker)) +suppressMessages(library(bsplus)) +suppressMessages(library(DT)) +suppressMessages(library(knitr)) +suppressMessages(library(kableExtra)) + +# global options +options(datatable.print.nrows = 20) +options(width = 80) +options(shiny.maxRequestSize = 500 * 1024 ^ 2) # maximal upload file size + +# for customized 'plotlyOutput' function ----- +widget_html <- function(name, package, id, style, class, inline = FALSE, ...) { + # attempt to lookup custom html function for widget + fn <- tryCatch(get(paste0(name, "_html"), + asNamespace(package), + inherits = FALSE), + error = function(e) NULL) + + # call the custom function if we have one, otherwise create a div + if (is.function(fn)) { + fn(id = id, style = style, class = class, ...) + } else if (inline) { + tags$span(id = id, style = style, class = class) + } else { + tags$div(id = id, style = style, class = class) + } +} + +checkShinyVersion <- function(error = TRUE) { + x <- utils::packageDescription('htmlwidgets', fields = 'Enhances') + r <- '^.*?shiny \\(>= ([0-9.]+)\\).*$' + if (is.na(x) || length(grep(r, x)) == 0 || system.file(package = 'shiny') == '') + return() + v <- gsub(r, '\\1', x) + f <- if (error) stop else packageStartupMessage + if (utils::packageVersion('shiny') < v) + f("Please upgrade the 'shiny' package to (at least) version ", v) +} + +widget_dependencies <- function(name, package){ + htmlwidgets::getDependency(name, package) +} + +plotlyOutput.IOHanalyzer <- function(outputId, width = '100%', aspect_ratio = 16/10) { + padding_bottom <- paste0(100 / aspect_ratio, '%') + reportSize <- TRUE + inline <- FALSE + + checkShinyVersion() + html <- htmltools::tagList( + widget_html('plotly', 'plotly', id = outputId, + class = paste0('plotly', " html-widget html-widget-output", + if (reportSize) + " shiny-report-size"), + style = sprintf("width:%s; height: 0; padding-bottom:%s; %s", + htmltools::validateCssUnit(width), + htmltools::validateCssUnit(padding_bottom), + if (inline) + "display: inline-block;" + else ""), + width = width, height = 0) + ) + dependencies <- widget_dependencies('plotly', 'plotly') + htmltools::attachDependencies(html, dependencies) +} + +# markers for plotly +symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", + "triangle-up-open", "triangle-down-open") + +# ploting settings for UI --------------------- +plotly_height <- "auto" +plotly_width <- "auto" +plotly_height2 <- "auto" +plotly_width2 <- "auto" + +IOHprofiler <- 'IOHprofiler' +COCO <- 'COCO' +TWO_COL <- 'TWO_COL' +AUTOMATIC <- 'AUTOMATIC' +BIBOJ_COCO <- 'BIBOJ_COCO' +NEVERGRAD <- 'NEVERGRAD' + +# directory where rds-data is stored +get_repo_location <- function() { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + if (file.exists(repo_dir)) repo_dir else '' +} + +print_html <- function(s, widget_id = 'process_data_promt') + shinyjs::html(widget_id, s, add = TRUE) + +# download file names: csv, image --------------------- +AUC_ECDF_aggr_name <- parse(text = "paste0('AUC_ECDF_MULTI.', input$RTECDF.AUC.Table.Format)") +overview_single_name <- parse(text = "paste0('Overview-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$Overview.Single.Format)") +overview_all_name <- parse(text = "paste0('Overview-All-', '.', input$Overview.All.Format)") +RT_csv_name <- parse(text = "paste0('RTStats-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RTSummary.Statistics.Format)") +RT_overview_name <- parse(text = "paste0('RTOverview-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RTSummary.Overview.Format)") +RTSample_csv_name <- parse(text = "paste0('RTSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RTSummary.Sample.Format)") +FV_csv_name <- parse(text = "paste0('FVStats-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FCESummary.Statistics.Format)") +FV_overview_name <- parse(text = "paste0('FVOverview-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FCESummary.Overview.Format)") +FVSample_csv_name <- parse(text = "paste0('FVSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FCESummary.Sample.FileFormat)") +FV_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Summary.Format)") +FV_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Sample.FileFormat)") +RT_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Summary.Format)") +RT_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Sample.FileFormat)") +ERT_multi_func_name <- parse(text = "paste0('MultiERT-', paste0(input$Overall.Dim, 'D'), + '.', input$ERTPlot.Aggr.TableFormat)") +ERT_multi_dim_name <- parse(text = "paste0('MultiERT-', paste0('F', input$Overall.Funcid), + '.', input$ERTPlot.Aggr_Dim.TableFormat)") +FCE_multi_func_name <- parse(text = "paste0('MultiFCE-', paste0(input$Overall.Dim, 'D'), + '.', input$FCEPlot.Aggr.TableFormat)") +RT_Glicko2_table_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.TableFormat)") +RT_Glicko2_figure_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.Format)") + +RT_DSC_table_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.TableFormat)") +RT_DSC_figure_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.Format)") +RT_DSC_figure_name_rank <- parse(text = "paste0('RT_DSC_PerformViz', '.', input$RT_Stats.DSC.Format_rank)") +RT_DSC_table_name_rank <- parse(text = "paste0('RT_DSC_Rank', '.', input$RT_Stats.DSC.TableFormat_rank)") + +FV_DSC_table_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.TableFormat)") +FV_DSC_figure_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.Format)") +FV_DSC_figure_name_rank <- parse(text = "paste0('FV_DSC_PerformViz', '.', input$FV_Stats.DSC.Format_rank)") +FV_DSC_table_name_rank <- parse(text = "paste0('FV_DSC_Rank', '.', input$FV_Stats.DSC.TableFormat_rank)") + +RT_Stats_table_name <- parse(text = "paste0('RT_Stat_Comp-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.TableFormat)") +RT_Stats_heatmap_name <- parse(text = "paste0('RT_Stat_Heatmap-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") +RT_Stats_network_name <- parse(text = "paste0('RT_Stat_Network-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") +RT_multifunc_ERT <- parse(text = "paste0('ERT_Table_Multi', '.', input$RT.MultiERT.Format)") +RT_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi', '.', input$RT.MultiSample.Format)") +FV_multifunc_FV <- parse(text = "paste0('FV_Table_Multi', '.', input$FV.MultiFV.Format)") +FV_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi_FV', '.', input$FV.MultiSample.Format)") +# max_samples <- 100 + +FIG_NAME_ERT_PER_FUN <- parse(text = "paste0('ERT-', Sys.Date(), '.', input$ERTPlot.Format)") +FIG_NAME_ERT_PER_FUN_MULTI <- parse(text = "paste0('ERT_Mult-', Sys.Date(), '.', input$ERTPlot.Multi.Format)") +FIG_NAME_ERT_AGGR <- parse(text = "paste0('ERT_Aggr-', Sys.Date(), '.', input$ERTPlot.Aggr.Format)") +FIG_NAME_ERT_AGGR_DIM <- parse(text = "paste0('ERT_Aggr_Dim-', Sys.Date(), '.', input$ERTPlot.Aggr_Dim.Format)") +FIG_NAME_RT_PMF <- parse(text = "paste0('RT_PMF-', Sys.Date(), '.', input$RTPMF.Bar.Format)") +FIG_NAME_RT_HIST <- parse(text = "paste0('RT_HIST-', Sys.Date(), '.', input$RTPMF.Hist.Format)") +FIG_NAME_RT_ECDF_AGGR <- parse(text = "paste0('RT_ECDF_AGGR-', Sys.Date(), '.', input$RTECDF.Multi.Format)") +FIG_NAME_RT_ECDF_MULT <- parse(text = "paste0('RT_ECDF_MULT-', Sys.Date(), '.', input$RTECDF.Aggr.Format)") +FIG_NAME_RT_AUC <- parse(text = "paste0('RT_AUC-', Sys.Date(), '.', input$RTECDF.AUC.Format)") + +FIG_NAME_FV_PER_FUN <- parse(text = "paste0('FV-', Sys.Date(), '.', input$FCEPlot.Format)") +FIG_NAME_FV_PER_FUN_MULTI <- parse(text = "paste0('FCE_Mult-', Sys.Date(), '.', input$FCEPlot.Multi.Format)") +FIG_NAME_FV_AGGR <- parse(text = "paste0('FCE_Aggr-', Sys.Date(), '.', input$FCEPlot.Aggr.Format)") +FIG_NAME_FV_PDF <- parse(text = "paste0('FV_PMF-', Sys.Date(), '.', input$FCEPDF.Bar.Format)") +FIG_NAME_FV_HIST <- parse(text = "paste0('FV_HIST-', Sys.Date(), '.', input$FCEPDF.Hist.Format)") +FIG_NAME_FV_ECDF_AGGR <- parse(text = "paste0('FV_ECDF_AGGR-', Sys.Date(), '.', input$FCEECDF.Mult.Format)") +FIG_NAME_FV_AUC <- parse(text = "paste0('FV_AUC-', Sys.Date(), '.', input$FCEECDF.AUC.Format)") + +FIG_NAME_RT_PAR_PER_FUN <- parse(text = "paste0('RT_PAR-', Sys.Date(), '.', input$RT_PAR.Plot.Format)") +FIG_NAME_FV_PAR_PER_FUN <- parse(text = "paste0('FV_PAR-', Sys.Date(), '.', input$FV_PAR.Plot.Format)") + +FIG_NAME_RT_SHAPLEY <- parse(text = "paste0('RT-Shapley-', Sys.Date(), '.', input$RTportfolio.Shapley.Format)") + +# ID of the control widget, whose current value should de always recorded and restored ---- +# those control widget are switched on and off +widget_id <- c('RTSummary.Statistics.Min', + 'RTSummary.Statistics.Max', + 'RTSummary.Statistics.Step', + 'RTSummary.Sample.Min', + 'RTSummary.Sample.Max', + 'RTSummary.Sample.Step', + 'RTECDF.Multi.Min', + 'RTECDF.Multi.Max', + 'RTECDF.Multi.Step', + 'RTECDF.Single.Target', + 'RTPMF.Bar.Target', + 'RTPMF.Hist.Target', + 'ERTPlot.Min', + 'ERTPlot.Max', + 'ERTPlot.Aggr.Targets', + 'RTECDF.AUC.Min', + 'RTECDF.AUC.Max', + 'RTECDF.AUC.Step', + 'FV_PAR.Plot.Min', + 'FV_PAR.Plot.Max', + 'FV_PAR.Summary.Min', + 'FV_PAR.Summary.Max', + 'FV_PAR.Summary.Step', + 'FV_PAR.Sample.Min', + 'FV_PAR.Sample.Max', + 'FV_PAR.Sample.Step', + 'RT_PAR.Plot.Min', + 'RT_PAR.Plot.Max', + 'RT_PAR.Summary.Min', + 'RT_PAR.Summary.Max', + 'RT_PAR.Summary.Step', + 'RT_PAR.Sample.Min', + 'RT_PAR.Sample.Max', + 'RT_PAR.Sample.Step', + 'FCESummary.Statistics.Min', + 'FCESummary.Statistics.Max', + 'FCESummary.Statistics.Step', + 'FCESummary.Sample.Min', + 'FCESummary.Sample.Max', + 'FCESummary.Sample.Step', + 'FCEPDF.Hist.Runtime', + 'FCEPDF.Bar.Runtime', + 'FCEPlot.Min', + 'FCEPlot.Max', + 'FCEECDF.Mult.Min', + 'FCEECDF.Mult.Max', + 'FCEECDF.Mult.Step', + 'FCEECDF.AUC.Min', + 'FCEECDF.AUC.Max', + 'FCEECDF.AUC.Step', + 'FCEECDF.Single.Target') + +eventExpr <- parse(text = paste0('{', paste(paste0('input$', widget_id), collapse = "\n"), '}')) + +# Supported formats table +supported_table_format <- c("csv", "tex", "md", "html") + +# token needed for mapbox, which is again needed for ocra... ------ +supported_fig_format <- c('pdf', 'png', 'eps', 'svg') +Sys.setenv('MAPBOX_TOKEN' = 'pk.eyJ1Ijoid2FuZ3JvbmluIiwiYSI6ImNqcmIzemhvMDBudnYzeWxoejh5c2Y5cXkifQ.9XGMWTDOsgi3-b5qG594kQ') + +sanity_check_id <- function(input) { + for (id in widget_id) { + tryCatch(eval(parse(text = paste0('input$', id))), + error = function(e) { + cat(paste('widget', id, 'does not exist!\n')) + }) + } +} + + + + + + diff --git a/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd index 64a818c8..cc4fbe0a 100644 --- a/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd @@ -1,9 +1,9 @@ -This table summarizes for each algorithm and each runtime value chosen on the left: - - * runs: the number of runs where non-missing parameter values are observed, for each required runtime value, - * mean: the average value of the specified __parameter__ after the specified amount of function evaluations. - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values - -When not all runs used the specified runtime value, the statistics hold only for those runs that did. That is, the mean value is the mean of the applicable runs. Same for the quantiles. - - +This table summarizes for each algorithm and each runtime value chosen on the left: + + * runs: the number of runs where non-missing parameter values are observed, for each required runtime value, + * mean: the average value of the specified __parameter__ after the specified amount of function evaluations. + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values + +When not all runs used the specified runtime value, the statistics hold only for those runs that did. That is, the mean value is the mean of the applicable runs. Same for the quantiles. + + diff --git a/inst/shiny-server/markdown/INSTALL.Rmd b/inst/shiny-server/markdown/INSTALL.Rmd index d5f1fa86..7e6d6178 100644 --- a/inst/shiny-server/markdown/INSTALL.Rmd +++ b/inst/shiny-server/markdown/INSTALL.Rmd @@ -1,39 +1,39 @@ ---- -title: "Installation" -output: - word_document: default - pdf_document: default - html_document: default ---- - -To install __IOHProfiler__ locally, you need to install R environment first. Please download the windows version from (https://cran.r-project.org/bin/windows/base/R-3.5.1-win.exe). Please install the __R-3.5.1-win.exe__ file (the latest, as of 11 July, 2018): just like installing a normal windows software. - -For linux/MacOS users, please download the corresponding version of R. - -After installation, you should see a R console icon on your desktop. On windows 10, it is very likely that there is no such an icon on the desktop. But, it can be found in all programs (select the __Start__ button, and then scroll through the alphabetical list on the left). An example is shown in the following screenshot: - -![](../logo/screenshot.jpg) - -Please open the R console now. This is where we will run the post-processing application. -Then, some dependences need to be installed. Please run the following commands in the R console (copy and paste the following commands in the R console). -```{r wrapper=TRUE, eval=FALSE} -install.packages('shiny') -install.packages('shinyjs') -install.packages('shinydashboard') -install.packages('magrittr') -install.packages('dplyr') -install.packages('reshape2') -install.packages('data.table') -install.packages('markdown') -install.packages('devtools') -devtools::install_github("ropensci/plotly") -``` - -Now, extract the zip of containing the source code to your preferred location, e.g., your desktop. -After this step, please just type the following command in the R console: -```{r wrapper=TRUE, eval=FALSE} -shiny::runApp('path/to/the/code/folder') -``` -Please make sure that you replace the path __'path/to/the/code/folder'__ to the path where the source code is located on your computer. For example, if the source code folder is located on your desktop, this path should look like __C:\\\\Users\\(username)\\\\Desktop\\\\post-processing__. On Linux/MacOS, this path might look like __/home/(username)/post-processing__ .If there is no error in the installation, a webpage should automatically pop-up in your web browser! - - +--- +title: "Installation" +output: + word_document: default + pdf_document: default + html_document: default +--- + +To install __IOHProfiler__ locally, you need to install R environment first. Please download the windows version from (https://cran.r-project.org/bin/windows/base/R-3.5.1-win.exe). Please install the __R-3.5.1-win.exe__ file (the latest, as of 11 July, 2018): just like installing a normal windows software. + +For linux/MacOS users, please download the corresponding version of R. + +After installation, you should see a R console icon on your desktop. On windows 10, it is very likely that there is no such an icon on the desktop. But, it can be found in all programs (select the __Start__ button, and then scroll through the alphabetical list on the left). An example is shown in the following screenshot: + +![](../logo/screenshot.jpg) + +Please open the R console now. This is where we will run the post-processing application. +Then, some dependences need to be installed. Please run the following commands in the R console (copy and paste the following commands in the R console). +```{r wrapper=TRUE, eval=FALSE} +install.packages('shiny') +install.packages('shinyjs') +install.packages('shinydashboard') +install.packages('magrittr') +install.packages('dplyr') +install.packages('reshape2') +install.packages('data.table') +install.packages('markdown') +install.packages('devtools') +devtools::install_github("ropensci/plotly") +``` + +Now, extract the zip of containing the source code to your preferred location, e.g., your desktop. +After this step, please just type the following command in the R console: +```{r wrapper=TRUE, eval=FALSE} +shiny::runApp('path/to/the/code/folder') +``` +Please make sure that you replace the path __'path/to/the/code/folder'__ to the path where the source code is located on your computer. For example, if the source code folder is located on your desktop, this path should look like __C:\\\\Users\\(username)\\\\Desktop\\\\post-processing__. On Linux/MacOS, this path might look like __/home/(username)/post-processing__ .If there is no error in the installation, a webpage should automatically pop-up in your web browser! + + diff --git a/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd index a18490eb..fbf89f15 100644 --- a/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd @@ -1,9 +1,9 @@ -This table summarizes for each algorithm and each target value chosen on the left: - - * runs: the number of runs where non-missing parameter values are observed, for each required target value $f(x)$, - * mean: the average value of the specified __parameter__ when target value $f(x)$ is hit. - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values - -When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. - - +This table summarizes for each algorithm and each target value chosen on the left: + + * runs: the number of runs where non-missing parameter values are observed, for each required target value $f(x)$, + * mean: the average value of the specified __parameter__ when target value $f(x)$ is hit. + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values + +When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. + + diff --git a/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd b/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd index 05d6b4a9..4bbadda4 100644 --- a/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd +++ b/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd @@ -1,10 +1,10 @@ -This table provides an overview on function values for all algorithms chosen on the left: - - * worst recorded: the worst $f(x)$ value ever recorded across _all iterations_, - * worst reached: the worst $f(x)$ value reached in the _last iterations_, - * best reached: the best $f(x)$ value reached in the _last iterations_, - * mean reached: the mean $f(x)$ value reached in the _last iterations_, - * median reached: the median $f(x)$ value reached in the _last iterations_, - * succ: the number of runs which successfully hit the best reached $f(x)$. - - +This table provides an overview on function values for all algorithms chosen on the left: + + * worst recorded: the worst $f(x)$ value ever recorded across _all iterations_, + * worst reached: the worst $f(x)$ value reached in the _last iterations_, + * best reached: the best $f(x)$ value reached in the _last iterations_, + * mean reached: the mean $f(x)$ value reached in the _last iterations_, + * median reached: the median $f(x)$ value reached in the _last iterations_, + * succ: the number of runs which successfully hit the best reached $f(x)$. + + diff --git a/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd index 4dd529a3..57b08a7c 100644 --- a/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd @@ -1,9 +1,9 @@ -This table summarizes for each algorithm and each target value chosen on the left: - - * runs: the number of runs that have found at least one solution of the required target quality $f(x)$, - * mean: the average number of function evaluations needed to find a solution of function value at least $f(x)$ - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these first-hitting times - -When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. - - +This table summarizes for each algorithm and each target value chosen on the left: + + * runs: the number of runs that have found at least one solution of the required target quality $f(x)$, + * mean: the average number of function evaluations needed to find a solution of function value at least $f(x)$ + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these first-hitting times + +When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. + + diff --git a/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd b/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd index fac70ce9..5bdf95fa 100644 --- a/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd @@ -1,40 +1,40 @@ ---- -title: "ECDF_AUC" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$RTECDF.AUC.Min[name]) - fmax <- as.numeric(REG$RTECDF.AUC.Max[name]) - fstep <- as.numeric(REG$RTECDF.AUC.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) - if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(fvs) - fmax <- max(fvs) - fstep <- NULL - } -) -p <- Plot.RT.ECDF_AUC(dsl_sub, fmin, fmax, fstep) -save_plotly(p, paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "ECDF_AUC" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$RTECDF.AUC.Min[name]) + fmax <- as.numeric(REG$RTECDF.AUC.Max[name]) + fstep <- as.numeric(REG$RTECDF.AUC.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) + if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(fvs) + fmax <- max(fvs) + fstep <- NULL + } +) +p <- Plot.RT.ECDF_AUC(dsl_sub, fmin, fmax, fstep) +save_plotly(p, paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd b/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd index a82955f6..5aced466 100644 --- a/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd @@ -1,27 +1,27 @@ ---- -title: "ECDF_Aggregated" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsList <- subset(dsl, algId %in% algs) - -if (!input$RTECDF.Aggr.Func) - dsList <- subset(dsList, funcId == input$Overall.Funcid) -if (!input$RTECDF.Aggr.Dim) - dsList <- subset(dsList, DIM == input$Overall.Dim) -#TODO: Add support for the custom targets -p <- Plot.RT.ECDF_Multi_Func(dsList, - scale.xlog = input$RTECDF.Aggr.Logx) -save_plotly(p, 'ECDF_Aggr.pdf', 'pdf', width = 1000, height = 500) -``` -## Aggregated ECDF plot on `r if(input$RTECDF.Aggr.Func) paste("all functions ") ` `r if(input$RTECDF.Aggr.Dim) paste("all dimensions ") ` - -```{r aggregated_ecdf, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics('ECDF_Aggr.pdf') -``` +--- +title: "ECDF_Aggregated" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsList <- subset(dsl, algId %in% algs) + +if (!input$RTECDF.Aggr.Func) + dsList <- subset(dsList, funcId == input$Overall.Funcid) +if (!input$RTECDF.Aggr.Dim) + dsList <- subset(dsList, DIM == input$Overall.Dim) +#TODO: Add support for the custom targets +p <- Plot.RT.ECDF_Multi_Func(dsList, + scale.xlog = input$RTECDF.Aggr.Logx) +save_plotly(p, 'ECDF_Aggr.pdf', 'pdf', width = 1000, height = 500) +``` +## Aggregated ECDF plot on `r if(input$RTECDF.Aggr.Func) paste("all functions ") ` `r if(input$RTECDF.Aggr.Dim) paste("all dimensions ") ` + +```{r aggregated_ecdf, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics('ECDF_Aggr.pdf') +``` diff --git a/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd b/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd index bee167d6..bb4332be 100644 --- a/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd @@ -1,40 +1,40 @@ ---- -title: "ECDF_Single_Function" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$RTECDF.Multi.Min[name]) - fmax <- as.numeric(REG$RTECDF.Multi.Max[name]) - fstep <- as.numeric(REG$RTECDF.Multi.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) - if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(fvs) - fmax <- max(fvs) - fstep <- NULL - } -) -p <- Plot.RT.ECDF_Single_Func(dsl_sub, fmin, fmax, fstep, - scale.xlog = input$RTECDF.Multi.Logx) -save_plotly(p, paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "ECDF_Single_Function" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$RTECDF.Multi.Min[name]) + fmax <- as.numeric(REG$RTECDF.Multi.Max[name]) + fstep <- as.numeric(REG$RTECDF.Multi.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) + if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(fvs) + fmax <- max(fvs) + fstep <- NULL + } +) +p <- Plot.RT.ECDF_Single_Func(dsl_sub, fmin, fmax, fstep, + scale.xlog = input$RTECDF.Multi.Logx) +save_plotly(p, paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd b/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd index 09d51b1c..3c271788 100644 --- a/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd @@ -1,31 +1,31 @@ ---- -title: "ECDF_Single_Target" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$RTECDF.Single.Target[name]) - if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) -}, -error = function(e) { - ftarget <- median(fvs) - } -) -p <- Plot.RT.ECDF_Per_Target(dsl_sub, ftarget, scale.xlog = input$RTECDF.Single.Logx) -save_plotly(p, paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "ECDF_Single_Target" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$RTECDF.Single.Target[name]) + if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) +}, +error = function(e) { + ftarget <- median(fvs) + } +) +p <- Plot.RT.ECDF_Per_Target(dsl_sub, ftarget, scale.xlog = input$RTECDF.Single.Logx) +save_plotly(p, paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd b/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd index 9e9c3460..993088e9 100644 --- a/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd +++ b/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd @@ -1,40 +1,40 @@ ---- -title: "FV_ECDF_AUC" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$FCEECDF.AUC.Min[name]) - fmax <- as.numeric(REG$FCEECDF.AUC.Max[name]) - fstep <- as.numeric(REG$FCEECDF.AUC.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) - if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(fvs) - fmax <- max(fvs) - fstep <- NULL - } -) -p <- Plot.FV.ECDF_AUC(dsl_sub, fmin, fmax, fstep) -save_plotly(p, paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_ECDF_AUC" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$FCEECDF.AUC.Min[name]) + fmax <- as.numeric(REG$FCEECDF.AUC.Max[name]) + fstep <- as.numeric(REG$FCEECDF.AUC.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) + if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(fvs) + fmax <- max(fvs) + fstep <- NULL + } +) +p <- Plot.FV.ECDF_AUC(dsl_sub, fmin, fmax, fstep) +save_plotly(p, paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd index 0d09f732..0f2ef0c9 100644 --- a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd +++ b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd @@ -1,42 +1,42 @@ ---- -title: "FV_ECDF_Single_Function" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$FCEECDF.Mult.Min[name]) - fmax <- as.numeric(REG$FCEECDF.Mult.Max[name]) - fstep <- as.numeric(REG$FCEECDF.Mult.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(rts) - if (is.null(fmax) || is.na(fmax)) fmax <- max(rts) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(rts) - fmax <- max(rts) - fstep <- NULL - } -) -p <- Plot.FV.ECDF_Single_Func(dsl_sub, rt_min = fmin, - rt_max = fmax, rt_step = fstep, - scale.xlog = input$FCEECDF.Mult.Logx, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_ECDF_Single_Function" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$FCEECDF.Mult.Min[name]) + fmax <- as.numeric(REG$FCEECDF.Mult.Max[name]) + fstep <- as.numeric(REG$FCEECDF.Mult.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(rts) + if (is.null(fmax) || is.na(fmax)) fmax <- max(rts) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(rts) + fmax <- max(rts) + fstep <- NULL + } +) +p <- Plot.FV.ECDF_Single_Func(dsl_sub, rt_min = fmin, + rt_max = fmax, rt_step = fstep, + scale.xlog = input$FCEECDF.Mult.Logx, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd index 5198357b..aa0a1227 100644 --- a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd +++ b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd @@ -1,32 +1,32 @@ ---- -title: "FV_ECDF_Single_Target" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$FCEECDF.Single.Target[name]) - if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(rts) -}, -error = function(e) { - ftarget <- median(rts) - } -) -p <- Plot.FV.ECDF_Per_Target(dsl_sub, as.integer(ftarget), scale.xlog = input$FCEECDF.Single.Logx, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_ECDF_Single_Target" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$FCEECDF.Single.Target[name]) + if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(rts) +}, +error = function(e) { + ftarget <- median(rts) + } +) +p <- Plot.FV.ECDF_Per_Target(dsl_sub, as.integer(ftarget), scale.xlog = input$FCEECDF.Single.Logx, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_Histogram.Rmd b/inst/shiny-server/markdown/Report/FV_Histogram.Rmd index 0693aae9..b6856928 100644 --- a/inst/shiny-server/markdown/Report/FV_Histogram.Rmd +++ b/inst/shiny-server/markdown/Report/FV_Histogram.Rmd @@ -1,32 +1,32 @@ ---- -title: "FV_Histogram" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - rttarget <- as.numeric(REG$FCEPDF.Hist.Runtime[name]) - if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) -}, -error = function(e) { - rttarget <- median(rts) - } -) -p <- Plot.FV.Histogram(dsl_sub, runtime = rttarget, plot_mode = input$FCEPDF.Hist.Mode, - use.equal.bins = input$FCEPDF.Hist.Equal) -save_plotly(p, paste0('FV_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Histogram of target values hit of F`r paste(fid)` on `r paste(dim)`D to at runtime `r paste(rttarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_Hist_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_Histogram" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + rttarget <- as.numeric(REG$FCEPDF.Hist.Runtime[name]) + if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) +}, +error = function(e) { + rttarget <- median(rts) + } +) +p <- Plot.FV.Histogram(dsl_sub, runtime = rttarget, plot_mode = input$FCEPDF.Hist.Mode, + use.equal.bins = input$FCEPDF.Hist.Equal) +save_plotly(p, paste0('FV_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Histogram of target values hit of F`r paste(fid)` on `r paste(dim)`D to at runtime `r paste(rttarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_Hist_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_PMF.Rmd b/inst/shiny-server/markdown/Report/FV_PMF.Rmd index 021f8f1f..6394965b 100644 --- a/inst/shiny-server/markdown/Report/FV_PMF.Rmd +++ b/inst/shiny-server/markdown/Report/FV_PMF.Rmd @@ -1,32 +1,32 @@ ---- -title: "FV_PMF" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - rttarget <- as.numeric(REG$FCEPDF.Bar.Runtime[name]) - if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) -}, -error = function(e) { - rttarget <- median(rts) - } -) -p <- Plot.FV.PDF(dsl_sub, rttarget, show.sample = input$FCEPDF.Bar.Samples, - scale.ylog = input$FCEPDF.Bar.Logy ) -save_plotly(p, paste0('FV_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Probability denisty function of target values hit of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(rttarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_PMF_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_PMF" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + rttarget <- as.numeric(REG$FCEPDF.Bar.Runtime[name]) + if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) +}, +error = function(e) { + rttarget <- median(rts) + } +) +p <- Plot.FV.PDF(dsl_sub, rttarget, show.sample = input$FCEPDF.Bar.Samples, + scale.ylog = input$FCEPDF.Bar.Logy ) +save_plotly(p, paste0('FV_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Probability denisty function of target values hit of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(rttarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_PMF_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_Rank.Rmd b/inst/shiny-server/markdown/Report/FV_Rank.Rmd index 605fb4dc..169e4ef1 100644 --- a/inst/shiny-server/markdown/Report/FV_Rank.Rmd +++ b/inst/shiny-server/markdown/Report/FV_Rank.Rmd @@ -1,23 +1,23 @@ ---- -title: "FV_Rank" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.FV.Aggregated(dsl_sub, plot_mode = input$FCEPlot.Aggr.Mode, - scale.ylog = input$FCEPlot.Aggr.Logy, - use_rank = input$FCEPlot.Aggr.Ranking, - aggr_on = ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM')) -save_plotly(p, paste0('FV_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Ranking plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('FV_Rank_', dim, 'D.pdf')) +--- +title: "FV_Rank" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.FV.Aggregated(dsl_sub, plot_mode = input$FCEPlot.Aggr.Mode, + scale.ylog = input$FCEPlot.Aggr.Logy, + use_rank = input$FCEPlot.Aggr.Ranking, + aggr_on = ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM')) +save_plotly(p, paste0('FV_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Ranking plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('FV_Rank_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Multi_ERT.Rmd b/inst/shiny-server/markdown/Report/Multi_ERT.Rmd index 14359ed1..c6c6b428 100644 --- a/inst/shiny-server/markdown/Report/Multi_ERT.Rmd +++ b/inst/shiny-server/markdown/Report/Multi_ERT.Rmd @@ -1,23 +1,23 @@ ---- -title: "Multi_ERT" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.RT.Multi_Func(dsl_sub, - scale.xlog = input$ERTPlot.Multi.Logx, - scale.ylog = input$ERTPlot.Multi.Logy, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('ERT_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) -``` -## ERT plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('ERT_multi_', dim, 'D.pdf')) +--- +title: "Multi_ERT" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.RT.Multi_Func(dsl_sub, + scale.xlog = input$ERTPlot.Multi.Logx, + scale.ylog = input$ERTPlot.Multi.Logy, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('ERT_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) +``` +## ERT plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('ERT_multi_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Multi_FCE.Rmd b/inst/shiny-server/markdown/Report/Multi_FCE.Rmd index 5c116e57..bb502a7f 100644 --- a/inst/shiny-server/markdown/Report/Multi_FCE.Rmd +++ b/inst/shiny-server/markdown/Report/Multi_FCE.Rmd @@ -1,22 +1,22 @@ ---- -title: "Multi_FCE" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.FV.Multi_Func(dsl_sub, - scale.xlog = input$FCEPlot.Multi.Logx, - scale.ylog = input$FCEPlot.Multi.Logy) -save_plotly(p, paste0('FCE_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) -``` -## ERT plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('FCE_multi_', dim, 'D.pdf')) +--- +title: "Multi_FCE" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.FV.Multi_Func(dsl_sub, + scale.xlog = input$FCEPlot.Multi.Logx, + scale.ylog = input$FCEPlot.Multi.Logy) +save_plotly(p, paste0('FCE_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) +``` +## ERT plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('FCE_multi_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Param_plot.Rmd b/inst/shiny-server/markdown/Report/Param_plot.Rmd index 2488e7d6..e1fc3617 100644 --- a/inst/shiny-server/markdown/Report/Param_plot.Rmd +++ b/inst/shiny-server/markdown/Report/Param_plot.Rmd @@ -1,39 +1,39 @@ ---- -title: "Param_plot" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -tryCatch( - { - fv_min <- as.numeric(REG$FV_PAR.Plot.Min[name]) - if (is.na(fv_min)) fv_min <- NULL - fv_max <- as.numeric(REG$FV_PAR.Plot.Max[name]) - if (is.na(fv_max)) fv_max <- NULL -}, -error = function(e) { - fv_min <- NULL - fv_max <- NULL - } -) -p <- Plot.Parameters(dsl_sub, fv_min, fv_max, - show.mean = (input$FV_PAR.Plot.show.mean == 'mean'), - show.median = (input$FV_PAR.Plot.show.mean == 'median'), - scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - show.CI = input$FV_PAR.Plot.CI, - par_name = input$FV_PAR.Plot.Params) -save_plotly(p, paste0('Param_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Plot of parameters of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('Param_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "Param_plot" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +tryCatch( + { + fv_min <- as.numeric(REG$FV_PAR.Plot.Min[name]) + if (is.na(fv_min)) fv_min <- NULL + fv_max <- as.numeric(REG$FV_PAR.Plot.Max[name]) + if (is.na(fv_max)) fv_max <- NULL +}, +error = function(e) { + fv_min <- NULL + fv_max <- NULL + } +) +p <- Plot.Parameters(dsl_sub, fv_min, fv_max, + show.mean = (input$FV_PAR.Plot.show.mean == 'mean'), + show.median = (input$FV_PAR.Plot.show.mean == 'median'), + scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + show.CI = input$FV_PAR.Plot.CI, + par_name = input$FV_PAR.Plot.Params) +save_plotly(p, paste0('Param_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Plot of parameters of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('Param_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/RT_Histogram.Rmd b/inst/shiny-server/markdown/Report/RT_Histogram.Rmd index 6bcb3dd3..5ebb70fd 100644 --- a/inst/shiny-server/markdown/Report/RT_Histogram.Rmd +++ b/inst/shiny-server/markdown/Report/RT_Histogram.Rmd @@ -1,32 +1,32 @@ ---- -title: "RT_Histogram" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$RTPMF.Hist.Target[name]) - if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) -}, -error = function(e) { - ftarget <- median(fvs) - } -) -p <- Plot.RT.Histogram(dsl_sub, ftarget = ftarget, plot_mode = input$RTPMF.Hist.Mode, - use.equal.bins = input$RTPMF.Hist.Equal) -save_plotly(p, paste0('RT_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Histogram of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('RT_Hist_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "RT_Histogram" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$RTPMF.Hist.Target[name]) + if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) +}, +error = function(e) { + ftarget <- median(fvs) + } +) +p <- Plot.RT.Histogram(dsl_sub, ftarget = ftarget, plot_mode = input$RTPMF.Hist.Mode, + use.equal.bins = input$RTPMF.Hist.Equal) +save_plotly(p, paste0('RT_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Histogram of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('RT_Hist_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/RT_PMF.Rmd b/inst/shiny-server/markdown/Report/RT_PMF.Rmd index a7adb8d3..f69ec98f 100644 --- a/inst/shiny-server/markdown/Report/RT_PMF.Rmd +++ b/inst/shiny-server/markdown/Report/RT_PMF.Rmd @@ -1,32 +1,32 @@ ---- -title: "RT_PMF" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$RTPMF.Bar.Target[name]) - if (is.na(ftarget)) ftarget <- median(fvs) -}, -error = function(e) { - ftarget <- median(fvs) - } -) -p <- Plot.RT.PMF(dsl_sub, ftarget = ftarget, show.sample = input$RTPMF.Bar.Sample, - scale.ylog = input$RTPMF.Bar.Logy) -save_plotly(p, paste0('RT_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Probability mass function of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('RT_PMF_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "RT_PMF" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$RTPMF.Bar.Target[name]) + if (is.na(ftarget)) ftarget <- median(fvs) +}, +error = function(e) { + ftarget <- median(fvs) + } +) +p <- Plot.RT.PMF(dsl_sub, ftarget = ftarget, show.sample = input$RTPMF.Bar.Sample, + scale.ylog = input$RTPMF.Bar.Logy) +save_plotly(p, paste0('RT_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Probability mass function of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('RT_PMF_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/RT_Rank.Rmd b/inst/shiny-server/markdown/Report/RT_Rank.Rmd index 16fea2e5..8548af62 100644 --- a/inst/shiny-server/markdown/Report/RT_Rank.Rmd +++ b/inst/shiny-server/markdown/Report/RT_Rank.Rmd @@ -1,24 +1,24 @@ ---- -title: "RT_Rank" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.RT.Aggregated(dsl_sub, - plot_mode = input$ERTPlot.Aggr.Mode, - scale.ylog = input$ERTPlot.Aggr.Logy, - maximize = attr(dsl_sub[[1]],'maximization'), - use_rank = input$ERTPlot.Aggr.Ranking) -save_plotly(p, paste0('RT_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Ranking plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('RT_Rank_', dim, 'D.pdf')) +--- +title: "RT_Rank" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.RT.Aggregated(dsl_sub, + plot_mode = input$ERTPlot.Aggr.Mode, + scale.ylog = input$ERTPlot.Aggr.Logy, + maximize = attr(dsl_sub[[1]],'maximization'), + use_rank = input$ERTPlot.Aggr.Ranking) +save_plotly(p, paste0('RT_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Ranking plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('RT_Rank_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Single_ERT.Rmd b/inst/shiny-server/markdown/Report/Single_ERT.Rmd index a0dbe480..9584ae97 100644 --- a/inst/shiny-server/markdown/Report/Single_ERT.Rmd +++ b/inst/shiny-server/markdown/Report/Single_ERT.Rmd @@ -1,40 +1,40 @@ ---- -title: "Single_ERT" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -tryCatch( - { - fv_min <- as.numeric(REG$ERTPlot.Min[name]) - if (is.na(fv_min)) fv_min <- NULL - fv_max <- as.numeric(REG$ERTPlot.Max[name]) - if (is.na(fv_max)) fv_max <- NULL -}, -error = function(e) { - fv_min <- NULL - fv_max <- NULL - } -) -p <- Plot.RT.Single_Func(dsl_sub, Fstart = fv_min, Fstop = fv_max, - show.CI = input$ERTPlot.show.CI, - show.ERT = input$ERTPlot.show.ERT, - show.mean = input$ERTPlot.show.mean, - show.median = input$ERTPlot.show.median, - scale.xlog = input$ERTPlot.semilogx, - scale.ylog = input$ERTPlot.semilogy, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('ERT_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ERT plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ERT_single_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "Single_ERT" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +tryCatch( + { + fv_min <- as.numeric(REG$ERTPlot.Min[name]) + if (is.na(fv_min)) fv_min <- NULL + fv_max <- as.numeric(REG$ERTPlot.Max[name]) + if (is.na(fv_max)) fv_max <- NULL +}, +error = function(e) { + fv_min <- NULL + fv_max <- NULL + } +) +p <- Plot.RT.Single_Func(dsl_sub, Fstart = fv_min, Fstop = fv_max, + show.CI = input$ERTPlot.show.CI, + show.ERT = input$ERTPlot.show.ERT, + show.mean = input$ERTPlot.show.mean, + show.median = input$ERTPlot.show.median, + scale.xlog = input$ERTPlot.semilogx, + scale.ylog = input$ERTPlot.semilogy, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('ERT_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ERT plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ERT_single_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/Single_FCE.Rmd b/inst/shiny-server/markdown/Report/Single_FCE.Rmd index a806b69a..6bad996b 100644 --- a/inst/shiny-server/markdown/Report/Single_FCE.Rmd +++ b/inst/shiny-server/markdown/Report/Single_FCE.Rmd @@ -1,35 +1,35 @@ ---- -title: "Single_ERT" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -tryCatch( - { - rt_min <- as.numeric(REG$FCEPlot.Min[name]) - if (is.na(rt_min)) rt_min <- NULL - rt_max <- as.numeric(REG$FCEPlot.Max[name]) - if (is.na(rt_max)) rt_max <- NULL -}, -error = function(e) { - fv_min <- NULL - fv_max <- NULL - } -) -p <- Plot.FV.Single_Func(dsl_sub, RTstart = rt_min, RTstop = rt_max, show.CI = input$FCEPlot.show.CI, - show.mean = input$FCEPlot.show.mean, show.median = input$FCEPlot.show.median, - scale.xlog = isolate(input$FCEPlot.semilogx), scale.ylog = isolate(input$FCEPlot.semilogy)) -save_plotly(p, paste0('FCE_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ERT plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FCE_single_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "Single_ERT" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +tryCatch( + { + rt_min <- as.numeric(REG$FCEPlot.Min[name]) + if (is.na(rt_min)) rt_min <- NULL + rt_max <- as.numeric(REG$FCEPlot.Max[name]) + if (is.na(rt_max)) rt_max <- NULL +}, +error = function(e) { + fv_min <- NULL + fv_max <- NULL + } +) +p <- Plot.FV.Single_Func(dsl_sub, RTstart = rt_min, RTstop = rt_max, show.CI = input$FCEPlot.show.CI, + show.mean = input$FCEPlot.show.mean, show.median = input$FCEPlot.show.median, + scale.xlog = isolate(input$FCEPlot.semilogx), scale.ylog = isolate(input$FCEPlot.semilogy)) +save_plotly(p, paste0('FCE_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ERT plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FCE_single_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/Template_test.Rmd b/inst/shiny-server/markdown/Report/Template_test.Rmd index 628b0ffc..b9c78dc9 100644 --- a/inst/shiny-server/markdown/Report/Template_test.Rmd +++ b/inst/shiny-server/markdown/Report/Template_test.Rmd @@ -1,465 +1,465 @@ ---- -title: "Automatic report" -author: "IOHprofiler" -date: "6/28/2019" -output: - pdf_document: - keep_tex: true - fig_caption: yes -params: - input: NA - dsl: NA - figure_folder: NA - REG: NA -bibliography: bibliography.bib ---- - -```{r setup, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -input <- params$input -dsl <- params$dsl -figure_folder <- params$figure_folder -get_data_id <- function(dsList) { - if (is.null(dsList) | length(dsList) == 0) - return(NULL) - - paste(get_funcId(dsList), get_dim(dsList), sep = '-') -} -REG <- params$REG -``` - -```{asis, echo=input$Report.Introduction} -# Introduction - -This report has been automatically generated by the IOHanalyzer [@IOHprofiler]. - -``` - -# Fixed-target view - -```{r overview, results='asis', eval=input$Report.RT.Overview} -asis_output("### Overview of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.RT.Overview-DIM` ) { - for (fid in input$`Report.RT.Overview-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Overview_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Overview-Alg` ) - p <- kable(get_RT_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r statistics, results='asis', eval=input$Report.RT.Statistics} -asis_output("### Runtime statistics of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.RT.Statistics-DIM` ) { - for (fid in input$`Report.RT.Statistics-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Statistics_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Statistics-Alg`) - name <- get_data_id(dsl_sub) - fvs <- get_funvals(dsl_sub) - tryCatch( - { - fv_seq <- seq_FV(fvs, as.numeric(REG$RTSummary.Statistics.Min[[name]]), as.numeric(REG$RTSummary.Statistics.Max[[name]]), - by = as.numeric(REG$RTSummary.Statistics.Step[[name]])) - }, - error = function(e) { fv_seq <- seq_RT(fvs, length.out = 10)} - ) - p <- kable(get_RT_summary(dsl_sub, fv_seq), - caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), - format = 'latex') %>% - kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r single_erts, eval=input$Report.RT.Single_ERT, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.Single_ERT-DIM` ) { - for (fid in input$`Report.RT.Single_ERT-FuncId` ) { - name_figure <- paste0('ERT_single_F', fid, '-', dim, 'D') - algs = input$`Report.RT.Single_ERT-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_ERT.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.Single_ERT) paste(out, collapse='\n')` - -```{r single_erts_text, eval=input$Report.RT.Single_ERT, results='asis'} -asis_output(paste0("This section shows the ERTs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - - -```{r multi_erts, eval=input$Report.RT.Multi_ERT, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.Multi_ERT-DIM` ) { - name_figure <- paste0('ERT_multi_', dim, 'D') - algs = input$`Report.RT.Multi_ERT-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_ERT.Rmd")))) -} -``` - -`r if(input$Report.RT.Multi_ERT) paste(out, collapse='\n')` - -```{r ranking_erts, eval=input$Report.RT.Rank, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.Rank-DIM` ) { - name_figure <- paste0('RT_Rank_', dim, 'D') - algs = input$`Report.RT.Rank-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Rank.Rmd")))) -} -``` - -`r if(input$Report.RT.Rank) paste(out, collapse='\n')` - -```{r rt_hist, eval=input$Report.RT.Histogram, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.Histogram-DIM` ) { - for (fid in input$`Report.RT.Histogram-FuncId` ) { - name_figure <- paste0('RT_Hist_F', fid, '-', dim, 'D') - algs = input$`Report.RT.Histogram-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Histogram.Rmd")))) - } -} -``` - -`r if(input$Report.RT.Histogram) paste(out, collapse='\n')` - -```{r rt_pmf, eval=input$Report.RT.PMF, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.PMF-DIM` ) { - for (fid in input$`Report.RT.PMF-FuncId` ) { - name_figure <- paste0('RT_PMF_F', fid, '-', dim, 'D') - algs = input$`Report.RT.PMF-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_PMF.Rmd")))) - } -} -``` - -`r if(input$Report.RT.PMF) paste(out, collapse='\n')` - -```{r ecdf_single_target, eval=input$Report.RT.ECDF_Single_Target, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.ECDF_Single_Target-DIM` ) { - for (fid in input$`Report.RT.ECDF_Single_Target-FuncId` ) { - name_figure <- paste0('ECDF_Single_Target', fid, '-', dim, 'D') - algs = input$`Report.RT.ECDF_Single_Target-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Target.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.ECDF_Single_Target) paste(out, collapse='\n')` - -```{r ecdf_single_target_text, eval=input$Report.RT.ECDF_Single_Target, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r ecdf_single_function, eval=input$Report.RT.ECDF_Single_Function, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.ECDF_Single_Function-DIM` ) { - for (fid in input$`Report.RT.ECDF_Single_Function-FuncId` ) { - name_figure <- paste0('ECDF_Single_Function', fid, '-', dim, 'D') - algs = input$`Report.RT.ECDF_Single_Function-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Function.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.ECDF_Single_Function) paste(out, collapse='\n')` - -```{r ecdf_single_function_text, eval=input$Report.RT.ECDF_Single_Function, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r ecdf_aggr, eval=input$Report.RT.ECDF_Aggregated, results='hide', include=F} -algs = input$`Report.RT.ECDF_Aggregated-Alg` -env = new.env() -out <- knit_child(text = knit_expand(file = "ECDF_Aggregated.Rmd")) - -``` - -`r if(input$Report.RT.ECDF_Aggregated) paste(out, collapse='\n')` - -```{r ecdf_aggr_text, eval=input$Report.RT.ECDF_Aggregated, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figure \\ref{fig:aggregated_ecdf}")) -``` - -```{r ecdf_auc, eval=input$Report.RT.ECDF_AUC, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.ECDF_AUC-DIM` ) { - for (fid in input$`Report.RT.ECDF_AUC-FuncId` ) { - name_figure <- paste0('ECDF_AUC', fid, '-', dim, 'D') - algs = input$`Report.RT.ECDF_AUC-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_AUC.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.ECDF_AUC) paste(out, collapse='\n')` - -```{r ecdf_auc_text, eval=input$Report.RT.ECDF_AUC, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -# Fixed-budget view - -```{r fv_overview, results='asis', eval=input$Report.FV.Overview} -asis_output("### Overview of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.FV.Overview-DIM` ) { - for (fid in input$`Report.FV.Overview-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Overview_FV_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Overview-Alg` ) - p <- kable(get_FV_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r fv_statistics, results='asis', eval=input$Report.FV.Statistics} -asis_output("### Runtime statistics of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.FV.Statistics-DIM` ) { - for (fid in input$`Report.FV.Statistics-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Statistics_FV_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Statistics-Alg`) - name <- get_data_id(dsl_sub) - rts <- get_runtimes(dsl_sub) - tryCatch( - { - rt_seq <- seq_RT(rts, as.numeric(REG$FCESummary.Statistics.Min[[name]]), as.numeric(REG$FCESummary.Statistics.Max[[name]]), - by = as.numeric(REG$FCESummary.Statistics.Step[[name]])) - }, - error = function(e) { rt_seq <- seq_RT(rts, length.out = 10)} - ) - p <- kable(get_FV_summary(dsl_sub, rt_seq), - caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), - format = 'latex') %>% - kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r single_fce, eval=input$Report.FV.Single_FCE, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.Single_FCE-DIM` ) { - for (fid in input$`Report.FV.Single_FCE-FuncId` ) { - name_figure <- paste0('FCE_single_F', fid, '-', dim, 'D') - algs = input$`Report.FV.Single_FCE-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_FCE.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.Single_FCE) paste(out, collapse='\n')` - -```{r single_fce_text, eval=input$Report.FV.Single_FCE, results='asis'} -asis_output(paste0("This section shows the FCEs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r multi_fces, eval=input$Report.FV.Multi_FCE, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.Multi_FCE-DIM` ) { - name_figure <- paste0('FCE_multi_', dim, 'D') - algs = input$`Report.FV.Multi_FCE-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_FCE.Rmd")))) -} -``` - -`r if(input$Report.FV.Multi_FCE) paste(out, collapse='\n')` - -```{r ranking_FVs, eval=input$Report.FV.Rank, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.Rank-DIM` ) { - name_figure <- paste0('FV_Rank_', dim, 'D') - algs = input$`Report.FV.Rank-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Rank.Rmd")))) -} -``` - -`r if(input$Report.FV.Rank) paste(out, collapse='\n')` - -```{r fv_hist, eval=input$Report.FV.Histogram, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.Histogram-DIM` ) { - for (fid in input$`Report.FV.Histogram-FuncId` ) { - name_figure <- paste0('FV_Hist_F', fid, '-', dim, 'D') - algs = input$`Report.FV.Histogram-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Histogram.Rmd")))) - } -} -``` - -`r if(input$Report.FV.Histogram) paste(out, collapse='\n')` - -```{r fv_pmf, eval=input$Report.FV.PMF, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.PMF-DIM` ) { - for (fid in input$`Report.FV.PMF-FuncId` ) { - name_figure <- paste0('FV_PMF_F', fid, '-', dim, 'D') - algs = input$`Report.FV.PMF-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_PMF.Rmd")))) - } -} -``` - -`r if(input$Report.FV.PMF) paste(out, collapse='\n')` - -```{r fv_ecdf_single_target, eval=input$Report.FV.ECDF_Single_Target, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.ECDF_Single_Target-DIM` ) { - for (fid in input$`Report.FV.ECDF_Single_Target-FuncId` ) { - name_figure <- paste0('FV_ECDF_Single_Target', fid, '-', dim, 'D') - algs = input$`Report.FV.ECDF_Single_Target-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Target.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.ECDF_Single_Target) paste(out, collapse='\n')` - -```{r fv_ecdf_single_target_text, eval=input$Report.FV.ECDF_Single_Target, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r fv_ecdf_single_function, eval=input$Report.FV.ECDF_Single_Function, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.ECDF_Single_Function-DIM` ) { - for (fid in input$`Report.FV.ECDF_Single_Function-FuncId` ) { - name_figure <- paste0('FV_ECDF_Single_Function', fid, '-', dim, 'D') - algs = input$`Report.FV.ECDF_Single_Function-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Function.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.ECDF_Single_Function) paste(out, collapse='\n')` - -```{r fv_ecdf_single_function_text, eval=input$Report.FV.ECDF_Single_Function, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r fv_ecdf_auc, eval=input$Report.FV.ECDF_AUC, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.ECDF_AUC-DIM` ) { - for (fid in input$`Report.FV.ECDF_AUC-FuncId` ) { - name_figure <- paste0('FV_ECDF_AUC', fid, '-', dim, 'D') - algs = input$`Report.FV.ECDF_AUC-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_AUC.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.ECDF_AUC) paste(out, collapse='\n')` - -```{r fv_ecdf_auc_text, eval=input$Report.FV.ECDF_AUC, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r param_statistics, results='asis', eval=input$Report.Param.Statistics} -asis_output("### Statistics of parameters of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.Param.Statistics-DIM` ) { - for (fid in input$`Report.Param.Statistics-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Statistics_Param_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.Param.Statistics-Alg`) - name <- get_data_id(dsl_sub) - fvs <- get_funvals(dsl_sub) - tryCatch( - { - fv_seq <- seq_FV(fvs, as.numeric(REG$PAR.Summary.Min[[name]]), as.numeric(REG$PAR.Summary.Max[[name]]), - by = as.numeric(REG$PAR.Summary.Step[[name]])) - }, - error = function(e) { fv_seq <- seq_FV(fvs, length.out = 10) } - ) - p <- kable(get_PAR_summary(dsl_sub, fv_seq), - caption = paste0("Statistics of parameters of selected algorithms on F", fid, " in ", dim, "D"), - format = 'latex') %>% - kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - } -} -``` - -```{r param_plot, eval=input$Report.Param.Plot, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.Param.Plot-DIM` ) { - for (fid in input$`Report.Param.Plot-FuncId` ) { - name_figure <- paste0('Param_Plot', fid, '-', dim, 'D') - algs = input$`Report.Param.Plot-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Param_plot.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.Param.Plot) paste(out, collapse='\n')` - -```{r param_plot_text, eval=input$Report.Param.Plot, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -# References +--- +title: "Automatic report" +author: "IOHprofiler" +date: "6/28/2019" +output: + pdf_document: + keep_tex: true + fig_caption: yes +params: + input: NA + dsl: NA + figure_folder: NA + REG: NA +bibliography: bibliography.bib +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +input <- params$input +dsl <- params$dsl +figure_folder <- params$figure_folder +get_data_id <- function(dsList) { + if (is.null(dsList) | length(dsList) == 0) + return(NULL) + + paste(get_funcId(dsList), get_dim(dsList), sep = '-') +} +REG <- params$REG +``` + +```{asis, echo=input$Report.Introduction} +# Introduction + +This report has been automatically generated by the IOHanalyzer [@IOHprofiler]. + +``` + +# Fixed-target view + +```{r overview, results='asis', eval=input$Report.RT.Overview} +asis_output("### Overview of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.RT.Overview-DIM` ) { + for (fid in input$`Report.RT.Overview-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Overview_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Overview-Alg` ) + p <- kable(get_RT_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r statistics, results='asis', eval=input$Report.RT.Statistics} +asis_output("### Runtime statistics of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.RT.Statistics-DIM` ) { + for (fid in input$`Report.RT.Statistics-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Statistics_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Statistics-Alg`) + name <- get_data_id(dsl_sub) + fvs <- get_funvals(dsl_sub) + tryCatch( + { + fv_seq <- seq_FV(fvs, as.numeric(REG$RTSummary.Statistics.Min[[name]]), as.numeric(REG$RTSummary.Statistics.Max[[name]]), + by = as.numeric(REG$RTSummary.Statistics.Step[[name]])) + }, + error = function(e) { fv_seq <- seq_RT(fvs, length.out = 10)} + ) + p <- kable(get_RT_summary(dsl_sub, fv_seq), + caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), + format = 'latex') %>% + kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r single_erts, eval=input$Report.RT.Single_ERT, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.Single_ERT-DIM` ) { + for (fid in input$`Report.RT.Single_ERT-FuncId` ) { + name_figure <- paste0('ERT_single_F', fid, '-', dim, 'D') + algs = input$`Report.RT.Single_ERT-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_ERT.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.Single_ERT) paste(out, collapse='\n')` + +```{r single_erts_text, eval=input$Report.RT.Single_ERT, results='asis'} +asis_output(paste0("This section shows the ERTs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + + +```{r multi_erts, eval=input$Report.RT.Multi_ERT, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.Multi_ERT-DIM` ) { + name_figure <- paste0('ERT_multi_', dim, 'D') + algs = input$`Report.RT.Multi_ERT-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_ERT.Rmd")))) +} +``` + +`r if(input$Report.RT.Multi_ERT) paste(out, collapse='\n')` + +```{r ranking_erts, eval=input$Report.RT.Rank, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.Rank-DIM` ) { + name_figure <- paste0('RT_Rank_', dim, 'D') + algs = input$`Report.RT.Rank-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Rank.Rmd")))) +} +``` + +`r if(input$Report.RT.Rank) paste(out, collapse='\n')` + +```{r rt_hist, eval=input$Report.RT.Histogram, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.Histogram-DIM` ) { + for (fid in input$`Report.RT.Histogram-FuncId` ) { + name_figure <- paste0('RT_Hist_F', fid, '-', dim, 'D') + algs = input$`Report.RT.Histogram-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Histogram.Rmd")))) + } +} +``` + +`r if(input$Report.RT.Histogram) paste(out, collapse='\n')` + +```{r rt_pmf, eval=input$Report.RT.PMF, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.PMF-DIM` ) { + for (fid in input$`Report.RT.PMF-FuncId` ) { + name_figure <- paste0('RT_PMF_F', fid, '-', dim, 'D') + algs = input$`Report.RT.PMF-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_PMF.Rmd")))) + } +} +``` + +`r if(input$Report.RT.PMF) paste(out, collapse='\n')` + +```{r ecdf_single_target, eval=input$Report.RT.ECDF_Single_Target, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.ECDF_Single_Target-DIM` ) { + for (fid in input$`Report.RT.ECDF_Single_Target-FuncId` ) { + name_figure <- paste0('ECDF_Single_Target', fid, '-', dim, 'D') + algs = input$`Report.RT.ECDF_Single_Target-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Target.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.ECDF_Single_Target) paste(out, collapse='\n')` + +```{r ecdf_single_target_text, eval=input$Report.RT.ECDF_Single_Target, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r ecdf_single_function, eval=input$Report.RT.ECDF_Single_Function, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.ECDF_Single_Function-DIM` ) { + for (fid in input$`Report.RT.ECDF_Single_Function-FuncId` ) { + name_figure <- paste0('ECDF_Single_Function', fid, '-', dim, 'D') + algs = input$`Report.RT.ECDF_Single_Function-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Function.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.ECDF_Single_Function) paste(out, collapse='\n')` + +```{r ecdf_single_function_text, eval=input$Report.RT.ECDF_Single_Function, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r ecdf_aggr, eval=input$Report.RT.ECDF_Aggregated, results='hide', include=F} +algs = input$`Report.RT.ECDF_Aggregated-Alg` +env = new.env() +out <- knit_child(text = knit_expand(file = "ECDF_Aggregated.Rmd")) + +``` + +`r if(input$Report.RT.ECDF_Aggregated) paste(out, collapse='\n')` + +```{r ecdf_aggr_text, eval=input$Report.RT.ECDF_Aggregated, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figure \\ref{fig:aggregated_ecdf}")) +``` + +```{r ecdf_auc, eval=input$Report.RT.ECDF_AUC, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.ECDF_AUC-DIM` ) { + for (fid in input$`Report.RT.ECDF_AUC-FuncId` ) { + name_figure <- paste0('ECDF_AUC', fid, '-', dim, 'D') + algs = input$`Report.RT.ECDF_AUC-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_AUC.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.ECDF_AUC) paste(out, collapse='\n')` + +```{r ecdf_auc_text, eval=input$Report.RT.ECDF_AUC, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +# Fixed-budget view + +```{r fv_overview, results='asis', eval=input$Report.FV.Overview} +asis_output("### Overview of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.FV.Overview-DIM` ) { + for (fid in input$`Report.FV.Overview-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Overview_FV_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Overview-Alg` ) + p <- kable(get_FV_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r fv_statistics, results='asis', eval=input$Report.FV.Statistics} +asis_output("### Runtime statistics of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.FV.Statistics-DIM` ) { + for (fid in input$`Report.FV.Statistics-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Statistics_FV_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Statistics-Alg`) + name <- get_data_id(dsl_sub) + rts <- get_runtimes(dsl_sub) + tryCatch( + { + rt_seq <- seq_RT(rts, as.numeric(REG$FCESummary.Statistics.Min[[name]]), as.numeric(REG$FCESummary.Statistics.Max[[name]]), + by = as.numeric(REG$FCESummary.Statistics.Step[[name]])) + }, + error = function(e) { rt_seq <- seq_RT(rts, length.out = 10)} + ) + p <- kable(get_FV_summary(dsl_sub, rt_seq), + caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), + format = 'latex') %>% + kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r single_fce, eval=input$Report.FV.Single_FCE, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.Single_FCE-DIM` ) { + for (fid in input$`Report.FV.Single_FCE-FuncId` ) { + name_figure <- paste0('FCE_single_F', fid, '-', dim, 'D') + algs = input$`Report.FV.Single_FCE-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_FCE.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.Single_FCE) paste(out, collapse='\n')` + +```{r single_fce_text, eval=input$Report.FV.Single_FCE, results='asis'} +asis_output(paste0("This section shows the FCEs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r multi_fces, eval=input$Report.FV.Multi_FCE, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.Multi_FCE-DIM` ) { + name_figure <- paste0('FCE_multi_', dim, 'D') + algs = input$`Report.FV.Multi_FCE-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_FCE.Rmd")))) +} +``` + +`r if(input$Report.FV.Multi_FCE) paste(out, collapse='\n')` + +```{r ranking_FVs, eval=input$Report.FV.Rank, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.Rank-DIM` ) { + name_figure <- paste0('FV_Rank_', dim, 'D') + algs = input$`Report.FV.Rank-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Rank.Rmd")))) +} +``` + +`r if(input$Report.FV.Rank) paste(out, collapse='\n')` + +```{r fv_hist, eval=input$Report.FV.Histogram, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.Histogram-DIM` ) { + for (fid in input$`Report.FV.Histogram-FuncId` ) { + name_figure <- paste0('FV_Hist_F', fid, '-', dim, 'D') + algs = input$`Report.FV.Histogram-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Histogram.Rmd")))) + } +} +``` + +`r if(input$Report.FV.Histogram) paste(out, collapse='\n')` + +```{r fv_pmf, eval=input$Report.FV.PMF, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.PMF-DIM` ) { + for (fid in input$`Report.FV.PMF-FuncId` ) { + name_figure <- paste0('FV_PMF_F', fid, '-', dim, 'D') + algs = input$`Report.FV.PMF-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_PMF.Rmd")))) + } +} +``` + +`r if(input$Report.FV.PMF) paste(out, collapse='\n')` + +```{r fv_ecdf_single_target, eval=input$Report.FV.ECDF_Single_Target, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.ECDF_Single_Target-DIM` ) { + for (fid in input$`Report.FV.ECDF_Single_Target-FuncId` ) { + name_figure <- paste0('FV_ECDF_Single_Target', fid, '-', dim, 'D') + algs = input$`Report.FV.ECDF_Single_Target-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Target.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.ECDF_Single_Target) paste(out, collapse='\n')` + +```{r fv_ecdf_single_target_text, eval=input$Report.FV.ECDF_Single_Target, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r fv_ecdf_single_function, eval=input$Report.FV.ECDF_Single_Function, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.ECDF_Single_Function-DIM` ) { + for (fid in input$`Report.FV.ECDF_Single_Function-FuncId` ) { + name_figure <- paste0('FV_ECDF_Single_Function', fid, '-', dim, 'D') + algs = input$`Report.FV.ECDF_Single_Function-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Function.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.ECDF_Single_Function) paste(out, collapse='\n')` + +```{r fv_ecdf_single_function_text, eval=input$Report.FV.ECDF_Single_Function, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r fv_ecdf_auc, eval=input$Report.FV.ECDF_AUC, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.ECDF_AUC-DIM` ) { + for (fid in input$`Report.FV.ECDF_AUC-FuncId` ) { + name_figure <- paste0('FV_ECDF_AUC', fid, '-', dim, 'D') + algs = input$`Report.FV.ECDF_AUC-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_AUC.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.ECDF_AUC) paste(out, collapse='\n')` + +```{r fv_ecdf_auc_text, eval=input$Report.FV.ECDF_AUC, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r param_statistics, results='asis', eval=input$Report.Param.Statistics} +asis_output("### Statistics of parameters of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.Param.Statistics-DIM` ) { + for (fid in input$`Report.Param.Statistics-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Statistics_Param_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.Param.Statistics-Alg`) + name <- get_data_id(dsl_sub) + fvs <- get_funvals(dsl_sub) + tryCatch( + { + fv_seq <- seq_FV(fvs, as.numeric(REG$PAR.Summary.Min[[name]]), as.numeric(REG$PAR.Summary.Max[[name]]), + by = as.numeric(REG$PAR.Summary.Step[[name]])) + }, + error = function(e) { fv_seq <- seq_FV(fvs, length.out = 10) } + ) + p <- kable(get_PAR_summary(dsl_sub, fv_seq), + caption = paste0("Statistics of parameters of selected algorithms on F", fid, " in ", dim, "D"), + format = 'latex') %>% + kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + } +} +``` + +```{r param_plot, eval=input$Report.Param.Plot, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.Param.Plot-DIM` ) { + for (fid in input$`Report.Param.Plot-FuncId` ) { + name_figure <- paste0('Param_Plot', fid, '-', dim, 'D') + algs = input$`Report.Param.Plot-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Param_plot.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.Param.Plot) paste(out, collapse='\n')` + +```{r param_plot_text, eval=input$Report.Param.Plot, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +# References diff --git a/inst/shiny-server/markdown/Report/bibliography.bib b/inst/shiny-server/markdown/Report/bibliography.bib index 5456cc1b..a3c40f98 100644 --- a/inst/shiny-server/markdown/Report/bibliography.bib +++ b/inst/shiny-server/markdown/Report/bibliography.bib @@ -1,11 +1,11 @@ -@ARTICLE{IOHprofiler, - author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, - title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, - journal = {arXiv e-prints:1810.05281}, - archivePrefix = "arXiv", - eprint = {1810.05281}, - year = 2018, - month = oct, - keywords = {Computer Science - Neural and Evolutionary Computing}, - url = {https://arxiv.org/abs/1810.05281} +@ARTICLE{IOHprofiler, + author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, + title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, + journal = {arXiv e-prints:1810.05281}, + archivePrefix = "arXiv", + eprint = {1810.05281}, + year = 2018, + month = oct, + keywords = {Computer Science - Neural and Evolutionary Computing}, + url = {https://arxiv.org/abs/1810.05281} } \ No newline at end of file diff --git a/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd index 19bbe555..ab79b4c6 100644 --- a/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd @@ -1,7 +1,7 @@ -This table summarizes for each algorithm and each __budget__ $B$ chosen on the left: - - * runs: the number of runs that have performed at least $B$ evaluations, - * mean: the average best-so-far function value obtain within a budget of $B$ evaluations, - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of the best function values found within the first $B$ evaluations. - -When not all runs evaluated at least $B$ search points, the statistics hold for the subset of runs that did. Alternative statistics using simulated restarted algorithms are in preparation. +This table summarizes for each algorithm and each __budget__ $B$ chosen on the left: + + * runs: the number of runs that have performed at least $B$ evaluations, + * mean: the average best-so-far function value obtain within a budget of $B$ evaluations, + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of the best function values found within the first $B$ evaluations. + +When not all runs evaluated at least $B$ search points, the statistics hold for the subset of runs that did. Alternative statistics using simulated restarted algorithms are in preparation. diff --git a/inst/shiny-server/markdown/about.md b/inst/shiny-server/markdown/about.md index 3b5ff9c9..574923e3 100644 --- a/inst/shiny-server/markdown/about.md +++ b/inst/shiny-server/markdown/about.md @@ -1,97 +1,97 @@ -
- - - - -
-
- -# Iterative Optimization Heuristics Profiler - -The __performance analyzer__ for Iterative Optimization Heuristics (IOHs). -It provides: - -* a web-based interface to analyze and visualize the empirical performance of IOHs -* interactive plot -* statistical evaluation -* report generation -* `R` programming interfaces in the backend - -## Development Team - -* [Hao Wang](https://www.universiteitleiden.nl/en/staffmembers/hao-wang), Leiden Institute of Advanced Computer Science. -* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science. -* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science. -* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), CNRS and Sorbonne University. -* [Ofer Shir](https://www.migal.org.il/Ofer-Shir) Migal - The Galilee Research Institute, Tel-Hai College. -* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science. - -## Installation - -### Software dependency - -* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). -* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. -* [optional] `inkscape` required to support pdf and eps figure format in downloading. Please visit the Inskscape Wiki page [http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape](http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape) for the detail. - -## Host it online yourself? - -We provide docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. - -## Reference -When using IOHprofiler and parts thereof, please kindly cite this work as - -Carola Doerr, Hao Wang, Furong Ye, Sander van Rijn, Thomas Bäck: IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics, arXiv e-prints:1810.05281, 2018. - -```bibtex -@ARTICLE{IOHprofiler, - author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, - title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, - journal = {arXiv e-prints:1810.05281}, - archivePrefix = "arXiv", - eprint = {1810.05281}, - year = 2018, - month = oct, - keywords = {Computer Science - Neural and Evolutionary Computing}, - url = {https://arxiv.org/abs/1810.05281} -} -``` - -## Acknowledgements -This work was supported by the Chinese scholarship council (CSC No. 201706310143), a public grant as part of the -Investissement d’avenir project, reference ANR-11-LABX-0056-LMH, LabEx LMH, in a joint call with the Gaspard Monge -Program for optimization, operations research, and their interactions with data sciences, by Paris Ile-de-France Region, -and by COST Action CA15140 “Improving Applicability of Nature-Inspired Optimisation by Joining Theory and Practice (ImAppNIO)”. - -## License -This application is governed by the __BSD 3-Clause license__. - -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +
+ + + + +
+
+ +# Iterative Optimization Heuristics Profiler + +The __performance analyzer__ for Iterative Optimization Heuristics (IOHs). +It provides: + +* a web-based interface to analyze and visualize the empirical performance of IOHs +* interactive plot +* statistical evaluation +* report generation +* `R` programming interfaces in the backend + +## Development Team + +* [Hao Wang](https://www.universiteitleiden.nl/en/staffmembers/hao-wang), Leiden Institute of Advanced Computer Science. +* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science. +* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science. +* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), CNRS and Sorbonne University. +* [Ofer Shir](https://www.migal.org.il/Ofer-Shir) Migal - The Galilee Research Institute, Tel-Hai College. +* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science. + +## Installation + +### Software dependency + +* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). +* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. +* [optional] `inkscape` required to support pdf and eps figure format in downloading. Please visit the Inskscape Wiki page [http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape](http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape) for the detail. + +## Host it online yourself? + +We provide docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. + +## Reference +When using IOHprofiler and parts thereof, please kindly cite this work as + +Carola Doerr, Hao Wang, Furong Ye, Sander van Rijn, Thomas Bäck: IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics, arXiv e-prints:1810.05281, 2018. + +```bibtex +@ARTICLE{IOHprofiler, + author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, + title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, + journal = {arXiv e-prints:1810.05281}, + archivePrefix = "arXiv", + eprint = {1810.05281}, + year = 2018, + month = oct, + keywords = {Computer Science - Neural and Evolutionary Computing}, + url = {https://arxiv.org/abs/1810.05281} +} +``` + +## Acknowledgements +This work was supported by the Chinese scholarship council (CSC No. 201706310143), a public grant as part of the +Investissement d’avenir project, reference ANR-11-LABX-0056-LMH, LabEx LMH, in a joint call with the Gaspard Monge +Program for optimization, operations research, and their interactions with data sciences, by Paris Ile-de-France Region, +and by COST Action CA15140 “Improving Applicability of Nature-Inspired Optimisation by Joining Theory and Practice (ImAppNIO)”. + +## License +This application is governed by the __BSD 3-Clause license__. + +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/inst/shiny-server/markdown/dataformat.md b/inst/shiny-server/markdown/dataformat.md index de3b2355..38829ec3 100644 --- a/inst/shiny-server/markdown/dataformat.md +++ b/inst/shiny-server/markdown/dataformat.md @@ -1,105 +1,105 @@ -# Supported Data Format - -The user is encouraged to analyze your own benchmark data using **IOHanalyzer**. For this purpose, users have prepare their own data set according to the supported data formats. At the time of writing, three data formats are supported by **IOHanalyzer**: - -* **COCO** data format as regulated in [1]. -* **IOHprofiler** data format, which is motivated and modified from **COCO** data format. -* A **two-column** format that is simplified from **IOHprofiler** format. - -When loading the data in programming interface (and in the graphical user interface as well), it is not necessary to specified which format the user took as **IOHanalyzer** detects the format automatically. For all three data formats, data files are organized in the same manner within the file system. The structure of data files exemplified as follows: - - - -Generally, in the folder (e.g., `./` here) that contains the data set, the following files are mandatory for **IOHanalyzer**: - -* _Meta-data_ (e.g., `IOHprofiler_f1_i1.info`) summarizes the algorithmic performance for each problem instance, with naming the following naming convention: `IOHprofiler_f[:num:]_i[:num:].info` for problem $f1$ and instance $1$. Note that one meta-data file can consist of several dimensions. Please see the detail below. -* _Raw-data_ (e.g., `IOHprofiler_f1_DIM100_i1.dat`) are CSV-based files that contain performance information indexed by the running time. Raw-data files are named in the similar manner as with the meta-data, for example, `IOHprofiler_f1_DIM100_i1.dat` for problem $f1$, dimension $100$ and instance $1$. By default, the data files are organized in the group of test functions and stored in folders with naming convention: `data_[function ID]/`. It is important to note that those three data formats only differ in structure of the raw-data files. - -## Meta-data - -When benchmarking, it is common to specify a number of different dimensions, functions and instances, resulting in a quite large number of data files (e.g., `*.dat` files). It would make the data organization more structured if some meta data are provided. Here, the meta data are implemented in a format that is very similar to that in the well-known **COCO** environment. The meta data are indicated with suffix \verb|.info|. An small example is provided as follows: - -```{bash} -suite = 'PBO', funcId = 10, DIM = 100, algId = '(1+1) fGA' -% -data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, -1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, -1:1953125|5.49000e+02 -suite = 'PBO', funcId = 10, DIM = 625, algId = '(1+1) fGA' -% -data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, -1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, -1:1953125|5.49000e+02 -... -``` - -Note that, as this meta information is also used in **IOHanalyzer** when loading the data, it is **crucial** to give an attention to the format of the meta data, if the user would convert their own data sets. - -* **The first line** stores some meta-information of the experiment as (name, value) pairs. Note that, such pairs are separated by commas and three names, `funcId`, `DIM` and `algId` are _case-sensitive_ and _mandatory_. -* **The second line** always starts with a single `%`, indicating what follows this symbol should be the general comments from the user on this experiment. By default, it is left empty. -* **The third line** starts with the relative path to the actual data file, followed by the meta-information obtained on each instance, with the following format: $$\underbrace{1}_{\text{instance ID}}:\underbrace{1953125}_{\text{running time}}|\;\underbrace{5.59000e+02}_{\text{best-so-far f(x)}}$$ - -## Raw-data - -Despite different events are used for those four types of data file, those files take the same format, which is adapted from CSV format to accommodate multiple runs/instances. Typically, this format is illustrated by the example below (with dummy data records): - -```{bash} -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... -1 +2.95000e+02 +2.95000e+02 +2.95000e+02 +2.95000e+02 0.000000 ... -2 +2.96000e+02 +2.96000e+02 +2.96000e+02 +2.96000e+02 0.001600 ... -4 +3.07000e+02 +3.07000e+02 +3.07000e+02 +3.07000e+02 0.219200 ... -9 +3.11000e+02 +3.11000e+02 +3.11000e+02 +3.11000e+02 0.006400 ... -12 +3.12000e+02 +3.12000e+02 +3.12000e+02 +3.12000e+02 0.001600 ... -16 +3.16000e+02 +3.16000e+02 +3.16000e+02 +3.16000e+02 0.006400 ... -20 +3.17000e+02 +3.17000e+02 +3.17000e+02 +3.17000e+02 0.001600 ... -23 +3.28000e+02 +3.28000e+02 +3.28000e+02 +3.28000e+02 0.027200 ... -27 +3.39000e+02 +3.39000e+02 +3.39000e+02 +3.39000e+02 0.059200 ... -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... -1 +3.20000e+02 +3.20000e+02 +3.20000e+02 +3.20000e+02 1.000000 ... -24 +3.44000e+02 +3.44000e+02 +3.44000e+02 +3.44000e+02 2.000000 ... -60 +3.64000e+02 +3.64000e+02 +3.64000e+02 +3.64000e+02 3.000000 ... -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... -... ... ... ... ... ... ... -``` - -Note that, - -1. **[mandatory]** each _separation line_ (line that starts with `"function evaluation"`) serves as a separator among different independent runs of the same algorithm. Therefore, it is clear that the data block between two separation lines corresponds to a single run a triplet of (dimension, function, instance). -2. **[mandatory]** `"function evaluation"` the current number of function evaluations. -3. **[mandatory]** `"best-so-far f(x)"` keeps track of the best function value observed since the beginning of one run. -4. **[optional]** `"current f(x)"` stands for the function value observed when the corresponding number of function evaluation is consumed. -5. **[optional]** The value stored under `"current af(x)+b"` and `"best af(x)+b"`, are so-called _transformed_ function values, obtained on each function instances that are generated by translating the orginal function in its domain and co-domain. -6. **[optional]** In addition, a parameter value (named `"parameter"`) is also tracked in this example and recording more parameter value is also facilitated (see below). - -## Two-Column Format - -The raw data file, in its simplest **two-column** format should resemble the following example: - -```{bash} -"function evaluation" "best-so-far f(x)" -1 +2.95000e+02 -2 +2.96000e+02 -4 +3.07000e+02 -23 +3.28000e+02 -27 +3.39000e+02 -"function evaluation" "best-so-far f(x)" -1 +3.20000e+02 -... ... -``` - -This format is regulated as follows: - -* The _double quotation_ (`"`) in the separation line shall always be kept and it cannot be replace with single quotation (`'`). -* The numbers in the record can either be written in the plain or scientific notation. -* To separate the column, _a single space or tab_ can be used (only one of them should be used consistently in a single data file). -* In the target-based tracking file (`*.dat`) each block of records (as divided by the separation line) **must** end with the last function evaluation. -* Each data line should contain a complete record. Incomplete data lines will be dropped when loading the data into **IOHanalyzer**. -* In case the quotation mark is needed in the parameter name, please use the single quotation (`'`). - -## Reference - -[1] Hansen N, Auger A, Finck S, Ros R (2009a). "Real-Parameter Black-Box OptimizationBenchmarking 2009: Experimental Setup." Research Report RR-6828, INRIA. URL +# Supported Data Format + +The user is encouraged to analyze your own benchmark data using **IOHanalyzer**. For this purpose, users have prepare their own data set according to the supported data formats. At the time of writing, three data formats are supported by **IOHanalyzer**: + +* **COCO** data format as regulated in [1]. +* **IOHprofiler** data format, which is motivated and modified from **COCO** data format. +* A **two-column** format that is simplified from **IOHprofiler** format. + +When loading the data in programming interface (and in the graphical user interface as well), it is not necessary to specified which format the user took as **IOHanalyzer** detects the format automatically. For all three data formats, data files are organized in the same manner within the file system. The structure of data files exemplified as follows: + + + +Generally, in the folder (e.g., `./` here) that contains the data set, the following files are mandatory for **IOHanalyzer**: + +* _Meta-data_ (e.g., `IOHprofiler_f1_i1.info`) summarizes the algorithmic performance for each problem instance, with naming the following naming convention: `IOHprofiler_f[:num:]_i[:num:].info` for problem $f1$ and instance $1$. Note that one meta-data file can consist of several dimensions. Please see the detail below. +* _Raw-data_ (e.g., `IOHprofiler_f1_DIM100_i1.dat`) are CSV-based files that contain performance information indexed by the running time. Raw-data files are named in the similar manner as with the meta-data, for example, `IOHprofiler_f1_DIM100_i1.dat` for problem $f1$, dimension $100$ and instance $1$. By default, the data files are organized in the group of test functions and stored in folders with naming convention: `data_[function ID]/`. It is important to note that those three data formats only differ in structure of the raw-data files. + +## Meta-data + +When benchmarking, it is common to specify a number of different dimensions, functions and instances, resulting in a quite large number of data files (e.g., `*.dat` files). It would make the data organization more structured if some meta data are provided. Here, the meta data are implemented in a format that is very similar to that in the well-known **COCO** environment. The meta data are indicated with suffix \verb|.info|. An small example is provided as follows: + +```{bash} +suite = 'PBO', funcId = 10, DIM = 100, algId = '(1+1) fGA' +% +data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, +1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, +1:1953125|5.49000e+02 +suite = 'PBO', funcId = 10, DIM = 625, algId = '(1+1) fGA' +% +data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, +1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, +1:1953125|5.49000e+02 +... +``` + +Note that, as this meta information is also used in **IOHanalyzer** when loading the data, it is **crucial** to give an attention to the format of the meta data, if the user would convert their own data sets. + +* **The first line** stores some meta-information of the experiment as (name, value) pairs. Note that, such pairs are separated by commas and three names, `funcId`, `DIM` and `algId` are _case-sensitive_ and _mandatory_. +* **The second line** always starts with a single `%`, indicating what follows this symbol should be the general comments from the user on this experiment. By default, it is left empty. +* **The third line** starts with the relative path to the actual data file, followed by the meta-information obtained on each instance, with the following format: $$\underbrace{1}_{\text{instance ID}}:\underbrace{1953125}_{\text{running time}}|\;\underbrace{5.59000e+02}_{\text{best-so-far f(x)}}$$ + +## Raw-data + +Despite different events are used for those four types of data file, those files take the same format, which is adapted from CSV format to accommodate multiple runs/instances. Typically, this format is illustrated by the example below (with dummy data records): + +```{bash} +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... +1 +2.95000e+02 +2.95000e+02 +2.95000e+02 +2.95000e+02 0.000000 ... +2 +2.96000e+02 +2.96000e+02 +2.96000e+02 +2.96000e+02 0.001600 ... +4 +3.07000e+02 +3.07000e+02 +3.07000e+02 +3.07000e+02 0.219200 ... +9 +3.11000e+02 +3.11000e+02 +3.11000e+02 +3.11000e+02 0.006400 ... +12 +3.12000e+02 +3.12000e+02 +3.12000e+02 +3.12000e+02 0.001600 ... +16 +3.16000e+02 +3.16000e+02 +3.16000e+02 +3.16000e+02 0.006400 ... +20 +3.17000e+02 +3.17000e+02 +3.17000e+02 +3.17000e+02 0.001600 ... +23 +3.28000e+02 +3.28000e+02 +3.28000e+02 +3.28000e+02 0.027200 ... +27 +3.39000e+02 +3.39000e+02 +3.39000e+02 +3.39000e+02 0.059200 ... +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... +1 +3.20000e+02 +3.20000e+02 +3.20000e+02 +3.20000e+02 1.000000 ... +24 +3.44000e+02 +3.44000e+02 +3.44000e+02 +3.44000e+02 2.000000 ... +60 +3.64000e+02 +3.64000e+02 +3.64000e+02 +3.64000e+02 3.000000 ... +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... +... ... ... ... ... ... ... +``` + +Note that, + +1. **[mandatory]** each _separation line_ (line that starts with `"function evaluation"`) serves as a separator among different independent runs of the same algorithm. Therefore, it is clear that the data block between two separation lines corresponds to a single run a triplet of (dimension, function, instance). +2. **[mandatory]** `"function evaluation"` the current number of function evaluations. +3. **[mandatory]** `"best-so-far f(x)"` keeps track of the best function value observed since the beginning of one run. +4. **[optional]** `"current f(x)"` stands for the function value observed when the corresponding number of function evaluation is consumed. +5. **[optional]** The value stored under `"current af(x)+b"` and `"best af(x)+b"`, are so-called _transformed_ function values, obtained on each function instances that are generated by translating the orginal function in its domain and co-domain. +6. **[optional]** In addition, a parameter value (named `"parameter"`) is also tracked in this example and recording more parameter value is also facilitated (see below). + +## Two-Column Format + +The raw data file, in its simplest **two-column** format should resemble the following example: + +```{bash} +"function evaluation" "best-so-far f(x)" +1 +2.95000e+02 +2 +2.96000e+02 +4 +3.07000e+02 +23 +3.28000e+02 +27 +3.39000e+02 +"function evaluation" "best-so-far f(x)" +1 +3.20000e+02 +... ... +``` + +This format is regulated as follows: + +* The _double quotation_ (`"`) in the separation line shall always be kept and it cannot be replace with single quotation (`'`). +* The numbers in the record can either be written in the plain or scientific notation. +* To separate the column, _a single space or tab_ can be used (only one of them should be used consistently in a single data file). +* In the target-based tracking file (`*.dat`) each block of records (as divided by the separation line) **must** end with the last function evaluation. +* Each data line should contain a complete record. Incomplete data lines will be dropped when loading the data into **IOHanalyzer**. +* In case the quotation mark is needed in the parameter name, please use the single quotation (`'`). + +## Reference + +[1] Hansen N, Auger A, Finck S, Ros R (2009a). "Real-Parameter Black-Box OptimizationBenchmarking 2009: Experimental Setup." Research Report RR-6828, INRIA. URL diff --git a/inst/shiny-server/markdown/docs.Rmd b/inst/shiny-server/markdown/docs.Rmd index 3cfe8676..146cb229 100644 --- a/inst/shiny-server/markdown/docs.Rmd +++ b/inst/shiny-server/markdown/docs.Rmd @@ -1,26 +1,26 @@ ---- -title: "Untitled" -output: html_document ---- - -**Remark**: This is just a mockup, to show that we could put some important/concise documents here. - -## R Markdown - -This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see . - -When you click the **Knit** button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this: - -```{r cars} -summary(cars) -``` - -## Including Plots - -You can also embed plots, for example: - -```{r pressure, echo=FALSE} -plot(pressure) -``` - -Note that the `echo = FALSE` parameter was added to the code chunk to prevent printing of the R code that generated the plot. +--- +title: "Untitled" +output: html_document +--- + +**Remark**: This is just a mockup, to show that we could put some important/concise documents here. + +## R Markdown + +This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see . + +When you click the **Knit** button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this: + +```{r cars} +summary(cars) +``` + +## Including Plots + +You can also embed plots, for example: + +```{r pressure, echo=FALSE} +plot(pressure) +``` + +Note that the `echo = FALSE` parameter was added to the code chunk to prevent printing of the R code that generated the plot. diff --git a/inst/shiny-server/markdown/quantile.Rmd b/inst/shiny-server/markdown/quantile.Rmd index b8be0e1f..757e2c1d 100644 --- a/inst/shiny-server/markdown/quantile.Rmd +++ b/inst/shiny-server/markdown/quantile.Rmd @@ -1,9 +1,9 @@ ---- -title: "quantile" -output: html_document ---- - -**Remark**: all sample quantiles are defined as weighted averages of consecutive order statistics. Let $\{x_1, x_2, \ldots, x_N\}$ stands for i.i.d. runtime samples. Sample quantiles are defined as: -$$Q(p) = (1 - \alpha) x_{i:N}+ \alpha x_{i+1:N},$$ -**TODO** this is just a example showing we could explain some detials on the calculations. - +--- +title: "quantile" +output: html_document +--- + +**Remark**: all sample quantiles are defined as weighted averages of consecutive order statistics. Let $\{x_1, x_2, \ldots, x_N\}$ stands for i.i.d. runtime samples. Sample quantiles are defined as: +$$Q(p) = (1 - \alpha) x_{i:N}+ \alpha x_{i+1:N},$$ +**TODO** this is just a example showing we could explain some detials on the calculations. + diff --git a/inst/shiny-server/markdown/welcome.md b/inst/shiny-server/markdown/welcome.md index 2ec7b243..54f9cf72 100644 --- a/inst/shiny-server/markdown/welcome.md +++ b/inst/shiny-server/markdown/welcome.md @@ -1,45 +1,45 @@ -This the **analyzer** for the empirical performance of _Iterative Optimization Heuristics_ (IOHs). It provides _two perspectives_ to look at the empirical performance: - -* Fixed-Target running time: focuses on the distribution of **running time** (a.k.a. the number of function evaluations) at various given target values (thus called "fixed-target"), functions and dimensions. -* Fixed-Budget results: focuses on the distribution of the **best function value** at various given budget values (thus called "fixed-budget"), functions and dimensions. - -To get started, you could - -* **upload** your own dataset using the _Upload Data_ box on the bottom left. The supported format is specified in the data format tab. -* or **choose** one of the pre-loaded datasets in the _Load Data from Repository_ box on the bottom right and directly explore the analyzer. - -In details, the following functionalities are provided: - - - - - - - - - - - - - - - - - - - - - - - - -
Data SummaryExpected ValuesProbability Density/Mass FunctionEmpirical Cumulative Distribution (ECDF)
Fixed-Target running timeSummary of running timesExpected Running Time (ERT)Probability Mass Function of running timeECDF of running times
Fixed-Budget resultsSummary of function valuesExpected Target ValueProbability Density Function of target valuesECDF of targets times
-
- -For more information on the features of IOHanalyzer and how to use them, -as well as a full specification of the accepted data formats, please visit our wiki. +This the **analyzer** for the empirical performance of _Iterative Optimization Heuristics_ (IOHs). It provides _two perspectives_ to look at the empirical performance: + +* Fixed-Target running time: focuses on the distribution of **running time** (a.k.a. the number of function evaluations) at various given target values (thus called "fixed-target"), functions and dimensions. +* Fixed-Budget results: focuses on the distribution of the **best function value** at various given budget values (thus called "fixed-budget"), functions and dimensions. + +To get started, you could + +* **upload** your own dataset using the _Upload Data_ box on the bottom left. The supported format is specified in the data format tab. +* or **choose** one of the pre-loaded datasets in the _Load Data from Repository_ box on the bottom right and directly explore the analyzer. + +In details, the following functionalities are provided: + + + + + + + + + + + + + + + + + + + + + + + + +
Data SummaryExpected ValuesProbability Density/Mass FunctionEmpirical Cumulative Distribution (ECDF)
Fixed-Target running timeSummary of running timesExpected Running Time (ERT)Probability Mass Function of running timeECDF of running times
Fixed-Budget resultsSummary of function valuesExpected Target ValueProbability Density Function of target valuesECDF of targets times
+
+ +For more information on the features of IOHanalyzer and how to use them, +as well as a full specification of the accepted data formats, please visit our wiki. diff --git a/inst/shiny-server/server.R b/inst/shiny-server/server.R index 520a8ceb..d6a1ace2 100644 --- a/inst/shiny-server/server.R +++ b/inst/shiny-server/server.R @@ -1,65 +1,65 @@ - -repo_dir <- '' # repository directory -repo_data <- NULL # repository data -has_rendered_ERT_per_fct <- FALSE - -# Formatter for function values. Somewhat overkill with as.numeric, but this prevents unneeded precision -format_FV <- function(v) as.numeric(format(v, digits = getOption("IOHanalyzer.precision", 2), - nsmall = getOption("IOHanalyzer.precision", 2))) -format_RT <- function(v) as.integer(v) - -# directory where data are extracted from the zip file -exdir <- file.path(tempdir(), 'data') - -rand_strings <- function(n = 10) { - a <- do.call(paste0, replicate(5, sample(LETTERS, n, TRUE), FALSE)) - paste0(a, sprintf("%04d", sample(9999, n, TRUE)), sample(LETTERS, n, TRUE)) -} - -setTextInput <- function(session, id, name, alternative) { - v <- REG[[id]] - if (name %in% names(v)) - updateTextInput(session, id, value = v[[name]]) - else - updateTextInput(session, id, value = alternative) -} - -# register previous text inputs, which is used to restore those values -REG <- lapply(widget_id, function(x) list()) - -# TODO: maybe give this function a better name -# get the current 'id' of the selected data: funcID + DIM -get_data_id <- function(dsList) { - if (is.null(dsList) | length(dsList) == 0) - return(NULL) - - paste(get_funcId(dsList), get_dim(dsList), sep = '-') -} - -# Define server logic required to draw a histogram -shinyServer(function(input, output, session) { - - # clean up the temporarsy files on server when exiting - session$onSessionEnded(function() { - # close_connection() - unlink(exdir, recursive = T) - }) - - for (f in list.files('server', pattern = '.R', full.names = T)) { - source(f, local = TRUE) - } - - output$VersionBox <- renderValueBox({ - infoBox(title = "IOHanalyzer version:", value = paste0('v', packageVersion("IOHanalyzer")), icon = icon("code-branch"), - color = "blue", width = 12, fill = T, href = "https://github.com/IOHprofiler/IOHanalyzer", subtitle = "View source on Github" - )}) - output$WikiBox <- renderValueBox({ - infoBox(title = "Wiki Page", value = "For full documentation of IOHprofiler", icon = icon("wikipedia-w"), - color = "blue", width = 12, fill = T, href = "https://iohprofiler.github.io/" - )}) - output$ContactBox <- renderValueBox({ - infoBox(title = "Contact us by email:", value = "iohprofiler@liacs.leidenuniv.nl", icon = icon("envelope"), - color = "blue", width = 14, fill = T, href = "mailto:iohprofiler@liacs.leidenuniv.nl" - ) - }) -}) + +repo_dir <- '' # repository directory +repo_data <- NULL # repository data +has_rendered_ERT_per_fct <- FALSE + +# Formatter for function values. Somewhat overkill with as.numeric, but this prevents unneeded precision +format_FV <- function(v) as.numeric(format(v, digits = getOption("IOHanalyzer.precision", 2), + nsmall = getOption("IOHanalyzer.precision", 2))) +format_RT <- function(v) as.integer(v) + +# directory where data are extracted from the zip file +exdir <- file.path(tempdir(), 'data') + +rand_strings <- function(n = 10) { + a <- do.call(paste0, replicate(5, sample(LETTERS, n, TRUE), FALSE)) + paste0(a, sprintf("%04d", sample(9999, n, TRUE)), sample(LETTERS, n, TRUE)) +} + +setTextInput <- function(session, id, name, alternative) { + v <- REG[[id]] + if (name %in% names(v)) + updateTextInput(session, id, value = v[[name]]) + else + updateTextInput(session, id, value = alternative) +} + +# register previous text inputs, which is used to restore those values +REG <- lapply(widget_id, function(x) list()) + +# TODO: maybe give this function a better name +# get the current 'id' of the selected data: funcID + DIM +get_data_id <- function(dsList) { + if (is.null(dsList) | length(dsList) == 0) + return(NULL) + + paste(get_funcId(dsList), get_dim(dsList), sep = '-') +} + +# Define server logic required to draw a histogram +shinyServer(function(input, output, session) { + + # clean up the temporarsy files on server when exiting + session$onSessionEnded(function() { + # close_connection() + unlink(exdir, recursive = T) + }) + + for (f in list.files('server', pattern = '.R', full.names = T)) { + source(f, local = TRUE) + } + + output$VersionBox <- renderValueBox({ + infoBox(title = "IOHanalyzer version:", value = paste0('v', packageVersion("IOHanalyzer")), icon = icon("code-branch"), + color = "blue", width = 12, fill = T, href = "https://github.com/IOHprofiler/IOHanalyzer", subtitle = "View source on Github" + )}) + output$WikiBox <- renderValueBox({ + infoBox(title = "Wiki Page", value = "For full documentation of IOHprofiler", icon = icon("wikipedia-w"), + color = "blue", width = 12, fill = T, href = "https://iohprofiler.github.io/" + )}) + output$ContactBox <- renderValueBox({ + infoBox(title = "Contact us by email:", value = "iohprofiler@liacs.leidenuniv.nl", icon = icon("envelope"), + color = "blue", width = 14, fill = T, href = "mailto:iohprofiler@liacs.leidenuniv.nl" + ) + }) +}) diff --git a/inst/shiny-server/server/ERTPlot.R b/inst/shiny-server/server/ERTPlot.R index f6fa7b16..eeead43f 100644 --- a/inst/shiny-server/server/ERTPlot.R +++ b/inst/shiny-server/server/ERTPlot.R @@ -1,195 +1,195 @@ -# The expected runtime plot --------------------- -# TODO: wrapp this as a separate function for DataSet class -# TODO: add very brief explanation on each function here - -output$ERT_PER_FUN <- renderPlotly({ - render_ert_per_fct() -}) - -get_data_ERT_PER_FUN <- reactive({ - req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) - selected_algs <- input$ERTPlot.Algs - data <- subset(DATA(), ID %in% input$ERTPlot.Algs) - fstart <- input$ERTPlot.Min %>% as.numeric - fstop <- input$ERTPlot.Max %>% as.numeric - budget <- input$ERTPlot.Additional.Budget %>% as.numeric - generate_data.Single_Function(data, fstart, fstop, input$ERTPlot.semilogx, - 'by_RT', include_opts = input$ERTPlot.inclueOpts, budget = budget) -}) - -render_ert_per_fct <- reactive({ - withProgress({ - y_attrs <- c() - if (input$ERTPlot.show.ERT) y_attrs <- c(y_attrs, 'ERT') - if (input$ERTPlot.show.mean) y_attrs <- c(y_attrs, 'mean') - if (input$ERTPlot.show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'ID', show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - else - p <- NULL - if (input$ERTPlot.show.CI) { - p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - if (input$ERTPlot.show.Quantiles) { - quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') - p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'median', - type = 'ribbon', legend_attr = 'ID', lower_attr = quantiles[[1]], - upper_attr = quantiles[[length(quantiles)]], p = p, - show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - if (input$ERTPlot.show.runs) { - fstart <- isolate(input$ERTPlot.Min %>% as.numeric) - fstop <- isolate(input$ERTPlot.Max %>% as.numeric) - data <- isolate(subset(DATA(), ID %in% input$ERTPlot.Algs)) - dt <- get_RT_sample(data, seq_FV(get_funvals(data), from = fstart, to = fstop, length.out = 50, - scale = ifelse(isolate(input$ERTPlot.semilogx), 'log', 'linear'))) - nr_runs <- ncol(dt) - 4 - for (i in seq_len(nr_runs)) { - p <- plot_general_data(dt, x_attr = 'target', y_attr = paste0('run.', i), type = 'line', - legend_attr = 'ID', p = p, show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - } - p - }, - message = "Creating plot" - ) -}) - -output$ERTPlot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_PER_FUN) - }, - content = function(file) { - save_plotly(render_ert_per_fct(), file) - }, - contentType = paste0('image/', input$ERTPlot.Format) -) - -# update_ert_per_fct_axis <- observe({ -# plotlyProxy("ERT_PER_FUN", session) %>% -# plotlyProxyInvoke( -# "restyle", -# list( -# xaxis = list(title = 'best-so-far-f(x)-value', -# type = ifelse(input$ERTPlot.semilogx, 'log', 'linear')), -# yaxis = list(title = 'function evaluations', -# type = ifelse(input$ERTPlot.semilogy, 'log', 'linear')) -# ) -# ) -# }) -# -# render_ert_per_fct <- reactive({ -# withProgress({ -# req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) -# selected_algs <- input$ERTPlot.Algs -# -# data <- subset(DATA(), algId %in% input$ERTPlot.Algs) -# req(length(data) > 0) -# -# fstart <- input$ERTPlot.Min %>% as.numeric -# fstop <- input$ERTPlot.Max %>% as.numeric -# -# Plot.RT.Single_Func(data, Fstart = fstart, Fstop = fstop, -# show.CI = input$ERTPlot.show.CI, -# show.ERT = input$ERTPlot.show.ERT, -# show.mean = input$ERTPlot.show.mean, -# show.median = input$ERTPlot.show.median, -# scale.xlog = input$ERTPlot.semilogx, -# scale.ylog = isolate(input$ERTPlot.semilogy), -# includeOpts = input$ERTPlot.inclueOpts, -# scale.reverse = !attr(data, 'maximization')) -# }, -# message = "Creating plot" -# ) -# }) - -output$ERTPlot.Multi.Plot <- renderPlotly( - render_ERTPlot_multi_plot() -) - -get_data_ERT_multi_func_bulk <- reactive({ - data <- subset(DATA_RAW(), - DIM == input$Overall.Dim) - if (length(get_id(data)) < 20) { #Arbitrary limit for the time being - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, - which = 'by_RT') - })) - } - else - NULL -}) - -get_data_ERT_multi_func <- reactive({ - req(isolate(input$ERTPlot.Multi.Algs)) - input$ERTPlot.Multi.PlotButton - data <- subset(DATA_RAW(), - DIM == input$Overall.Dim) - if (length(get_id(data)) < 20) { - get_data_ERT_multi_func_bulk()[ID %in% isolate(input$ERTPlot.Multi.Algs), ] - } - else { - selected_algs <- isolate(input$ERTPlot.Multi.Algs) - data <- subset(DATA_RAW(), - ID %in% selected_algs, - DIM == input$Overall.Dim) - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, - which = 'by_RT') - })) - } -}) - -render_ERTPlot_multi_plot <- reactive({ - req(isolate(input$ERTPlot.Multi.Algs)) - withProgress({ - dt <- get_data_ERT_multi_func() - if (is.null(dt)) - return(NULL) - plot_general_data(dt, x_attr = 'target', y_attr = 'ERT', - subplot_attr = 'funcId', type = 'line', scale.xlog = input$ERTPlot.Multi.Logx, - scale.ylog = input$ERTPlot.Multi.Logy, x_title = 'Best-so-far f(x)', - y_title = 'ERT', show.legend = T, - scale.reverse = !attr(DATA(), 'maximization')) - }, - message = "Creating plot") -}) - -output$ERTPlot.Multi.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_PER_FUN_MULTI) - }, - content = function(file) { - save_plotly(render_ERTPlot_multi_plot(), file) - }, - contentType = paste0('image/', input$ERTPlot.Multi.Format) -) - - - - +# The expected runtime plot --------------------- +# TODO: wrapp this as a separate function for DataSet class +# TODO: add very brief explanation on each function here + +output$ERT_PER_FUN <- renderPlotly({ + render_ert_per_fct() +}) + +get_data_ERT_PER_FUN <- reactive({ + req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) + selected_algs <- input$ERTPlot.Algs + data <- subset(DATA(), ID %in% input$ERTPlot.Algs) + fstart <- input$ERTPlot.Min %>% as.numeric + fstop <- input$ERTPlot.Max %>% as.numeric + budget <- input$ERTPlot.Additional.Budget %>% as.numeric + generate_data.Single_Function(data, fstart, fstop, input$ERTPlot.semilogx, + 'by_RT', include_opts = input$ERTPlot.inclueOpts, budget = budget) +}) + +render_ert_per_fct <- reactive({ + withProgress({ + y_attrs <- c() + if (input$ERTPlot.show.ERT) y_attrs <- c(y_attrs, 'ERT') + if (input$ERTPlot.show.mean) y_attrs <- c(y_attrs, 'mean') + if (input$ERTPlot.show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + else + p <- NULL + if (input$ERTPlot.show.CI) { + p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + if (input$ERTPlot.show.Quantiles) { + quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') + p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'median', + type = 'ribbon', legend_attr = 'ID', lower_attr = quantiles[[1]], + upper_attr = quantiles[[length(quantiles)]], p = p, + show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + if (input$ERTPlot.show.runs) { + fstart <- isolate(input$ERTPlot.Min %>% as.numeric) + fstop <- isolate(input$ERTPlot.Max %>% as.numeric) + data <- isolate(subset(DATA(), ID %in% input$ERTPlot.Algs)) + dt <- get_RT_sample(data, seq_FV(get_funvals(data), from = fstart, to = fstop, length.out = 50, + scale = ifelse(isolate(input$ERTPlot.semilogx), 'log', 'linear'))) + nr_runs <- ncol(dt) - 4 + for (i in seq_len(nr_runs)) { + p <- plot_general_data(dt, x_attr = 'target', y_attr = paste0('run.', i), type = 'line', + legend_attr = 'ID', p = p, show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + } + p + }, + message = "Creating plot" + ) +}) + +output$ERTPlot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_PER_FUN) + }, + content = function(file) { + save_plotly(render_ert_per_fct(), file) + }, + contentType = paste0('image/', input$ERTPlot.Format) +) + +# update_ert_per_fct_axis <- observe({ +# plotlyProxy("ERT_PER_FUN", session) %>% +# plotlyProxyInvoke( +# "restyle", +# list( +# xaxis = list(title = 'best-so-far-f(x)-value', +# type = ifelse(input$ERTPlot.semilogx, 'log', 'linear')), +# yaxis = list(title = 'function evaluations', +# type = ifelse(input$ERTPlot.semilogy, 'log', 'linear')) +# ) +# ) +# }) +# +# render_ert_per_fct <- reactive({ +# withProgress({ +# req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) +# selected_algs <- input$ERTPlot.Algs +# +# data <- subset(DATA(), algId %in% input$ERTPlot.Algs) +# req(length(data) > 0) +# +# fstart <- input$ERTPlot.Min %>% as.numeric +# fstop <- input$ERTPlot.Max %>% as.numeric +# +# Plot.RT.Single_Func(data, Fstart = fstart, Fstop = fstop, +# show.CI = input$ERTPlot.show.CI, +# show.ERT = input$ERTPlot.show.ERT, +# show.mean = input$ERTPlot.show.mean, +# show.median = input$ERTPlot.show.median, +# scale.xlog = input$ERTPlot.semilogx, +# scale.ylog = isolate(input$ERTPlot.semilogy), +# includeOpts = input$ERTPlot.inclueOpts, +# scale.reverse = !attr(data, 'maximization')) +# }, +# message = "Creating plot" +# ) +# }) + +output$ERTPlot.Multi.Plot <- renderPlotly( + render_ERTPlot_multi_plot() +) + +get_data_ERT_multi_func_bulk <- reactive({ + data <- subset(DATA_RAW(), + DIM == input$Overall.Dim) + if (length(get_id(data)) < 20) { #Arbitrary limit for the time being + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, + which = 'by_RT') + })) + } + else + NULL +}) + +get_data_ERT_multi_func <- reactive({ + req(isolate(input$ERTPlot.Multi.Algs)) + input$ERTPlot.Multi.PlotButton + data <- subset(DATA_RAW(), + DIM == input$Overall.Dim) + if (length(get_id(data)) < 20) { + get_data_ERT_multi_func_bulk()[ID %in% isolate(input$ERTPlot.Multi.Algs), ] + } + else { + selected_algs <- isolate(input$ERTPlot.Multi.Algs) + data <- subset(DATA_RAW(), + ID %in% selected_algs, + DIM == input$Overall.Dim) + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, + which = 'by_RT') + })) + } +}) + +render_ERTPlot_multi_plot <- reactive({ + req(isolate(input$ERTPlot.Multi.Algs)) + withProgress({ + dt <- get_data_ERT_multi_func() + if (is.null(dt)) + return(NULL) + plot_general_data(dt, x_attr = 'target', y_attr = 'ERT', + subplot_attr = 'funcId', type = 'line', scale.xlog = input$ERTPlot.Multi.Logx, + scale.ylog = input$ERTPlot.Multi.Logy, x_title = 'Best-so-far f(x)', + y_title = 'ERT', show.legend = T, + scale.reverse = !attr(DATA(), 'maximization')) + }, + message = "Creating plot") +}) + +output$ERTPlot.Multi.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_PER_FUN_MULTI) + }, + content = function(file) { + save_plotly(render_ERTPlot_multi_plot(), file) + }, + contentType = paste0('image/', input$ERTPlot.Multi.Format) +) + + + + diff --git a/inst/shiny-server/server/ERT_aggr_dim_plot.R b/inst/shiny-server/server/ERT_aggr_dim_plot.R index 3e26b77e..765ce438 100644 --- a/inst/shiny-server/server/ERT_aggr_dim_plot.R +++ b/inst/shiny-server/server/ERT_aggr_dim_plot.R @@ -1,133 +1,133 @@ -### Generate ERT table -get_data_ERT_aggr_dim <- reactive({ - input$ERTPlot.Aggr_Dim.Refresh - req(length(DATA_RAW()) > 0) - data <- ERTPlot.Aggr_Dim.data() - if (is.null(data)) return(NULL) - aggr_on <- 'DIM' - targets <- isolate(ERTPlot.Aggr_Dim.Targets_obj) - dt <- generate_data.Aggr(data, aggr_on, targets) - dt -}) - -### Plot the data -render_ERTPlot_aggr_plot_dim <- reactive({ - withProgress({ - data <- get_data_ERT_aggr_dim() - req(data) - y_attr <- if (input$ERTPlot.Aggr_Dim.Ranking) 'rank' else 'value' - y_title <- if (input$ERTPlot.Aggr_Dim.Ranking) 'Rank' else 'ERT' - reverse_scale <- input$ERTPlot.Aggr_Dim.Mode == 'radar' - - plot_general_data(data, type = input$ERTPlot.Aggr_Dim.Mode, x_attr = 'DIM', - y_attr = y_attr, x_title = "DIM", y_title = y_title, show.legend = T, - scale.ylog = input$ERTPlot.Aggr_Dim.Logy, scale.xlog = T, - scale.reverse = reverse_scale) - }, - message = "Creating plot") -}) - -### Gather relevant datasetlist -ERTPlot.Aggr_Dim.data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr_Dim.Algs)) - if (length(data) == 0) return(NULL) - data <- subset(data, funcId == input$Overall.Funcid) - - if (length(unique(get_dim(data))) <= 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple dimensions for the selected function") - return(NULL) - } - - if (length(unique(get_id(data))) <= 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple IDs for the selected function") - return(NULL) - } - data -} - -### format table for display -ert_multi_dim <- function(){ - dt <- get_data_ERT_aggr_dim() - dt <- dcast(dt, DIM~ID, value.var = 'value') - format(dt, digits = 4) -} - -### Table with default targets -default_targets_table_dim <- reactive({ - data <- ERTPlot.Aggr_Dim.data() - if (is.null(data)) return(NULL) - targets <- get_target_dt(data, 'by_FV') - targets <- targets[, c('DIM','target')] -}) - - -### Target table object -ERTPlot.Aggr_Dim.Targets_obj <- NULL - -### Target table proxy -proxy_ERTPlot.Aggr_Dim.Targets <- dataTableProxy('ERTPlot.Aggr_Dim.Targets') - -### Target table print -output$ERTPlot.Aggr_Dim.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - ERTPlot.Aggr_Dim.Targets_obj <<- default_targets_table_dim() - ERTPlot.Aggr_Dim.Targets_obj -}, editable = TRUE, rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - - -### Target table edit -observeEvent(input$ERTPlot.Aggr_Dim.Targets_cell_edit, { - info <- input$ERTPlot.Aggr_Dim.Targets_cell_edit - i <- info$row - req(i > 0) - j <- info$col + 1 - v <- info$value - ERTPlot.Aggr_Dim.Targets_obj[i, j] <<- - DT::coerceValue(v, ERTPlot.Aggr_Dim.Targets_obj[['target']][[i]]) - replaceData(proxy, ERTPlot.Aggr_Dim.Targets_obj, resetPaging = FALSE, rownames = FALSE) -}) - -### Table output -output$ERTPlot.Aggr_Dim.ERTTable <- DT::renderDataTable({ - input$ERTPlot.Aggr_Dim.Refresh - req(length(DATA_RAW()) > 0) - - withProgress({ - ert_multi_dim() - }, - message = "Creating table") -}, editable = FALSE, rownames = TRUE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - - -### plot output -output$ERTPlot.Aggr_Dim.Plot <- renderPlotly( - render_ERTPlot_aggr_plot_dim() -) - -### Download table -output$ERTPlot.Aggr_Dim.DownloadTable <- downloadHandler( - filename = function() { - eval(ERT_multi_dim_name) - }, - content = function(file) { - df <- ert_multi_dim() - save_table(df, file) - } -) - -### Download plot -output$ERTPlot.Aggr_Dim.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_AGGR_DIM) - }, - content = function(file) { - save_plotly(render_ERTPlot_aggr_plot_dim(), file) - }, - contentType = paste0('image/', input$ERTPlot.Aggr_Dim.Format) -) +### Generate ERT table +get_data_ERT_aggr_dim <- reactive({ + input$ERTPlot.Aggr_Dim.Refresh + req(length(DATA_RAW()) > 0) + data <- ERTPlot.Aggr_Dim.data() + if (is.null(data)) return(NULL) + aggr_on <- 'DIM' + targets <- isolate(ERTPlot.Aggr_Dim.Targets_obj) + dt <- generate_data.Aggr(data, aggr_on, targets) + dt +}) + +### Plot the data +render_ERTPlot_aggr_plot_dim <- reactive({ + withProgress({ + data <- get_data_ERT_aggr_dim() + req(data) + y_attr <- if (input$ERTPlot.Aggr_Dim.Ranking) 'rank' else 'value' + y_title <- if (input$ERTPlot.Aggr_Dim.Ranking) 'Rank' else 'ERT' + reverse_scale <- input$ERTPlot.Aggr_Dim.Mode == 'radar' + + plot_general_data(data, type = input$ERTPlot.Aggr_Dim.Mode, x_attr = 'DIM', + y_attr = y_attr, x_title = "DIM", y_title = y_title, show.legend = T, + scale.ylog = input$ERTPlot.Aggr_Dim.Logy, scale.xlog = T, + scale.reverse = reverse_scale) + }, + message = "Creating plot") +}) + +### Gather relevant datasetlist +ERTPlot.Aggr_Dim.data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr_Dim.Algs)) + if (length(data) == 0) return(NULL) + data <- subset(data, funcId == input$Overall.Funcid) + + if (length(unique(get_dim(data))) <= 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple dimensions for the selected function") + return(NULL) + } + + if (length(unique(get_id(data))) <= 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected function") + return(NULL) + } + data +} + +### format table for display +ert_multi_dim <- function(){ + dt <- get_data_ERT_aggr_dim() + dt <- dcast(dt, DIM~ID, value.var = 'value') + format(dt, digits = 4) +} + +### Table with default targets +default_targets_table_dim <- reactive({ + data <- ERTPlot.Aggr_Dim.data() + if (is.null(data)) return(NULL) + targets <- get_target_dt(data, 'by_FV') + targets <- targets[, c('DIM','target')] +}) + + +### Target table object +ERTPlot.Aggr_Dim.Targets_obj <- NULL + +### Target table proxy +proxy_ERTPlot.Aggr_Dim.Targets <- dataTableProxy('ERTPlot.Aggr_Dim.Targets') + +### Target table print +output$ERTPlot.Aggr_Dim.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + ERTPlot.Aggr_Dim.Targets_obj <<- default_targets_table_dim() + ERTPlot.Aggr_Dim.Targets_obj +}, editable = TRUE, rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + + +### Target table edit +observeEvent(input$ERTPlot.Aggr_Dim.Targets_cell_edit, { + info <- input$ERTPlot.Aggr_Dim.Targets_cell_edit + i <- info$row + req(i > 0) + j <- info$col + 1 + v <- info$value + ERTPlot.Aggr_Dim.Targets_obj[i, j] <<- + DT::coerceValue(v, ERTPlot.Aggr_Dim.Targets_obj[['target']][[i]]) + replaceData(proxy, ERTPlot.Aggr_Dim.Targets_obj, resetPaging = FALSE, rownames = FALSE) +}) + +### Table output +output$ERTPlot.Aggr_Dim.ERTTable <- DT::renderDataTable({ + input$ERTPlot.Aggr_Dim.Refresh + req(length(DATA_RAW()) > 0) + + withProgress({ + ert_multi_dim() + }, + message = "Creating table") +}, editable = FALSE, rownames = TRUE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + + +### plot output +output$ERTPlot.Aggr_Dim.Plot <- renderPlotly( + render_ERTPlot_aggr_plot_dim() +) + +### Download table +output$ERTPlot.Aggr_Dim.DownloadTable <- downloadHandler( + filename = function() { + eval(ERT_multi_dim_name) + }, + content = function(file) { + df <- ert_multi_dim() + save_table(df, file) + } +) + +### Download plot +output$ERTPlot.Aggr_Dim.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_AGGR_DIM) + }, + content = function(file) { + save_plotly(render_ERTPlot_aggr_plot_dim(), file) + }, + contentType = paste0('image/', input$ERTPlot.Aggr_Dim.Format) +) diff --git a/inst/shiny-server/server/ERT_aggr_plot.R b/inst/shiny-server/server/ERT_aggr_plot.R index 9d8a59b8..33185043 100644 --- a/inst/shiny-server/server/ERT_aggr_plot.R +++ b/inst/shiny-server/server/ERT_aggr_plot.R @@ -1,134 +1,134 @@ -### Generate ERT table -ERTPlot.Aggr.ERTs_obj <- reactive({ - input$ERTPlot.Aggr.Refresh - dsList <- isolate(ERTPlot.Aggr.data()) - if (is.null(dsList)) return(NULL) - aggr_on <- 'funcId' - targets <- isolate(ERTPlot.Aggr.Targets_obj) - dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) - dt -}) - -### Plot the data -render_ERTPlot_aggr_plot <- reactive({ - withProgress({ - y_attr <- if (input$ERTPlot.Aggr.Ranking) 'rank' else 'value' - y_title <- if (input$ERTPlot.Aggr.Ranking) 'Rank' else 'ERT' - reverse_scale <- input$ERTPlot.Aggr.Mode == 'radar' - dt <- ERTPlot.Aggr.ERTs_obj() - if (is.null(dt)) - return(NULL) - dt <- dt[funcId %in% isolate(input$ERTPlot.Aggr.Funcs), ] - plot_general_data(dt, type = input$ERTPlot.Aggr.Mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = input$ERTPlot.Aggr.Logy, scale.reverse = reverse_scale, - inf.action = 'jitter') - }, - message = "Creating plot") -}) - -### Gather relevant datasetlist -ERTPlot.Aggr.data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr.Algs)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM == input$Overall.Dim) - - if (length(unique(get_funcId(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple functions for the selected dimension.") - return(NULL) - } - - if (length(unique(get_id(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple IDs for the selected dimension.") - return(NULL) - } - data -} - -### format table for display -ert_multi_function <- function() { - dt <- ERTPlot.Aggr.ERTs_obj() - dt <- dcast(dt, funcId~ID, value.var = 'value') - format(dt, digits = 4) -} - -### Table with default targets -default_targets_table <- reactive({ - data <- ERTPlot.Aggr.data() - if (is.null(data)) return(NULL) - targets <- get_target_dt(data) - targets <- targets[, c('funcId','target')] - # %>% melt(id.vars = 'funcId') %>% dcast(variable ~ funcId) - # targets <- targets[, 'variable' = NULL] - # rownames(targets) <- 'funcId' - # targets -}) - -### Target table object -ERTPlot.Aggr.Targets_obj <- NULL - -### Target table proxy -proxy_ERTPlot.Aggr.Targets <- dataTableProxy('ERTPlot.Aggr.Targets') - -### Target table print -output$ERTPlot.Aggr.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - ERTPlot.Aggr.Targets_obj <<- default_targets_table() - ERTPlot.Aggr.Targets_obj -}, editable = TRUE, rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -### Target table edit -observeEvent(input$ERTPlot.Aggr.Targets_cell_edit, { - info <- input$ERTPlot.Aggr.Targets_cell_edit - i <- info$row - req(i > 0) - j <- info$col + 1 - v <- info$value - ERTPlot.Aggr.Targets_obj[i, j] <<- - DT::coerceValue(v, ERTPlot.Aggr.Targets_obj[['target']][[i]]) - replaceData(proxy, ERTPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) -}) - -### Table output -output$ERTPlot.Aggr.ERTTable <- DT::renderDataTable({ - input$ERTPlot.Aggr.Refresh - req(length(DATA_RAW()) > 0) - - withProgress({ - dt <- ert_multi_function() - }, - message = "Creating table") - dt -}, editable = FALSE, rownames = TRUE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - -### plot output -output$ERTPlot.Aggr.Plot <- renderPlotly( - render_ERTPlot_aggr_plot() -) - -### Download table -output$ERTPlot.Aggr.DownloadTable <- downloadHandler( - filename = function() { - eval(ERT_multi_func_name) - }, - content = function(file) { - df <- ert_multi_function() - save_table(df, file) - } -) - -### Download plot -output$ERTPlot.Aggr.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_AGGR) - }, - content = function(file) { - save_plotly(render_ERTPlot_aggr_plot(), file) - }, - contentType = paste0('image/', input$ERTPlot.Aggr.Format) -) +### Generate ERT table +ERTPlot.Aggr.ERTs_obj <- reactive({ + input$ERTPlot.Aggr.Refresh + dsList <- isolate(ERTPlot.Aggr.data()) + if (is.null(dsList)) return(NULL) + aggr_on <- 'funcId' + targets <- isolate(ERTPlot.Aggr.Targets_obj) + dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) + dt +}) + +### Plot the data +render_ERTPlot_aggr_plot <- reactive({ + withProgress({ + y_attr <- if (input$ERTPlot.Aggr.Ranking) 'rank' else 'value' + y_title <- if (input$ERTPlot.Aggr.Ranking) 'Rank' else 'ERT' + reverse_scale <- input$ERTPlot.Aggr.Mode == 'radar' + dt <- ERTPlot.Aggr.ERTs_obj() + if (is.null(dt)) + return(NULL) + dt <- dt[funcId %in% isolate(input$ERTPlot.Aggr.Funcs), ] + plot_general_data(dt, type = input$ERTPlot.Aggr.Mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = input$ERTPlot.Aggr.Logy, scale.reverse = reverse_scale, + inf.action = 'jitter') + }, + message = "Creating plot") +}) + +### Gather relevant datasetlist +ERTPlot.Aggr.data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr.Algs)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM == input$Overall.Dim) + + if (length(unique(get_funcId(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple functions for the selected dimension.") + return(NULL) + } + + if (length(unique(get_id(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected dimension.") + return(NULL) + } + data +} + +### format table for display +ert_multi_function <- function() { + dt <- ERTPlot.Aggr.ERTs_obj() + dt <- dcast(dt, funcId~ID, value.var = 'value') + format(dt, digits = 4) +} + +### Table with default targets +default_targets_table <- reactive({ + data <- ERTPlot.Aggr.data() + if (is.null(data)) return(NULL) + targets <- get_target_dt(data) + targets <- targets[, c('funcId','target')] + # %>% melt(id.vars = 'funcId') %>% dcast(variable ~ funcId) + # targets <- targets[, 'variable' = NULL] + # rownames(targets) <- 'funcId' + # targets +}) + +### Target table object +ERTPlot.Aggr.Targets_obj <- NULL + +### Target table proxy +proxy_ERTPlot.Aggr.Targets <- dataTableProxy('ERTPlot.Aggr.Targets') + +### Target table print +output$ERTPlot.Aggr.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + ERTPlot.Aggr.Targets_obj <<- default_targets_table() + ERTPlot.Aggr.Targets_obj +}, editable = TRUE, rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +### Target table edit +observeEvent(input$ERTPlot.Aggr.Targets_cell_edit, { + info <- input$ERTPlot.Aggr.Targets_cell_edit + i <- info$row + req(i > 0) + j <- info$col + 1 + v <- info$value + ERTPlot.Aggr.Targets_obj[i, j] <<- + DT::coerceValue(v, ERTPlot.Aggr.Targets_obj[['target']][[i]]) + replaceData(proxy, ERTPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) +}) + +### Table output +output$ERTPlot.Aggr.ERTTable <- DT::renderDataTable({ + input$ERTPlot.Aggr.Refresh + req(length(DATA_RAW()) > 0) + + withProgress({ + dt <- ert_multi_function() + }, + message = "Creating table") + dt +}, editable = FALSE, rownames = TRUE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + +### plot output +output$ERTPlot.Aggr.Plot <- renderPlotly( + render_ERTPlot_aggr_plot() +) + +### Download table +output$ERTPlot.Aggr.DownloadTable <- downloadHandler( + filename = function() { + eval(ERT_multi_func_name) + }, + content = function(file) { + df <- ert_multi_function() + save_table(df, file) + } +) + +### Download plot +output$ERTPlot.Aggr.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_AGGR) + }, + content = function(file) { + save_plotly(render_ERTPlot_aggr_plot(), file) + }, + contentType = paste0('image/', input$ERTPlot.Aggr.Format) +) diff --git a/inst/shiny-server/server/ERT_summary.R b/inst/shiny-server/server/ERT_summary.R index a662e7e2..3f825687 100644 --- a/inst/shiny-server/server/ERT_summary.R +++ b/inst/shiny-server/server/ERT_summary.R @@ -1,136 +1,136 @@ -# Data summary for Fixed-Target Runtime (ERT) -------------- -runtime_summary_condensed <- reactive({ - data <- DATA() - req(length(data) > 0) - # fall <- get_funvals(data) - data <- subset(data, ID %in% input$RTSummary.Overview.ID) - df <- get_FV_overview(data) - df$budget %<>% as.numeric - df$runs %<>% as.integer - df$funcId %<>% as.integer - df$DIM %<>% as.integer - df$succ %<>% as.integer - df$"worst recorded" <- format_FV(df$"worst recorded") - df$"worst reached" <- format_FV(df$"worst reached") - df$"mean reached" <- format_FV(df$"mean reached") - df$"median reached" <- format_FV(df$"median reached") - df$"best reached" <- format_FV(df$"best reached") - df -}) - -output$table_RT_overview <- DT::renderDataTable({ - req(input$RTSummary.Overview.ID) - runtime_summary_condensed() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RTSummary.Overview.Download <- downloadHandler( - filename = function() { - eval(RT_overview_name) - }, - content = function(file) { - df <- runtime_summary_condensed() - df <- df[input[["table_RT_overview_rows_all"]]] - save_table(df, file) - } -) - -# Data summary for Fixed-Target Runtime (ERT) -------------- -runtime_summary <- reactive({ - req(input$RTSummary.Statistics.Min, - input$RTSummary.Statistics.Max, - input$RTSummary.Statistics.Step) - - fstart <- format_FV(input$RTSummary.Statistics.Min) - fstop <- format_FV(input$RTSummary.Statistics.Max) - fstep <- format_FV(input$RTSummary.Statistics.Step) - data <- DATA() - data <- subset(data, ID %in% input$RTSummary.Statistics.ID) - if (!input$RTSummary.Statistics.Single) { - req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, fstep) - req(fseq) - } else { - fseq <- fstart - } - - df <- get_RT_summary(data, fseq) - df <- df[, c('DIM', 'funcId') := NULL] - df$target <- format_FV(df$target) %>% as.numeric - - # format the integers - probs <- getOption("IOHanalyzer.quantiles") - for (p in paste0(probs * 100, '%')) { - df[[p]] %<>% as.integer - } - - df$mean <- round(df$mean, digits = 2) - df$sd <- round(df$sd, digits = 2) - df$ERT <- round(df$ERT, digits = 2) - df$ps <- round(df$ps, digits = 2) - - df$target <- format_FV(df$target) - df -}) - -output$table_RT_summary <- DT::renderDataTable({ - runtime_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) - -output$RTSummary.Statistics.Download <- downloadHandler( - filename = function() { - eval(RT_csv_name) - }, - content = function(file) { - df <- runtime_summary() - df <- df[input[["table_RT_summary_rows_all"]]] - save_table(df, file) - } -) - -get_RT <- reactive({ - req(input$RTSummary.Sample.Min, - input$RTSummary.Sample.Max, - input$RTSummary.Sample.Step) - - fstart <- format_FV(input$RTSummary.Sample.Min) - fstop <- format_FV(input$RTSummary.Sample.Max) - fstep <- format_FV(input$RTSummary.Sample.Step) - data <- DATA() - data <- subset(data, ID %in% input$RTSummary.Sample.ID) - if (!input$RTSummary.Sample.Single) { - req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, fstep) - req(fseq) - } - else{ - fseq <- fstart - } - - df <- get_RT_sample(data, ftarget = fseq, - output = input$RTSummary.Sample.DownloadFormat) - df$target <- format_FV(df$target) - df[is.na(df)] <- 'NA' - df -}) - -output$RTSummary.Sample.Download <- downloadHandler( - filename = function() { - eval(RTSample_csv_name) - }, - content = function(file) { - df <- get_RT() - df <- df[input[["table_RT_sample_rows_all"]]] - save_table(df, file) - } -) - -output$table_RT_sample <- DT::renderDataTable({ - df <- get_RT() - df[is.na(df)] <- 'NA' - df -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) +# Data summary for Fixed-Target Runtime (ERT) -------------- +runtime_summary_condensed <- reactive({ + data <- DATA() + req(length(data) > 0) + # fall <- get_funvals(data) + data <- subset(data, ID %in% input$RTSummary.Overview.ID) + df <- get_FV_overview(data) + df$budget %<>% as.numeric + df$runs %<>% as.integer + df$funcId %<>% as.integer + df$DIM %<>% as.integer + df$succ %<>% as.integer + df$"worst recorded" <- format_FV(df$"worst recorded") + df$"worst reached" <- format_FV(df$"worst reached") + df$"mean reached" <- format_FV(df$"mean reached") + df$"median reached" <- format_FV(df$"median reached") + df$"best reached" <- format_FV(df$"best reached") + df +}) + +output$table_RT_overview <- DT::renderDataTable({ + req(input$RTSummary.Overview.ID) + runtime_summary_condensed() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RTSummary.Overview.Download <- downloadHandler( + filename = function() { + eval(RT_overview_name) + }, + content = function(file) { + df <- runtime_summary_condensed() + df <- df[input[["table_RT_overview_rows_all"]]] + save_table(df, file) + } +) + +# Data summary for Fixed-Target Runtime (ERT) -------------- +runtime_summary <- reactive({ + req(input$RTSummary.Statistics.Min, + input$RTSummary.Statistics.Max, + input$RTSummary.Statistics.Step) + + fstart <- format_FV(input$RTSummary.Statistics.Min) + fstop <- format_FV(input$RTSummary.Statistics.Max) + fstep <- format_FV(input$RTSummary.Statistics.Step) + data <- DATA() + data <- subset(data, ID %in% input$RTSummary.Statistics.ID) + if (!input$RTSummary.Statistics.Single) { + req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, fstep) + req(fseq) + } else { + fseq <- fstart + } + + df <- get_RT_summary(data, fseq) + df <- df[, c('DIM', 'funcId') := NULL] + df$target <- format_FV(df$target) %>% as.numeric + + # format the integers + probs <- getOption("IOHanalyzer.quantiles") + for (p in paste0(probs * 100, '%')) { + df[[p]] %<>% as.integer + } + + df$mean <- round(df$mean, digits = 2) + df$sd <- round(df$sd, digits = 2) + df$ERT <- round(df$ERT, digits = 2) + df$ps <- round(df$ps, digits = 2) + + df$target <- format_FV(df$target) + df +}) + +output$table_RT_summary <- DT::renderDataTable({ + runtime_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) + +output$RTSummary.Statistics.Download <- downloadHandler( + filename = function() { + eval(RT_csv_name) + }, + content = function(file) { + df <- runtime_summary() + df <- df[input[["table_RT_summary_rows_all"]]] + save_table(df, file) + } +) + +get_RT <- reactive({ + req(input$RTSummary.Sample.Min, + input$RTSummary.Sample.Max, + input$RTSummary.Sample.Step) + + fstart <- format_FV(input$RTSummary.Sample.Min) + fstop <- format_FV(input$RTSummary.Sample.Max) + fstep <- format_FV(input$RTSummary.Sample.Step) + data <- DATA() + data <- subset(data, ID %in% input$RTSummary.Sample.ID) + if (!input$RTSummary.Sample.Single) { + req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, fstep) + req(fseq) + } + else{ + fseq <- fstart + } + + df <- get_RT_sample(data, ftarget = fseq, + output = input$RTSummary.Sample.DownloadFormat) + df$target <- format_FV(df$target) + df[is.na(df)] <- 'NA' + df +}) + +output$RTSummary.Sample.Download <- downloadHandler( + filename = function() { + eval(RTSample_csv_name) + }, + content = function(file) { + df <- get_RT() + df <- df[input[["table_RT_sample_rows_all"]]] + save_table(df, file) + } +) + +output$table_RT_sample <- DT::renderDataTable({ + df <- get_RT() + df[is.na(df)] <- 'NA' + df +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) diff --git a/inst/shiny-server/server/FCEPDF.R b/inst/shiny-server/server/FCEPDF.R index a99295b4..57469cc3 100644 --- a/inst/shiny-server/server/FCEPDF.R +++ b/inst/shiny-server/server/FCEPDF.R @@ -1,70 +1,70 @@ -# empirical p.d.f. of the target value -get_data_FV_PMF <- reactive({ - ftarget <- input$FCEPDF.Bar.Runtime %>% as.numeric - data <- subset(DATA(), ID %in% input$FCEPDF.Bar.Algs) - generate_data.PMF(data, ftarget, 'by_FV') -}) - -render_FV_PDF <- reactive({ - withProgress({ - plot_general_data(get_data_FV_PMF(), 'ID', 'f(x)', scale.ylog = input$FCEPDF.Bar.Logy, - x_title = "Algorithm", y_title = "Target Value") - # ftarget <- input$RTPMF.Bar.Target %>% as.numeric - # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) - # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, - # scale.ylog = input$RTPMF.Bar.Logy) - }, - message = "Creating plot") -}) - -output$FCEPDF.Bar.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PDF) - }, - content = function(file) { - save_plotly(render_FV_PDF(), file) - }, - contentType = paste0('image/', input$FCEPDF.Bar.Format) -) - -output$FCE_PDF <- renderPlotly({ - render_FV_PDF() -}) - -# historgram of the target values ----------- - - -get_data_FV_HIST <- reactive({ - ftarget <- input$FCEPDF.Hist.Runtime %>% as.numeric - data <- subset(DATA(), ID %in% input$FCEPDF.Hist.Algs) - generate_data.hist(data, ftarget, input$FCEPDF.Hist.Equal, 'by_FV') -}) - - -render_FV_HIST <- reactive({ - req(input$FCEPDF.Hist.Runtime != "", length(DATA()) > 0) # require non-empty input - withProgress({ - subplot_attr <- if (input$FCEPDF.Hist.Mode == 'subplot') 'ID' else NULL - plot_general_data(get_data_FV_HIST(), 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Target Values", - y_title = "Runs") - # runtime <- input$FCEPDF.Hist.Runtime %>% as.numeric - # data <- subset(DATA(), algId %in% input$FCEPDF.Hist.Algs) - # Plot.FV.Histogram(data, runtime, plot_mode = input$FCEPDF.Hist.Mode, use.equal.bins = input$FCEPDF.Hist.Equal) - }, - message = "Creating plot") -}) - -output$FCEPDF.Hist.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_HIST) - }, - content = function(file) { - save_plotly(render_FV_HIST(), file) - }, - contentType = paste0('image/', input$FCEPDF.Hist.Format) -) - -output$FCE_HIST <- renderPlotly({ - render_FV_HIST() -}) +# empirical p.d.f. of the target value +get_data_FV_PMF <- reactive({ + ftarget <- input$FCEPDF.Bar.Runtime %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEPDF.Bar.Algs) + generate_data.PMF(data, ftarget, 'by_FV') +}) + +render_FV_PDF <- reactive({ + withProgress({ + plot_general_data(get_data_FV_PMF(), 'ID', 'f(x)', scale.ylog = input$FCEPDF.Bar.Logy, + x_title = "Algorithm", y_title = "Target Value") + # ftarget <- input$RTPMF.Bar.Target %>% as.numeric + # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) + # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, + # scale.ylog = input$RTPMF.Bar.Logy) + }, + message = "Creating plot") +}) + +output$FCEPDF.Bar.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PDF) + }, + content = function(file) { + save_plotly(render_FV_PDF(), file) + }, + contentType = paste0('image/', input$FCEPDF.Bar.Format) +) + +output$FCE_PDF <- renderPlotly({ + render_FV_PDF() +}) + +# historgram of the target values ----------- + + +get_data_FV_HIST <- reactive({ + ftarget <- input$FCEPDF.Hist.Runtime %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEPDF.Hist.Algs) + generate_data.hist(data, ftarget, input$FCEPDF.Hist.Equal, 'by_FV') +}) + + +render_FV_HIST <- reactive({ + req(input$FCEPDF.Hist.Runtime != "", length(DATA()) > 0) # require non-empty input + withProgress({ + subplot_attr <- if (input$FCEPDF.Hist.Mode == 'subplot') 'ID' else NULL + plot_general_data(get_data_FV_HIST(), 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Target Values", + y_title = "Runs") + # runtime <- input$FCEPDF.Hist.Runtime %>% as.numeric + # data <- subset(DATA(), algId %in% input$FCEPDF.Hist.Algs) + # Plot.FV.Histogram(data, runtime, plot_mode = input$FCEPDF.Hist.Mode, use.equal.bins = input$FCEPDF.Hist.Equal) + }, + message = "Creating plot") +}) + +output$FCEPDF.Hist.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_HIST) + }, + content = function(file) { + save_plotly(render_FV_HIST(), file) + }, + contentType = paste0('image/', input$FCEPDF.Hist.Format) +) + +output$FCE_HIST <- renderPlotly({ + render_FV_HIST() +}) diff --git a/inst/shiny-server/server/FCEPlot.R b/inst/shiny-server/server/FCEPlot.R index f564d17a..b28ef15e 100644 --- a/inst/shiny-server/server/FCEPlot.R +++ b/inst/shiny-server/server/FCEPlot.R @@ -1,140 +1,140 @@ -# Expected Target Value Convergence -output$FCE_PER_FUN <- renderPlotly({ - req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) - render_FV_PER_FUN() -}) - -get_data_FCE_PER_FUN <- reactive({ - req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) - data <- subset(DATA(), ID %in% input$FCEPlot.Algs) - fstart <- input$FCEPlot.Min %>% as.numeric - fstop <- input$FCEPlot.Max %>% as.numeric - generate_data.Single_Function(data, fstart, fstop, input$FCEPlot.semilogx, - 'by_FV') -}) - -render_FV_PER_FUN <- reactive({ - withProgress({ - y_attrs <- c() - if (input$FCEPlot.show.mean) y_attrs <- c(y_attrs, 'mean') - if (input$FCEPlot.show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'ID', show.legend = show_legend, - scale.ylog = isolate(input$FCEPlot.semilogy), - scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", - y_title = "Best-so-far f(x)-value") - show_legend <- F - } - else - p <- NULL - if (input$FCEPlot.show.CI) { - p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend) - show_legend <- F - } - - else if (input$FCEPlot.show.IQR) { - IOHanalyzer.quantiles.bk <- getOption("IOHanalyzer.quantiles") - options(IOHanalyzer.quantiles = c(0.25, 0.75)) - p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'ID', lower_attr = '25%', - upper_attr = '75%', p = p, show.legend = show_legend) - show_legend <- F - options(IOHanalyzer.quantiles = IOHanalyzer.quantiles.bk) - } - if (input$FCEPlot.show.runs) { - fstart <- isolate(input$FCEPlot.Min %>% as.numeric) - fstop <- isolate(input$FCEPlot.Max %>% as.numeric) - data <- isolate(subset(DATA(), ID %in% input$FCEPlot.Algs)) - dt <- get_FV_sample(data, seq_RT(c(fstart, fstop), from = fstart, to = fstop, length.out = 50, - scale = ifelse(isolate(input$FCEPlot.semilogx), 'log', 'linear'))) - nr_runs <- ncol(dt) - 4 - for (i in seq_len(nr_runs)) { - p <- plot_general_data(dt, x_attr = 'runtime', y_attr = paste0('run.', i), type = 'line', - legend_attr = 'ID', p = p, show.legend = show_legend, - scale.ylog = input$FCEPlot.semilogy, - scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", - y_title = "Best-so-far f(x)-value") - show_legend <- F - } - } - p - }, - message = "Creating plot" - ) -}) - -output$FCEPlot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PER_FUN) - }, - content = function(file) { - save_plotly(render_FV_PER_FUN(), file) - }, - contentType = paste0('image/', input$FCEPlot.Format) -) - - -update_fv_per_fct_axis <- observe({ - plotlyProxy("FCE_PER_FUN", session) %>% - plotlyProxyInvoke("relayout", list(yaxis = list(title = 'best-so-far-f(x)-value', type = ifelse(input$FCEPlot.semilogy, 'log', 'linear')))) -}) - - -output$FCEPlot.Multi.Plot <- renderPlotly( - render_FCEPlot_multi_plot() -) - -get_data_FCE_multi_func_bulk <- reactive({ - data <- subset(DATA_RAW(), - DIM == input$Overall.Dim) - if (length(get_id(data)) < 20) { #Arbitrary limit for the time being - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, - which = 'by_FV') - })) - } - else - NULL -}) - -get_data_FCEPlot_multi <- reactive({ - req(isolate(input$FCEPlot.Multi.Algs)) - input$FCEPlot.Multi.PlotButton - if (length(get_id(data)) < 20) { - get_data_FCE_multi_func_bulk()[ID %in% isolate(input$FCEPlot.Multi.Algs), ] - } - else { - data <- subset(DATA_RAW(), - ID %in% isolate(input$FCEPlot.Multi.Algs), - DIM == input$Overall.Dim) - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, - which = 'by_FV') - })) - } -}) - -render_FCEPlot_multi_plot <- reactive({ - withProgress({ - plot_general_data(get_data_FCEPlot_multi(), x_attr = 'runtime', y_attr = 'mean', - subplot_attr = 'funcId', type = 'line', scale.xlog = input$FCEPlot.Multi.Logx, - scale.ylog = input$FCEPlot.Multi.Logy, x_title = 'Runtime', - y_title = 'Best-so-far f(x)', show.legend = T) - }, - message = "Creating plot") -}) - -output$FCEPlot.Multi.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PER_FUN_MULTI) - }, - content = function(file) { - save_plotly(render_FCEPlot_multi_plot(), file) - }, - contentType = paste0('image/', input$FCEPlot.Multi.Format) -) - +# Expected Target Value Convergence +output$FCE_PER_FUN <- renderPlotly({ + req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) + render_FV_PER_FUN() +}) + +get_data_FCE_PER_FUN <- reactive({ + req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) + data <- subset(DATA(), ID %in% input$FCEPlot.Algs) + fstart <- input$FCEPlot.Min %>% as.numeric + fstop <- input$FCEPlot.Max %>% as.numeric + generate_data.Single_Function(data, fstart, fstop, input$FCEPlot.semilogx, + 'by_FV') +}) + +render_FV_PER_FUN <- reactive({ + withProgress({ + y_attrs <- c() + if (input$FCEPlot.show.mean) y_attrs <- c(y_attrs, 'mean') + if (input$FCEPlot.show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = isolate(input$FCEPlot.semilogy), + scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", + y_title = "Best-so-far f(x)-value") + show_legend <- F + } + else + p <- NULL + if (input$FCEPlot.show.CI) { + p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend) + show_legend <- F + } + + else if (input$FCEPlot.show.IQR) { + IOHanalyzer.quantiles.bk <- getOption("IOHanalyzer.quantiles") + options(IOHanalyzer.quantiles = c(0.25, 0.75)) + p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = '25%', + upper_attr = '75%', p = p, show.legend = show_legend) + show_legend <- F + options(IOHanalyzer.quantiles = IOHanalyzer.quantiles.bk) + } + if (input$FCEPlot.show.runs) { + fstart <- isolate(input$FCEPlot.Min %>% as.numeric) + fstop <- isolate(input$FCEPlot.Max %>% as.numeric) + data <- isolate(subset(DATA(), ID %in% input$FCEPlot.Algs)) + dt <- get_FV_sample(data, seq_RT(c(fstart, fstop), from = fstart, to = fstop, length.out = 50, + scale = ifelse(isolate(input$FCEPlot.semilogx), 'log', 'linear'))) + nr_runs <- ncol(dt) - 4 + for (i in seq_len(nr_runs)) { + p <- plot_general_data(dt, x_attr = 'runtime', y_attr = paste0('run.', i), type = 'line', + legend_attr = 'ID', p = p, show.legend = show_legend, + scale.ylog = input$FCEPlot.semilogy, + scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", + y_title = "Best-so-far f(x)-value") + show_legend <- F + } + } + p + }, + message = "Creating plot" + ) +}) + +output$FCEPlot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PER_FUN) + }, + content = function(file) { + save_plotly(render_FV_PER_FUN(), file) + }, + contentType = paste0('image/', input$FCEPlot.Format) +) + + +update_fv_per_fct_axis <- observe({ + plotlyProxy("FCE_PER_FUN", session) %>% + plotlyProxyInvoke("relayout", list(yaxis = list(title = 'best-so-far-f(x)-value', type = ifelse(input$FCEPlot.semilogy, 'log', 'linear')))) +}) + + +output$FCEPlot.Multi.Plot <- renderPlotly( + render_FCEPlot_multi_plot() +) + +get_data_FCE_multi_func_bulk <- reactive({ + data <- subset(DATA_RAW(), + DIM == input$Overall.Dim) + if (length(get_id(data)) < 20) { #Arbitrary limit for the time being + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, + which = 'by_FV') + })) + } + else + NULL +}) + +get_data_FCEPlot_multi <- reactive({ + req(isolate(input$FCEPlot.Multi.Algs)) + input$FCEPlot.Multi.PlotButton + if (length(get_id(data)) < 20) { + get_data_FCE_multi_func_bulk()[ID %in% isolate(input$FCEPlot.Multi.Algs), ] + } + else { + data <- subset(DATA_RAW(), + ID %in% isolate(input$FCEPlot.Multi.Algs), + DIM == input$Overall.Dim) + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, + which = 'by_FV') + })) + } +}) + +render_FCEPlot_multi_plot <- reactive({ + withProgress({ + plot_general_data(get_data_FCEPlot_multi(), x_attr = 'runtime', y_attr = 'mean', + subplot_attr = 'funcId', type = 'line', scale.xlog = input$FCEPlot.Multi.Logx, + scale.ylog = input$FCEPlot.Multi.Logy, x_title = 'Runtime', + y_title = 'Best-so-far f(x)', show.legend = T) + }, + message = "Creating plot") +}) + +output$FCEPlot.Multi.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PER_FUN_MULTI) + }, + content = function(file) { + save_plotly(render_FCEPlot_multi_plot(), file) + }, + contentType = paste0('image/', input$FCEPlot.Multi.Format) +) + diff --git a/inst/shiny-server/server/FCESummary.R b/inst/shiny-server/server/FCESummary.R index 0dc65bd5..8527fe0a 100644 --- a/inst/shiny-server/server/FCESummary.R +++ b/inst/shiny-server/server/FCESummary.R @@ -1,128 +1,128 @@ -# TODO: rename 'FCE'... -# Data summary for Fixed-Budget target (FCE) -------------- -FCE_runtime_summary_condensed <- reactive({ - data <- DATA() - data <- subset(data, ID %in% input$FCESummary.Overview.ID) - # fall <- get_funvals(data) - df <- get_RT_overview(data) - df$"runs" %<>% as.integer - df$"Budget" %<>% as.numeric - df$"miminal runtime" %<>% as.numeric - df$"maximal runtime" %<>% as.numeric - - df -}) - -output$table_FV_overview <- DT::renderDataTable({ - req(input$FCESummary.Overview.ID) - FCE_runtime_summary_condensed() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FCESummary.Overview.Download <- downloadHandler( - filename = function() { - eval(FV_overview_name) - }, - content = function(file) { - df <- FCE_runtime_summary_condensed() - df <- df[input[["FCE_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -get_FCE_summary <- reactive({ - req(input$FCESummary.Statistics.Min, input$FCESummary.Statistics.Max, input$FCESummary.Statistics.Step, length(DATA()) > 0) - - rt_min <- input$FCESummary.Statistics.Min %>% as.numeric - rt_max <- input$FCESummary.Statistics.Max %>% as.numeric - rt_step <- input$FCESummary.Statistics.Step %>% as.numeric - - data <- DATA() - data <- subset(data, ID %in% input$FCESummary.Statistics.ID) - - - if (!input$FCESummary.Statistics.Single) { - req(rt_min <= rt_max, rt_step <= rt_max - rt_min) - rt <- get_runtimes(data) - rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) - req(rt_seq) - } - else{ - rt_seq <- rt_min - } - - df <- get_FV_summary(data, rt_seq)[ - , c('DIM', 'funcId') := NULL - ] - df$runs %<>% as.integer - # df$runs.MaxFunvals %<>% as.integer - df$median %<>% format(format = 'e', digits = 3) - df$mean %<>% format(format = 'e', digits = 3) - df$runtime %<>% as.numeric - - probs <- getOption("IOHanalyzer.quantiles") - - # format the integers - for (p in paste0(probs * 100, '%')) { - df[[p]] %<>% format(format = 'e', digits = 3) - } - df$sd <- round(df$sd, 2) - df -}) - -output$FCE_SUMMARY <- DT::renderDataTable({ - get_FCE_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FCESummary.Statistics.Download <- downloadHandler( - filename = function() { - eval(FV_csv_name) - }, - content = function(file) { - df <- get_FCE_summary() - df <- df[input[["FCE_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -get_FCE <- reactive({ - req(input$FCESummary.Sample.Min, input$FCESummary.Sample.Max, input$FCESummary.Sample.Step, length(DATA()) > 0) - rt_min <- input$FCESummary.Sample.Min %>% as.numeric - rt_max <- input$FCESummary.Sample.Max %>% as.numeric - rt_step <- input$FCESummary.Sample.Step %>% as.numeric - - data <- DATA() - data <- subset(data, ID %in% input$FCESummary.Sample.ID) - - if (!input$FCESummary.Sample.Single) { - req(rt_min <= rt_max, rt_step <= rt_max - rt_min) - rt <- get_runtimes(data) - rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) - req(rt_seq) - } - else{ - rt_seq <- rt_min - } - - get_FV_sample(data, rt_seq, - output = input$FCESummary.Sample.Format) -}) - -output$FCESummary.Sample.Download <- downloadHandler( - filename = function() { - eval(FVSample_csv_name) - }, - content = function(file) { - df <- get_FCE() - df <- df[input[["FCE_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -output$FCE_SAMPLE <- DT::renderDataTable({ - df <- get_FCE() - df[is.na(df)] <- 'NA' - df -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) +# TODO: rename 'FCE'... +# Data summary for Fixed-Budget target (FCE) -------------- +FCE_runtime_summary_condensed <- reactive({ + data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Overview.ID) + # fall <- get_funvals(data) + df <- get_RT_overview(data) + df$"runs" %<>% as.integer + df$"Budget" %<>% as.numeric + df$"miminal runtime" %<>% as.numeric + df$"maximal runtime" %<>% as.numeric + + df +}) + +output$table_FV_overview <- DT::renderDataTable({ + req(input$FCESummary.Overview.ID) + FCE_runtime_summary_condensed() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FCESummary.Overview.Download <- downloadHandler( + filename = function() { + eval(FV_overview_name) + }, + content = function(file) { + df <- FCE_runtime_summary_condensed() + df <- df[input[["FCE_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +get_FCE_summary <- reactive({ + req(input$FCESummary.Statistics.Min, input$FCESummary.Statistics.Max, input$FCESummary.Statistics.Step, length(DATA()) > 0) + + rt_min <- input$FCESummary.Statistics.Min %>% as.numeric + rt_max <- input$FCESummary.Statistics.Max %>% as.numeric + rt_step <- input$FCESummary.Statistics.Step %>% as.numeric + + data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Statistics.ID) + + + if (!input$FCESummary.Statistics.Single) { + req(rt_min <= rt_max, rt_step <= rt_max - rt_min) + rt <- get_runtimes(data) + rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) + req(rt_seq) + } + else{ + rt_seq <- rt_min + } + + df <- get_FV_summary(data, rt_seq)[ + , c('DIM', 'funcId') := NULL + ] + df$runs %<>% as.integer + # df$runs.MaxFunvals %<>% as.integer + df$median %<>% format(format = 'e', digits = 3) + df$mean %<>% format(format = 'e', digits = 3) + df$runtime %<>% as.numeric + + probs <- getOption("IOHanalyzer.quantiles") + + # format the integers + for (p in paste0(probs * 100, '%')) { + df[[p]] %<>% format(format = 'e', digits = 3) + } + df$sd <- round(df$sd, 2) + df +}) + +output$FCE_SUMMARY <- DT::renderDataTable({ + get_FCE_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FCESummary.Statistics.Download <- downloadHandler( + filename = function() { + eval(FV_csv_name) + }, + content = function(file) { + df <- get_FCE_summary() + df <- df[input[["FCE_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +get_FCE <- reactive({ + req(input$FCESummary.Sample.Min, input$FCESummary.Sample.Max, input$FCESummary.Sample.Step, length(DATA()) > 0) + rt_min <- input$FCESummary.Sample.Min %>% as.numeric + rt_max <- input$FCESummary.Sample.Max %>% as.numeric + rt_step <- input$FCESummary.Sample.Step %>% as.numeric + + data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Sample.ID) + + if (!input$FCESummary.Sample.Single) { + req(rt_min <= rt_max, rt_step <= rt_max - rt_min) + rt <- get_runtimes(data) + rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) + req(rt_seq) + } + else{ + rt_seq <- rt_min + } + + get_FV_sample(data, rt_seq, + output = input$FCESummary.Sample.Format) +}) + +output$FCESummary.Sample.Download <- downloadHandler( + filename = function() { + eval(FVSample_csv_name) + }, + content = function(file) { + df <- get_FCE() + df <- df[input[["FCE_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +output$FCE_SAMPLE <- DT::renderDataTable({ + df <- get_FCE() + df[is.na(df)] <- 'NA' + df +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) diff --git a/inst/shiny-server/server/FCE_ECDF.R b/inst/shiny-server/server/FCE_ECDF.R index 8dacc421..4f2a2449 100644 --- a/inst/shiny-server/server/FCE_ECDF.R +++ b/inst/shiny-server/server/FCE_ECDF.R @@ -1,121 +1,121 @@ -# The ECDF plots for the target value ---------------- -output$FCE_ECDF_PER_TARGET <- renderPlotly({ - render_ecdf_per_target() -}) - -get_data_FV_ECDF_Single <- reactive({ - req(input$FCEECDF.Single.Target) - ftargets <- as.numeric(format_FV(input$FCEECDF.Single.Target)) - data <- subset(DATA(), ID %in% input$FCEECDF.Single.Algs) - generate_data.ECDF(data, ftargets, input$FCEECDF.Single.Logx, which = 'by_FV') -}) - -render_ecdf_per_target <- reactive({ - withProgress({ - plot_general_data(get_data_FV_ECDF_Single(), 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of runs", scale.xlog = input$FCEECDF.Single.Logx, - show.legend = T, - scale.reverse = !attr(DATA()[[1]], 'maximization')) - }, - message = "Creating plot") -}) - -output$FCE_RT_GRID <- renderPrint({ - req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) - - rt_min <- input$FCEECDF.Mult.Min %>% as.numeric - rt_max <- input$FCEECDF.Mult.Max %>% as.numeric - rt_step <- input$FCEECDF.Mult.Step %>% as.numeric - - req(rt_min <= rt_max, rt_step <= rt_max - rt_min) - data <- DATA() - rt <- get_runtimes(data) - - seq_RT(rt, from = rt_min, to = rt_max, by = rt_step) %>% cat -}) - -get_data_FV_ECDF_AGGR <- reactive({ - req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) - fstart <- format_FV(input$FCEECDF.Mult.Min) %>% as.numeric - fstop <- format_FV(input$FCEECDF.Mult.Max) %>% as.numeric - fstep <- format_FV(input$FCEECDF.Mult.Step) %>% as.numeric - data <- subset(DATA(), ID %in% input$FCEECDF.Mult.Algs) - targets <- seq_RT(get_funvals(data), fstart, fstop, fstep) - generate_data.ECDF(data, targets, input$FCEECDF.Mult.Logx, which = 'by_FV') -}) - -render_FV_ECDF_AGGR <- reactive({ - withProgress({ - # rt_min <- input$FCEECDF.Mult.Min %>% as.numeric - # rt_max <- input$FCEECDF.Mult.Max %>% as.numeric - # rt_step <- input$FCEECDF.Mult.Step %>% as.numeric - # data <- subset(DATA(), algId %in% input$FCEECDF.Mult.Algs) - # - # Plot.FV.ECDF_Single_Func(data,rt_min = rt_min, - # rt_max = rt_max, rt_step = rt_step, - # scale.xlog = input$FCEECDF.Mult.Logx, - # # show.per_target = input$FCEECDF.Mult.Targets, - # scale.reverse = !attr(DATA()[[1]],'maximization')) - plot_general_data(get_data_FV_ECDF_AGGR(), 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of (run, target) pairs", - scale.xlog = input$FCEECDF.Mult.Logx, - scale.reverse = !attr(DATA()[[1]], 'maximization'), show.legend = T) - }, - message = "Creating plot") -}) - -output$FCEECDF.Mult.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_ECDF_AGGR) - }, - content = function(file) { - save_plotly(render_FV_ECDF_AGGR(), file) - }, - contentType = paste0('image/', input$FCEECDF.Mult.Format) -) - -output$FCE_ECDF_AGGR <- renderPlotly({ - render_FV_ECDF_AGGR() -}) - -get_data_FV_AUC <- reactive({ - req(input$FCEECDF.AUC.Min, input$FCEECDF.AUC.Max, input$FCEECDF.AUC.Step, length(DATA()) > 0) - - rt_min <- input$FCEECDF.AUC.Min %>% as.numeric - rt_max <- input$FCEECDF.AUC.Max %>% as.numeric - rt_step <- input$FCEECDF.AUC.Step %>% as.numeric - data <- subset(DATA(), ID %in% input$FCEECDF.AUC.Algs) - targets <- seq_RT(get_runtimes(data), rt_min, rt_max, rt_step, length.out = 10) - generate_data.AUC(data, targets, which = 'by_FV') -}) - - -render_FV_AUC <- reactive({ - withProgress({ - # rt_min <- input$FCEECDF.AUC.Min %>% as.numeric - # rt_max <- input$FCEECDF.AUC.Max %>% as.numeric - # rt_step <- input$FCEECDF.AUC.Step %>% as.numeric - # data <- subset(DATA(), algId %in% input$FCEECDF.AUC.Algs) - # - # Plot.FV.ECDF_AUC(data, rt_min = rt_min, - # rt_max = rt_max, rt_step = rt_step) - plot_general_data(get_data_FV_AUC(), 'x', 'AUC', 'radar') - }, - message = "Creating plot") -}) - -output$FCEECDF.AUC.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_AUC) - }, - content = function(file) { - save_plotly(render_FV_AUC(), file) - }, - contentType = paste0('image/', input$FCEECDF.AUC.Format) -) - -output$FCE_AUC <- renderPlotly({ - render_FV_AUC() -}) +# The ECDF plots for the target value ---------------- +output$FCE_ECDF_PER_TARGET <- renderPlotly({ + render_ecdf_per_target() +}) + +get_data_FV_ECDF_Single <- reactive({ + req(input$FCEECDF.Single.Target) + ftargets <- as.numeric(format_FV(input$FCEECDF.Single.Target)) + data <- subset(DATA(), ID %in% input$FCEECDF.Single.Algs) + generate_data.ECDF(data, ftargets, input$FCEECDF.Single.Logx, which = 'by_FV') +}) + +render_ecdf_per_target <- reactive({ + withProgress({ + plot_general_data(get_data_FV_ECDF_Single(), 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of runs", scale.xlog = input$FCEECDF.Single.Logx, + show.legend = T, + scale.reverse = !attr(DATA()[[1]], 'maximization')) + }, + message = "Creating plot") +}) + +output$FCE_RT_GRID <- renderPrint({ + req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) + + rt_min <- input$FCEECDF.Mult.Min %>% as.numeric + rt_max <- input$FCEECDF.Mult.Max %>% as.numeric + rt_step <- input$FCEECDF.Mult.Step %>% as.numeric + + req(rt_min <= rt_max, rt_step <= rt_max - rt_min) + data <- DATA() + rt <- get_runtimes(data) + + seq_RT(rt, from = rt_min, to = rt_max, by = rt_step) %>% cat +}) + +get_data_FV_ECDF_AGGR <- reactive({ + req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) + fstart <- format_FV(input$FCEECDF.Mult.Min) %>% as.numeric + fstop <- format_FV(input$FCEECDF.Mult.Max) %>% as.numeric + fstep <- format_FV(input$FCEECDF.Mult.Step) %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEECDF.Mult.Algs) + targets <- seq_RT(get_funvals(data), fstart, fstop, fstep) + generate_data.ECDF(data, targets, input$FCEECDF.Mult.Logx, which = 'by_FV') +}) + +render_FV_ECDF_AGGR <- reactive({ + withProgress({ + # rt_min <- input$FCEECDF.Mult.Min %>% as.numeric + # rt_max <- input$FCEECDF.Mult.Max %>% as.numeric + # rt_step <- input$FCEECDF.Mult.Step %>% as.numeric + # data <- subset(DATA(), algId %in% input$FCEECDF.Mult.Algs) + # + # Plot.FV.ECDF_Single_Func(data,rt_min = rt_min, + # rt_max = rt_max, rt_step = rt_step, + # scale.xlog = input$FCEECDF.Mult.Logx, + # # show.per_target = input$FCEECDF.Mult.Targets, + # scale.reverse = !attr(DATA()[[1]],'maximization')) + plot_general_data(get_data_FV_ECDF_AGGR(), 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of (run, target) pairs", + scale.xlog = input$FCEECDF.Mult.Logx, + scale.reverse = !attr(DATA()[[1]], 'maximization'), show.legend = T) + }, + message = "Creating plot") +}) + +output$FCEECDF.Mult.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_ECDF_AGGR) + }, + content = function(file) { + save_plotly(render_FV_ECDF_AGGR(), file) + }, + contentType = paste0('image/', input$FCEECDF.Mult.Format) +) + +output$FCE_ECDF_AGGR <- renderPlotly({ + render_FV_ECDF_AGGR() +}) + +get_data_FV_AUC <- reactive({ + req(input$FCEECDF.AUC.Min, input$FCEECDF.AUC.Max, input$FCEECDF.AUC.Step, length(DATA()) > 0) + + rt_min <- input$FCEECDF.AUC.Min %>% as.numeric + rt_max <- input$FCEECDF.AUC.Max %>% as.numeric + rt_step <- input$FCEECDF.AUC.Step %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEECDF.AUC.Algs) + targets <- seq_RT(get_runtimes(data), rt_min, rt_max, rt_step, length.out = 10) + generate_data.AUC(data, targets, which = 'by_FV') +}) + + +render_FV_AUC <- reactive({ + withProgress({ + # rt_min <- input$FCEECDF.AUC.Min %>% as.numeric + # rt_max <- input$FCEECDF.AUC.Max %>% as.numeric + # rt_step <- input$FCEECDF.AUC.Step %>% as.numeric + # data <- subset(DATA(), algId %in% input$FCEECDF.AUC.Algs) + # + # Plot.FV.ECDF_AUC(data, rt_min = rt_min, + # rt_max = rt_max, rt_step = rt_step) + plot_general_data(get_data_FV_AUC(), 'x', 'AUC', 'radar') + }, + message = "Creating plot") +}) + +output$FCEECDF.AUC.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_AUC) + }, + content = function(file) { + save_plotly(render_FV_AUC(), file) + }, + contentType = paste0('image/', input$FCEECDF.AUC.Format) +) + +output$FCE_AUC <- renderPlotly({ + render_FV_AUC() +}) diff --git a/inst/shiny-server/server/FCE_aggr_plot.R b/inst/shiny-server/server/FCE_aggr_plot.R index 58e741d7..5e231676 100644 --- a/inst/shiny-server/server/FCE_aggr_plot.R +++ b/inst/shiny-server/server/FCE_aggr_plot.R @@ -1,147 +1,147 @@ -### Generate FCE table -FCEPlot.Aggr.FCEs_obj <- reactive({ - input$FCEPlot.Aggr.Refresh - dsList <- isolate(FCEPlot.Aggr.data()) - if (is.null(dsList)) return(NULL) - aggr_on <- ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM') - targets <- isolate(FCEPlot.Aggr.Targets_obj) - dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, - which = 'by_FV') - dt -}) - -### Plot the data -render_FCEPlot_aggr_plot <- reactive({ - withProgress({ - y_attr <- if (input$FCEPlot.Aggr.Ranking) 'rank' else 'value' - y_title <- if (input$FCEPlot.Aggr.Ranking) 'Rank' else 'Best-so-far f(x)' - reverse_scale <- input$FCEPlot.Aggr.Mode == 'radar' - dt <- FCEPlot.Aggr.FCEs_obj() - plot_general_data(dt, type = input$FCEPlot.Aggr.Mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = input$FCEPlot.Aggr.Logy, scale.reverse = reverse_scale) - }, - message = "Creating plot") -}) - - -### Gather relevant datasetlist -FCEPlot.Aggr.data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$FCEPlot.Aggr.Algs)) - if (length(data) == 0) return(NULL) - if (input$FCEPlot.Aggr.Aggregator == 'Functions') { - data <- subset(data, DIM == input$Overall.Dim) - if (length(unique(get_funcId(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple functions for the selected dimension.") - return(NULL) - } - } - else{ - data <- subset(data, funcId == input$Overall.Funcid) - if (length(unique(get_dim(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple dimensions for the selected function") - return(NULL) - } - } - if (length(unique(get_id(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple IDs for the selected dimension.") - return(NULL) - } - data -} - - -### format table for display -FCE_multi_function <- function() { - dt <- FCEPlot.Aggr.FCEs_obj() - if (input$FCEPlot.Aggr.Aggregator == 'Functions') - dt <- dcast(dt, funcId~ID, value.var = 'value') - else - dt <- dcast(dt, DIM~ID, value.var = 'value') - dt -} - -### Table with default targets -default_runtimes_table <- reactive({ - data <- FCEPlot.Aggr.data() - if (is.null(data)) return(NULL) - targets <- get_target_dt(data, 'by_FV') - if (input$FCEPlot.Aggr.Aggregator == 'Functions') - targets <- targets[, c('funcId', 'target')] - else - targets <- targets[, c('DIM', 'target')] -}) - -### Target table object -FCEPlot.Aggr.Targets_obj <- NULL - -### Target table proxy -proxy_FCEPlot.Aggr.Targets <- dataTableProxy('FCEPlot.Aggr.Targets') - -### Target table print -output$FCEPlot.Aggr.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - FCEPlot.Aggr.Targets_obj <<- default_runtimes_table() - FCEPlot.Aggr.Targets_obj -}, editable = TRUE, rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - - - -### Target table edit -observeEvent(input$FCEPlot.Aggr.Targets_cell_edit, { - info <- input$FCEPlot.Aggr.Targets_cell_edit - i <- info$row - req(i > 0) - j <- info$col + 1 - v <- info$value - FCEPlot.Aggr.Targets_obj[i, j] <<- - DT::coerceValue(v, FCEPlot.Aggr.Targets_obj[['target']][[i]]) - replaceData(proxy, FCEPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) -}) - -### Table output -output$FCEPlot.Aggr.FCETable <- DT::renderDataTable({ - input$FCEPlot.Aggr.Refresh - req(length(DATA_RAW()) > 0) - - withProgress({ - dt <- FCE_multi_function() - }, - message = "Creating table") - dt -}, editable = FALSE, rownames = TRUE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - -### plot output -output$FCEPlot.Aggr.Plot <- renderPlotly( - render_FCEPlot_aggr_plot() -) - - - -### Download table -output$FCEPlot.Aggr.DownloadTable <- downloadHandler( - filename = function() { - eval(FCE_multi_func_name) - }, - content = function(file) { - df <- FCE_multi_function() - save_table(df, file) - } -) - -### Download plot -output$FCEPlot.Aggr.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_AGGR) - }, - content = function(file) { - save_plotly(render_FCEPlot_aggr_plot(), file) - }, - contentType = paste0('image/', input$FCEPlot.Aggr.Format) -) - - - +### Generate FCE table +FCEPlot.Aggr.FCEs_obj <- reactive({ + input$FCEPlot.Aggr.Refresh + dsList <- isolate(FCEPlot.Aggr.data()) + if (is.null(dsList)) return(NULL) + aggr_on <- ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM') + targets <- isolate(FCEPlot.Aggr.Targets_obj) + dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, + which = 'by_FV') + dt +}) + +### Plot the data +render_FCEPlot_aggr_plot <- reactive({ + withProgress({ + y_attr <- if (input$FCEPlot.Aggr.Ranking) 'rank' else 'value' + y_title <- if (input$FCEPlot.Aggr.Ranking) 'Rank' else 'Best-so-far f(x)' + reverse_scale <- input$FCEPlot.Aggr.Mode == 'radar' + dt <- FCEPlot.Aggr.FCEs_obj() + plot_general_data(dt, type = input$FCEPlot.Aggr.Mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = input$FCEPlot.Aggr.Logy, scale.reverse = reverse_scale) + }, + message = "Creating plot") +}) + + +### Gather relevant datasetlist +FCEPlot.Aggr.data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$FCEPlot.Aggr.Algs)) + if (length(data) == 0) return(NULL) + if (input$FCEPlot.Aggr.Aggregator == 'Functions') { + data <- subset(data, DIM == input$Overall.Dim) + if (length(unique(get_funcId(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains multiple functions for the selected dimension.") + return(NULL) + } + } + else{ + data <- subset(data, funcId == input$Overall.Funcid) + if (length(unique(get_dim(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains multiple dimensions for the selected function") + return(NULL) + } + } + if (length(unique(get_id(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains multiple IDs for the selected dimension.") + return(NULL) + } + data +} + + +### format table for display +FCE_multi_function <- function() { + dt <- FCEPlot.Aggr.FCEs_obj() + if (input$FCEPlot.Aggr.Aggregator == 'Functions') + dt <- dcast(dt, funcId~ID, value.var = 'value') + else + dt <- dcast(dt, DIM~ID, value.var = 'value') + dt +} + +### Table with default targets +default_runtimes_table <- reactive({ + data <- FCEPlot.Aggr.data() + if (is.null(data)) return(NULL) + targets <- get_target_dt(data, 'by_FV') + if (input$FCEPlot.Aggr.Aggregator == 'Functions') + targets <- targets[, c('funcId', 'target')] + else + targets <- targets[, c('DIM', 'target')] +}) + +### Target table object +FCEPlot.Aggr.Targets_obj <- NULL + +### Target table proxy +proxy_FCEPlot.Aggr.Targets <- dataTableProxy('FCEPlot.Aggr.Targets') + +### Target table print +output$FCEPlot.Aggr.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + FCEPlot.Aggr.Targets_obj <<- default_runtimes_table() + FCEPlot.Aggr.Targets_obj +}, editable = TRUE, rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + + + +### Target table edit +observeEvent(input$FCEPlot.Aggr.Targets_cell_edit, { + info <- input$FCEPlot.Aggr.Targets_cell_edit + i <- info$row + req(i > 0) + j <- info$col + 1 + v <- info$value + FCEPlot.Aggr.Targets_obj[i, j] <<- + DT::coerceValue(v, FCEPlot.Aggr.Targets_obj[['target']][[i]]) + replaceData(proxy, FCEPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) +}) + +### Table output +output$FCEPlot.Aggr.FCETable <- DT::renderDataTable({ + input$FCEPlot.Aggr.Refresh + req(length(DATA_RAW()) > 0) + + withProgress({ + dt <- FCE_multi_function() + }, + message = "Creating table") + dt +}, editable = FALSE, rownames = TRUE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + +### plot output +output$FCEPlot.Aggr.Plot <- renderPlotly( + render_FCEPlot_aggr_plot() +) + + + +### Download table +output$FCEPlot.Aggr.DownloadTable <- downloadHandler( + filename = function() { + eval(FCE_multi_func_name) + }, + content = function(file) { + df <- FCE_multi_function() + save_table(df, file) + } +) + +### Download plot +output$FCEPlot.Aggr.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_AGGR) + }, + content = function(file) { + save_plotly(render_FCEPlot_aggr_plot(), file) + }, + contentType = paste0('image/', input$FCEPlot.Aggr.Format) +) + + + diff --git a/inst/shiny-server/server/FV_DSCinterface.R b/inst/shiny-server/server/FV_DSCinterface.R index 944dea5a..13aa224b 100644 --- a/inst/shiny-server/server/FV_DSCinterface.R +++ b/inst/shiny-server/server/FV_DSCinterface.R @@ -1,225 +1,225 @@ -FV_DSC_rank_result <- reactive({ - req(input$FV_Stats.DSC.Create_rank) - - isolate({ - withProgress({ - data <- FV_DSC_data() - req(length(data) > 0) - # if (input$FV_Stats.DSC.Value_type != 'Raw') { - # shinyjs::alert("This feature is not yet implemented in the current version of IOHanalyzer.") - # } - test_type <- if (input$FV_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" - rank_res <- get_dsc_rank(data, FV_stats_DSC_targets_obj, which = 'by_FV', - alpha = input$FV_Stats.DSC.Alpha_rank, - test_type = test_type, - monte_carlo_iterations = input$FV_Stats.DSC.MCsamples_rank, - epsilon = input$FV_Stats.DSC.Epsilon_rank) - if (is.null(rank_res)) { - shinyjs::alert("There was an error getting the ranking data from DSCtool. Please - select different settings and try again") - return(NULL) - } - updateSelectInput(session, 'FV_Stats.DSC.Omni_options', choices = rank_res$valid_methods, - selected = rank_res$valid_methods[[1]]) - - - rank_res - }, message = "Getting DSC ranking data") - }) -}) - -FV_render_performviz <- reactive({ - rank_res <- FV_DSC_rank_result() - if (is.null(rank_res)) {return(NULL)} - Plot.Performviz(rank_res) -}) - -output$FV_Stats.DSC.PerformViz <- renderPlot({ - FV_render_performviz() -}) - -output$FV_Stats.DSC.Download_rank_table <- downloadHandler( - filename = function() { - eval(FV_DSC_table_name_rank) - }, - content = function(file) { - mlist <- FV_DSC_rank_result()$ranked_matrix - df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { - df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - function(x) { - list(algorithm = x$algorithm, rank = x$rank) - })) - df_temp[, problem := mlist[[problem_idx]]$problem] - })) - df <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') - save_table(df, file) - } -) - -output$FV_Stats.DSC.Download_rank <- downloadHandler( - filename = function() { - eval(FV_DSC_figure_name_rank) - }, - content = function(file) { - #Temporary settings, will replace plotting from complexheatmap to - #Some other plotly-compatible library later, and use save_plotly - suppressWarnings({ - pdf(file = file, width = 16, height = 9) - RT_render_performviz() - dev.off() - }) - }, - contentType = paste0('image/', input$FV_Stats.DSC.Format_rank) -) - -FV_DSC_omni_result <- reactive({ - input$FV_Stats.DSC.Create_omni - - isolate({ - withProgress({ - rank_res <- FV_DSC_rank_result() - req(rank_res) - omni_res <- get_dsc_omnibus(rank_res, method = input$FV_Stats.DSC.Omni_options, - alpha = input$FV_Stats.DSC.Alpha_omni) - - }, message = "Creating Comparison, this might take a while") - }) -}) - - -output$FV_Stats.DSC.Output_omni <- renderText({ - res_omni <- FV_DSC_omni_result() - req(res_omni) - paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") -}) - - -FV_DSC_posthoc_result <- reactive({ - input$FV_Stats.DSC.Create_posthoc - - isolate({ - withProgress({ - omni_res <- FV_DSC_omni_result() - req(omni_res) - data <- FV_DSC_data() - req(length(data) > 0) - df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ - posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), - nrow(FV_stats_DSC_targets_obj), - alpha = input$FV_Stats.DSC.Alpha_posthoc, - base_algorithm = algname, - method = input$FV_Stats.DSC.Posthoc_test) - if (is.null(posthoc_res)) { return(NULL)} - values <- lapply(seq(4), function(method_idx) { - lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, - function(x) { - x$value - }) - }) - - algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, - function(x) { - x$algorithm - }) - df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], - "Holm" = values[[3]], "Hochberg" = values[[4]]) - - })) - as.data.table(format(df_posthoc, digits = 3)) - }, message = "Creating Comparison, this might take a while") - }) -}) - -output$FV_Stats.DSC.PosthocTable <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - FV_DSC_posthoc_result() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -FV_render_DSC_plot <- reactive({ - dt <- FV_DSC_posthoc_result() - if (is.null(dt) || nrow(dt) < 1) {return(NULL)} - # plot_general_data(dt[method == input$FV_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', - # legend_attr = 'algId') - dt <- dt[,c('baseline', 'compared alg', input$FV_Stats.DSC.Posthoc_method), with = F] - p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$FV_Stats.DSC.Posthoc_method) - storage.mode(p_matrix) <- "numeric" - y <- p_matrix <= input$FV_Stats.DSC.Alpha_posthoc - colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), - col = c('blue', 'blue', 'white', 'white', 'red', 'red') - ) - heatmap <- y - t(y) - - p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', - xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) - p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), - xaxis = list(tickangle = 45)) - p -}) - -output$FV_Stats.DSC.PosthocViz <- renderPlotly({ - FV_render_DSC_plot() -}) - -output$FV_Stats.DSC.DownloadTable <- downloadHandler( - filename = function() { - eval(FV_DSC_table_name) - }, - content = function(file) { - df <- FV_DSC_posthoc_result() - save_table(df, file) - } -) - -output$FV_Stats.DSC.Download <- downloadHandler( - filename = function() { - eval(FV_DSC_figure_name) - }, - content = function(file) { - save_plotly(FV_render_DSC_plot(), file) - }, - contentType = paste0('image/', input$FV_Stats.DSC.Format) -) - - -FV_DSC_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.DSC.ID)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$FV_Stats.DSC.Dim) - data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) - if (length(unique(get_id(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple IDs for the selected functions and dimensions.") - return(NULL) - } - data -} - -FV_stats_DSC_targets <- reactive({ - data <- FV_DSC_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_FV") -}) - -FV_stats_DSC_targets_obj <- NULL - -proxy_FV_Stats.DSC.Targets <- dataTableProxy('FV_Stats.DSC.Targets') - -output$FV_Stats.DSC.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - FV_stats_DSC_targets_obj <<- FV_stats_DSC_targets() - FV_stats_DSC_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$FV_Stats.DSC.Targets_cell_edit, { - info <- input$FV_Stats.DSC.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- FV_DSC_data() - if (is.null(data)) return(NULL) - FV_stats_DSC_targets_obj$target[[i]] <<- v - replaceData(proxy, FV_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) +FV_DSC_rank_result <- reactive({ + req(input$FV_Stats.DSC.Create_rank) + + isolate({ + withProgress({ + data <- FV_DSC_data() + req(length(data) > 0) + # if (input$FV_Stats.DSC.Value_type != 'Raw') { + # shinyjs::alert("This feature is not yet implemented in the current version of IOHanalyzer.") + # } + test_type <- if (input$FV_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" + rank_res <- get_dsc_rank(data, FV_stats_DSC_targets_obj, which = 'by_FV', + alpha = input$FV_Stats.DSC.Alpha_rank, + test_type = test_type, + monte_carlo_iterations = input$FV_Stats.DSC.MCsamples_rank, + epsilon = input$FV_Stats.DSC.Epsilon_rank) + if (is.null(rank_res)) { + shinyjs::alert("There was an error getting the ranking data from DSCtool. Please + select different settings and try again") + return(NULL) + } + updateSelectInput(session, 'FV_Stats.DSC.Omni_options', choices = rank_res$valid_methods, + selected = rank_res$valid_methods[[1]]) + + + rank_res + }, message = "Getting DSC ranking data") + }) +}) + +FV_render_performviz <- reactive({ + rank_res <- FV_DSC_rank_result() + if (is.null(rank_res)) {return(NULL)} + Plot.Performviz(rank_res) +}) + +output$FV_Stats.DSC.PerformViz <- renderPlot({ + FV_render_performviz() +}) + +output$FV_Stats.DSC.Download_rank_table <- downloadHandler( + filename = function() { + eval(FV_DSC_table_name_rank) + }, + content = function(file) { + mlist <- FV_DSC_rank_result()$ranked_matrix + df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { + df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + function(x) { + list(algorithm = x$algorithm, rank = x$rank) + })) + df_temp[, problem := mlist[[problem_idx]]$problem] + })) + df <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') + save_table(df, file) + } +) + +output$FV_Stats.DSC.Download_rank <- downloadHandler( + filename = function() { + eval(FV_DSC_figure_name_rank) + }, + content = function(file) { + #Temporary settings, will replace plotting from complexheatmap to + #Some other plotly-compatible library later, and use save_plotly + suppressWarnings({ + pdf(file = file, width = 16, height = 9) + RT_render_performviz() + dev.off() + }) + }, + contentType = paste0('image/', input$FV_Stats.DSC.Format_rank) +) + +FV_DSC_omni_result <- reactive({ + input$FV_Stats.DSC.Create_omni + + isolate({ + withProgress({ + rank_res <- FV_DSC_rank_result() + req(rank_res) + omni_res <- get_dsc_omnibus(rank_res, method = input$FV_Stats.DSC.Omni_options, + alpha = input$FV_Stats.DSC.Alpha_omni) + + }, message = "Creating Comparison, this might take a while") + }) +}) + + +output$FV_Stats.DSC.Output_omni <- renderText({ + res_omni <- FV_DSC_omni_result() + req(res_omni) + paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") +}) + + +FV_DSC_posthoc_result <- reactive({ + input$FV_Stats.DSC.Create_posthoc + + isolate({ + withProgress({ + omni_res <- FV_DSC_omni_result() + req(omni_res) + data <- FV_DSC_data() + req(length(data) > 0) + df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ + posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), + nrow(FV_stats_DSC_targets_obj), + alpha = input$FV_Stats.DSC.Alpha_posthoc, + base_algorithm = algname, + method = input$FV_Stats.DSC.Posthoc_test) + if (is.null(posthoc_res)) { return(NULL)} + values <- lapply(seq(4), function(method_idx) { + lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, + function(x) { + x$value + }) + }) + + algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, + function(x) { + x$algorithm + }) + df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], + "Holm" = values[[3]], "Hochberg" = values[[4]]) + + })) + as.data.table(format(df_posthoc, digits = 3)) + }, message = "Creating Comparison, this might take a while") + }) +}) + +output$FV_Stats.DSC.PosthocTable <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + FV_DSC_posthoc_result() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +FV_render_DSC_plot <- reactive({ + dt <- FV_DSC_posthoc_result() + if (is.null(dt) || nrow(dt) < 1) {return(NULL)} + # plot_general_data(dt[method == input$FV_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', + # legend_attr = 'algId') + dt <- dt[,c('baseline', 'compared alg', input$FV_Stats.DSC.Posthoc_method), with = F] + p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$FV_Stats.DSC.Posthoc_method) + storage.mode(p_matrix) <- "numeric" + y <- p_matrix <= input$FV_Stats.DSC.Alpha_posthoc + colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), + col = c('blue', 'blue', 'white', 'white', 'red', 'red') + ) + heatmap <- y - t(y) + + p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', + xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) + p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), + xaxis = list(tickangle = 45)) + p +}) + +output$FV_Stats.DSC.PosthocViz <- renderPlotly({ + FV_render_DSC_plot() +}) + +output$FV_Stats.DSC.DownloadTable <- downloadHandler( + filename = function() { + eval(FV_DSC_table_name) + }, + content = function(file) { + df <- FV_DSC_posthoc_result() + save_table(df, file) + } +) + +output$FV_Stats.DSC.Download <- downloadHandler( + filename = function() { + eval(FV_DSC_figure_name) + }, + content = function(file) { + save_plotly(FV_render_DSC_plot(), file) + }, + contentType = paste0('image/', input$FV_Stats.DSC.Format) +) + + +FV_DSC_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.DSC.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$FV_Stats.DSC.Dim) + data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +FV_stats_DSC_targets <- reactive({ + data <- FV_DSC_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_FV") +}) + +FV_stats_DSC_targets_obj <- NULL + +proxy_FV_Stats.DSC.Targets <- dataTableProxy('FV_Stats.DSC.Targets') + +output$FV_Stats.DSC.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + FV_stats_DSC_targets_obj <<- FV_stats_DSC_targets() + FV_stats_DSC_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$FV_Stats.DSC.Targets_cell_edit, { + info <- input$FV_Stats.DSC.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- FV_DSC_data() + if (is.null(data)) return(NULL) + FV_stats_DSC_targets_obj$target[[i]] <<- v + replaceData(proxy, FV_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) }) \ No newline at end of file diff --git a/inst/shiny-server/server/FV_PAR.R b/inst/shiny-server/server/FV_PAR.R index a59d9b6b..86878303 100644 --- a/inst/shiny-server/server/FV_PAR.R +++ b/inst/shiny-server/server/FV_PAR.R @@ -1,182 +1,182 @@ -# Expected Evolution of parameters in the algorithm -get_data_FV_PAR_PER_FUN <- reactive({ - data <- subset(DATA(), ID %in% input$FV_PAR.Plot.Algs) - generate_data.Parameters(data, scale_log = input$FV_PAR.Plot.Logx, which = 'by_RT') -}) - -render_FV_PAR_PER_FUN <- reactive({ - req(input$FV_PAR.Plot.Min, input$FV_PAR.Plot.Max) - withProgress({ - #TODO: use these parameters - rt_min <- as.numeric(input$FV_PAR.Plot.Min) - rt_max <- as.numeric(input$FV_PAR.Plot.Max) - - dt <- get_data_FV_PAR_PER_FUN() - dt <- dt[parId %in% input$FV_PAR.Plot.Params] - sub_attr <- if (length(input$FV_PAR.Plot.Params) > 1) 'parId' else NULL - - lower <- 'lower' - upper <- 'upper' - if (input$FV_PAR.Plot.CI == 'None') type <- 'line' - else { - type <- 'line+ribbon' - if (input$FV_PAR.Plot.CI == 'Outer Quantiles') { - quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') - lower <- quantiles[[1]] - upper <- quantiles[[length(quantiles)]] - } - } - if (is.null(sub_attr) && type == 'line+ribbon') { - p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'line', - subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = T) - p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'ribbon', - subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = F, p = p) - } - else { - p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, type, - subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper) - } - p - }, - message = "Creating plot") -}) - -output$FV_PAR.Plot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PAR_PER_FUN) - }, - content = function(file) { - save_plotly(render_FV_PAR_PER_FUN(), file) - }, - contentType = paste0('image/', input$FV_PAR.Plot.Format) -) - -output$FV_PAR.Plot.Figure <- renderPlotly({ - render_FV_PAR_PER_FUN() -}) - -# TODO: add ks test for ECDF later -# output$ks <- renderPrint({ -# target <- input$target %>% as.numeric -# df.aligneds <- aligned() -# -# running_time <- list() -# for (i in seq_along(df.aligneds)) { -# df <- df.aligneds[[i]] -# v <- rownames(df) %>% as.numeric -# idx <- order(abs(target - v))[1] -# running_time[[i]] <- df[idx, ] %>% as.vector -# } -# algorithm1 <- running_time[[1]] -# algorithm2 <- running_time[[2]] -# a <- ks.test(algorithm1, algorithm2, alternative = 'less') -# print(a) -# }) -# - -fv_parameter_summary <- reactive({ - req(input$FV_PAR.Summary.Min, input$FV_PAR.Summary.Max, input$FV_PAR.Summary.Step) - - rtstart <- as.numeric(input$FV_PAR.Summary.Min) - rtstop <- as.numeric(input$FV_PAR.Summary.Max) - rtstep <- as.numeric(input$FV_PAR.Summary.Step) - data <- DATA() - data <- subset(data, ID %in% input$FV_PAR.Summary.ID) - - if (!input$FV_PAR.Summary.Single) { - req(rtstart <= rtstop, rtstep <= rtstop - rtstart) - rtall <- get_runtimes(data) - rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) - req(rtseq) - } - else - rtseq <- rtstart - - dt <- get_PAR_summary(data, rtseq, parId = input$FV_PAR.Summary.Param, which = 'by_RT') - req(length(dt) != 0) - - dt$runs %<>% as.integer - dt$mean %<>% format(digits = 2, nsmall = 2) - dt$median %<>% format(digits = 2, nsmall = 2) - dt$sd %<>% format(digits = 2, nsmall = 2) - - probs <- getOption("IOHanalyzer.quantiles") - - # format the integers - for (p in paste0(probs * 100, '%')) { - dt[[p]] %<>% format(digits = 2, nsmall = 2) - } - dt -}) - -fv_parameter_sample <- reactive({ - req(input$FV_PAR.Sample.ID, input$FV_PAR.Sample.Max, - input$FV_PAR.Sample.Step, input$FV_PAR.Sample.Min, - input$FV_PAR.Sample.Param) - - rtstart <- as.numeric(input$FV_PAR.Sample.Min) - rtstop <- as.numeric(input$FV_PAR.Sample.Max) - rtstep <- as.numeric(input$FV_PAR.Sample.Step) - data <- DATA() - data <- subset(data, ID %in% input$FV_PAR.Sample.ID) - - if (!input$FV_PAR.Sample.Single) { - req(rtstart <= rtstop, rtstep <= rtstop - rtstart) - rtall <- get_runtimes(data) - rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) - req(rtseq) - } - else - rtseq <- rtstart - - df <- get_PAR_sample(data, idxValue = rtseq, - parId = input$FV_PAR.Sample.Param, - output = input$FV_PAR.Sample.Format, - which = 'by_RT') - - for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) - df[[p]] %<>% format(digits = 2, nsmall = 2) - - df -}) - -output$table_FV_PAR_SAMPLE <- DT::renderDataTable({ - dt <- fv_parameter_sample() - req(length(dt) != 0) - dt[is.na(dt)] <- 'NA' - dt -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$table_FV_PAR_summary <- DT::renderDataTable({ - fv_parameter_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) - -output$FV_PAR.Sample.Download <- downloadHandler( - filename = function() { - eval(FV_PARSample_csv_name) - }, - content = function(file) { - df <- parameter_sample() - df <- df[input[["table_FV_PAR_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -output$FV_PAR.Summary.Download <- downloadHandler( - filename = function() { - eval(FV_PAR_csv_name) - }, - content = function(file) { - df <- fv_parameter_summary() - df <- df[input[["table_FV_PAR_summary_rows_all"]]] - save_table(df, file) - } +# Expected Evolution of parameters in the algorithm +get_data_FV_PAR_PER_FUN <- reactive({ + data <- subset(DATA(), ID %in% input$FV_PAR.Plot.Algs) + generate_data.Parameters(data, scale_log = input$FV_PAR.Plot.Logx, which = 'by_RT') +}) + +render_FV_PAR_PER_FUN <- reactive({ + req(input$FV_PAR.Plot.Min, input$FV_PAR.Plot.Max) + withProgress({ + #TODO: use these parameters + rt_min <- as.numeric(input$FV_PAR.Plot.Min) + rt_max <- as.numeric(input$FV_PAR.Plot.Max) + + dt <- get_data_FV_PAR_PER_FUN() + dt <- dt[parId %in% input$FV_PAR.Plot.Params] + sub_attr <- if (length(input$FV_PAR.Plot.Params) > 1) 'parId' else NULL + + lower <- 'lower' + upper <- 'upper' + if (input$FV_PAR.Plot.CI == 'None') type <- 'line' + else { + type <- 'line+ribbon' + if (input$FV_PAR.Plot.CI == 'Outer Quantiles') { + quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') + lower <- quantiles[[1]] + upper <- quantiles[[length(quantiles)]] + } + } + if (is.null(sub_attr) && type == 'line+ribbon') { + p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'line', + subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = T) + p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'ribbon', + subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = F, p = p) + } + else { + p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, type, + subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper) + } + p + }, + message = "Creating plot") +}) + +output$FV_PAR.Plot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PAR_PER_FUN) + }, + content = function(file) { + save_plotly(render_FV_PAR_PER_FUN(), file) + }, + contentType = paste0('image/', input$FV_PAR.Plot.Format) +) + +output$FV_PAR.Plot.Figure <- renderPlotly({ + render_FV_PAR_PER_FUN() +}) + +# TODO: add ks test for ECDF later +# output$ks <- renderPrint({ +# target <- input$target %>% as.numeric +# df.aligneds <- aligned() +# +# running_time <- list() +# for (i in seq_along(df.aligneds)) { +# df <- df.aligneds[[i]] +# v <- rownames(df) %>% as.numeric +# idx <- order(abs(target - v))[1] +# running_time[[i]] <- df[idx, ] %>% as.vector +# } +# algorithm1 <- running_time[[1]] +# algorithm2 <- running_time[[2]] +# a <- ks.test(algorithm1, algorithm2, alternative = 'less') +# print(a) +# }) +# + +fv_parameter_summary <- reactive({ + req(input$FV_PAR.Summary.Min, input$FV_PAR.Summary.Max, input$FV_PAR.Summary.Step) + + rtstart <- as.numeric(input$FV_PAR.Summary.Min) + rtstop <- as.numeric(input$FV_PAR.Summary.Max) + rtstep <- as.numeric(input$FV_PAR.Summary.Step) + data <- DATA() + data <- subset(data, ID %in% input$FV_PAR.Summary.ID) + + if (!input$FV_PAR.Summary.Single) { + req(rtstart <= rtstop, rtstep <= rtstop - rtstart) + rtall <- get_runtimes(data) + rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) + req(rtseq) + } + else + rtseq <- rtstart + + dt <- get_PAR_summary(data, rtseq, parId = input$FV_PAR.Summary.Param, which = 'by_RT') + req(length(dt) != 0) + + dt$runs %<>% as.integer + dt$mean %<>% format(digits = 2, nsmall = 2) + dt$median %<>% format(digits = 2, nsmall = 2) + dt$sd %<>% format(digits = 2, nsmall = 2) + + probs <- getOption("IOHanalyzer.quantiles") + + # format the integers + for (p in paste0(probs * 100, '%')) { + dt[[p]] %<>% format(digits = 2, nsmall = 2) + } + dt +}) + +fv_parameter_sample <- reactive({ + req(input$FV_PAR.Sample.ID, input$FV_PAR.Sample.Max, + input$FV_PAR.Sample.Step, input$FV_PAR.Sample.Min, + input$FV_PAR.Sample.Param) + + rtstart <- as.numeric(input$FV_PAR.Sample.Min) + rtstop <- as.numeric(input$FV_PAR.Sample.Max) + rtstep <- as.numeric(input$FV_PAR.Sample.Step) + data <- DATA() + data <- subset(data, ID %in% input$FV_PAR.Sample.ID) + + if (!input$FV_PAR.Sample.Single) { + req(rtstart <= rtstop, rtstep <= rtstop - rtstart) + rtall <- get_runtimes(data) + rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) + req(rtseq) + } + else + rtseq <- rtstart + + df <- get_PAR_sample(data, idxValue = rtseq, + parId = input$FV_PAR.Sample.Param, + output = input$FV_PAR.Sample.Format, + which = 'by_RT') + + for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) + df[[p]] %<>% format(digits = 2, nsmall = 2) + + df +}) + +output$table_FV_PAR_SAMPLE <- DT::renderDataTable({ + dt <- fv_parameter_sample() + req(length(dt) != 0) + dt[is.na(dt)] <- 'NA' + dt +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$table_FV_PAR_summary <- DT::renderDataTable({ + fv_parameter_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) + +output$FV_PAR.Sample.Download <- downloadHandler( + filename = function() { + eval(FV_PARSample_csv_name) + }, + content = function(file) { + df <- parameter_sample() + df <- df[input[["table_FV_PAR_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +output$FV_PAR.Summary.Download <- downloadHandler( + filename = function() { + eval(FV_PAR_csv_name) + }, + content = function(file) { + df <- fv_parameter_summary() + df <- df[input[["table_FV_PAR_summary_rows_all"]]] + save_table(df, file) + } ) \ No newline at end of file diff --git a/inst/shiny-server/server/RTPMF.R b/inst/shiny-server/server/RTPMF.R index df5ce7c2..62d86126 100644 --- a/inst/shiny-server/server/RTPMF.R +++ b/inst/shiny-server/server/RTPMF.R @@ -1,73 +1,73 @@ -# empirical p.m.f. of the runtime -output$RT_PMF <- renderPlotly({ - render_RT_PMF() -}) - -output$RTPMF.Bar.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_PMF) - }, - content = function(file) { - save_plotly(render_RT_PMF(), file) - }, - contentType = paste0('image/', input$RTPMF.Bar.Format) -) - -get_data_RT_PMF <- reactive({ - ftarget <- input$RTPMF.Bar.Target %>% as.numeric - data <- subset(DATA(), ID %in% input$RTPMF.Bar.Algs) - generate_data.PMF(data, ftarget, 'by_RT') -}) - -render_RT_PMF <- reactive({ - withProgress({ - plot_general_data(get_data_RT_PMF(), 'ID', 'RT', scale.ylog = input$RTPMF.Bar.Logy, - x_title = "Algorithm", y_title = "Function Evaluations") - # ftarget <- input$RTPMF.Bar.Target %>% as.numeric - # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) - # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, - # scale.ylog = input$RTPMF.Bar.Logy) - }, - message = "Creating plot") -}) - -# historgram of the running time -output$RT_HIST <- renderPlotly({ - req(input$RTPMF.Bar.Target) - render_RT_HIST() -}) - -output$RTPMF.Hist.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_HIST) - }, - content = function(file) { - save_plotly(render_RT_HIST(), file) - }, - contentType = paste0('image/', input$RTPMF.Hist.Format) -) - -get_data_RT_HIST <- reactive({ - ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric - data <- subset(DATA(), ID %in% input$RTPMF.Hist.Algs) - generate_data.hist(data, ftarget, input$RTPMF.Hist.Equal, 'by_RT') -}) - -render_RT_HIST <- reactive({ - req(input$RTPMF.Hist.Target) - - withProgress({ - # ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric - # - # # TODO: remove 'DataSetList' in the future - # data <- subset(DATA(), algId %in% input$RTPMF.Hist.Algs) - # - # Plot.RT.Histogram(data, ftarget, plot_mode = input$RTPMF.Hist.Mode, - # use.equal.bins = input$RTPMF.Hist.Equal) - subplot_attr <- if (input$RTPMF.Hist.Mode == 'subplot') 'ID' else NULL - plot_general_data(get_data_RT_HIST(), 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Function Evaluations", - y_title = "Runs") - }, - message = "Creating plot") -}) +# empirical p.m.f. of the runtime +output$RT_PMF <- renderPlotly({ + render_RT_PMF() +}) + +output$RTPMF.Bar.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_PMF) + }, + content = function(file) { + save_plotly(render_RT_PMF(), file) + }, + contentType = paste0('image/', input$RTPMF.Bar.Format) +) + +get_data_RT_PMF <- reactive({ + ftarget <- input$RTPMF.Bar.Target %>% as.numeric + data <- subset(DATA(), ID %in% input$RTPMF.Bar.Algs) + generate_data.PMF(data, ftarget, 'by_RT') +}) + +render_RT_PMF <- reactive({ + withProgress({ + plot_general_data(get_data_RT_PMF(), 'ID', 'RT', scale.ylog = input$RTPMF.Bar.Logy, + x_title = "Algorithm", y_title = "Function Evaluations") + # ftarget <- input$RTPMF.Bar.Target %>% as.numeric + # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) + # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, + # scale.ylog = input$RTPMF.Bar.Logy) + }, + message = "Creating plot") +}) + +# historgram of the running time +output$RT_HIST <- renderPlotly({ + req(input$RTPMF.Bar.Target) + render_RT_HIST() +}) + +output$RTPMF.Hist.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_HIST) + }, + content = function(file) { + save_plotly(render_RT_HIST(), file) + }, + contentType = paste0('image/', input$RTPMF.Hist.Format) +) + +get_data_RT_HIST <- reactive({ + ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric + data <- subset(DATA(), ID %in% input$RTPMF.Hist.Algs) + generate_data.hist(data, ftarget, input$RTPMF.Hist.Equal, 'by_RT') +}) + +render_RT_HIST <- reactive({ + req(input$RTPMF.Hist.Target) + + withProgress({ + # ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric + # + # # TODO: remove 'DataSetList' in the future + # data <- subset(DATA(), algId %in% input$RTPMF.Hist.Algs) + # + # Plot.RT.Histogram(data, ftarget, plot_mode = input$RTPMF.Hist.Mode, + # use.equal.bins = input$RTPMF.Hist.Equal) + subplot_attr <- if (input$RTPMF.Hist.Mode == 'subplot') 'ID' else NULL + plot_general_data(get_data_RT_HIST(), 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Function Evaluations", + y_title = "Runs") + }, + message = "Creating plot") +}) diff --git a/inst/shiny-server/server/RT_DSCinterface.R b/inst/shiny-server/server/RT_DSCinterface.R index d188cb08..33a766a5 100644 --- a/inst/shiny-server/server/RT_DSCinterface.R +++ b/inst/shiny-server/server/RT_DSCinterface.R @@ -1,224 +1,224 @@ -RT_DSC_rank_result <- reactive({ - req(input$RT_Stats.DSC.Create_rank) - - isolate({ - withProgress({ - data <- RT_DSC_data() - req(length(data) > 0) - test_type <- if (input$RT_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" - rank_res <- get_dsc_rank(data, RT_stats_DSC_targets_obj, which = 'by_RT', - alpha = input$RT_Stats.DSC.Alpha_rank, - test_type = test_type, - monte_carlo_iterations = input$RT_Stats.DSC.MCsamples_rank, - epsilon = input$RT_Stats.DSC.Epsilon_rank, - na.correction = input$RT_Stats.DSC.Value_type) - if (is.null(rank_res)) { - shinyjs::alert("There was an error getting the ranking data from DSCtool. Please - select different settings and try again") - return(NULL) - } - updateSelectInput(session, 'RT_Stats.DSC.Omni_options', choices = rank_res$valid_methods, - selected = rank_res$valid_methods[[1]]) - - - rank_res - }, message = "Getting DSC ranking data") - }) -}) - -RT_render_performviz <- reactive({ - rank_res <- RT_DSC_rank_result() - if (is.null(rank_res)) {return(NULL)} - Plot.Performviz(rank_res) -}) - -output$RT_Stats.DSC.PerformViz <- renderPlot({ - RT_render_performviz() -}) - -output$RT_Stats.DSC.Download_rank_table <- downloadHandler( - filename = function() { - eval(RT_DSC_table_name_rank) - }, - content = function(file) { - mlist <- RT_DSC_rank_result()$ranked_matrix - df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { - df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - function(x) { - list(algorithm = x$algorithm, rank = x$rank) - })) - df_temp[, problem := mlist[[problem_idx]]$problem] - })) - df2 <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') - save_table(df, file) - } -) - -output$RT_Stats.DSC.Download_rank <- downloadHandler( - filename = function() { - eval(RT_DSC_figure_name_rank) - }, - content = function(file) { - #Temporary settings, will replace plotting from complexheatmap to - #Some other plotly-compatible library later, and use save_plotly - suppressWarnings({ - pdf(file = file, width = 16, height = 9) - RT_render_performviz() - dev.off() - }) - }, - contentType = paste0('image/', input$RT_Stats.DSC.Format_rank) -) - - -RT_DSC_omni_result <- reactive({ - input$RT_Stats.DSC.Create_omni - - isolate({ - withProgress({ - rank_res <- RT_DSC_rank_result() - req(rank_res) - omni_res <- get_dsc_omnibus(rank_res, method = input$RT_Stats.DSC.Omni_options, - alpha = input$RT_Stats.DSC.Alpha_omni) - - }, message = "Creating Comparison, this might take a while") - }) -}) - - -output$RT_Stats.DSC.Output_omni <- renderText({ - res_omni <- RT_DSC_omni_result() - req(res_omni) - paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") -}) - - -RT_DSC_posthoc_result <- reactive({ - input$RT_Stats.DSC.Create_posthoc - - isolate({ - withProgress({ - omni_res <- RT_DSC_omni_result() - req(omni_res) - data <- RT_DSC_data() - req(length(data) > 0) - df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ - posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), - nrow(RT_stats_DSC_targets_obj), - alpha = input$RT_Stats.DSC.Alpha_posthoc, - base_algorithm = algname, - method = input$RT_Stats.DSC.Posthoc_test) - if (is.null(posthoc_res)) { return(NULL)} - values <- lapply(seq(4), function(method_idx) { - lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, - function(x) { - x$value - }) - }) - - algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, - function(x) { - x$algorithm - }) - df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], - "Holm" = values[[3]], "Hochberg" = values[[4]]) - - })) - as.data.table(format(df_posthoc, digits = 3)) - }, message = "Creating Comparison, this might take a while") - }) -}) - -output$RT_Stats.DSC.PosthocTable <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - RT_DSC_posthoc_result() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -RT_render_DSC_plot <- reactive({ - dt <- RT_DSC_posthoc_result() - if (is.null(dt) || nrow(dt) < 1) {return(NULL)} - # plot_general_data(dt[method == input$RT_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', - # legend_attr = 'algId') - dt <- dt[,c('baseline', 'compared alg', input$RT_Stats.DSC.Posthoc_method), with = F] - p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$RT_Stats.DSC.Posthoc_method) - storage.mode(p_matrix) <- "numeric" - y <- p_matrix <= input$RT_Stats.DSC.Alpha_posthoc - colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), - col = c('blue', 'blue', 'white', 'white', 'red', 'red') - ) - heatmap <- y - t(y) - - p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', - xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) - p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), - xaxis = list(tickangle = 45)) - p -}) - -output$RT_Stats.DSC.PosthocViz <- renderPlotly({ - RT_render_DSC_plot() -}) - -output$RT_Stats.DSC.DownloadTable <- downloadHandler( - filename = function() { - eval(RT_DSC_table_name) - }, - content = function(file) { - df <- RT_DSC_posthoc_result() - save_table(df, file) - } -) - -output$RT_Stats.DSC.Download <- downloadHandler( - filename = function() { - eval(RT_DSC_figure_name) - }, - content = function(file) { - save_plotly(RT_render_DSC_plot(), file) - }, - contentType = paste0('image/', input$RT_Stats.DSC.Format) -) - - -RT_DSC_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.DSC.ID)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$RT_Stats.DSC.Dim) - data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) - if (length(unique(get_id(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple IDs for the selected functions and dimensions.") - return(NULL) - } - data -} - -RT_stats_DSC_targets <- reactive({ - data <- RT_DSC_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_RT") -}) - -RT_stats_DSC_targets_obj <- NULL - -proxy_RT_Stats.DSC.Targets <- dataTableProxy('RT_Stats.DSC.Targets') - -output$RT_Stats.DSC.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - RT_stats_DSC_targets_obj <<- RT_stats_DSC_targets() - RT_stats_DSC_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$RT_Stats.DSC.Targets_cell_edit, { - info <- input$RT_Stats.DSC.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- RT_DSC_data() - if (is.null(data)) return(NULL) - RT_stats_DSC_targets_obj$target[[i]] <<- v - replaceData(proxy, RT_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) +RT_DSC_rank_result <- reactive({ + req(input$RT_Stats.DSC.Create_rank) + + isolate({ + withProgress({ + data <- RT_DSC_data() + req(length(data) > 0) + test_type <- if (input$RT_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" + rank_res <- get_dsc_rank(data, RT_stats_DSC_targets_obj, which = 'by_RT', + alpha = input$RT_Stats.DSC.Alpha_rank, + test_type = test_type, + monte_carlo_iterations = input$RT_Stats.DSC.MCsamples_rank, + epsilon = input$RT_Stats.DSC.Epsilon_rank, + na.correction = input$RT_Stats.DSC.Value_type) + if (is.null(rank_res)) { + shinyjs::alert("There was an error getting the ranking data from DSCtool. Please + select different settings and try again") + return(NULL) + } + updateSelectInput(session, 'RT_Stats.DSC.Omni_options', choices = rank_res$valid_methods, + selected = rank_res$valid_methods[[1]]) + + + rank_res + }, message = "Getting DSC ranking data") + }) +}) + +RT_render_performviz <- reactive({ + rank_res <- RT_DSC_rank_result() + if (is.null(rank_res)) {return(NULL)} + Plot.Performviz(rank_res) +}) + +output$RT_Stats.DSC.PerformViz <- renderPlot({ + RT_render_performviz() +}) + +output$RT_Stats.DSC.Download_rank_table <- downloadHandler( + filename = function() { + eval(RT_DSC_table_name_rank) + }, + content = function(file) { + mlist <- RT_DSC_rank_result()$ranked_matrix + df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { + df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + function(x) { + list(algorithm = x$algorithm, rank = x$rank) + })) + df_temp[, problem := mlist[[problem_idx]]$problem] + })) + df2 <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') + save_table(df, file) + } +) + +output$RT_Stats.DSC.Download_rank <- downloadHandler( + filename = function() { + eval(RT_DSC_figure_name_rank) + }, + content = function(file) { + #Temporary settings, will replace plotting from complexheatmap to + #Some other plotly-compatible library later, and use save_plotly + suppressWarnings({ + pdf(file = file, width = 16, height = 9) + RT_render_performviz() + dev.off() + }) + }, + contentType = paste0('image/', input$RT_Stats.DSC.Format_rank) +) + + +RT_DSC_omni_result <- reactive({ + input$RT_Stats.DSC.Create_omni + + isolate({ + withProgress({ + rank_res <- RT_DSC_rank_result() + req(rank_res) + omni_res <- get_dsc_omnibus(rank_res, method = input$RT_Stats.DSC.Omni_options, + alpha = input$RT_Stats.DSC.Alpha_omni) + + }, message = "Creating Comparison, this might take a while") + }) +}) + + +output$RT_Stats.DSC.Output_omni <- renderText({ + res_omni <- RT_DSC_omni_result() + req(res_omni) + paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") +}) + + +RT_DSC_posthoc_result <- reactive({ + input$RT_Stats.DSC.Create_posthoc + + isolate({ + withProgress({ + omni_res <- RT_DSC_omni_result() + req(omni_res) + data <- RT_DSC_data() + req(length(data) > 0) + df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ + posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), + nrow(RT_stats_DSC_targets_obj), + alpha = input$RT_Stats.DSC.Alpha_posthoc, + base_algorithm = algname, + method = input$RT_Stats.DSC.Posthoc_test) + if (is.null(posthoc_res)) { return(NULL)} + values <- lapply(seq(4), function(method_idx) { + lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, + function(x) { + x$value + }) + }) + + algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, + function(x) { + x$algorithm + }) + df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], + "Holm" = values[[3]], "Hochberg" = values[[4]]) + + })) + as.data.table(format(df_posthoc, digits = 3)) + }, message = "Creating Comparison, this might take a while") + }) +}) + +output$RT_Stats.DSC.PosthocTable <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + RT_DSC_posthoc_result() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +RT_render_DSC_plot <- reactive({ + dt <- RT_DSC_posthoc_result() + if (is.null(dt) || nrow(dt) < 1) {return(NULL)} + # plot_general_data(dt[method == input$RT_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', + # legend_attr = 'algId') + dt <- dt[,c('baseline', 'compared alg', input$RT_Stats.DSC.Posthoc_method), with = F] + p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$RT_Stats.DSC.Posthoc_method) + storage.mode(p_matrix) <- "numeric" + y <- p_matrix <= input$RT_Stats.DSC.Alpha_posthoc + colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), + col = c('blue', 'blue', 'white', 'white', 'red', 'red') + ) + heatmap <- y - t(y) + + p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', + xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) + p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), + xaxis = list(tickangle = 45)) + p +}) + +output$RT_Stats.DSC.PosthocViz <- renderPlotly({ + RT_render_DSC_plot() +}) + +output$RT_Stats.DSC.DownloadTable <- downloadHandler( + filename = function() { + eval(RT_DSC_table_name) + }, + content = function(file) { + df <- RT_DSC_posthoc_result() + save_table(df, file) + } +) + +output$RT_Stats.DSC.Download <- downloadHandler( + filename = function() { + eval(RT_DSC_figure_name) + }, + content = function(file) { + save_plotly(RT_render_DSC_plot(), file) + }, + contentType = paste0('image/', input$RT_Stats.DSC.Format) +) + + +RT_DSC_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.DSC.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$RT_Stats.DSC.Dim) + data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +RT_stats_DSC_targets <- reactive({ + data <- RT_DSC_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_RT") +}) + +RT_stats_DSC_targets_obj <- NULL + +proxy_RT_Stats.DSC.Targets <- dataTableProxy('RT_Stats.DSC.Targets') + +output$RT_Stats.DSC.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + RT_stats_DSC_targets_obj <<- RT_stats_DSC_targets() + RT_stats_DSC_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$RT_Stats.DSC.Targets_cell_edit, { + info <- input$RT_Stats.DSC.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- RT_DSC_data() + if (is.null(data)) return(NULL) + RT_stats_DSC_targets_obj$target[[i]] <<- v + replaceData(proxy, RT_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) }) \ No newline at end of file diff --git a/inst/shiny-server/server/RT_ECDF.R b/inst/shiny-server/server/RT_ECDF.R index 23414f92..74855e99 100644 --- a/inst/shiny-server/server/RT_ECDF.R +++ b/inst/shiny-server/server/RT_ECDF.R @@ -1,303 +1,303 @@ -output$RT_ECDF_MULT <- renderPlotly({ - req(length(DATA_RAW()) > 0) - render_RT_ECDF_MULT() -}) - -get_data_RT_ECDF_MULT <- reactive({ - req(input$RTECDF.Aggr.Func || input$RTECDF.Aggr.Dim) - input$RTECDF.Aggr.Refresh - dsList <- subset(DATA_RAW(), ID %in% input$RTECDF.Aggr.Algs) - - if (!input$RTECDF.Aggr.Func) - dsList <- subset(dsList, funcId == input$Overall.Funcid) - - if (!input$RTECDF.Aggr.Dim) - dsList <- subset(dsList, DIM == input$Overall.Dim) - - if (length(dsList) <= 1) { - shinyjs::alert("This is an invalid configuration for this plot. \n - Please ensure that the dataset contains multiple functions / - dimensions to aggregate over.") - return(NULL) - } - - isolate({ - targets <- RT_ECDF_MULTI_TABLE() - }) - generate_data.ECDF(dsList, targets, input$RTECDF.Aggr.Logx) -}) - -render_RT_ECDF_MULT <- reactive({ - withProgress({ - plot_general_data(get_data_RT_ECDF_MULT(), 'x', 'mean', 'line', - scale.xlog = input$RTECDF.Aggr.Logx, - scale.ylog = input$RTECDF.Aggr.Logy, - x_title = "Function Evaluations", - y_title = "Proportion of (run, target, ...) pairs", - show.legend = T) - }, - message = "Creating plot") -}) - -output$RTECDF.Aggr.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_ECDF_MULT) - }, - content = function(file) { - save_plotly(render_RT_ECDF_MULT(), file) - }, - contentType = paste0('image/', input$RTECDF.Aggr.Format) -) - -RT_ECDF_MULTI_TABLE <- reactiveVal(NULL) -trigger_renderDT <- reactiveVal(NULL) -proxy <- dataTableProxy('RT_GRID_GENERATED') - -observe({ - alg <- input$RTECDF.Aggr.Algs - funcid <- input$Overall.Funcid - data <- DATA_RAW() - dim <- input$Overall.Dim - - req(length(data) > 0) - req(alg) - req(funcid) - req(dim) - # req(data$suite != NEVERGRAD) - - dsList <- subset(data, ID %in% alg) - req(length(dsList) > 0) - - if (!input$RTECDF.Aggr.Func) - dsList <- subset(dsList, funcId == input$Overall.Funcid) - - if (!input$RTECDF.Aggr.Dim) - dsList <- subset(dsList, DIM == input$Overall.Dim) - - if (length(dsList) == 0) return(NULL) - - targets <- get_ECDF_targets(dsList, input$RTECDF.Aggr.Target_type, input$RTECDF.Aggr.Target_number) - - RT_ECDF_MULTI_TABLE(targets) # add values to `RT_ECDF_MULTI_TABLE` - trigger_renderDT(rnorm(1)) -}) - -output$RTECDF.AUC.Table.Download <- downloadHandler( - filename = function() { - eval(AUC_ECDF_aggr_name) - }, - content = function(file) { - save_table(auc_grid_table(), file) - } -) - -auc_grid_table <- reactive({ - dt_ecdf <- get_data_RT_ECDF_MULT() - generate_data.AUC(NULL, NULL, dt_ecdf = dt_ecdf) -}) - -output$AUC_GRID_GENERATED <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - isolate({auc_grid_table()}) - }, - editable = FALSE, - rownames = FALSE, - options = list( - pageLength = 10, - lengthMenu = c(5, 10, 25, -1), - scrollX = T, - server = T, - columnDefs = list( - list( - className = 'dt-right', targets = "_all" - ) - ) - ), - filter = 'top' -) - -output$RT_GRID_GENERATED <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - trigger_renderDT() - isolate({RT_ECDF_MULTI_TABLE()}) - }, - editable = TRUE, - rownames = FALSE, - options = list( - pageLength = 10, - lengthMenu = c(5, 10, 25, -1), - scrollX = T, - server = T, - columnDefs = list( - list( - className = 'dt-right', targets = "_all" - ) - ) - ), - filter = 'top' -) - -observeEvent(input$RT_GRID_GENERATED_cell_edit, { - info <- input$RT_GRID_GENERATED_cell_edit - i <- info$row - j <- info$col + 1 - v <- info$value - - suppressWarnings({ - df <- RT_ECDF_MULTI_TABLE() - set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) - RT_ECDF_MULTI_TABLE(df) - }) - replaceData(proxy, RT_ECDF_MULTI_TABLE(), - resetPaging = FALSE, rownames = FALSE) -}) - -observeEvent(input$RTECDF.Aggr.Table.Upload, { - if (!is.null(input$RTECDF.Aggr.Table.Upload)) { - df <- read.csv2(input$RTECDF.Aggr.Table.Upload$datapath, header = T) %>% - as.data.table - - if (ncol(df) == ncol(RT_ECDF_MULTI_TABLE())) { - RT_ECDF_MULTI_TABLE(df) - replaceData(proxy, RT_ECDF_MULTI_TABLE(), - resetPaging = FALSE, - rownames = FALSE) - } else { - RT_ECDF_MULTI_TABLE(df) - trigger_renderDT(rnorm(1)) - } - } else - NULL -}) - -output$RTECDF.Aggr.Table.Download <- downloadHandler( - filename = 'Example_ECDF_TARGETS.csv', - content = function(file) { - write.csv2(RT_ECDF_MULTI_TABLE(), file, row.names = F) - }, - contentType = "text/csv" -) - -# The ECDF plots for the runtime ---------------- - -get_data_RT_ECDF_Single <- reactive({ - ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) - data <- subset(DATA(), ID %in% input$RTECDF.Single.Algs) - generate_data.ECDF(data, ftargets, input$RTECDF.Single.Logx) -}) - -output$RT_ECDF <- renderPlotly({ - req(input$RTECDF.Single.Target) - plot_general_data(get_data_RT_ECDF_Single(), 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of runs", scale.xlog = input$RTECDF.Single.Logx, show.legend = T) - # ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) - # data <- subset(DATA(), algId %in% input$RTECDF.Single.Algs) - # Plot.RT.ECDF_Per_Target(data, ftargets, scale.xlog = input$RTECDF.Single.Logx) -}) - -output$AUC_GRID_GENERATED_SINGLE <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_Single()) -}) - -output$AUC_GRID_GENERATED_FUNC <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - input$RTECDF.Aggr.Refresh - generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_AGGR()) -}) - - -output$RT_GRID <- renderPrint({ - req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) - - fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric - fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric - fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric - - req(fstart <= fstop, fstep <= fstop - fstart) - data <- DATA() - fall <- get_funvals(data) - - cat(seq_FV(fall, fstart, fstop, by = fstep)) -}) - -output$RT_ECDF_AGGR <- renderPlotly({ - render_RT_ECDF_AGGR() -}) - -output$RTECDF.Multi.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_ECDF_AGGR) - }, - content = function(file) { - save_plotly(render_RT_ECDF_AGGR(), file) - }, - contentType = paste0('image/', input$RTECDF.Multi.Format) -) - -get_data_RT_ECDF_AGGR <- reactive({ - req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) - fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric - fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric - fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric - data <- subset(DATA(), ID %in% input$RTECDF.Multi.Algs) - targets <- seq_FV(get_funvals(data), fstart, fstop, fstep) - generate_data.ECDF(data, targets, input$RTECDF.Multi.Logx) -}) - -render_RT_ECDF_AGGR <- reactive({ - withProgress({ - - plot_general_data(get_data_RT_ECDF_AGGR(), 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of (run, target) pairs", - scale.xlog = input$RTECDF.Multi.Logx, show.legend = T) - - # Plot.RT.ECDF_Single_Func( - # data, fstart, fstop, fstep, - # scale.xlog = input$RTECDF.Multi.Logx - # ) - }, - message = "Creating plot") -}) - -# # evaluation rake of all courses -# output$RT_AUC <- renderPlotly({ -# render_RT_AUC() -# }) -# -# output$RTECDF.AUC.Download <- downloadHandler( -# filename = function() { -# eval(FIG_NAME_RT_AUC) -# }, -# content = function(file) { -# save_plotly(render_RT_AUC(), file) -# }, -# contentType = paste0('image/', input$RTECDF.AUC.Format) -# ) -# -# get_data_RT_AUC <- reactive({ -# req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) -# -# fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric -# fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric -# fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric -# data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) -# targets <- seq_FV(get_funvals(data), fstart, fstop, fstep, length.out = 10) -# generate_data.AUC(data, targets) -# }) -# -# render_RT_AUC <- reactive({ -# # req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) -# # -# # fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric -# # fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric -# # fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric -# # data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) -# # -# # Plot.RT.ECDF_AUC( -# # data, fstart, fstop, fstep, fval_formatter = format_FV -# # ) -# plot_general_data(get_data_RT_AUC(), 'x', 'AUC', 'radar') -# }) +output$RT_ECDF_MULT <- renderPlotly({ + req(length(DATA_RAW()) > 0) + render_RT_ECDF_MULT() +}) + +get_data_RT_ECDF_MULT <- reactive({ + req(input$RTECDF.Aggr.Func || input$RTECDF.Aggr.Dim) + input$RTECDF.Aggr.Refresh + dsList <- subset(DATA_RAW(), ID %in% input$RTECDF.Aggr.Algs) + + if (!input$RTECDF.Aggr.Func) + dsList <- subset(dsList, funcId == input$Overall.Funcid) + + if (!input$RTECDF.Aggr.Dim) + dsList <- subset(dsList, DIM == input$Overall.Dim) + + if (length(dsList) <= 1) { + shinyjs::alert("This is an invalid configuration for this plot. \n + Please ensure that the dataset contains multiple functions / + dimensions to aggregate over.") + return(NULL) + } + + isolate({ + targets <- RT_ECDF_MULTI_TABLE() + }) + generate_data.ECDF(dsList, targets, input$RTECDF.Aggr.Logx) +}) + +render_RT_ECDF_MULT <- reactive({ + withProgress({ + plot_general_data(get_data_RT_ECDF_MULT(), 'x', 'mean', 'line', + scale.xlog = input$RTECDF.Aggr.Logx, + scale.ylog = input$RTECDF.Aggr.Logy, + x_title = "Function Evaluations", + y_title = "Proportion of (run, target, ...) pairs", + show.legend = T) + }, + message = "Creating plot") +}) + +output$RTECDF.Aggr.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_ECDF_MULT) + }, + content = function(file) { + save_plotly(render_RT_ECDF_MULT(), file) + }, + contentType = paste0('image/', input$RTECDF.Aggr.Format) +) + +RT_ECDF_MULTI_TABLE <- reactiveVal(NULL) +trigger_renderDT <- reactiveVal(NULL) +proxy <- dataTableProxy('RT_GRID_GENERATED') + +observe({ + alg <- input$RTECDF.Aggr.Algs + funcid <- input$Overall.Funcid + data <- DATA_RAW() + dim <- input$Overall.Dim + + req(length(data) > 0) + req(alg) + req(funcid) + req(dim) + # req(data$suite != NEVERGRAD) + + dsList <- subset(data, ID %in% alg) + req(length(dsList) > 0) + + if (!input$RTECDF.Aggr.Func) + dsList <- subset(dsList, funcId == input$Overall.Funcid) + + if (!input$RTECDF.Aggr.Dim) + dsList <- subset(dsList, DIM == input$Overall.Dim) + + if (length(dsList) == 0) return(NULL) + + targets <- get_ECDF_targets(dsList, input$RTECDF.Aggr.Target_type, input$RTECDF.Aggr.Target_number) + + RT_ECDF_MULTI_TABLE(targets) # add values to `RT_ECDF_MULTI_TABLE` + trigger_renderDT(rnorm(1)) +}) + +output$RTECDF.AUC.Table.Download <- downloadHandler( + filename = function() { + eval(AUC_ECDF_aggr_name) + }, + content = function(file) { + save_table(auc_grid_table(), file) + } +) + +auc_grid_table <- reactive({ + dt_ecdf <- get_data_RT_ECDF_MULT() + generate_data.AUC(NULL, NULL, dt_ecdf = dt_ecdf) +}) + +output$AUC_GRID_GENERATED <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + isolate({auc_grid_table()}) + }, + editable = FALSE, + rownames = FALSE, + options = list( + pageLength = 10, + lengthMenu = c(5, 10, 25, -1), + scrollX = T, + server = T, + columnDefs = list( + list( + className = 'dt-right', targets = "_all" + ) + ) + ), + filter = 'top' +) + +output$RT_GRID_GENERATED <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + trigger_renderDT() + isolate({RT_ECDF_MULTI_TABLE()}) + }, + editable = TRUE, + rownames = FALSE, + options = list( + pageLength = 10, + lengthMenu = c(5, 10, 25, -1), + scrollX = T, + server = T, + columnDefs = list( + list( + className = 'dt-right', targets = "_all" + ) + ) + ), + filter = 'top' +) + +observeEvent(input$RT_GRID_GENERATED_cell_edit, { + info <- input$RT_GRID_GENERATED_cell_edit + i <- info$row + j <- info$col + 1 + v <- info$value + + suppressWarnings({ + df <- RT_ECDF_MULTI_TABLE() + set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) + RT_ECDF_MULTI_TABLE(df) + }) + replaceData(proxy, RT_ECDF_MULTI_TABLE(), + resetPaging = FALSE, rownames = FALSE) +}) + +observeEvent(input$RTECDF.Aggr.Table.Upload, { + if (!is.null(input$RTECDF.Aggr.Table.Upload)) { + df <- read.csv2(input$RTECDF.Aggr.Table.Upload$datapath, header = T) %>% + as.data.table + + if (ncol(df) == ncol(RT_ECDF_MULTI_TABLE())) { + RT_ECDF_MULTI_TABLE(df) + replaceData(proxy, RT_ECDF_MULTI_TABLE(), + resetPaging = FALSE, + rownames = FALSE) + } else { + RT_ECDF_MULTI_TABLE(df) + trigger_renderDT(rnorm(1)) + } + } else + NULL +}) + +output$RTECDF.Aggr.Table.Download <- downloadHandler( + filename = 'Example_ECDF_TARGETS.csv', + content = function(file) { + write.csv2(RT_ECDF_MULTI_TABLE(), file, row.names = F) + }, + contentType = "text/csv" +) + +# The ECDF plots for the runtime ---------------- + +get_data_RT_ECDF_Single <- reactive({ + ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) + data <- subset(DATA(), ID %in% input$RTECDF.Single.Algs) + generate_data.ECDF(data, ftargets, input$RTECDF.Single.Logx) +}) + +output$RT_ECDF <- renderPlotly({ + req(input$RTECDF.Single.Target) + plot_general_data(get_data_RT_ECDF_Single(), 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of runs", scale.xlog = input$RTECDF.Single.Logx, show.legend = T) + # ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) + # data <- subset(DATA(), algId %in% input$RTECDF.Single.Algs) + # Plot.RT.ECDF_Per_Target(data, ftargets, scale.xlog = input$RTECDF.Single.Logx) +}) + +output$AUC_GRID_GENERATED_SINGLE <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_Single()) +}) + +output$AUC_GRID_GENERATED_FUNC <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + input$RTECDF.Aggr.Refresh + generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_AGGR()) +}) + + +output$RT_GRID <- renderPrint({ + req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) + + fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric + fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric + fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric + + req(fstart <= fstop, fstep <= fstop - fstart) + data <- DATA() + fall <- get_funvals(data) + + cat(seq_FV(fall, fstart, fstop, by = fstep)) +}) + +output$RT_ECDF_AGGR <- renderPlotly({ + render_RT_ECDF_AGGR() +}) + +output$RTECDF.Multi.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_ECDF_AGGR) + }, + content = function(file) { + save_plotly(render_RT_ECDF_AGGR(), file) + }, + contentType = paste0('image/', input$RTECDF.Multi.Format) +) + +get_data_RT_ECDF_AGGR <- reactive({ + req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) + fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric + fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric + fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric + data <- subset(DATA(), ID %in% input$RTECDF.Multi.Algs) + targets <- seq_FV(get_funvals(data), fstart, fstop, fstep) + generate_data.ECDF(data, targets, input$RTECDF.Multi.Logx) +}) + +render_RT_ECDF_AGGR <- reactive({ + withProgress({ + + plot_general_data(get_data_RT_ECDF_AGGR(), 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of (run, target) pairs", + scale.xlog = input$RTECDF.Multi.Logx, show.legend = T) + + # Plot.RT.ECDF_Single_Func( + # data, fstart, fstop, fstep, + # scale.xlog = input$RTECDF.Multi.Logx + # ) + }, + message = "Creating plot") +}) + +# # evaluation rake of all courses +# output$RT_AUC <- renderPlotly({ +# render_RT_AUC() +# }) +# +# output$RTECDF.AUC.Download <- downloadHandler( +# filename = function() { +# eval(FIG_NAME_RT_AUC) +# }, +# content = function(file) { +# save_plotly(render_RT_AUC(), file) +# }, +# contentType = paste0('image/', input$RTECDF.AUC.Format) +# ) +# +# get_data_RT_AUC <- reactive({ +# req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) +# +# fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric +# fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric +# fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric +# data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) +# targets <- seq_FV(get_funvals(data), fstart, fstop, fstep, length.out = 10) +# generate_data.AUC(data, targets) +# }) +# +# render_RT_AUC <- reactive({ +# # req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) +# # +# # fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric +# # fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric +# # fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric +# # data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) +# # +# # Plot.RT.ECDF_AUC( +# # data, fstart, fstop, fstep, fval_formatter = format_FV +# # ) +# plot_general_data(get_data_RT_AUC(), 'x', 'AUC', 'radar') +# }) diff --git a/inst/shiny-server/server/RT_Par.R b/inst/shiny-server/server/RT_Par.R index 47cd746d..232b4f66 100644 --- a/inst/shiny-server/server/RT_Par.R +++ b/inst/shiny-server/server/RT_Par.R @@ -1,181 +1,181 @@ - -get_data_RT_PAR_PER_FUN <- reactive({ - data <- subset(DATA(), ID %in% input$RT_PAR.Plot.Algs) - generate_data.Parameters(data, scale_log = input$RT_PAR.Plot.Logx, which = 'by_FV') -}) - -# Expected Evolution of parameters in the algorithm -render_RT_PAR_PER_FUN <- reactive({ - req(input$RT_PAR.Plot.Min, input$RT_PAR.Plot.Max) - withProgress({ - f_min <- as.numeric(format_FV(input$RT_PAR.Plot.Min)) - f_max <- as.numeric(format_FV(input$RT_PAR.Plot.Max)) - - dt <- get_data_RT_PAR_PER_FUN() - dt <- dt[parId %in% input$RT_PAR.Plot.Params] - sub_attr <- if (length(input$RT_PAR.Plot.Params) > 1) 'parId' else NULL - - lower <- 'lower' - upper <- 'upper' - if (input$RT_PAR.Plot.CI == 'None') type <- 'line' - else { - type <- 'line+ribbon' - if (input$RT_PAR.Plot.CI == 'Outer Quantiles') { - quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') - lower <- quantiles[[1]] - upper <- quantiles[[length(quantiles)]] - } - } - if (is.null(sub_attr) && type == 'line+ribbon') { - p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'line', - subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, - scale.ylog = input$RT_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = T) - p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'ribbon', - subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, - scale.ylog = input$RT_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = F, p = p) - } - else { - p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, type, - subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, - scale.ylog = input$RT_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = T) - } - p - }, - message = "Creating plot") -}) - -output$RT_PAR.Plot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_PAR_PER_FUN) - }, - content = function(file) { - save_plotly(render_RT_PAR_PER_FUN(), file) - }, - contentType = paste0('image/', input$RT_PAR.Plot.Format) -) - -output$RT_PAR.Plot.Figure <- renderPlotly({ - render_RT_PAR_PER_FUN() -}) - -# TODO: add ks test for ECDF later -# output$ks <- renderPrint({ -# target <- input$target %>% as.numeric -# df.aligneds <- aligned() -# -# running_time <- list() -# for (i in seq_along(df.aligneds)) { -# df <- df.aligneds[[i]] -# v <- rownames(df) %>% as.numeric -# idx <- order(abs(target - v))[1] -# running_time[[i]] <- df[idx, ] %>% as.vector -# } -# algorithm1 <- running_time[[1]] -# algorithm2 <- running_time[[2]] -# a <- ks.test(algorithm1, algorithm2, alternative = 'less') -# print(a) -# }) -# - -rt_parameter_summary <- reactive({ - req(input$RT_PAR.Summary.Min, input$RT_PAR.Summary.Max, input$RT_PAR.Summary.Step) - - fstart <- format_FV(input$RT_PAR.Summary.Min) %>% as.numeric - fstop <- format_FV(input$RT_PAR.Summary.Max) %>% as.numeric - fstep <- format_FV(input$RT_PAR.Summary.Step) %>% as.numeric - data <- DATA() - data <- subset(data, ID %in% input$RT_PAR.Summary.ID) - - if (!input$RT_PAR.Summary.Single) { - req(fstart <= fstop, fstep <= fstop - fstart) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, by = fstep) - req(fseq) - } - else - fseq <- fstart - - dt <- get_PAR_summary(data, fseq, parId = input$RT_PAR.Summary.Param) - req(length(dt) != 0) - - dt$runs %<>% as.integer - dt$mean %<>% format(digits = 2, nsmall = 2) - dt$median %<>% format(digits = 2, nsmall = 2) - dt$sd %<>% format(digits = 2, nsmall = 2) - - probs <- getOption("IOHanalyzer.quantiles") - - # format the integers - for (p in paste0(probs * 100, '%')) { - dt[[p]] %<>% format(digits = 2, nsmall = 2) - } - dt -}) - -rt_parameter_sample <- reactive({ - req(input$RT_PAR.Sample.ID, input$RT_PAR.Sample.Max, - input$RT_PAR.Sample.Step, input$RT_PAR.Sample.Min, - input$RT_PAR.Sample.Param) - - fstart <- format_FV(input$RT_PAR.Sample.Min) %>% as.numeric - fstop <- format_FV(input$RT_PAR.Sample.Max) %>% as.numeric - fstep <- format_FV(input$RT_PAR.Sample.Step) %>% as.numeric - data <- DATA() - data <- subset(data, ID %in% input$RT_PAR.Sample.ID) - - if (!input$RT_PAR.Sample.Single) { - req(fstart <= fstop, fstep <= fstop - fstart) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, by = fstep) - req(fseq) - } - else - fseq <- fstart - - df <- get_PAR_sample(data, idxValue = fseq, - parId = input$RT_PAR.Sample.Param, - output = input$RT_PAR.Sample.Format) - - for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) - df[[p]] %<>% format(digits = 2, nsmall = 2) - - df -}) - -output$table_RT_PAR_SAMPLE <- DT::renderDataTable({ - dt <- rt_parameter_sample() - req(length(dt) != 0) - dt[is.na(dt)] <- 'NA' - dt -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$table_RT_PAR_summary <- DT::renderDataTable({ - rt_parameter_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) - -output$RT_PAR.Sample.Download <- downloadHandler( - filename = function() { - eval(RT_PARSample_csv_name) - }, - content = function(file) { - df <- parameter_sample() - df <- df[input[["table_RT_PAR_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -output$RT_PAR.Summary.Download <- downloadHandler( - filename = function() { - eval(RT_PAR_csv_name) - }, - content = function(file) { - df <- rt_parameter_summary() - df <- df[input[["table_RT_PAR_summary_rows_all"]]] - save_table(df, file) - } + +get_data_RT_PAR_PER_FUN <- reactive({ + data <- subset(DATA(), ID %in% input$RT_PAR.Plot.Algs) + generate_data.Parameters(data, scale_log = input$RT_PAR.Plot.Logx, which = 'by_FV') +}) + +# Expected Evolution of parameters in the algorithm +render_RT_PAR_PER_FUN <- reactive({ + req(input$RT_PAR.Plot.Min, input$RT_PAR.Plot.Max) + withProgress({ + f_min <- as.numeric(format_FV(input$RT_PAR.Plot.Min)) + f_max <- as.numeric(format_FV(input$RT_PAR.Plot.Max)) + + dt <- get_data_RT_PAR_PER_FUN() + dt <- dt[parId %in% input$RT_PAR.Plot.Params] + sub_attr <- if (length(input$RT_PAR.Plot.Params) > 1) 'parId' else NULL + + lower <- 'lower' + upper <- 'upper' + if (input$RT_PAR.Plot.CI == 'None') type <- 'line' + else { + type <- 'line+ribbon' + if (input$RT_PAR.Plot.CI == 'Outer Quantiles') { + quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') + lower <- quantiles[[1]] + upper <- quantiles[[length(quantiles)]] + } + } + if (is.null(sub_attr) && type == 'line+ribbon') { + p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'line', + subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, + scale.ylog = input$RT_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = T) + p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'ribbon', + subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, + scale.ylog = input$RT_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = F, p = p) + } + else { + p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, type, + subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, + scale.ylog = input$RT_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = T) + } + p + }, + message = "Creating plot") +}) + +output$RT_PAR.Plot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_PAR_PER_FUN) + }, + content = function(file) { + save_plotly(render_RT_PAR_PER_FUN(), file) + }, + contentType = paste0('image/', input$RT_PAR.Plot.Format) +) + +output$RT_PAR.Plot.Figure <- renderPlotly({ + render_RT_PAR_PER_FUN() +}) + +# TODO: add ks test for ECDF later +# output$ks <- renderPrint({ +# target <- input$target %>% as.numeric +# df.aligneds <- aligned() +# +# running_time <- list() +# for (i in seq_along(df.aligneds)) { +# df <- df.aligneds[[i]] +# v <- rownames(df) %>% as.numeric +# idx <- order(abs(target - v))[1] +# running_time[[i]] <- df[idx, ] %>% as.vector +# } +# algorithm1 <- running_time[[1]] +# algorithm2 <- running_time[[2]] +# a <- ks.test(algorithm1, algorithm2, alternative = 'less') +# print(a) +# }) +# + +rt_parameter_summary <- reactive({ + req(input$RT_PAR.Summary.Min, input$RT_PAR.Summary.Max, input$RT_PAR.Summary.Step) + + fstart <- format_FV(input$RT_PAR.Summary.Min) %>% as.numeric + fstop <- format_FV(input$RT_PAR.Summary.Max) %>% as.numeric + fstep <- format_FV(input$RT_PAR.Summary.Step) %>% as.numeric + data <- DATA() + data <- subset(data, ID %in% input$RT_PAR.Summary.ID) + + if (!input$RT_PAR.Summary.Single) { + req(fstart <= fstop, fstep <= fstop - fstart) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, by = fstep) + req(fseq) + } + else + fseq <- fstart + + dt <- get_PAR_summary(data, fseq, parId = input$RT_PAR.Summary.Param) + req(length(dt) != 0) + + dt$runs %<>% as.integer + dt$mean %<>% format(digits = 2, nsmall = 2) + dt$median %<>% format(digits = 2, nsmall = 2) + dt$sd %<>% format(digits = 2, nsmall = 2) + + probs <- getOption("IOHanalyzer.quantiles") + + # format the integers + for (p in paste0(probs * 100, '%')) { + dt[[p]] %<>% format(digits = 2, nsmall = 2) + } + dt +}) + +rt_parameter_sample <- reactive({ + req(input$RT_PAR.Sample.ID, input$RT_PAR.Sample.Max, + input$RT_PAR.Sample.Step, input$RT_PAR.Sample.Min, + input$RT_PAR.Sample.Param) + + fstart <- format_FV(input$RT_PAR.Sample.Min) %>% as.numeric + fstop <- format_FV(input$RT_PAR.Sample.Max) %>% as.numeric + fstep <- format_FV(input$RT_PAR.Sample.Step) %>% as.numeric + data <- DATA() + data <- subset(data, ID %in% input$RT_PAR.Sample.ID) + + if (!input$RT_PAR.Sample.Single) { + req(fstart <= fstop, fstep <= fstop - fstart) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, by = fstep) + req(fseq) + } + else + fseq <- fstart + + df <- get_PAR_sample(data, idxValue = fseq, + parId = input$RT_PAR.Sample.Param, + output = input$RT_PAR.Sample.Format) + + for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) + df[[p]] %<>% format(digits = 2, nsmall = 2) + + df +}) + +output$table_RT_PAR_SAMPLE <- DT::renderDataTable({ + dt <- rt_parameter_sample() + req(length(dt) != 0) + dt[is.na(dt)] <- 'NA' + dt +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$table_RT_PAR_summary <- DT::renderDataTable({ + rt_parameter_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) + +output$RT_PAR.Sample.Download <- downloadHandler( + filename = function() { + eval(RT_PARSample_csv_name) + }, + content = function(file) { + df <- parameter_sample() + df <- df[input[["table_RT_PAR_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +output$RT_PAR.Summary.Download <- downloadHandler( + filename = function() { + eval(RT_PAR_csv_name) + }, + content = function(file) { + df <- rt_parameter_summary() + df <- df[input[["table_RT_PAR_summary_rows_all"]]] + save_table(df, file) + } ) \ No newline at end of file diff --git a/inst/shiny-server/server/Shapley_computations.R b/inst/shiny-server/server/Shapley_computations.R index 2e9c65a5..9602ca14 100644 --- a/inst/shiny-server/server/Shapley_computations.R +++ b/inst/shiny-server/server/Shapley_computations.R @@ -1,123 +1,123 @@ -output$RT_SHAPLEY <- renderPlotly({ - req(length(DATA_RAW()) > 0) - render_RT_SHAPLEY() -}) - -get_data_RT_SHAPLEY <- reactive({ - input$RTportfolio.Shapley.Refresh - dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & - funcId %in% input$RTportfolio.Shapley.Funcs & - DIM %in% input$RTportfolio.Shapley.Dims) - - - if (length(get_id(dsList)) <= 1) { - shinyjs::alert("This is an invalid configuration for this plot. \n - Please ensure that the dataset contains multiple IDs.") - return(NULL) - } - - isolate({ - targets <- RT_SHAPLEY_TARGETS_TABLE() - }) - get_shapley_values(dsList, targets, input$RTportfolio.Shapley.Logx, input$RTportfolio.Shapley.Groupsize, - input$RTportfolio.Shapley.Permsize) -}) - -render_RT_SHAPLEY <- reactive({ - withProgress({ - dt <- get_data_RT_SHAPLEY() - plot_general_data(dt, x_attr = 'ID', y_attr = 'shapley', type = 'bar', - legend_attr = 'ID', show.legend = T) - }, - message = "Creating plot") -}) - -output$RTportfolio.Shapley.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_SHAPLEY) - }, - content = function(file) { - save_plotly(render_RT_SHAPLEY(), file) - }, - contentType = paste0('image/', input$RTportfolio.Shapley.Format) -) - -RT_SHAPLEY_TARGETS_TABLE <- reactiveVal(NULL) -trigger_renderDT_shap <- reactiveVal(NULL) -proxy <- dataTableProxy('RT_SHAPLEY_TARGETS_GENERATED') - -observe({ - req(length(DATA_RAW()) > 0) - dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & - funcId %in% input$RTportfolio.Shapley.Funcs & - DIM %in% input$RTportfolio.Shapley.Dims) - req(length(dsList) > 0) - - targets <- get_ECDF_targets(dsList, input$RTportfolio.Shapley.Target_type, input$RTportfolio.Shapley.Target_number) - - RT_SHAPLEY_TARGETS_TABLE(targets) # add values to `RT_SHAPLEY_TARGETS_TABLE` - trigger_renderDT_shap(rnorm(1)) -}) - - -output$RT_SHAPLEY_TARGETS_GENERATED <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - trigger_renderDT_shap() - isolate({RT_SHAPLEY_TARGETS_TABLE()}) -}, -editable = TRUE, -rownames = FALSE, -options = list( - pageLength = 10, - lengthMenu = c(5, 10, 25, -1), - scrollX = T, - server = T, - columnDefs = list( - list( - className = 'dt-right', targets = "_all" - ) - ) -), -filter = 'top' -) - -observeEvent(input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit, { - info <- input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit - i <- info$row - j <- info$col + 1 - v <- info$value - - suppressWarnings({ - df <- RT_SHAPLEY_TARGETS_TABLE() - set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) - RT_SHAPLEY_TARGETS_TABLE(df) - }) - replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), - resetPaging = FALSE, rownames = FALSE) -}) - -observeEvent(input$RTportfolio.Shapley.Table.Upload, { - if (!is.null(input$RTportfolio.Shapley.Table.Upload)) { - df <- read.csv2(input$RTportfolio.Shapley.Table.Upload$datapath, header = T, row.names = F) %>% - as.data.table - - if (ncol(df) == ncol(RT_SHAPLEY_TARGETS_TABLE())) { - RT_SHAPLEY_TARGETS_TABLE(df) - replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), - resetPaging = FALSE, - rownames = FALSE) - } else { - RT_SHAPLEY_TARGETS_TABLE(df) - trigger_renderDT_shap(rnorm(1)) - } - } else - NULL -}) - -output$RTportfolio.Shapley.Table.Download <- downloadHandler( - filename = 'Example_ECDF_TARGETS.csv', - content = function(file) { - write.csv2(RT_SHAPLEY_TARGETS_TABLE(), file, row.names = F) - }, - contentType = "text/csv" +output$RT_SHAPLEY <- renderPlotly({ + req(length(DATA_RAW()) > 0) + render_RT_SHAPLEY() +}) + +get_data_RT_SHAPLEY <- reactive({ + input$RTportfolio.Shapley.Refresh + dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & + funcId %in% input$RTportfolio.Shapley.Funcs & + DIM %in% input$RTportfolio.Shapley.Dims) + + + if (length(get_id(dsList)) <= 1) { + shinyjs::alert("This is an invalid configuration for this plot. \n + Please ensure that the dataset contains multiple IDs.") + return(NULL) + } + + isolate({ + targets <- RT_SHAPLEY_TARGETS_TABLE() + }) + get_shapley_values(dsList, targets, input$RTportfolio.Shapley.Logx, input$RTportfolio.Shapley.Groupsize, + input$RTportfolio.Shapley.Permsize) +}) + +render_RT_SHAPLEY <- reactive({ + withProgress({ + dt <- get_data_RT_SHAPLEY() + plot_general_data(dt, x_attr = 'ID', y_attr = 'shapley', type = 'bar', + legend_attr = 'ID', show.legend = T) + }, + message = "Creating plot") +}) + +output$RTportfolio.Shapley.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_SHAPLEY) + }, + content = function(file) { + save_plotly(render_RT_SHAPLEY(), file) + }, + contentType = paste0('image/', input$RTportfolio.Shapley.Format) +) + +RT_SHAPLEY_TARGETS_TABLE <- reactiveVal(NULL) +trigger_renderDT_shap <- reactiveVal(NULL) +proxy <- dataTableProxy('RT_SHAPLEY_TARGETS_GENERATED') + +observe({ + req(length(DATA_RAW()) > 0) + dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & + funcId %in% input$RTportfolio.Shapley.Funcs & + DIM %in% input$RTportfolio.Shapley.Dims) + req(length(dsList) > 0) + + targets <- get_ECDF_targets(dsList, input$RTportfolio.Shapley.Target_type, input$RTportfolio.Shapley.Target_number) + + RT_SHAPLEY_TARGETS_TABLE(targets) # add values to `RT_SHAPLEY_TARGETS_TABLE` + trigger_renderDT_shap(rnorm(1)) +}) + + +output$RT_SHAPLEY_TARGETS_GENERATED <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + trigger_renderDT_shap() + isolate({RT_SHAPLEY_TARGETS_TABLE()}) +}, +editable = TRUE, +rownames = FALSE, +options = list( + pageLength = 10, + lengthMenu = c(5, 10, 25, -1), + scrollX = T, + server = T, + columnDefs = list( + list( + className = 'dt-right', targets = "_all" + ) + ) +), +filter = 'top' +) + +observeEvent(input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit, { + info <- input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit + i <- info$row + j <- info$col + 1 + v <- info$value + + suppressWarnings({ + df <- RT_SHAPLEY_TARGETS_TABLE() + set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) + RT_SHAPLEY_TARGETS_TABLE(df) + }) + replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), + resetPaging = FALSE, rownames = FALSE) +}) + +observeEvent(input$RTportfolio.Shapley.Table.Upload, { + if (!is.null(input$RTportfolio.Shapley.Table.Upload)) { + df <- read.csv2(input$RTportfolio.Shapley.Table.Upload$datapath, header = T, row.names = F) %>% + as.data.table + + if (ncol(df) == ncol(RT_SHAPLEY_TARGETS_TABLE())) { + RT_SHAPLEY_TARGETS_TABLE(df) + replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), + resetPaging = FALSE, + rownames = FALSE) + } else { + RT_SHAPLEY_TARGETS_TABLE(df) + trigger_renderDT_shap(rnorm(1)) + } + } else + NULL +}) + +output$RTportfolio.Shapley.Table.Download <- downloadHandler( + filename = 'Example_ECDF_TARGETS.csv', + content = function(file) { + write.csv2(RT_SHAPLEY_TARGETS_TABLE(), file, row.names = F) + }, + contentType = "text/csv" ) \ No newline at end of file diff --git a/inst/shiny-server/server/fv_stat_tests.R b/inst/shiny-server/server/fv_stat_tests.R index 6e6cf72b..f04ad7ff 100644 --- a/inst/shiny-server/server/fv_stat_tests.R +++ b/inst/shiny-server/server/fv_stat_tests.R @@ -1,161 +1,161 @@ -fv_render_heatmap <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) - Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), - bootstrap.size = 0, which = 'by_RT') - }, - message = "Creating plot") -}) - - -output$FV_Stats.Overview.Heatmap <- renderPlotly( - fv_render_heatmap() -) - -fv_create_stats_table <- reactive({ - req(length(DATA()) > 0) - req(length(get_id(DATA())) > 1) - data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) - target <- as.numeric(input$FV_Stats.Overview.Target) - df <- pairwise.test(data, target, bootstrap.size = 0, which = 'by_RT') - df <- format(df, digits = 3) - df -}) - -output$FV_Stats.Overview.Pmatrix <- DT::renderDataTable({ - fv_create_stats_table() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FV_Stats.Overview.Graph <- renderPlot({ - fv_render_graph() -}) - -fv_render_graph <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) - Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), - bootstrap.size = 0, which = 'by_RT') - }, - message = "Creating plot") -}) - - -output$FV_Stats.Overview.DownloadTable <- downloadHandler( - filename = function() { - eval(FV_Stats_table_name) - }, - content = function(file) { - df <- create_stats_table() - save_table(df, file) - } -) - -output$FV_Stats.Overview.DownloadHeatmap <- downloadHandler( - filename = function() { - eval(FV_Stats_heatmap_name) - }, - content = function(file) { - save_plotly(render_heatmap(), file) - }, - contentType = paste0('image/', input$FV_Stats.Overview.Format) -) - -fv_data_table_glicko2 <- reactive({ - input$FV_Stats.Glicko.Create - isolate({ - withProgress({ - data <- FV_glicko_data() - req(length(data) > 0 && length(get_id(data)) > 0) - nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) - df <- glicko2_ranking(data, nr_games, which = 'by_RT', - target_dt = FV_stats_glicko_targets_obj)$ratings - format(df, digits = 3) - }, message = "Creating Ranking, this might take a while") - }) -}) - -output$FV_Stats.Glicko.Dataframe <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - fv_data_table_glicko2() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -fv_render_glico2_plot <- reactive({ - isolate({ - data <- FV_glicko_data() - nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) - }) - Plot.Stats.Glicko2_Candlestick(data, nr_games, fv_data_table_glicko2(), which = 'by_RT', - target_dt = FV_stats_glicko_targets_obj) -}) - -output$FV_Stats.Glicko.Candlestick <- renderPlotly({ - fv_render_glico2_plot() -}) - -output$FV_Stats.Glicko.DownloadTable <- downloadHandler( - filename = function() { - eval(FV_Glicko2_table_name) - }, - content = function(file) { - df <- fv_data_table_glicko2() - save_table(df, file) - } -) - -output$FV_Stats.Glicko.Download <- downloadHandler( - filename = function() { - eval(FV_Glicko2_figure_name) - }, - content = function(file) { - save_plotly(fv_render_glico2_plot(), file) - }, - contentType = paste0('image/', input$FV_Stats.Glicko.Format) -) - - -FV_glicko_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.Glicko.ID)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$FV_Stats.Glicko.Dim) - data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) - if (length(unique(get_id(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple IDs for the selected functions and dimensions.") - return(NULL) - } - data -} - -FV_stats_glicko_targets <- reactive({ - data <- FV_glicko_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_RT") -}) - -FV_stats_glicko_targets_obj <- NULL - -proxy_FV_Stats.Glicko.Targets <- dataTableProxy('FV_Stats.Glicko.Targets') - -output$FV_Stats.Glicko.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - FV_stats_glicko_targets_obj <<- FV_stats_glicko_targets() - FV_stats_glicko_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$FV_Stats.Glicko.Targets_cell_edit, { - info <- input$FV_Stats.Glicko.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- FV_glicko_data() - if (is.null(data)) return(NULL) - FV_stats_glicko_targets_obj$target[[i]] <<- v - replaceData(proxy, FV_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) +fv_render_heatmap <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$FV_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) + Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), + bootstrap.size = 0, which = 'by_RT') + }, + message = "Creating plot") +}) + + +output$FV_Stats.Overview.Heatmap <- renderPlotly( + fv_render_heatmap() +) + +fv_create_stats_table <- reactive({ + req(length(DATA()) > 0) + req(length(get_id(DATA())) > 1) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) + target <- as.numeric(input$FV_Stats.Overview.Target) + df <- pairwise.test(data, target, bootstrap.size = 0, which = 'by_RT') + df <- format(df, digits = 3) + df +}) + +output$FV_Stats.Overview.Pmatrix <- DT::renderDataTable({ + fv_create_stats_table() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FV_Stats.Overview.Graph <- renderPlot({ + fv_render_graph() +}) + +fv_render_graph <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$FV_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) + Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), + bootstrap.size = 0, which = 'by_RT') + }, + message = "Creating plot") +}) + + +output$FV_Stats.Overview.DownloadTable <- downloadHandler( + filename = function() { + eval(FV_Stats_table_name) + }, + content = function(file) { + df <- create_stats_table() + save_table(df, file) + } +) + +output$FV_Stats.Overview.DownloadHeatmap <- downloadHandler( + filename = function() { + eval(FV_Stats_heatmap_name) + }, + content = function(file) { + save_plotly(render_heatmap(), file) + }, + contentType = paste0('image/', input$FV_Stats.Overview.Format) +) + +fv_data_table_glicko2 <- reactive({ + input$FV_Stats.Glicko.Create + isolate({ + withProgress({ + data <- FV_glicko_data() + req(length(data) > 0 && length(get_id(data)) > 0) + nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) + df <- glicko2_ranking(data, nr_games, which = 'by_RT', + target_dt = FV_stats_glicko_targets_obj)$ratings + format(df, digits = 3) + }, message = "Creating Ranking, this might take a while") + }) +}) + +output$FV_Stats.Glicko.Dataframe <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + fv_data_table_glicko2() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +fv_render_glico2_plot <- reactive({ + isolate({ + data <- FV_glicko_data() + nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) + }) + Plot.Stats.Glicko2_Candlestick(data, nr_games, fv_data_table_glicko2(), which = 'by_RT', + target_dt = FV_stats_glicko_targets_obj) +}) + +output$FV_Stats.Glicko.Candlestick <- renderPlotly({ + fv_render_glico2_plot() +}) + +output$FV_Stats.Glicko.DownloadTable <- downloadHandler( + filename = function() { + eval(FV_Glicko2_table_name) + }, + content = function(file) { + df <- fv_data_table_glicko2() + save_table(df, file) + } +) + +output$FV_Stats.Glicko.Download <- downloadHandler( + filename = function() { + eval(FV_Glicko2_figure_name) + }, + content = function(file) { + save_plotly(fv_render_glico2_plot(), file) + }, + contentType = paste0('image/', input$FV_Stats.Glicko.Format) +) + + +FV_glicko_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.Glicko.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$FV_Stats.Glicko.Dim) + data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +FV_stats_glicko_targets <- reactive({ + data <- FV_glicko_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_RT") +}) + +FV_stats_glicko_targets_obj <- NULL + +proxy_FV_Stats.Glicko.Targets <- dataTableProxy('FV_Stats.Glicko.Targets') + +output$FV_Stats.Glicko.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + FV_stats_glicko_targets_obj <<- FV_stats_glicko_targets() + FV_stats_glicko_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$FV_Stats.Glicko.Targets_cell_edit, { + info <- input$FV_Stats.Glicko.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- FV_glicko_data() + if (is.null(data)) return(NULL) + FV_stats_glicko_targets_obj$target[[i]] <<- v + replaceData(proxy, FV_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) }) \ No newline at end of file diff --git a/inst/shiny-server/server/general_overview.R b/inst/shiny-server/server/general_overview.R index 12a80da5..7c78d2d6 100644 --- a/inst/shiny-server/server/general_overview.R +++ b/inst/shiny-server/server/general_overview.R @@ -1,62 +1,62 @@ -overview_table_single <- reactive({ - data <- DATA() - req(length(data) > 0) - df <- get_overview(subset(data, ID %in% input$Overview.Single.ID)) - - df$budget %<>% as.numeric - df$runs %<>% as.integer - df$funcId %<>% as.integer - df$DIM %<>% as.integer - df$succ %<>% as.integer - df$"worst recorded" <- format_FV(df$"worst recorded") - df$"worst reached" <- format_FV(df$"worst reached") - df$"mean reached" <- format_FV(df$"mean reached") - df$"median reached" <- format_FV(df$"median reached") - df$"best reached" <- format_FV(df$"best reached") - df$"max evals used" %<>% as.numeric - df -}) - -output$Overview.Single.Table <- DT::renderDataTable({ - req(input$Overview.Single.ID) - overview_table_single() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$Overview.Single.Download <- downloadHandler( - filename = function() { - eval(overview_single_name) - }, - content = function(file) { - df <- overview_table_single() - df <- df[input[["Overview.Single.Table_rows_all"]]] - save_table(df, file) - } -) - -overview_table_all <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - df <- get_overview(data) - df$"worst recorded" <- format_FV(df$"worst recorded") - df$"worst reached" <- format_FV(df$"worst reached") - df$"mean reached" <- format_FV(df$"mean reached") - df$"best reached" <- format_FV(df$"best reached") - df -}) - -output$Overview.All.Table <- DT::renderDataTable({ - overview_table_all() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$Overview.All.Download <- downloadHandler( - filename = function() { - eval(overview_all_name) - }, - content = function(file) { - df <- overview_table_all() - df <- df[input[["Overview.All.Table_rows_all"]]] - save_table(df, file) - } +overview_table_single <- reactive({ + data <- DATA() + req(length(data) > 0) + df <- get_overview(subset(data, ID %in% input$Overview.Single.ID)) + + df$budget %<>% as.numeric + df$runs %<>% as.integer + df$funcId %<>% as.integer + df$DIM %<>% as.integer + df$succ %<>% as.integer + df$"worst recorded" <- format_FV(df$"worst recorded") + df$"worst reached" <- format_FV(df$"worst reached") + df$"mean reached" <- format_FV(df$"mean reached") + df$"median reached" <- format_FV(df$"median reached") + df$"best reached" <- format_FV(df$"best reached") + df$"max evals used" %<>% as.numeric + df +}) + +output$Overview.Single.Table <- DT::renderDataTable({ + req(input$Overview.Single.ID) + overview_table_single() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$Overview.Single.Download <- downloadHandler( + filename = function() { + eval(overview_single_name) + }, + content = function(file) { + df <- overview_table_single() + df <- df[input[["Overview.Single.Table_rows_all"]]] + save_table(df, file) + } +) + +overview_table_all <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + df <- get_overview(data) + df$"worst recorded" <- format_FV(df$"worst recorded") + df$"worst reached" <- format_FV(df$"worst reached") + df$"mean reached" <- format_FV(df$"mean reached") + df$"best reached" <- format_FV(df$"best reached") + df +}) + +output$Overview.All.Table <- DT::renderDataTable({ + overview_table_all() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$Overview.All.Download <- downloadHandler( + filename = function() { + eval(overview_all_name) + }, + content = function(file) { + df <- overview_table_all() + df <- df[input[["Overview.All.Table_rows_all"]]] + save_table(df, file) + } ) \ No newline at end of file diff --git a/inst/shiny-server/server/parrallel_coord.R b/inst/shiny-server/server/parrallel_coord.R index 1570f497..de990d9e 100644 --- a/inst/shiny-server/server/parrallel_coord.R +++ b/inst/shiny-server/server/parrallel_coord.R @@ -1,39 +1,39 @@ -render_parallel_coord <- reactive({ - data <- DATA() - req(attr(data, 'suite') == "SOS") - data <- subset(data, ID == input$ParCoordPlot.Algs) - withProgress({ - p <- IOH_plot_ly_default("Parallel coordinate plot", "Coordinate", "Value") - - pos <- rbindlist(lapply(data, function(ds) { - final_pos_dt <- rbindlist(lapply(ds$PAR$final_position, function(pos) - {dt <- as.data.table(t(pos))})) - colnames(final_pos_dt) <- as.character(seq_len(ncol(final_pos_dt))) - final_pos_dt[, ID := attr(ds, 'ID')] - final_pos_dt[, funcId := attr(ds, 'funcId')] - final_pos_dt[, fval_log := log10(attr(ds, 'finalFV'))] - })) - - pos2 <- pos %>% melt(id.vars = c('fval_log', 'funcId', 'ID')) - p %>% add_trace(data = pos2, x = ~variable, y = ~value, type = 'scatter', - mode = 'markers', marker = list(color = ~fval_log, colorscale = 'Viridis', - colorbar = list(title = 'Log best f(x)'), - symbol = 'cross'), - legendgroup = ~ID, opacity = 0.9, showlegend = F) %>% - layout(yaxis = list(range = c(-0.02,1.02)), xaxis = list(range = c(-0.5, get_dim(dsl) + 0.5))) - }) -}) - -output$Parallel_Coord_Plot <- renderPlotly({ - render_parallel_coord() -}) - -output$ParCoordPlot.Download <- downloadHandler( - filename = function() { - paste0('Par_coord_', input$ParCoordPlot.Algs, '.', input$ParCoordPlot.Format ) - }, - content = function(file) { - save_plotly(render_parallel_coord(), file) - }, - contentType = paste0('image/', input$ParCoordPlot.Format) +render_parallel_coord <- reactive({ + data <- DATA() + req(attr(data, 'suite') == "SOS") + data <- subset(data, ID == input$ParCoordPlot.Algs) + withProgress({ + p <- IOH_plot_ly_default("Parallel coordinate plot", "Coordinate", "Value") + + pos <- rbindlist(lapply(data, function(ds) { + final_pos_dt <- rbindlist(lapply(ds$PAR$final_position, function(pos) + {dt <- as.data.table(t(pos))})) + colnames(final_pos_dt) <- as.character(seq_len(ncol(final_pos_dt))) + final_pos_dt[, ID := attr(ds, 'ID')] + final_pos_dt[, funcId := attr(ds, 'funcId')] + final_pos_dt[, fval_log := log10(attr(ds, 'finalFV'))] + })) + + pos2 <- pos %>% melt(id.vars = c('fval_log', 'funcId', 'ID')) + p %>% add_trace(data = pos2, x = ~variable, y = ~value, type = 'scatter', + mode = 'markers', marker = list(color = ~fval_log, colorscale = 'Viridis', + colorbar = list(title = 'Log best f(x)'), + symbol = 'cross'), + legendgroup = ~ID, opacity = 0.9, showlegend = F) %>% + layout(yaxis = list(range = c(-0.02,1.02)), xaxis = list(range = c(-0.5, get_dim(dsl) + 0.5))) + }) +}) + +output$Parallel_Coord_Plot <- renderPlotly({ + render_parallel_coord() +}) + +output$ParCoordPlot.Download <- downloadHandler( + filename = function() { + paste0('Par_coord_', input$ParCoordPlot.Algs, '.', input$ParCoordPlot.Format ) + }, + content = function(file) { + save_plotly(render_parallel_coord(), file) + }, + contentType = paste0('image/', input$ParCoordPlot.Format) ) \ No newline at end of file diff --git a/inst/shiny-server/server/report_generation.R b/inst/shiny-server/server/report_generation.R index 41b105aa..15051706 100644 --- a/inst/shiny-server/server/report_generation.R +++ b/inst/shiny-server/server/report_generation.R @@ -1,40 +1,40 @@ -output$Report.Generate <- downloadHandler( - filename = "report.pdf", - content = function(file) { - generate_report(file) - }, - contentType = "pdf" -) - -output$Report.Generate.Tar <- downloadHandler( - filename = "report.tar", - content = function(file) { - get_tarbal_report(file) - }, - contentType = "tar" -) - -generate_report <- function(file){ - dir_loc <- tempdir() - dir.create(file.path(dir_loc, "Report"), showWarnings = F) - file.copy("markdown/Report/", file.path(dir_loc), overwrite = TRUE, recursive = TRUE) - tempReport <- file.path(file.path(dir_loc, "Report"), "Template_test.Rmd") - params <- list(input = input, dsl = DATA_RAW(), figure_folder = dir_loc, REG = REG) - rmarkdown::render(tempReport, output_file = file, - params = params, - envir = new.env(parent = globalenv())) -} - -get_tarbal_report <- function(file){ - dir_loc <- tempdir() - if(length(list.files(dir_loc, pattern = ".tex")) == 0){ - generate_report(tempfile(fileext = "pdf")) - } - dir.create(file.path(dir_loc, "Report_files"), showWarnings = F) - file.copy(list.files(dir_loc, pattern = ".tex", full.names = T), file.path(dir_loc, "Report_files")) - file.copy(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T), file.path(dir_loc, "Report_files")) - tar(tarfile = file, files = file.path(dir_loc, "Report_files")) - file.remove(list.files(file.path(dir_loc, "Report_files"), full.names = T)) - file.remove(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T)) - file.remove(file.path(dir_loc, list.files(dir_loc, pattern = ".tex"))) +output$Report.Generate <- downloadHandler( + filename = "report.pdf", + content = function(file) { + generate_report(file) + }, + contentType = "pdf" +) + +output$Report.Generate.Tar <- downloadHandler( + filename = "report.tar", + content = function(file) { + get_tarbal_report(file) + }, + contentType = "tar" +) + +generate_report <- function(file){ + dir_loc <- tempdir() + dir.create(file.path(dir_loc, "Report"), showWarnings = F) + file.copy("markdown/Report/", file.path(dir_loc), overwrite = TRUE, recursive = TRUE) + tempReport <- file.path(file.path(dir_loc, "Report"), "Template_test.Rmd") + params <- list(input = input, dsl = DATA_RAW(), figure_folder = dir_loc, REG = REG) + rmarkdown::render(tempReport, output_file = file, + params = params, + envir = new.env(parent = globalenv())) +} + +get_tarbal_report <- function(file){ + dir_loc <- tempdir() + if(length(list.files(dir_loc, pattern = ".tex")) == 0){ + generate_report(tempfile(fileext = "pdf")) + } + dir.create(file.path(dir_loc, "Report_files"), showWarnings = F) + file.copy(list.files(dir_loc, pattern = ".tex", full.names = T), file.path(dir_loc, "Report_files")) + file.copy(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T), file.path(dir_loc, "Report_files")) + tar(tarfile = file, files = file.path(dir_loc, "Report_files")) + file.remove(list.files(file.path(dir_loc, "Report_files"), full.names = T)) + file.remove(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T)) + file.remove(file.path(dir_loc, list.files(dir_loc, pattern = ".tex"))) } \ No newline at end of file diff --git a/inst/shiny-server/server/settings_page.R b/inst/shiny-server/server/settings_page.R index 52c17d1c..9686c15b 100644 --- a/inst/shiny-server/server/settings_page.R +++ b/inst/shiny-server/server/settings_page.R @@ -1,231 +1,231 @@ -observe({ - if (input$Settings.Color.Scheme != "Custom") { - set_color_scheme(input$Settings.Color.Scheme, get_id(DATA()), NULL) - } -}) - -output$Settings.Color.Example <- downloadHandler( - filename = function() { - "Example_Colorfile" - }, - content = function(file) { - write.csv2(get_color_scheme_dt(), file, row.names = F) - } -) - -output$Settings.Color.Plot <- renderPlotly({ - plot_color_example() -}) - -plot_color_example <- function(){ - curr_settings <- c(input$Settings.Color.Bg, - input$Settings.Color.Grid, - input$Settings.Color.Tick, - input$Settings.Legend.Location, - input$Settings.Font.Title, - input$Settings.Font.Legend, - input$Settings.Font.Label, - input$Settings.Font.Tick, - input$Settings.Color.Linewidth, - input$Settings.Color.Markersize, - input$IOHanalyzer.custom_legend_x, - input$IOHanalyzer.custom_legend_y - ) - if (any(is.null(curr_settings))) return(NULL) - if (length(DATA_RAW()) > 0) { - algnames <- get_id(DATA_RAW()) - } - else algnames <- c("Alg 1", "Alg 2", "Alg 3", "Alg 4", "Alg 5") - colors <- get_color_scheme(algnames) - schemename <- input$Settings.Color.Scheme - if (schemename == "Custom" && !is.null(input$Settings.Color.Upload)) { - schemename <- paste0(schemename, ": ", input$Settings.Color.Upload$datapath) - } - - x <- c(rep(1, length(algnames)),rep(2, length(algnames))) - y <- seq_len(length(algnames)) - dt <- data.table(ID = rep(algnames, 2), x, y) - - p <- plot_general_data(dt, 'x', 'y', 'line', show.legend = T, - x_title = 'X-title', y_title = 'Y-title', plot_title = 'Plot Title') - - p -} - -selected_color_congfig <- observe({ - if (!is.null(input$Settings.Color.Upload)) { - datapath <- input$Settings.Color.Upload$datapath - tryCatch( - expr = { - set_color_scheme("Custom", path = datapath) - }, - error = function(e) { - shinyjs::alert("File could not be read, please upload a file in the same format as the example.") - } - ) - } -}) - -observe({ - input$Settings.General.Probs %>% - strsplit(.,',') %>% - .[[1]] %>% - as.numeric %>% - options("IOHanalyzer.quantiles" = .) -}) - -observe({ - options("IOHanalyzer.max_samples" = input$Settings.General.Max_samples) -}) - -observe({ - options("IOHanalyzer.backend" = input$Settings.General.Backend) -}) - -observe({ - options("IOHanalyzer.bgcolor" = input$Settings.Color.Bg) -}) - -observe({ - options("IOHanalyzer.gridcolor" = input$Settings.Color.Grid) -}) - -observe({ - options("IOHanalyzer.tickcolor" = input$Settings.Color.Tick) -}) - -observe({ - options("IOHanalyzer.linewidth" = input$Settings.Color.Linewidth) -}) - -observe({ - options("IOHanalyzer.markersize" = input$Settings.Color.Markersize) -}) - -observe({ - options("IOHanalyzer.figure_width" = input$Settings.Download.Width) -}) - -observe({ - options("IOHanalyzer.figure_height" = input$Settings.Download.Height) -}) - -observe({ - options("IOHanalyzer.custom_legend_x" = input$Settings.Legend.LocationX) -}) - -observe({ - options("IOHanalyzer.custom_legend_y" = input$Settings.Legend.LocationY) -}) - -observe({ - legend_loc <- input$Settings.Legend.Location - if (legend_loc == "Outside, right") legend_loc_str <- "outside_right" - else if (legend_loc == "Inside, right") legend_loc_str <- "inside_right" - else if (legend_loc == "Inside, left") legend_loc_str <- "inside_left" - else if (legend_loc == "Below") legend_loc_str <- "below" - else if (legend_loc == "Custom") legend_loc_str <- "custom" - options("IOHanalyzer.legend_location" = legend_loc_str) -}) - -observe({ - options("IOHanalyzer.tick_fontsize" = input$Settings.Font.Tick) -}) - -observe({ - options("IOHanalyzer.legend_fontsize" = input$Settings.Font.Legend) -}) - -observe({ - options("IOHanalyzer.title_fontsize" = input$Settings.Font.Title) -}) - -observe({ - options("IOHanalyzer.label_fontsize" = input$Settings.Font.Label) -}) - -observe({ - options("IOHanalyzer.precision" = input$Settings.General.Precision) -}) - -observe({ - req(input$Settings.ID.Variables) - withProgress({ - id_vars <- input$Settings.ID.Variables - if (!setequal(id_vars,getOption('IOHanalyzer.ID_vars', c('algId')))) { - options("IOHanalyzer.ID_vars" = input$Settings.ID.Variables) - DataList$data <- change_id(DataList$data, input$Settings.ID.Variables) - } - if ('algId' %in% id_vars) - shinyjs::hide(id = "overall_algid_box") - else - shinyjs::show(id = "overall_algid_box") -}, message = "Processing IDs") -}) - -observe({ - if (input$Settings.Use_Funcname) { - shinyjs::show(id = "overall_funcname_box") - shinyjs::hide(id = "overall_funcid_box") - options('IOHanalyzer.function_representation' = 'funcname') - } - else { - shinyjs::hide(id = "overall_funcname_box") - shinyjs::show(id = "overall_funcid_box") - options('IOHanalyzer.function_representation' = 'funcId') - } -}) - -observe({ - setting_preset <- input$Settings.Download.Preset - if (setting_preset == "Default") { - updateNumericInput(session, 'Settings.Download.Width', value = 1000) - updateNumericInput(session, 'Settings.Download.Height', value = 1000) - updateNumericInput(session, 'Settings.Font.Tick', value = 12) - updateNumericInput(session, 'Settings.Font.Legend', value = 13) - updateNumericInput(session, 'Settings.Font.Title', value = 16) - updateNumericInput(session, 'Settings.Font.Label', value = 16) - } - else if (setting_preset == "Paper-1col") { - updateNumericInput(session, 'Settings.Download.Width', value = 700) - updateNumericInput(session, 'Settings.Download.Height', value = 400) - updateNumericInput(session, 'Settings.Font.Tick', value = 9) - updateNumericInput(session, 'Settings.Font.Legend', value = 10) - updateNumericInput(session, 'Settings.Font.Title', value = 13) - updateNumericInput(session, 'Settings.Font.Label', value = 13) - } - else if (setting_preset == "Paper-2col") { - updateNumericInput(session, 'Settings.Download.Width', value = 900) - updateNumericInput(session, 'Settings.Download.Height', value = 600) - updateNumericInput(session, 'Settings.Font.Tick', value = 11) - updateNumericInput(session, 'Settings.Font.Legend', value = 12) - updateNumericInput(session, 'Settings.Font.Title', value = 16) - updateNumericInput(session, 'Settings.Font.Label', value = 15) - } -}) - -output$Settings.Download <- downloadHandler( - filename = "IOHanalyzer_settings.rds", - content = function(file){ - curr_opts <- options() - IOH_opts <- curr_opts[grep(names(curr_opts), pattern = "IOH")] - saveRDS(IOH_opts, file) - }, - contentType = "rds" -) - -observe({ - if (!is.null(input$Settings.Upload)) { - file <- input$Settings.Upload$datapath - IOH_opts <- readRDS(file) - options(IOH_opts[grep(names(IOH_opts), pattern = "IOH")]) #Ensure no other options get changed by the user - } -}) - -output$Settings.Plot.Download <- downloadHandler( - filename = "Sample_plot.pdf", - content = function(file) { - save_plotly(plot_color_example(), file) - }, - contentType = 'image/pdf' +observe({ + if (input$Settings.Color.Scheme != "Custom") { + set_color_scheme(input$Settings.Color.Scheme, get_id(DATA()), NULL) + } +}) + +output$Settings.Color.Example <- downloadHandler( + filename = function() { + "Example_Colorfile" + }, + content = function(file) { + write.csv2(get_color_scheme_dt(), file, row.names = F) + } +) + +output$Settings.Color.Plot <- renderPlotly({ + plot_color_example() +}) + +plot_color_example <- function(){ + curr_settings <- c(input$Settings.Color.Bg, + input$Settings.Color.Grid, + input$Settings.Color.Tick, + input$Settings.Legend.Location, + input$Settings.Font.Title, + input$Settings.Font.Legend, + input$Settings.Font.Label, + input$Settings.Font.Tick, + input$Settings.Color.Linewidth, + input$Settings.Color.Markersize, + input$IOHanalyzer.custom_legend_x, + input$IOHanalyzer.custom_legend_y + ) + if (any(is.null(curr_settings))) return(NULL) + if (length(DATA_RAW()) > 0) { + algnames <- get_id(DATA_RAW()) + } + else algnames <- c("Alg 1", "Alg 2", "Alg 3", "Alg 4", "Alg 5") + colors <- get_color_scheme(algnames) + schemename <- input$Settings.Color.Scheme + if (schemename == "Custom" && !is.null(input$Settings.Color.Upload)) { + schemename <- paste0(schemename, ": ", input$Settings.Color.Upload$datapath) + } + + x <- c(rep(1, length(algnames)),rep(2, length(algnames))) + y <- seq_len(length(algnames)) + dt <- data.table(ID = rep(algnames, 2), x, y) + + p <- plot_general_data(dt, 'x', 'y', 'line', show.legend = T, + x_title = 'X-title', y_title = 'Y-title', plot_title = 'Plot Title') + + p +} + +selected_color_congfig <- observe({ + if (!is.null(input$Settings.Color.Upload)) { + datapath <- input$Settings.Color.Upload$datapath + tryCatch( + expr = { + set_color_scheme("Custom", path = datapath) + }, + error = function(e) { + shinyjs::alert("File could not be read, please upload a file in the same format as the example.") + } + ) + } +}) + +observe({ + input$Settings.General.Probs %>% + strsplit(.,',') %>% + .[[1]] %>% + as.numeric %>% + options("IOHanalyzer.quantiles" = .) +}) + +observe({ + options("IOHanalyzer.max_samples" = input$Settings.General.Max_samples) +}) + +observe({ + options("IOHanalyzer.backend" = input$Settings.General.Backend) +}) + +observe({ + options("IOHanalyzer.bgcolor" = input$Settings.Color.Bg) +}) + +observe({ + options("IOHanalyzer.gridcolor" = input$Settings.Color.Grid) +}) + +observe({ + options("IOHanalyzer.tickcolor" = input$Settings.Color.Tick) +}) + +observe({ + options("IOHanalyzer.linewidth" = input$Settings.Color.Linewidth) +}) + +observe({ + options("IOHanalyzer.markersize" = input$Settings.Color.Markersize) +}) + +observe({ + options("IOHanalyzer.figure_width" = input$Settings.Download.Width) +}) + +observe({ + options("IOHanalyzer.figure_height" = input$Settings.Download.Height) +}) + +observe({ + options("IOHanalyzer.custom_legend_x" = input$Settings.Legend.LocationX) +}) + +observe({ + options("IOHanalyzer.custom_legend_y" = input$Settings.Legend.LocationY) +}) + +observe({ + legend_loc <- input$Settings.Legend.Location + if (legend_loc == "Outside, right") legend_loc_str <- "outside_right" + else if (legend_loc == "Inside, right") legend_loc_str <- "inside_right" + else if (legend_loc == "Inside, left") legend_loc_str <- "inside_left" + else if (legend_loc == "Below") legend_loc_str <- "below" + else if (legend_loc == "Custom") legend_loc_str <- "custom" + options("IOHanalyzer.legend_location" = legend_loc_str) +}) + +observe({ + options("IOHanalyzer.tick_fontsize" = input$Settings.Font.Tick) +}) + +observe({ + options("IOHanalyzer.legend_fontsize" = input$Settings.Font.Legend) +}) + +observe({ + options("IOHanalyzer.title_fontsize" = input$Settings.Font.Title) +}) + +observe({ + options("IOHanalyzer.label_fontsize" = input$Settings.Font.Label) +}) + +observe({ + options("IOHanalyzer.precision" = input$Settings.General.Precision) +}) + +observe({ + req(input$Settings.ID.Variables) + withProgress({ + id_vars <- input$Settings.ID.Variables + if (!setequal(id_vars,getOption('IOHanalyzer.ID_vars', c('algId')))) { + options("IOHanalyzer.ID_vars" = input$Settings.ID.Variables) + DataList$data <- change_id(DataList$data, input$Settings.ID.Variables) + } + if ('algId' %in% id_vars) + shinyjs::hide(id = "overall_algid_box") + else + shinyjs::show(id = "overall_algid_box") +}, message = "Processing IDs") +}) + +observe({ + if (input$Settings.Use_Funcname) { + shinyjs::show(id = "overall_funcname_box") + shinyjs::hide(id = "overall_funcid_box") + options('IOHanalyzer.function_representation' = 'funcname') + } + else { + shinyjs::hide(id = "overall_funcname_box") + shinyjs::show(id = "overall_funcid_box") + options('IOHanalyzer.function_representation' = 'funcId') + } +}) + +observe({ + setting_preset <- input$Settings.Download.Preset + if (setting_preset == "Default") { + updateNumericInput(session, 'Settings.Download.Width', value = 1000) + updateNumericInput(session, 'Settings.Download.Height', value = 1000) + updateNumericInput(session, 'Settings.Font.Tick', value = 12) + updateNumericInput(session, 'Settings.Font.Legend', value = 13) + updateNumericInput(session, 'Settings.Font.Title', value = 16) + updateNumericInput(session, 'Settings.Font.Label', value = 16) + } + else if (setting_preset == "Paper-1col") { + updateNumericInput(session, 'Settings.Download.Width', value = 700) + updateNumericInput(session, 'Settings.Download.Height', value = 400) + updateNumericInput(session, 'Settings.Font.Tick', value = 9) + updateNumericInput(session, 'Settings.Font.Legend', value = 10) + updateNumericInput(session, 'Settings.Font.Title', value = 13) + updateNumericInput(session, 'Settings.Font.Label', value = 13) + } + else if (setting_preset == "Paper-2col") { + updateNumericInput(session, 'Settings.Download.Width', value = 900) + updateNumericInput(session, 'Settings.Download.Height', value = 600) + updateNumericInput(session, 'Settings.Font.Tick', value = 11) + updateNumericInput(session, 'Settings.Font.Legend', value = 12) + updateNumericInput(session, 'Settings.Font.Title', value = 16) + updateNumericInput(session, 'Settings.Font.Label', value = 15) + } +}) + +output$Settings.Download <- downloadHandler( + filename = "IOHanalyzer_settings.rds", + content = function(file){ + curr_opts <- options() + IOH_opts <- curr_opts[grep(names(curr_opts), pattern = "IOH")] + saveRDS(IOH_opts, file) + }, + contentType = "rds" +) + +observe({ + if (!is.null(input$Settings.Upload)) { + file <- input$Settings.Upload$datapath + IOH_opts <- readRDS(file) + options(IOH_opts[grep(names(IOH_opts), pattern = "IOH")]) #Ensure no other options get changed by the user + } +}) + +output$Settings.Plot.Download <- downloadHandler( + filename = "Sample_plot.pdf", + content = function(file) { + save_plotly(plot_color_example(), file) + }, + contentType = 'image/pdf' ) \ No newline at end of file diff --git a/inst/shiny-server/server/statistical_significance.R b/inst/shiny-server/server/statistical_significance.R index eb08f95a..18c16835 100644 --- a/inst/shiny-server/server/statistical_significance.R +++ b/inst/shiny-server/server/statistical_significance.R @@ -1,159 +1,159 @@ -render_heatmap <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) - Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), - bootstrap.size = input$RT_Stats.Overview.Samples) - }, - message = "Creating plot") -}) - - -output$RT_Stats.Overview.Heatmap <- renderPlotly( - render_heatmap() -) - -create_stats_table <- reactive({ - req(length(DATA()) > 0) - req(length(get_id(DATA())) > 1) - data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) - target <- as.numeric(input$RT_Stats.Overview.Target) - df <- pairwise.test(data, target, bootstrap.size = input$RT_Stats.Overview.Samples) - df <- format(df, digits = 3) - df -}) - -output$RT_Stats.Overview.Pmatrix <- DT::renderDataTable({ - create_stats_table() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RT_Stats.Overview.Graph <- renderPlot({ - render_graph() -}) - -render_graph <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) - Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), - bootstrap.size = input$RT_Stats.Overview.Samples) - }, - message = "Creating plot") -}) - - -output$RT_Stats.Overview.DownloadTable <- downloadHandler( - filename = function() { - eval(RT_Stats_table_name) - }, - content = function(file) { - df <- create_stats_table() - save_table(df, file) - } -) - -output$RT_Stats.Overview.DownloadHeatmap <- downloadHandler( - filename = function() { - eval(RT_Stats_heatmap_name) - }, - content = function(file) { - save_plotly(render_heatmap(), file) - }, - contentType = paste0('image/', input$RT_Stats.Overview.Format) -) - -data_table_glicko2 <- reactive({ - input$RT_Stats.Glicko.Create - isolate({ - withProgress({ - data <- RT_glicko_data() - nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) - df <- glicko2_ranking(data, nr_games, target_dt = RT_stats_glicko_targets_obj, which = 'by_FV')$ratings - format(df, digits = 3) - }, message = "Creating Ranking, this might take a while") - }) -}) - -output$RT_Stats.Glicko.Dataframe <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - data_table_glicko2() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -render_glico2_plot <- reactive({ - isolate({ - data <- RT_glicko_data() - nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) - }) - Plot.Stats.Glicko2_Candlestick(data, nr_games, data_table_glicko2(), which = 'by_FV', - target_dt = RT_stats_glicko_targets_obj) -}) - -output$RT_Stats.Glicko.Candlestick <- renderPlotly({ - render_glico2_plot() -}) - -output$RT_Stats.Glicko.DownloadTable <- downloadHandler( - filename = function() { - eval(RT_Glicko2_table_name) - }, - content = function(file) { - df <- data_table_glicko2() - save_table(df, file) - } -) - -output$RT_Stats.Glicko.Download <- downloadHandler( - filename = function() { - eval(RT_Glicko2_figure_name) - }, - content = function(file) { - save_plotly(render_glico2_plot(), file) - }, - contentType = paste0('image/', input$RT_Stats.Glicko.Format) -) - - -RT_glicko_data <- function() { - data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.Glicko.ID)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$RT_Stats.Glicko.Dim) - data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) - if (length(unique(get_id(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple IDs for the selected functions and dimensions.") - return(NULL) - } - data -} - -RT_stats_glicko_targets <- reactive({ - data <- RT_glicko_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_FV") -}) - -RT_stats_glicko_targets_obj <- NULL - -proxy_RT_Stats.Glicko.Targets <- dataTableProxy('RT_Stats.Glicko.Targets') - -output$RT_Stats.Glicko.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - RT_stats_glicko_targets_obj <<- RT_stats_glicko_targets() - RT_stats_glicko_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$RT_Stats.Glicko.Targets_cell_edit, { - info <- input$RT_Stats.Glicko.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- RT_glicko_data() - if (is.null(data)) return(NULL) - RT_stats_glicko_targets_obj$target[[i]] <<- v - replaceData(proxy, RT_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) -}) +render_heatmap <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$RT_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) + Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), + bootstrap.size = input$RT_Stats.Overview.Samples) + }, + message = "Creating plot") +}) + + +output$RT_Stats.Overview.Heatmap <- renderPlotly( + render_heatmap() +) + +create_stats_table <- reactive({ + req(length(DATA()) > 0) + req(length(get_id(DATA())) > 1) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) + target <- as.numeric(input$RT_Stats.Overview.Target) + df <- pairwise.test(data, target, bootstrap.size = input$RT_Stats.Overview.Samples) + df <- format(df, digits = 3) + df +}) + +output$RT_Stats.Overview.Pmatrix <- DT::renderDataTable({ + create_stats_table() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RT_Stats.Overview.Graph <- renderPlot({ + render_graph() +}) + +render_graph <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$RT_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) + Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), + bootstrap.size = input$RT_Stats.Overview.Samples) + }, + message = "Creating plot") +}) + + +output$RT_Stats.Overview.DownloadTable <- downloadHandler( + filename = function() { + eval(RT_Stats_table_name) + }, + content = function(file) { + df <- create_stats_table() + save_table(df, file) + } +) + +output$RT_Stats.Overview.DownloadHeatmap <- downloadHandler( + filename = function() { + eval(RT_Stats_heatmap_name) + }, + content = function(file) { + save_plotly(render_heatmap(), file) + }, + contentType = paste0('image/', input$RT_Stats.Overview.Format) +) + +data_table_glicko2 <- reactive({ + input$RT_Stats.Glicko.Create + isolate({ + withProgress({ + data <- RT_glicko_data() + nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) + df <- glicko2_ranking(data, nr_games, target_dt = RT_stats_glicko_targets_obj, which = 'by_FV')$ratings + format(df, digits = 3) + }, message = "Creating Ranking, this might take a while") + }) +}) + +output$RT_Stats.Glicko.Dataframe <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + data_table_glicko2() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +render_glico2_plot <- reactive({ + isolate({ + data <- RT_glicko_data() + nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) + }) + Plot.Stats.Glicko2_Candlestick(data, nr_games, data_table_glicko2(), which = 'by_FV', + target_dt = RT_stats_glicko_targets_obj) +}) + +output$RT_Stats.Glicko.Candlestick <- renderPlotly({ + render_glico2_plot() +}) + +output$RT_Stats.Glicko.DownloadTable <- downloadHandler( + filename = function() { + eval(RT_Glicko2_table_name) + }, + content = function(file) { + df <- data_table_glicko2() + save_table(df, file) + } +) + +output$RT_Stats.Glicko.Download <- downloadHandler( + filename = function() { + eval(RT_Glicko2_figure_name) + }, + content = function(file) { + save_plotly(render_glico2_plot(), file) + }, + contentType = paste0('image/', input$RT_Stats.Glicko.Format) +) + + +RT_glicko_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.Glicko.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$RT_Stats.Glicko.Dim) + data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +RT_stats_glicko_targets <- reactive({ + data <- RT_glicko_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_FV") +}) + +RT_stats_glicko_targets_obj <- NULL + +proxy_RT_Stats.Glicko.Targets <- dataTableProxy('RT_Stats.Glicko.Targets') + +output$RT_Stats.Glicko.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + RT_stats_glicko_targets_obj <<- RT_stats_glicko_targets() + RT_stats_glicko_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$RT_Stats.Glicko.Targets_cell_edit, { + info <- input$RT_Stats.Glicko.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- RT_glicko_data() + if (is.null(data)) return(NULL) + RT_stats_glicko_targets_obj$target[[i]] <<- v + replaceData(proxy, RT_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) +}) diff --git a/inst/shiny-server/server/tables_multi_func.R b/inst/shiny-server/server/tables_multi_func.R index 7c745c8a..bc949590 100644 --- a/inst/shiny-server/server/tables_multi_func.R +++ b/inst/shiny-server/server/tables_multi_func.R @@ -1,102 +1,102 @@ -multi_function_data_summary <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - data <- subset(data, ID %in% input$RT.MultiERT.ID & - funcId %in% input$RT.MultiERT.FuncId & - DIM %in% input$RT.MultiERT.DIM) - - get_RT_summary(data, as.numeric(input$RT.MultiERT.Target)) -}) - -output$RT.MultiERT.Table <- DT::renderDataTable({ - req(input$RT.MultiERT.ID) - multi_function_data_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RT.MultiERT.Download <- downloadHandler( - filename = function() { - eval(RT_multifunc_ERT) - }, - content = function(file) { - save_table(multi_function_data_summary(), file) - } -) - -multi_function_data_sample <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - data <- subset(data, ID %in% input$RT.Multisample.ID & - funcId %in% input$RT.Multisample.FuncId & - DIM %in% input$RT.Multisample.DIM) - - get_RT_sample(data, as.numeric(input$RT.Multisample.Target), output = input$RT.Multisample.mode) -}) - -output$RT.Multisample.Table <- DT::renderDataTable({ - req(input$RT.Multisample.ID) - multi_function_data_sample() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RT.Multisample.Download <- downloadHandler( - filename = function() { - eval(RT_multifunc_sample) - }, - content = function(file) { - save_table(multi_function_data_sample(), file) - } -) - - -multi_function_data_summary_FV <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - req(as.numeric(input$FV.MultiFV.Target) > 0) - data <- subset(data, ID %in% input$FV.MultiFV.ID & - funcId %in% input$FV.MultiFV.FuncId & - DIM %in% input$FV.MultiFV.DIM) - - get_FV_summary(data, as.numeric(input$FV.MultiFV.Target)) -}) - -output$FV.MultiFV.Table <- DT::renderDataTable({ - req(input$FV.MultiFV.ID) - multi_function_data_summary_FV() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FV.MultiFV.Download <- downloadHandler( - filename = function() { - eval(FV_multifunc_FV) - }, - content = function(file) { - save_table(multi_function_data_summary_FV(), file) - } -) - -multi_function_data_sample_FV <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - data <- subset(data, ID %in% input$FV.Multisample.ID & - funcId %in% input$FV.Multisample.FuncId & - DIM %in% input$FV.Multisample.DIM) - - get_FV_sample(data, as.numeric(input$FV.Multisample.Target), output = input$FV.Multisample.mode) -}) - -output$FV.Multisample.Table <- DT::renderDataTable({ - req(input$FV.Multisample.ID) - multi_function_data_sample_FV() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FV.Multisample.Download <- downloadHandler( - filename = function() { - eval(FV_multifunc_sample) - }, - content = function(file) { - save_table(multi_function_data_sample_FV(), file) - } -) - +multi_function_data_summary <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + data <- subset(data, ID %in% input$RT.MultiERT.ID & + funcId %in% input$RT.MultiERT.FuncId & + DIM %in% input$RT.MultiERT.DIM) + + get_RT_summary(data, as.numeric(input$RT.MultiERT.Target)) +}) + +output$RT.MultiERT.Table <- DT::renderDataTable({ + req(input$RT.MultiERT.ID) + multi_function_data_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RT.MultiERT.Download <- downloadHandler( + filename = function() { + eval(RT_multifunc_ERT) + }, + content = function(file) { + save_table(multi_function_data_summary(), file) + } +) + +multi_function_data_sample <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + data <- subset(data, ID %in% input$RT.Multisample.ID & + funcId %in% input$RT.Multisample.FuncId & + DIM %in% input$RT.Multisample.DIM) + + get_RT_sample(data, as.numeric(input$RT.Multisample.Target), output = input$RT.Multisample.mode) +}) + +output$RT.Multisample.Table <- DT::renderDataTable({ + req(input$RT.Multisample.ID) + multi_function_data_sample() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RT.Multisample.Download <- downloadHandler( + filename = function() { + eval(RT_multifunc_sample) + }, + content = function(file) { + save_table(multi_function_data_sample(), file) + } +) + + +multi_function_data_summary_FV <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + req(as.numeric(input$FV.MultiFV.Target) > 0) + data <- subset(data, ID %in% input$FV.MultiFV.ID & + funcId %in% input$FV.MultiFV.FuncId & + DIM %in% input$FV.MultiFV.DIM) + + get_FV_summary(data, as.numeric(input$FV.MultiFV.Target)) +}) + +output$FV.MultiFV.Table <- DT::renderDataTable({ + req(input$FV.MultiFV.ID) + multi_function_data_summary_FV() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FV.MultiFV.Download <- downloadHandler( + filename = function() { + eval(FV_multifunc_FV) + }, + content = function(file) { + save_table(multi_function_data_summary_FV(), file) + } +) + +multi_function_data_sample_FV <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + data <- subset(data, ID %in% input$FV.Multisample.ID & + funcId %in% input$FV.Multisample.FuncId & + DIM %in% input$FV.Multisample.DIM) + + get_FV_sample(data, as.numeric(input$FV.Multisample.Target), output = input$FV.Multisample.mode) +}) + +output$FV.Multisample.Table <- DT::renderDataTable({ + req(input$FV.Multisample.ID) + multi_function_data_sample_FV() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FV.Multisample.Download <- downloadHandler( + filename = function() { + eval(FV_multifunc_sample) + }, + content = function(file) { + save_table(multi_function_data_sample_FV(), file) + } +) + diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index a32ba10e..328a4725 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -1,734 +1,734 @@ -# shared reactive variables -folderList <- reactiveValues(data = list()) -DataList <- reactiveValues(data = DataSetList()) - -observe({ - repo_dir <- get_repo_location() - dirs <- list.dirs(repo_dir, full.names = F) - if (length(dirs) == 0) { - shinyjs::alert("No repository directory found. To make use of the IOHProfiler-repository, - please create a folder called 'repository' in your home directory - and make sure it contains at least one '.rds'-file of a DataSetList-object, - such as the ones provided on the IOHProfiler github-page.") - } - else - updateSelectInput(session, 'repository.type', choices = dirs, selected = dirs[[1]]) -}) - -# set up list of datasets (scan the repository, looking for .rds files) -observe({ - req(input$repository.type) - repo_dir <- get_repo_location() - dir <- file.path(repo_dir, input$repository.type) - - rds_files <- list.files(dir, pattern = '.rds$') %>% sub('\\.rds$', '', .) - if (length(rds_files) != 0) { - updateSelectInput(session, 'repository.dataset', choices = rds_files, selected = rds_files[[1]]) - } else {# TODO: the alert msg should be updated - shinyjs::alert("No repository file found. To make use of the IOHProfiler-repository, - please create a folder called 'repository' in your home directory - and make sure it contains at least one '.rds'-file of a DataSetList-object, - such as the ones provided on the IOHProfiler github-page.") - updateSelectInput(session, 'repository.dataset', choices = NULL, selected = NULL) - } -}) - -# load repository that is selected -observeEvent(input$repository.dataset, { - req(input$repository.dataset) - repo_dir <- get_repo_location() - algs <- c() - dims <- c() - funcs <- c() - - for (f in input$repository.dataset) { - rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) - if (file.exists(paste0(rds_file, '_info'))) { - info <- readRDS(paste0(rds_file, '_info')) - } - else { - dsl <- readRDS(rds_file) - info = list(algId = get_algId(dsl), funcId = get_funcId(dsl), DIM = get_dim(dsl)) - } - algs <- c(algs, info$algId) - dims <- c(dims, info$DIM) - funcs <- c(funcs, info$funcId) - } - - algs <- unique(algs) - dims <- unique(dims) - funcs <- unique(funcs) - - updateSelectInput(session, 'repository.ID', choices = algs, selected = algs) - updateSelectInput(session, 'repository.dim', choices = dims, selected = dims) - updateSelectInput(session, 'repository.funcId', choices = funcs, selected = funcs) - shinyjs::enable('repository.load_button') -}) - -# add the data from repository -observeEvent(input$repository.load_button, { - data <- DataSetList() - repo_dir <- get_repo_location() - for (f in input$repository.dataset) { - rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) - data <- c(data, readRDS(rds_file)) - } - data <- subset(data, funcId %in% input$repository.funcId) - data <- subset(data, DIM %in% input$repository.dim) - data <- subset(data, algId %in% input$repository.ID) - - if (length(DataList$data) > 0 && attr(data, 'maximization') != attr(DataList$data, 'maximization')) { - shinyjs::alert(paste0("Attempting to add data from a different optimization type to the currently", - " loaded data.\nPlease either remove the currently loaded data or", - " choose a different dataset to load.")) - return(NULL) - } - - DataList$data <- c(DataList$data, data) - DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) - update_menu_visibility(attr(DataList$data, 'suite')) - # set_format_func(attr(DataList$data, 'suite')) - IDs <- get_id(DataList$data) - if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { - set_color_scheme("Default", IDs) - } -}) - -# decompress zip files recursively and return the root directory of extracted files -unzip_fct_recursive <- function(zipfile, exdir, print_fun = print, alert_fun = print, depth = 0) { - filetype <- basename(zipfile) %>% - strsplit('\\.') %>% `[[`(1) %>% - rev %>% - `[`(1) - folders <- list() - - if (filetype == 'zip') - unzip_fct <- unzip - else if (filetype %in% c('bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')) - unzip_fct <- untar - - files <- unzip_fct(zipfile, list = FALSE, exdir = file.path(exdir, rand_strings(1))) - if (length(files) == 0) { - alert_fun("An error occured while unzipping the provided files.\n - Please ensure no archives are corrupted and the filenames are - in base-64.") - return(NULL) - } - print_fun(paste0('

Succesfully unzipped ', basename(zipfile), '.
')) - - folders <- grep('*.info|csv|txt', files, value = T) %>% - dirname %>% - unique %>% - grep('__MACOSX', ., value = T, invert = T) %>% # to get rid of __MACOSX folder on MAC.. - c(folders) - - zip_files <- grep('.*zip|bz2|bz|gz|tar|tgz|tar\\.gz|xz', files, value = T, perl = T) %>% - grep('__MACOSX', ., value = T, invert = T) - - if (depth <= 3) { # only allow for 4 levels of recursions - for (zipfile in zip_files) { - .folders <- unzip_fct_recursive(zipfile, dirname(zipfile), alert_fun, - print_fun, depth + 1) - folders <- c(folders, .folders) - } - } - unique(folders) -} - -# upload the compressed the data file and uncompress them -selected_folders <- reactive({ - if (is.null(input$upload.add_zip)) return(NULL) - - tryCatch({ - datapath <- input$upload.add_zip$datapath - folders <- c() - - for (i in seq(datapath)) { - filetype <- basename(datapath[i]) %>% - strsplit('\\.') %>% `[[`(1) %>% - rev %>% - `[`(1) - - print_html(paste0('

Handling ', filetype, '-data.
')) - if (filetype == 'csv' || filetype == 'rds') { - # add the data path to the folders list direct - folders <- c(folders, datapath[[i]]) - next - } - - .folders <- unzip_fct_recursive(datapath[i], exdir, print_html) %>% unique - folders %<>% c(.folders) - } - unique(folders) - }, error = function(e) shinyjs::alert(paste0("The following error occured when processing the uploaded data: ", e)) - ) -}) - -# load, process the data folders, and update DataSetList -observeEvent(selected_folders(), { - withProgress({ - folders <- selected_folders() - req(length(folders) != 0) - - format_selected <- input$upload.data_format - maximization <- input$upload.maximization - - if (maximization == "AUTOMATIC") maximization <- NULL - - if (length(folderList$data) == 0) - folder_new <- folders - else - folder_new <- setdiff(folders, intersect(folderList$data, folders)) - - req(length(folder_new) != 0) - format_detected <- lapply(folder_new, check_format) %>% unique - - if (length(format_detected) > 1) - print_html(paste('

more than one format:
', - format_detected, - 'is detected in the uploaded data set... skip the uploaded data')) - else - format_detected <- format_detected[[1]] - - print_html(paste0('

Data processing of source type:', format_detected, '
')) - - for (folder in folder_new) { - indexFiles <- scan_index_file(folder) - - if (length(indexFiles) == 0 && !(format_detected %in% c(NEVERGRAD, "SOS", "RDS"))) - print_html(paste('

No .info-files detected in the - uploaded folder, while they were expected:

', folder)) - else { - folderList$data <- c(folderList$data, folder) - - if (format_detected == "RDS") { - new_data <- readRDS(folder) - if (!("DataSetList" %in% class(new_data))) - DataSetList() - } - else { - # read the data set and handle potential errors - new_data <- tryCatch( - DataSetList(folder, print_fun = print_html, - maximization = maximization, - format = format_detected, - subsampling = input$upload.subsampling), - error = function(e) { - print_html(paste('

The following error happened - when processing the data set:

')) - print_html(paste('

', e, '

')) - DataSetList() - } - ) - } - tryCatch( - DataList$data <- c(DataList$data, new_data), - error = function(e) { - print_html(paste('

The following error happened', - 'when adding the uploaded data set:

')) - print_html(paste('

', e, - '\nRemoving the old data.

')) - DataList$data <- new_data - } - ) - shinyjs::html("upload_data_promt", - sprintf('%d: %s\n', length(folderList$data), folder), - add = TRUE) - } - } - - DataList$data <- clean_DataSetList(DataList$data) - DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) - if (is.null(DataList$data)) { - shinyjs::alert("An error occurred when processing the uploaded data. - Please ensure the data is not corrupted.") - return(NULL) - } - - update_menu_visibility(attr(DataList$data, 'suite')) - # set_format_func(attr(DataList$data, 'suite')) - IDs <- get_id(DataList$data) - if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { - set_color_scheme("Default", IDs) - } - }, message = "Processing data, this might take some time") -}) - -update_menu_visibility <- function(suite){ - if (all(suite == NEVERGRAD)) { - #TODO: Better way of doing this such that these pages are not even populated with data instead of just being hidden - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "#shiny-tab-ERT")) - } - else{ - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "#shiny-tab-ERT")) - } - if (all(suite == "PBO")) { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "FCE_ECDF")) - } - else { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "FCE_ECDF")) - } - if (any(suite == "SOS")) { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "Positions")) - } - else { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "Positions")) - } - if (!is.null(get_funcName(DataList$data))) - shinyjs::enable("Settings.Use_Funcname") - else - shinyjs::disable("Settings.Use_Funcname") -} - -observeEvent(input$Upload.Add_to_repo, { - data <- DATA_RAW() - repo_dir <- get_repo_location() - if (!file.exists(file.path(repo_dir, "public_repo"))) return(NULL) - nr_skipped <- 0 - public_dir <- file.path(repo_dir, "public_repo") - for (algname in get_algId(data)) { - filename <- file.path(public_dir, paste0(algname, '.rds')) - if (file.exists(filename)) { - nr_skipped <- nr_skipped + 1 - next - } - dsl <- subset(data, algId == algname) - saveRDS(dsl, file = filename) - } - nr_success <- length(get_algId(data)) - nr_skipped - shinyjs::alert(paste0("Successfully added ", nr_success, " algorithms to the public repository.", - "A total of ", nr_skipped, " algorithms were not uploaded because an algorithm", - "of the same name already exists, and overwriting data in the public repository is not yet", - "supported.")) -}) - -# remove all uploaded data set -observeEvent(input$upload.remove_data, { - if (length(DataList$data) != 0) { - DataList$data <- DataSetList() # NOTE: this must be a `DataSetList` object - unlink(folderList$data, T) - folderList$data <- list() - - updateSelectInput(session, 'Overall.Dim', choices = c(), selected = '') - updateSelectInput(session, 'Overall.Funcid', choices = c(), selected = '') - updateSelectInput(session, 'Overall.Funcname', choices = c(), selected = '') - updateSelectInput(session, 'Overall.ID', choices = c(), selected = '') - - print_html('

all data are removed!

') - print_html('', 'upload_data_promt') - } -}) - -# show the detailed information on DataSetList -output$data_info <- renderDataTable({ - datasets <- DataList$data - if (length(datasets) != 0) - summary(datasets) - else - data.frame() -}, options = list(pageLength = 10, scrollX = F, autoWidth = TRUE, - columnDefs = list(list(width = '20px', targets = c(0, 1))))) - -# update the list of dimensionality, funcId and algId and parameter list -observe({ - data <- DataList$data - if (length(data) == 0) - return() - - # TODO: create reactive values for them - IDs <- get_id(data) - # algIds <- c(IDs, 'all') - parIds_ <- get_parId(data) - parIds <- c(parIds_, 'all') - funcIds <- get_funcId(data) - DIMs <- get_dim(data) - algIds <- get_algId(data) - - selected_ds <- data[[1]] - selected_f <- attr(selected_ds,'funcId') - selected_dim <- attr(selected_ds, 'DIM') - selected_alg <- attr(selected_ds, 'algId') - selected_ID <- get_id(selected_ds) - - updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) - if (input$Settings.Use_Funcname) - updateSelectInput(session, 'Overall.Funcname', choices = get_funcName(data), selected = get_funcName(data[1])) - - if ('algId' %in% input$Settings.ID.Variables) - updateSelectInput(session, 'Overall.ID', choices = NULL, selected = NULL) - else - updateSelectInput(session, 'Overall.ID', choices = algIds, selected = selected_alg) - - updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'Overview.Single.ID', choices = IDs, selected = IDs) - - # updateSelectInput(session, 'Report.RT.Overview-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Overview-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Overview-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.Statistics-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Statistics-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.Single_ERT-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Single_ERT-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Single_ERT-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.Multi_ERT-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Multi_ERT-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.Rank-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Rank-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.Histogram-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Histogram-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Histogram-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.PMF-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.PMF-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.PMF-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.RT.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_AUC-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.Overview-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Overview-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Overview-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.Statistics-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Statistics-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.Single_FCE-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Single_FCE-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Single_FCE-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.Multi_FCE-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Multi_FCE-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.Rank-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Rank-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.Histogram-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Histogram-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Histogram-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.PMF-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.PMF-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.PMF-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.FV.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_AUC-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.Param.Plot-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.Param.Plot-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.Param.Plot-Alg', choices = IDs, selected = IDs) - # - # updateSelectInput(session, 'Report.Param.Statistics-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.Param.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.Param.Statistics-Alg', choices = IDs, selected = IDs) - - updateSelectInput(session, 'RTportfolio.Shapley.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTportfolio.Shapley.Funcs', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RTportfolio.Shapley.Dims', choices = DIMs, selected = DIMs) - updateNumericInput(session, 'RTportfolio.Shapley.Permsize', min = 2, max = length(IDs), - value = min(10, length(IDs))) - - - updateSelectInput(session, 'RT.MultiERT.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT.MultiERT.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RT.MultiERT.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT.Multisample.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT.Multisample.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RT.Multisample.DIM', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'FV.MultiFV.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV.MultiFV.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'FV.MultiFV.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV.Multisample.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV.Multisample.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'FV.Multisample.DIM', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'RT_Stats.Glicko.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) - updateSelectInput(session, 'RT_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'RT_Stats.DSC.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RT_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - - updateSelectInput(session, 'FV_Stats.DSC.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'FV_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - - updateSelectInput(session, 'RT_Stats.Overview.ID', choices = IDs, selected = IDs) - - updateSelectInput(session, 'FV_Stats.Glicko.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) - updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'FV_Stats.Overview.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTSummary.Statistics.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTSummary.Overview.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCESummary.Overview.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTSummary.Sample.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCESummary.Statistics.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCESummary.Sample.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_PAR.Summary.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_PAR.Sample.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_PAR.Summary.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'RT_PAR.Sample.ID', choices = IDs, selected = IDs) - updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = IDs, selected = selected_ID) - updateSelectInput(session, 'ERTPlot.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'ERTPlot.Aggr_Dim.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEPlot.Multi.Algs', choices = IDs, selected = selected_ID) - updateSelectInput(session, 'FCEPlot.Aggr.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEPlot.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEPDF.Bar.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEPDF.Hist.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTPMF.Bar.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTPMF.Hist.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FV_PAR.Summary.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Sample.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Summary.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Sample.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RTECDF.Single.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTECDF.Aggr.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTECDF.AUC.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'RTECDF.Multi.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEECDF.Single.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEECDF.Mult.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'FCEECDF.AUC.Algs', choices = IDs, selected = IDs) - updateSelectInput(session, 'ParCoordPlot.Algs', choices = IDs, selected = IDs[[1]]) - updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) - updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) - - attr_choices <- get_static_attributes(data) - invalid_choices <- c('funcId', 'DIM', 'ID') - updateSelectInput(session, 'Settings.ID.Variables', choices = attr_choices[!attr_choices %in% invalid_choices], - selected = attr(data, 'ID_attributes')) -}) - - - -# update (filter) according to users selection DataSets -DATA <- reactive({ - dim <- input$Overall.Dim - req(dim) - - d <- subset(DataList$data, DIM == dim) - - if (!'algId' %in% input$Settings.ID.Variables) { - algid <- input$Overall.ID - if (!is.null(algid)) d <- subset(d, algId == algid) - } - - if (input$Settings.Use_Funcname) { - fname <- input$Overall.Funcname - if (!is.null(fname)) d <- subset(d, funcname == fname) - } - else { - fid <- input$Overall.Funcid - if (!is.null(fid)) d <- subset(d, funcId == fid) - } - - if (length(DataList$data) == 0) return(NULL) - - if (length(d) == 0 && dim != "") { - showNotification("There is no data available for this (dimension,function)-pair") - } - d -}) - -# TODO: give a different name for DATA and DATA_RAW -DATA_RAW <- reactive({ - DataList$data -}) - -# This observe statement tries to match funcid and funcname seletions so funcId can still be used internally. -# TODO: think of a better solution to ensure this matching doesn't break. -observe({ - req(length(DATA_RAW()) > 0) - fname <- input$Overall.Funcname - req(fname) - req(getOption('IOHanalyzer.function_representation', 'funcId') == 'funcname') - dsl_sub <- subset(DATA_RAW(), funcName == fname) - fids <- get_funcId(dsl_sub) - if (length(fids) == 1) { - updateSelectInput(session, 'Overall.Funcid', selected = fids) - } -}) - -MAX_ERTS_FUNC <- reactive({ - dim <- input$Overall.Dim - data <- subset(DataList$data, DIM == dim) - max_ERTs(data, aggr_on = 'funcId', maximize = attr(data, 'maximization')) -}) - -MAX_ERTS_DIM <- reactive({ - func <- input$Overall.Funcid - data <- subset(DataList$data, funcId == func) - max_ERTs(data, aggr_on = 'DIM', maximize = attr(data, 'maximization')) -}) - -MEAN_FVALS_FUNC <- reactive({ - dim <- input$Overall.Dim - data <- subset(DataList$data, DIM == dim) - mean_FVs(data, aggr_on = 'funcId') -}) - -MEAN_FVALS_DIM <- reactive({ - func <- input$Overall.Funcid - data <- subset(DataList$data, funcId == func) - mean_FVs(data, aggr_on = 'DIM') -}) - -# TODO: make this urgely snippet look better... -# register the TextInput and restore them when switching funcID and DIM -observeEvent(eval(eventExpr), { - data <- DATA() - name <- get_data_id(data) - - if (is.null(name)) - return() - - for (id in widget_id) { - REG[[id]][[name]] <<- input[[id]] - } -}) - -# update the values for the grid of target values -observe({ - data <- DATA() - v <- get_funvals(data) - name <- get_data_id(data) - req(v) - - # choose proper scaling for the function value - # v <- trans_funeval(v) # Do the scaling in seq_FV instead - q <- quantile(v, probs = c(.25, .5, .75), names = F) - - # TODO: we need to fix this for the general case!!! - length.out <- 10 - fseq <- seq_FV(v, length.out = length.out) - start <- fseq[1] - stop <- fseq[length.out] - - #TODO: Make more general - if (abs(2 * fseq[3] - fseq[2] - fseq[4]) < 1e-12) #arbitrary precision - step <- fseq[3] - fseq[2] - else - step <- log10(fseq[3]) - log10(fseq[2]) - - setTextInput(session, 'RTSummary.Statistics.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTSummary.Statistics.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTSummary.Statistics.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RTSummary.Sample.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTSummary.Sample.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTSummary.Sample.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RTECDF.Multi.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTECDF.Multi.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTECDF.Multi.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RTPMF.Bar.Target', name, alternative = format_FV(median(v))) - setTextInput(session, 'RTPMF.Hist.Target', name, alternative = format_FV(median(v))) - setTextInput(session, 'ERTPlot.Min', name, alternative = format_FV(start)) - setTextInput(session, 'ERTPlot.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'ERTPlot.Aggr.Targets', name, alternative = "") - setTextInput(session, 'FCEPlot.Aggr.Targets', name, alternative = "") - setTextInput(session, 'RTECDF.Single.Target', name, alternative = format_FV(q[2])) - setTextInput(session, 'RTECDF.AUC.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTECDF.AUC.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTECDF.AUC.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RT_PAR.Plot.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RT_PAR.Plot.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RT_PAR.Summary.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RT_PAR.Summary.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RT_PAR.Summary.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RT_PAR.Sample.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RT_PAR.Sample.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RT_PAR.Sample.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RT_Stats.Overview.Target', name, alternative = format_FV(stop)) - setTextInput(session, 'RT.Multisample.Target', name, alternative = format_FV(median(v))) - setTextInput(session, 'RT.MultiERT.Target', name, alternative = format_FV(median(v))) -}) - -# update the values for the grid of running times -observe({ - data <- DATA() - v <- get_runtimes(data) - name <- get_data_id(data) - - if (length(v) == 0) return() - - # TODO: this part should be made generic!!! - q <- quantile(v, probs = c(.25, .5, .75), names = F, type = 3) - - grid <- seq(min(v), max(v), length.out = 10) - step <- max(1, min(grid[-1] - grid[-length(grid)])) - - start <- min(v) - stop <- max(v) - setTextInput(session, 'FV.Multisample.Target', name, alternative = max(v)) - setTextInput(session, 'FV.MultiFV.Target', name, alternative = max(v)) - setTextInput(session, 'FCESummary.Statistics.Min', name, alternative = min(v)) - setTextInput(session, 'FCESummary.Statistics.Max', name, alternative = max(v)) - setTextInput(session, 'FCESummary.Statistics.Step', name, alternative = step) - setTextInput(session, 'FCESummary.Sample.Min', name, alternative = min(v)) - setTextInput(session, 'FCESummary.Sample.Max', name, alternative = max(v)) - setTextInput(session, 'FCESummary.Sample.Step', name, alternative = step) - setTextInput(session, 'FCEPDF.Hist.Runtime', name, alternative = median(v)) - setTextInput(session, 'FCEPDF.Bar.Runtime', name, alternative = median(v)) - setTextInput(session, 'FCEPlot.Min', name, alternative = start) - setTextInput(session, 'FCEPlot.Max', name, alternative = stop) - setTextInput(session, 'FCEECDF.Mult.Min', name, alternative = min(v)) - setTextInput(session, 'FCEECDF.Mult.Max', name, alternative = max(v)) - setTextInput(session, 'FCEECDF.Mult.Step', name, alternative = step) - setTextInput(session, 'FCEECDF.AUC.Min', name, alternative = min(v)) - setTextInput(session, 'FCEECDF.AUC.Max', name, alternative = max(v)) - setTextInput(session, 'FCEECDF.AUC.Step', name, alternative = step) - setTextInput(session, 'FV_PAR.Plot.Min', name, alternative = min(v)) - setTextInput(session, 'FV_PAR.Plot.Max', name, alternative = max(v)) - setTextInput(session, 'FV_PAR.Summary.Min', name, alternative = min(v)) - setTextInput(session, 'FV_PAR.Summary.Max', name, alternative = max(v)) - setTextInput(session, 'FV_PAR.Summary.Step', name, alternative = step) - setTextInput(session, 'FV_PAR.Sample.Min', name, alternative = min(v)) - setTextInput(session, 'FV_PAR.Sample.Max', name, alternative = max(v)) - setTextInput(session, 'FV_PAR.Sample.Step', name, alternative = step) - setTextInput(session, 'FV_Stats.Overview.Target', name, alternative = max(v)) - #TODO: remove q and replace by single number - setTextInput(session, 'FCEECDF.Single.Target', name, alternative = q[2]) -}) - -output$upload.Download_processed <- downloadHandler( - filename = "DataSetList.rds", - content = function(file) { - saveRDS(DATA_RAW(), file = file) - } -) - +# shared reactive variables +folderList <- reactiveValues(data = list()) +DataList <- reactiveValues(data = DataSetList()) + +observe({ + repo_dir <- get_repo_location() + dirs <- list.dirs(repo_dir, full.names = F) + if (length(dirs) == 0) { + shinyjs::alert("No repository directory found. To make use of the IOHProfiler-repository, + please create a folder called 'repository' in your home directory + and make sure it contains at least one '.rds'-file of a DataSetList-object, + such as the ones provided on the IOHProfiler github-page.") + } + else + updateSelectInput(session, 'repository.type', choices = dirs, selected = dirs[[1]]) +}) + +# set up list of datasets (scan the repository, looking for .rds files) +observe({ + req(input$repository.type) + repo_dir <- get_repo_location() + dir <- file.path(repo_dir, input$repository.type) + + rds_files <- list.files(dir, pattern = '.rds$') %>% sub('\\.rds$', '', .) + if (length(rds_files) != 0) { + updateSelectInput(session, 'repository.dataset', choices = rds_files, selected = rds_files[[1]]) + } else {# TODO: the alert msg should be updated + shinyjs::alert("No repository file found. To make use of the IOHProfiler-repository, + please create a folder called 'repository' in your home directory + and make sure it contains at least one '.rds'-file of a DataSetList-object, + such as the ones provided on the IOHProfiler github-page.") + updateSelectInput(session, 'repository.dataset', choices = NULL, selected = NULL) + } +}) + +# load repository that is selected +observeEvent(input$repository.dataset, { + req(input$repository.dataset) + repo_dir <- get_repo_location() + algs <- c() + dims <- c() + funcs <- c() + + for (f in input$repository.dataset) { + rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) + if (file.exists(paste0(rds_file, '_info'))) { + info <- readRDS(paste0(rds_file, '_info')) + } + else { + dsl <- readRDS(rds_file) + info = list(algId = get_algId(dsl), funcId = get_funcId(dsl), DIM = get_dim(dsl)) + } + algs <- c(algs, info$algId) + dims <- c(dims, info$DIM) + funcs <- c(funcs, info$funcId) + } + + algs <- unique(algs) + dims <- unique(dims) + funcs <- unique(funcs) + + updateSelectInput(session, 'repository.ID', choices = algs, selected = algs) + updateSelectInput(session, 'repository.dim', choices = dims, selected = dims) + updateSelectInput(session, 'repository.funcId', choices = funcs, selected = funcs) + shinyjs::enable('repository.load_button') +}) + +# add the data from repository +observeEvent(input$repository.load_button, { + data <- DataSetList() + repo_dir <- get_repo_location() + for (f in input$repository.dataset) { + rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) + data <- c(data, readRDS(rds_file)) + } + data <- subset(data, funcId %in% input$repository.funcId) + data <- subset(data, DIM %in% input$repository.dim) + data <- subset(data, algId %in% input$repository.ID) + + if (length(DataList$data) > 0 && attr(data, 'maximization') != attr(DataList$data, 'maximization')) { + shinyjs::alert(paste0("Attempting to add data from a different optimization type to the currently", + " loaded data.\nPlease either remove the currently loaded data or", + " choose a different dataset to load.")) + return(NULL) + } + + DataList$data <- c(DataList$data, data) + DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) + update_menu_visibility(attr(DataList$data, 'suite')) + # set_format_func(attr(DataList$data, 'suite')) + IDs <- get_id(DataList$data) + if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { + set_color_scheme("Default", IDs) + } +}) + +# decompress zip files recursively and return the root directory of extracted files +unzip_fct_recursive <- function(zipfile, exdir, print_fun = print, alert_fun = print, depth = 0) { + filetype <- basename(zipfile) %>% + strsplit('\\.') %>% `[[`(1) %>% + rev %>% + `[`(1) + folders <- list() + + if (filetype == 'zip') + unzip_fct <- unzip + else if (filetype %in% c('bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')) + unzip_fct <- untar + + files <- unzip_fct(zipfile, list = FALSE, exdir = file.path(exdir, rand_strings(1))) + if (length(files) == 0) { + alert_fun("An error occured while unzipping the provided files.\n + Please ensure no archives are corrupted and the filenames are + in base-64.") + return(NULL) + } + print_fun(paste0('

Succesfully unzipped ', basename(zipfile), '.
')) + + folders <- grep('*.info|csv|txt', files, value = T) %>% + dirname %>% + unique %>% + grep('__MACOSX', ., value = T, invert = T) %>% # to get rid of __MACOSX folder on MAC.. + c(folders) + + zip_files <- grep('.*zip|bz2|bz|gz|tar|tgz|tar\\.gz|xz', files, value = T, perl = T) %>% + grep('__MACOSX', ., value = T, invert = T) + + if (depth <= 3) { # only allow for 4 levels of recursions + for (zipfile in zip_files) { + .folders <- unzip_fct_recursive(zipfile, dirname(zipfile), alert_fun, + print_fun, depth + 1) + folders <- c(folders, .folders) + } + } + unique(folders) +} + +# upload the compressed the data file and uncompress them +selected_folders <- reactive({ + if (is.null(input$upload.add_zip)) return(NULL) + + tryCatch({ + datapath <- input$upload.add_zip$datapath + folders <- c() + + for (i in seq(datapath)) { + filetype <- basename(datapath[i]) %>% + strsplit('\\.') %>% `[[`(1) %>% + rev %>% + `[`(1) + + print_html(paste0('

Handling ', filetype, '-data.
')) + if (filetype == 'csv' || filetype == 'rds') { + # add the data path to the folders list direct + folders <- c(folders, datapath[[i]]) + next + } + + .folders <- unzip_fct_recursive(datapath[i], exdir, print_html) %>% unique + folders %<>% c(.folders) + } + unique(folders) + }, error = function(e) shinyjs::alert(paste0("The following error occured when processing the uploaded data: ", e)) + ) +}) + +# load, process the data folders, and update DataSetList +observeEvent(selected_folders(), { + withProgress({ + folders <- selected_folders() + req(length(folders) != 0) + + format_selected <- input$upload.data_format + maximization <- input$upload.maximization + + if (maximization == "AUTOMATIC") maximization <- NULL + + if (length(folderList$data) == 0) + folder_new <- folders + else + folder_new <- setdiff(folders, intersect(folderList$data, folders)) + + req(length(folder_new) != 0) + format_detected <- lapply(folder_new, check_format) %>% unique + + if (length(format_detected) > 1) + print_html(paste('

more than one format:
', + format_detected, + 'is detected in the uploaded data set... skip the uploaded data')) + else + format_detected <- format_detected[[1]] + + print_html(paste0('

Data processing of source type:', format_detected, '
')) + + for (folder in folder_new) { + indexFiles <- scan_index_file(folder) + + if (length(indexFiles) == 0 && !(format_detected %in% c(NEVERGRAD, "SOS", "RDS"))) + print_html(paste('

No .info-files detected in the + uploaded folder, while they were expected:

', folder)) + else { + folderList$data <- c(folderList$data, folder) + + if (format_detected == "RDS") { + new_data <- readRDS(folder) + if (!("DataSetList" %in% class(new_data))) + DataSetList() + } + else { + # read the data set and handle potential errors + new_data <- tryCatch( + DataSetList(folder, print_fun = print_html, + maximization = maximization, + format = format_detected, + subsampling = input$upload.subsampling), + error = function(e) { + print_html(paste('

The following error happened + when processing the data set:

')) + print_html(paste('

', e, '

')) + DataSetList() + } + ) + } + tryCatch( + DataList$data <- c(DataList$data, new_data), + error = function(e) { + print_html(paste('

The following error happened', + 'when adding the uploaded data set:

')) + print_html(paste('

', e, + '\nRemoving the old data.

')) + DataList$data <- new_data + } + ) + shinyjs::html("upload_data_promt", + sprintf('%d: %s\n', length(folderList$data), folder), + add = TRUE) + } + } + + DataList$data <- clean_DataSetList(DataList$data) + DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) + if (is.null(DataList$data)) { + shinyjs::alert("An error occurred when processing the uploaded data. + Please ensure the data is not corrupted.") + return(NULL) + } + + update_menu_visibility(attr(DataList$data, 'suite')) + # set_format_func(attr(DataList$data, 'suite')) + IDs <- get_id(DataList$data) + if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { + set_color_scheme("Default", IDs) + } + }, message = "Processing data, this might take some time") +}) + +update_menu_visibility <- function(suite){ + if (all(suite == NEVERGRAD)) { + #TODO: Better way of doing this such that these pages are not even populated with data instead of just being hidden + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "#shiny-tab-ERT")) + } + else{ + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "#shiny-tab-ERT")) + } + if (all(suite == "PBO")) { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "FCE_ECDF")) + } + else { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "FCE_ECDF")) + } + if (any(suite == "SOS")) { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "Positions")) + } + else { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "Positions")) + } + if (!is.null(get_funcName(DataList$data))) + shinyjs::enable("Settings.Use_Funcname") + else + shinyjs::disable("Settings.Use_Funcname") +} + +observeEvent(input$Upload.Add_to_repo, { + data <- DATA_RAW() + repo_dir <- get_repo_location() + if (!file.exists(file.path(repo_dir, "public_repo"))) return(NULL) + nr_skipped <- 0 + public_dir <- file.path(repo_dir, "public_repo") + for (algname in get_algId(data)) { + filename <- file.path(public_dir, paste0(algname, '.rds')) + if (file.exists(filename)) { + nr_skipped <- nr_skipped + 1 + next + } + dsl <- subset(data, algId == algname) + saveRDS(dsl, file = filename) + } + nr_success <- length(get_algId(data)) - nr_skipped + shinyjs::alert(paste0("Successfully added ", nr_success, " algorithms to the public repository.", + "A total of ", nr_skipped, " algorithms were not uploaded because an algorithm", + "of the same name already exists, and overwriting data in the public repository is not yet", + "supported.")) +}) + +# remove all uploaded data set +observeEvent(input$upload.remove_data, { + if (length(DataList$data) != 0) { + DataList$data <- DataSetList() # NOTE: this must be a `DataSetList` object + unlink(folderList$data, T) + folderList$data <- list() + + updateSelectInput(session, 'Overall.Dim', choices = c(), selected = '') + updateSelectInput(session, 'Overall.Funcid', choices = c(), selected = '') + updateSelectInput(session, 'Overall.Funcname', choices = c(), selected = '') + updateSelectInput(session, 'Overall.ID', choices = c(), selected = '') + + print_html('

all data are removed!

') + print_html('', 'upload_data_promt') + } +}) + +# show the detailed information on DataSetList +output$data_info <- renderDataTable({ + datasets <- DataList$data + if (length(datasets) != 0) + summary(datasets) + else + data.frame() +}, options = list(pageLength = 10, scrollX = F, autoWidth = TRUE, + columnDefs = list(list(width = '20px', targets = c(0, 1))))) + +# update the list of dimensionality, funcId and algId and parameter list +observe({ + data <- DataList$data + if (length(data) == 0) + return() + + # TODO: create reactive values for them + IDs <- get_id(data) + # algIds <- c(IDs, 'all') + parIds_ <- get_parId(data) + parIds <- c(parIds_, 'all') + funcIds <- get_funcId(data) + DIMs <- get_dim(data) + algIds <- get_algId(data) + + selected_ds <- data[[1]] + selected_f <- attr(selected_ds,'funcId') + selected_dim <- attr(selected_ds, 'DIM') + selected_alg <- attr(selected_ds, 'algId') + selected_ID <- get_id(selected_ds) + + updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) + updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) + if (input$Settings.Use_Funcname) + updateSelectInput(session, 'Overall.Funcname', choices = get_funcName(data), selected = get_funcName(data[1])) + + if ('algId' %in% input$Settings.ID.Variables) + updateSelectInput(session, 'Overall.ID', choices = NULL, selected = NULL) + else + updateSelectInput(session, 'Overall.ID', choices = algIds, selected = selected_alg) + + updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'Overview.Single.ID', choices = IDs, selected = IDs) + + # updateSelectInput(session, 'Report.RT.Overview-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Overview-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Overview-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Statistics-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Statistics-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Statistics-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Single_ERT-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Single_ERT-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Single_ERT-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Multi_ERT-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Multi_ERT-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Rank-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Rank-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Histogram-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Histogram-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Histogram-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.PMF-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.PMF-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.PMF-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.ECDF_AUC-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Overview-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Overview-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Overview-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Statistics-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Statistics-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Statistics-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Single_FCE-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Single_FCE-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Single_FCE-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Multi_FCE-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Multi_FCE-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Rank-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Rank-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Histogram-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Histogram-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Histogram-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.PMF-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.PMF-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.PMF-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.ECDF_AUC-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.Param.Plot-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.Param.Plot-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.Param.Plot-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.Param.Statistics-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.Param.Statistics-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.Param.Statistics-Alg', choices = IDs, selected = IDs) + + updateSelectInput(session, 'RTportfolio.Shapley.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTportfolio.Shapley.Funcs', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RTportfolio.Shapley.Dims', choices = DIMs, selected = DIMs) + updateNumericInput(session, 'RTportfolio.Shapley.Permsize', min = 2, max = length(IDs), + value = min(10, length(IDs))) + + + updateSelectInput(session, 'RT.MultiERT.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT.MultiERT.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RT.MultiERT.DIM', choices = DIMs, selected = selected_dim) + updateSelectInput(session, 'RT.Multisample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT.Multisample.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RT.Multisample.DIM', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'FV.MultiFV.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV.MultiFV.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'FV.MultiFV.DIM', choices = DIMs, selected = selected_dim) + updateSelectInput(session, 'FV.Multisample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV.Multisample.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'FV.Multisample.DIM', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'RT_Stats.Glicko.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) + updateSelectInput(session, 'RT_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'RT_Stats.DSC.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RT_Stats.DSC.Dim', choices = DIMs, selected = DIMs) + + updateSelectInput(session, 'FV_Stats.DSC.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'FV_Stats.DSC.Dim', choices = DIMs, selected = DIMs) + + updateSelectInput(session, 'RT_Stats.Overview.ID', choices = IDs, selected = IDs) + + updateSelectInput(session, 'FV_Stats.Glicko.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) + updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'FV_Stats.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Statistics.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Statistics.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Summary.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Summary.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = IDs, selected = selected_ID) + updateSelectInput(session, 'ERTPlot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Aggr_Dim.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPlot.Multi.Algs', choices = IDs, selected = selected_ID) + updateSelectInput(session, 'FCEPlot.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPlot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPDF.Bar.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPDF.Hist.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTPMF.Bar.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTPMF.Hist.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Summary.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'FV_PAR.Sample.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'RT_PAR.Summary.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'RT_PAR.Sample.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'RTECDF.Single.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.AUC.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.Multi.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.Single.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.Mult.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.AUC.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ParCoordPlot.Algs', choices = IDs, selected = IDs[[1]]) + updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) + updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) + + attr_choices <- get_static_attributes(data) + invalid_choices <- c('funcId', 'DIM', 'ID') + updateSelectInput(session, 'Settings.ID.Variables', choices = attr_choices[!attr_choices %in% invalid_choices], + selected = attr(data, 'ID_attributes')) +}) + + + +# update (filter) according to users selection DataSets +DATA <- reactive({ + dim <- input$Overall.Dim + req(dim) + + d <- subset(DataList$data, DIM == dim) + + if (!'algId' %in% input$Settings.ID.Variables) { + algid <- input$Overall.ID + if (!is.null(algid)) d <- subset(d, algId == algid) + } + + if (input$Settings.Use_Funcname) { + fname <- input$Overall.Funcname + if (!is.null(fname)) d <- subset(d, funcname == fname) + } + else { + fid <- input$Overall.Funcid + if (!is.null(fid)) d <- subset(d, funcId == fid) + } + + if (length(DataList$data) == 0) return(NULL) + + if (length(d) == 0 && dim != "") { + showNotification("There is no data available for this (dimension,function)-pair") + } + d +}) + +# TODO: give a different name for DATA and DATA_RAW +DATA_RAW <- reactive({ + DataList$data +}) + +# This observe statement tries to match funcid and funcname seletions so funcId can still be used internally. +# TODO: think of a better solution to ensure this matching doesn't break. +observe({ + req(length(DATA_RAW()) > 0) + fname <- input$Overall.Funcname + req(fname) + req(getOption('IOHanalyzer.function_representation', 'funcId') == 'funcname') + dsl_sub <- subset(DATA_RAW(), funcName == fname) + fids <- get_funcId(dsl_sub) + if (length(fids) == 1) { + updateSelectInput(session, 'Overall.Funcid', selected = fids) + } +}) + +MAX_ERTS_FUNC <- reactive({ + dim <- input$Overall.Dim + data <- subset(DataList$data, DIM == dim) + max_ERTs(data, aggr_on = 'funcId', maximize = attr(data, 'maximization')) +}) + +MAX_ERTS_DIM <- reactive({ + func <- input$Overall.Funcid + data <- subset(DataList$data, funcId == func) + max_ERTs(data, aggr_on = 'DIM', maximize = attr(data, 'maximization')) +}) + +MEAN_FVALS_FUNC <- reactive({ + dim <- input$Overall.Dim + data <- subset(DataList$data, DIM == dim) + mean_FVs(data, aggr_on = 'funcId') +}) + +MEAN_FVALS_DIM <- reactive({ + func <- input$Overall.Funcid + data <- subset(DataList$data, funcId == func) + mean_FVs(data, aggr_on = 'DIM') +}) + +# TODO: make this urgely snippet look better... +# register the TextInput and restore them when switching funcID and DIM +observeEvent(eval(eventExpr), { + data <- DATA() + name <- get_data_id(data) + + if (is.null(name)) + return() + + for (id in widget_id) { + REG[[id]][[name]] <<- input[[id]] + } +}) + +# update the values for the grid of target values +observe({ + data <- DATA() + v <- get_funvals(data) + name <- get_data_id(data) + req(v) + + # choose proper scaling for the function value + # v <- trans_funeval(v) # Do the scaling in seq_FV instead + q <- quantile(v, probs = c(.25, .5, .75), names = F) + + # TODO: we need to fix this for the general case!!! + length.out <- 10 + fseq <- seq_FV(v, length.out = length.out) + start <- fseq[1] + stop <- fseq[length.out] + + #TODO: Make more general + if (abs(2 * fseq[3] - fseq[2] - fseq[4]) < 1e-12) #arbitrary precision + step <- fseq[3] - fseq[2] + else + step <- log10(fseq[3]) - log10(fseq[2]) + + setTextInput(session, 'RTSummary.Statistics.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTSummary.Statistics.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTSummary.Statistics.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RTSummary.Sample.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTSummary.Sample.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTSummary.Sample.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RTECDF.Multi.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTECDF.Multi.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTECDF.Multi.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RTPMF.Bar.Target', name, alternative = format_FV(median(v))) + setTextInput(session, 'RTPMF.Hist.Target', name, alternative = format_FV(median(v))) + setTextInput(session, 'ERTPlot.Min', name, alternative = format_FV(start)) + setTextInput(session, 'ERTPlot.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'ERTPlot.Aggr.Targets', name, alternative = "") + setTextInput(session, 'FCEPlot.Aggr.Targets', name, alternative = "") + setTextInput(session, 'RTECDF.Single.Target', name, alternative = format_FV(q[2])) + setTextInput(session, 'RTECDF.AUC.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTECDF.AUC.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTECDF.AUC.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RT_PAR.Plot.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RT_PAR.Plot.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RT_PAR.Summary.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RT_PAR.Summary.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RT_PAR.Summary.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RT_PAR.Sample.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RT_PAR.Sample.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RT_PAR.Sample.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RT_Stats.Overview.Target', name, alternative = format_FV(stop)) + setTextInput(session, 'RT.Multisample.Target', name, alternative = format_FV(median(v))) + setTextInput(session, 'RT.MultiERT.Target', name, alternative = format_FV(median(v))) +}) + +# update the values for the grid of running times +observe({ + data <- DATA() + v <- get_runtimes(data) + name <- get_data_id(data) + + if (length(v) == 0) return() + + # TODO: this part should be made generic!!! + q <- quantile(v, probs = c(.25, .5, .75), names = F, type = 3) + + grid <- seq(min(v), max(v), length.out = 10) + step <- max(1, min(grid[-1] - grid[-length(grid)])) + + start <- min(v) + stop <- max(v) + setTextInput(session, 'FV.Multisample.Target', name, alternative = max(v)) + setTextInput(session, 'FV.MultiFV.Target', name, alternative = max(v)) + setTextInput(session, 'FCESummary.Statistics.Min', name, alternative = min(v)) + setTextInput(session, 'FCESummary.Statistics.Max', name, alternative = max(v)) + setTextInput(session, 'FCESummary.Statistics.Step', name, alternative = step) + setTextInput(session, 'FCESummary.Sample.Min', name, alternative = min(v)) + setTextInput(session, 'FCESummary.Sample.Max', name, alternative = max(v)) + setTextInput(session, 'FCESummary.Sample.Step', name, alternative = step) + setTextInput(session, 'FCEPDF.Hist.Runtime', name, alternative = median(v)) + setTextInput(session, 'FCEPDF.Bar.Runtime', name, alternative = median(v)) + setTextInput(session, 'FCEPlot.Min', name, alternative = start) + setTextInput(session, 'FCEPlot.Max', name, alternative = stop) + setTextInput(session, 'FCEECDF.Mult.Min', name, alternative = min(v)) + setTextInput(session, 'FCEECDF.Mult.Max', name, alternative = max(v)) + setTextInput(session, 'FCEECDF.Mult.Step', name, alternative = step) + setTextInput(session, 'FCEECDF.AUC.Min', name, alternative = min(v)) + setTextInput(session, 'FCEECDF.AUC.Max', name, alternative = max(v)) + setTextInput(session, 'FCEECDF.AUC.Step', name, alternative = step) + setTextInput(session, 'FV_PAR.Plot.Min', name, alternative = min(v)) + setTextInput(session, 'FV_PAR.Plot.Max', name, alternative = max(v)) + setTextInput(session, 'FV_PAR.Summary.Min', name, alternative = min(v)) + setTextInput(session, 'FV_PAR.Summary.Max', name, alternative = max(v)) + setTextInput(session, 'FV_PAR.Summary.Step', name, alternative = step) + setTextInput(session, 'FV_PAR.Sample.Min', name, alternative = min(v)) + setTextInput(session, 'FV_PAR.Sample.Max', name, alternative = max(v)) + setTextInput(session, 'FV_PAR.Sample.Step', name, alternative = step) + setTextInput(session, 'FV_Stats.Overview.Target', name, alternative = max(v)) + #TODO: remove q and replace by single number + setTextInput(session, 'FCEECDF.Single.Target', name, alternative = q[2]) +}) + +output$upload.Download_processed <- downloadHandler( + filename = "DataSetList.rds", + content = function(file) { + saveRDS(DATA_RAW(), file = file) + } +) + diff --git a/inst/shiny-server/theme.R b/inst/shiny-server/theme.R index c9d17cab..9ededc31 100644 --- a/inst/shiny-server/theme.R +++ b/inst/shiny-server/theme.R @@ -1,101 +1,101 @@ -theme_grey_light <- shinyDashboardThemeDIY( - - ### general - appFontFamily = "Arial" - ,appFontColor = "rgb(45,45,45)" - ,primaryFontColor = "rgb(255,255,255)" - ,infoFontColor = "rgb(15,15,15)" - ,successFontColor = "rgb(15,15,15)" - ,warningFontColor = "rgb(15,15,15)" - ,dangerFontColor = "rgb(15,15,15)" - ,bodyBackColor = "rgb(240,240,240)" - - ### header - ,logoBackColor = "rgb(0,17,88)" - - ,headerButtonBackColor = "rgb(0,17,88)" - ,headerButtonIconColor = "rgb(220,220,220)" - ,headerButtonBackColorHover = "rgb(0,17,88)" - ,headerButtonIconColorHover = "rgb(60,60,60)" - - ,headerBackColor = "rgb(0,17,88)" - ,headerBoxShadowColor = "#dfdfdf" - ,headerBoxShadowSize = "3px 5px 5px" - - ### sidebar - ,sidebarBackColor = "rgb(255,255,255)" - ,sidebarPadding = 0 - - ,sidebarMenuBackColor = "transparent" - ,sidebarMenuPadding = 0 - ,sidebarMenuBorderRadius = 0 - - ,sidebarShadowRadius = "3px 5px 5px" - ,sidebarShadowColor = "#dfdfdf" - - ,sidebarUserTextColor = "rgb(115,115,115)" - - ,sidebarSearchBackColor = "rgb(240,240,240)" - ,sidebarSearchIconColor = "rgb(100,100,100)" - ,sidebarSearchBorderColor = "rgb(220,220,220)" - - ,sidebarTabTextColor = "rgb(100,100,100)" - ,sidebarTabTextSize = 14 - ,sidebarTabBorderStyle = "none" - ,sidebarTabBorderColor = "none" - ,sidebarTabBorderWidth = 0 - - ,sidebarTabBackColorSelected = "rgb(230,230,230)" - ,sidebarTabTextColorSelected = "rgb(0,0,0)" - ,sidebarTabRadiusSelected = "0px" - - ,sidebarTabBackColorHover = "rgb(245,245,245)" - ,sidebarTabTextColorHover = "rgb(0,0,0)" - ,sidebarTabBorderStyleHover = "none solid none none" - ,sidebarTabBorderColorHover = "rgb(200,200,200)" - ,sidebarTabBorderWidthHover = 4 - ,sidebarTabRadiusHover = "0px" - - ,boxBackColor = "rgb(248,248,248)" - ,boxBorderRadius = 5 - ,boxShadowSize = "none" - ,boxShadowColor = "" - ,boxTitleSize = 18 - ,boxDefaultColor = "rgb(225,225,225)" - ,boxPrimaryColor = "rgb(0,17,88)" - ,boxInfoColor = "rgb(180,180,180)" - ,boxSuccessColor = "rgb(112,173,71)" - ,boxWarningColor = "rgb(237,125,49)" - ,boxDangerColor = "rgb(232,76,34)" - - ,tabBoxTabColor = "rgb(0,17,88)" - ,tabBoxTabTextSize = 14 - ,tabBoxTabTextColor = "rgb(0,0,0)" - ,tabBoxTabTextColorSelected = "rgb(0,0,0)" - ,tabBoxBackColor = "rgb(248,248,248)" - ,tabBoxHighlightColor = "rgb(200,200,200)" - ,tabBoxBorderRadius = 5 - - ### inputs - ,buttonBackColor = "rgb(215,215,215)" - ,buttonTextColor = "rgb(45,45,45)" - ,buttonBorderColor = "rgb(150,150,150)" - ,buttonBorderRadius = 5 - - ,buttonBackColorHover = "rgb(190,190,190)" - ,buttonTextColorHover = "rgb(0,0,0)" - ,buttonBorderColorHover = "rgb(150,150,150)" - - ,textboxBackColor = "rgb(255,255,255)" - ,textboxBorderColor = "rgb(118,118,118)" - ,textboxBorderRadius = 5 - ,textboxBackColorSelect = "rgb(245,245,245)" - ,textboxBorderColorSelect = "rgb(108,108,108)" - - ### tables - ,tableBackColor = "rgb(248,248,248)" - ,tableBorderColor = "rgb(238,238,238)" - ,tableBorderTopSize = 1 - ,tableBorderRowSize = 1 - +theme_grey_light <- shinyDashboardThemeDIY( + + ### general + appFontFamily = "Arial" + ,appFontColor = "rgb(45,45,45)" + ,primaryFontColor = "rgb(255,255,255)" + ,infoFontColor = "rgb(15,15,15)" + ,successFontColor = "rgb(15,15,15)" + ,warningFontColor = "rgb(15,15,15)" + ,dangerFontColor = "rgb(15,15,15)" + ,bodyBackColor = "rgb(240,240,240)" + + ### header + ,logoBackColor = "rgb(0,17,88)" + + ,headerButtonBackColor = "rgb(0,17,88)" + ,headerButtonIconColor = "rgb(220,220,220)" + ,headerButtonBackColorHover = "rgb(0,17,88)" + ,headerButtonIconColorHover = "rgb(60,60,60)" + + ,headerBackColor = "rgb(0,17,88)" + ,headerBoxShadowColor = "#dfdfdf" + ,headerBoxShadowSize = "3px 5px 5px" + + ### sidebar + ,sidebarBackColor = "rgb(255,255,255)" + ,sidebarPadding = 0 + + ,sidebarMenuBackColor = "transparent" + ,sidebarMenuPadding = 0 + ,sidebarMenuBorderRadius = 0 + + ,sidebarShadowRadius = "3px 5px 5px" + ,sidebarShadowColor = "#dfdfdf" + + ,sidebarUserTextColor = "rgb(115,115,115)" + + ,sidebarSearchBackColor = "rgb(240,240,240)" + ,sidebarSearchIconColor = "rgb(100,100,100)" + ,sidebarSearchBorderColor = "rgb(220,220,220)" + + ,sidebarTabTextColor = "rgb(100,100,100)" + ,sidebarTabTextSize = 14 + ,sidebarTabBorderStyle = "none" + ,sidebarTabBorderColor = "none" + ,sidebarTabBorderWidth = 0 + + ,sidebarTabBackColorSelected = "rgb(230,230,230)" + ,sidebarTabTextColorSelected = "rgb(0,0,0)" + ,sidebarTabRadiusSelected = "0px" + + ,sidebarTabBackColorHover = "rgb(245,245,245)" + ,sidebarTabTextColorHover = "rgb(0,0,0)" + ,sidebarTabBorderStyleHover = "none solid none none" + ,sidebarTabBorderColorHover = "rgb(200,200,200)" + ,sidebarTabBorderWidthHover = 4 + ,sidebarTabRadiusHover = "0px" + + ,boxBackColor = "rgb(248,248,248)" + ,boxBorderRadius = 5 + ,boxShadowSize = "none" + ,boxShadowColor = "" + ,boxTitleSize = 18 + ,boxDefaultColor = "rgb(225,225,225)" + ,boxPrimaryColor = "rgb(0,17,88)" + ,boxInfoColor = "rgb(180,180,180)" + ,boxSuccessColor = "rgb(112,173,71)" + ,boxWarningColor = "rgb(237,125,49)" + ,boxDangerColor = "rgb(232,76,34)" + + ,tabBoxTabColor = "rgb(0,17,88)" + ,tabBoxTabTextSize = 14 + ,tabBoxTabTextColor = "rgb(0,0,0)" + ,tabBoxTabTextColorSelected = "rgb(0,0,0)" + ,tabBoxBackColor = "rgb(248,248,248)" + ,tabBoxHighlightColor = "rgb(200,200,200)" + ,tabBoxBorderRadius = 5 + + ### inputs + ,buttonBackColor = "rgb(215,215,215)" + ,buttonTextColor = "rgb(45,45,45)" + ,buttonBorderColor = "rgb(150,150,150)" + ,buttonBorderRadius = 5 + + ,buttonBackColorHover = "rgb(190,190,190)" + ,buttonTextColorHover = "rgb(0,0,0)" + ,buttonBorderColorHover = "rgb(150,150,150)" + + ,textboxBackColor = "rgb(255,255,255)" + ,textboxBorderColor = "rgb(118,118,118)" + ,textboxBorderRadius = 5 + ,textboxBackColorSelect = "rgb(245,245,245)" + ,textboxBorderColorSelect = "rgb(108,108,108)" + + ### tables + ,tableBackColor = "rgb(248,248,248)" + ,tableBorderColor = "rgb(238,238,238)" + ,tableBorderTopSize = 1 + ,tableBorderRowSize = 1 + ) \ No newline at end of file diff --git a/inst/shiny-server/ui.R b/inst/shiny-server/ui.R index e23ff8c6..154bf62d 100644 --- a/inst/shiny-server/ui.R +++ b/inst/shiny-server/ui.R @@ -1,622 +1,622 @@ -for (f in list.files('ui', pattern = '.R', full.names = T)) { - source(f) -} - -# NOTE: this is copied from `shinydashboard` -#' Assert that a tag has specified properties -#' @param tag A tag object. -#' @param type The type of a tag, like "div", "a", "span". -#' @param class An HTML class. -#' @param allowUI If TRUE (the default), allow dynamic outputs generated by -#' \code{\link[shiny]{uiOutput}} or \code{\link[shiny]{htmlOutput}}. When a -#' dynamic output is provided, \code{tagAssert} won't try to validate the the -#' contents. -#' @keywords internal -tagAssert <- function(tag, type = NULL, class = NULL, allowUI = TRUE) { - if (!inherits(tag, "shiny.tag")) { - print(tag) - stop("Expected an object with class 'shiny.tag'.") - } - - # Skip dynamic output elements - if (allowUI && - (hasCssClass(tag, "shiny-html-output") || - hasCssClass(tag, "shinydashboard-menu-output"))) { - return() - } - - if (!is.null(type) && tag$name != type) { - stop("Expected tag to be of type ", type) - } - - if (!is.null(class)) { - if (is.null(tag$attribs$class)) { - stop("Expected tag to have class '", class, "'") - - } else { - tagClasses <- strsplit(tag$attribs$class, " ")[[1]] - if (!(class %in% tagClasses)) { - stop("Expected tag to have class '", class, "'") - } - } - } -} - -# Modified dashboard header from the original source for incorporating two `select input` widgets -.dashboardHeader <- function(..., title = NULL, titleWidth = NULL, disable = FALSE, .list = NULL) { - items <- c(list(...), .list) - lapply(items, tagAssert, type = "li", class = "dropdown") - - titleWidth <- validateCssUnit(titleWidth) - - # Set up custom CSS for custom width. - custom_css <- NULL - if (!is.null(titleWidth)) { - # This CSS is derived from the header-related instances of '230px' (the - # default sidebar width) from inst/AdminLTE/AdminLTE.css. One change is that - # instead making changes to the global settings, we've put them in a media - # query (min-width: 768px), so that it won't override other media queries - # (like max-width: 767px) that work for narrower screens. - custom_css <- tags$head( - tags$style( - HTML( - gsub( - "_WIDTH_", titleWidth, fixed = TRUE, ' - @media (min-width: 768px) { - .main-header > .navbar { - margin-left: _WIDTH_; - } - .main-header .logo { - width: _WIDTH_; - } - }' - ) - ) - ) - ) - } - - tags$header( - class = "main-header", - id = 'header', - custom_css, - style = if (disable) "display: none;", - span(class = "logo", title), - tags$nav( - class = "navbar navbar-static-top", role = "navigation", - # Embed hidden icon so that we get the font-awesome dependency - span(shiny::icon("bars"), style = "display:none;"), - # Sidebar toggle button - a( - href="#", class="sidebar-toggle", `data-toggle`="offcanvas", - role="button", span(class="sr-only", "Toggle navigation") - ), - - # select inputs for dimension and function/problem ID - # HTML(' - #
- #
- # - # - # - # - # - #
- # Dimension: - # - # - # Problem ID: - # - #
' - # ), - - div( - class = "navbar-custom-menu", - tags$ul(class = "nav navbar-nav", items) - ) - ) - ) -} - -header <- .dashboardHeader(title = HTML('
IOHanalyzer
')) - -# The side bar layout --------------------------------------------- -sidebar <- dashboardSidebar( - useShinyjs(), - sidebar_menu(), - hr(), - DIM_fID_panel() -) - -body <- dashboardBody( - tags$style( - HTML(' - .popover-title {color:black;} - .popover-content {color:black;} - .main-sidebar {z-index:auto;} - .fa-exclamation-triangle {color:#E87722} - .sticky { - position: fixed; - top: 0; - width: 100%; - } - - .sticky2 { - position: fixed; - } - - .table { - border-collapse: collapse; - width: 100%; - } - .table td, tr { - padding: 0px; - margin: 0px; - vertical-align: middle; - height: 45px; - } - .table th {height: 0px;}' - ) - ), - - # to show text on the header (heading banner) - tags$head( - tags$style( - HTML(' - .myClass { - font-size: 20px; - line-height: 50px; - text-align: left; - font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; - padding: 0 15px; - overflow: hidden; - color: white; - }' - ) - ) - ), - - tags$head( - tags$style( - HTML( - '.box-title { - font-size: 20px; - line-height: 50px; - text-align: left; - font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; - padding: 0 15px; - overflow: hidden; - color: white; - }' - ) - ) - ), - - tags$head( - tags$style( - HTML( - "label { font-size:120%; }" - ) - ) - ), - - tags$script( - HTML(' - $(document).ready(function() { - $("header").find("nav").append(\'Performance Evaluation for Iterative - Optimization Heuristics\'); - })' - ) - ), - - tags$script( - "Shiny.addCustomMessageHandler('background-color', - function(color) { - document.body.style.backgroundColor = color; - document.body.innerText = color; - });" - ), - - tags$script( - HTML(' - window.setInterval(function() { - var elem = document.getElementById("process_data_promt"); - if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; - }, 20);' - ) - ), - - tags$head( - tags$script( - HTML(" - Shiny.addCustomMessageHandler('manipulateMenuItem', function(message){ - var aNodeList = document.getElementsByTagName('a'); - - for (var i = 0; i < aNodeList.length; i++) { - if(aNodeList[i].getAttribute('data-value') == message.tabName || aNodeList[i].getAttribute('href') == message.tabName) { - if(message.action == 'hide'){ - aNodeList[i].setAttribute('style', 'display: none;'); - } else { - aNodeList[i].setAttribute('style', 'display: block;'); - }; - }; - } - }); - ") - ) - ), - - # make the data uploading prompt always scroll to the bottom - tags$script( - HTML(' - window.setInterval(function() { - var elem = document.getElementById("upload_data_promt"); - if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; - }, 20);' - ) - ), - - # render the header and the side bar 'sticky' - tags$script( - HTML( - '// When the user scrolls the page, execute myFunction - window.onscroll = function() {myFunction()}; - - // Get the header - var header = document.getElementById("header"); - - // Get the side bar - var sideBar = document.getElementById("sidebarCollapsed"); - sideBar.classList.add("sticky2"); - - // Get the offset position of the navbar - var sticky = header.offsetTop; - - // Add the sticky class to the header when you reach its scroll position. - // Remove "sticky" when you leave the scroll position - function myFunction() { - if (window.pageYOffset > sticky) { - header.classList.add("sticky"); - } else { - header.classList.remove("sticky"); - } - }' - ) - ), - - if (suppressWarnings(require("dashboardthemes", quietly = T))) { - shinyDashboardThemes( - theme = "grey_light" - ) - }, - - # load MathJax - # TODO: download MathJax and its license and include it in our package - HTML(""), - use_bs_tooltip(), - use_bs_popover(), - - # tabitems ---------------------- - tabItems( - tabItem(tabName = 'about', includeMarkdown('markdown/about.md')), - # tabItem(tabName = 'dataformat', includeMarkdown('markdown/dataformat.md')), - - # data uploading functionalities ----------------- - tabItem( - tabName = 'upload', - fluidRow( - welcome_bar(width = 12) - ), - - fluidRow( - column( - width = 6, - upload_box(collapsible = F) - ), - column( - width = 6, - repository_box(collapsible = F) - ) - ), - - fluidRow( - column( - width = 6, - upload_prompt_box(collapsible = F) - ), - column( - width = 6, - data_list_box(collapsible = F) - ) - ) - ), - - # General data overview ---------------------- - tabItem( - tabName = 'overview', - fluidRow( - column( - width = 12, - general_overview_box_all(collapsed = F), - general_overview_box_single(collapsed = F) - ) - ) - ), - - # RT (RunTime): Data Summary ----------------- - tabItem( - tabName = 'ERT_data', - fluidRow( - column( - width = 12, - rt_overview_box(collapsed = F), - rt_stats_box(collapsed = F), - rt_sample_box() - ) - ) - ), - - # RT: Expected Convergence Curve --------------------------------------------- - tabItem( - tabName = 'ERT_convergence_single', - fluidRow( - column( - width = 12, - ERT_box(collapsed = F), - ERT_comparison_box_dim() - ) - ) - ), - - tabItem( - tabName = 'ERT_convergence_aggr', - fluidRow( - column( - width = 12, - ERT_agg_box(collapsed = F), - ERT_comparison_box(collapsed = T) - ) - ) - ), - - # RT: histograms, violin plots ------------------------------------------ - tabItem( - tabName = 'RT_PMF', - fluidRow( - column( - width = 12, - rt_histogram_box(collapsed = F), - rt_pmf_box() - ) - ) - ), - - # RT ECDF ------------------------------------------ - tabItem( - tabName = 'RT_ECDF_single', - fluidRow( - column( - width = 12, - rt_ecdf_single_target_box(collapsed = F), - rt_ecdf_agg_targets_box() - # rt_ecdf_auc_box() - ) - ) - ), - - tabItem( - tabName = 'RT_ECDF_aggr', - fluidRow( - column( - width = 12, - rt_ecdf_agg_fct_box(collapsed = F) - ) - ) - ), - - # Parameter tab ------- - tabItem( - tabName = 'RT_PARAMETER', - fluidRow( - column( - width = 12, - rt_par_summary_box(collapsed = F), - rt_par_expected_value_box(), - rt_par_sample_box() - ) - ) - ), - - tabItem( - tabName = 'RT_Statistics_single', - fluidRow( - column( - width = 12, - rt_heatmap_box() - ) - ) - ), - tabItem(tabName = 'RT_DSC', - fluidRow( - column( - width = 12, - rt_dsc_box_rank(), - rt_dsc_box_omni(), - rt_dsc_box_posthoc() - ) - ) - ), - tabItem(tabName = 'RT_Statistics_aggr', - fluidRow( - column( - width = 12, - rt_glicko2_box(collapsed = F) - ) - ) - - ), - tabItem(tabName = 'RT_table_multi', - fluidRow( - column( - width = 12, - multi_function_ert_box(collapsed = F), - multi_function_sample_box(collapsed = T) - ) - ) - - ), - tabItem(tabName = 'RT_portfolio', - fluidRow( - column( - width = 12, - rt_shapleys_box(collapsed = F) - ) - ) - - ), - # FCE: Data Summary ----------------- - tabItem( - tabName = 'FCE_DATA', - fluidRow( - column( - width = 12, - fv_overview_box(collapsed = F), - fv_stats_box(collapsed = F), - fv_sample_box() - ) - ) - ), - - # FCE: Expected Convergence Curve --------------------------------------------- - tabItem( - tabName = 'FCE_convergence_single', - fluidRow( - column( - width = 12, - fv_per_fct_box(collapsed = F) - ) - ) - ), - - tabItem( - tabName = 'FCE_convergence_aggr', - fluidRow( - column( - width = 12, - fv_agg_box(collapsed = F), - fv_comparison_box() - ) - ) - ), - # FCE: historgrams, p.d.f. -------- - tabItem( - tabName = 'FCE_PDF', - fluidRow( - column( - width = 12, - fv_histgram_box(collapsed = F), - fv_pdf_box() - ) - ) - ), - - # FCE: empirical c.d.f. ------------------------------------------ - tabItem( - tabName = 'FCE_ECDF', - fluidRow( - column( - width = 12, - fv_ecdf_single_budget_box(collapsed = F), - fv_ecdf_agg_budgets_box(), - fv_ecdf_auc_box() - ) - ) - ), - - # Parameter tab ------- - tabItem( - tabName = 'FCE_PARAMETER', - fluidRow( - column( - width = 12, - fv_par_expected_value_box(collapsed = F), - fv_par_summary_box(), - fv_par_sample_box() - ) - ) - ), - - tabItem( - tabName = 'FCE_Statistics_single', - fluidRow( - column( - width = 12, - fv_heatmap_box(collapsed = F) - ) - ) - ), - tabItem(tabName = 'FV_table_multi', - fluidRow( - column( - width = 12, - multi_function_fv_box(collapsed = F), - multi_function_sample_box_fv(collapsed = T) - ) - ) - - ), - - tabItem(tabName = 'FCE_DSC', - fluidRow( - column( - width = 12, - fv_dsc_box_rank(), - fv_dsc_box_omni(), - fv_dsc_box_posthoc() - ) - ) - ), - - tabItem(tabName = 'FCE_Statistics_aggr', - fluidRow( - column( - width = 12, - fv_glicko2_box(collapsed = F) - ) - ) - ), - tabItem(tabName = 'Positions', - fluidRow( - column( - width = 12, - Par_coord_box() - ) - ) - ), - - tabItem( - tabName = 'Settings', - fluidRow( - column( - width = 12, - color_settings_box(), - general_settings_box() - ) - ) - ) - # tabItem(tabName = 'Report', - # fluidRow( - # column( - # width = 12, - # main_report_box() - # ) - # ) - # ) - ) -) - -# ----------------------------------------------------------- +for (f in list.files('ui', pattern = '.R', full.names = T)) { + source(f) +} + +# NOTE: this is copied from `shinydashboard` +#' Assert that a tag has specified properties +#' @param tag A tag object. +#' @param type The type of a tag, like "div", "a", "span". +#' @param class An HTML class. +#' @param allowUI If TRUE (the default), allow dynamic outputs generated by +#' \code{\link[shiny]{uiOutput}} or \code{\link[shiny]{htmlOutput}}. When a +#' dynamic output is provided, \code{tagAssert} won't try to validate the the +#' contents. +#' @keywords internal +tagAssert <- function(tag, type = NULL, class = NULL, allowUI = TRUE) { + if (!inherits(tag, "shiny.tag")) { + print(tag) + stop("Expected an object with class 'shiny.tag'.") + } + + # Skip dynamic output elements + if (allowUI && + (hasCssClass(tag, "shiny-html-output") || + hasCssClass(tag, "shinydashboard-menu-output"))) { + return() + } + + if (!is.null(type) && tag$name != type) { + stop("Expected tag to be of type ", type) + } + + if (!is.null(class)) { + if (is.null(tag$attribs$class)) { + stop("Expected tag to have class '", class, "'") + + } else { + tagClasses <- strsplit(tag$attribs$class, " ")[[1]] + if (!(class %in% tagClasses)) { + stop("Expected tag to have class '", class, "'") + } + } + } +} + +# Modified dashboard header from the original source for incorporating two `select input` widgets +.dashboardHeader <- function(..., title = NULL, titleWidth = NULL, disable = FALSE, .list = NULL) { + items <- c(list(...), .list) + lapply(items, tagAssert, type = "li", class = "dropdown") + + titleWidth <- validateCssUnit(titleWidth) + + # Set up custom CSS for custom width. + custom_css <- NULL + if (!is.null(titleWidth)) { + # This CSS is derived from the header-related instances of '230px' (the + # default sidebar width) from inst/AdminLTE/AdminLTE.css. One change is that + # instead making changes to the global settings, we've put them in a media + # query (min-width: 768px), so that it won't override other media queries + # (like max-width: 767px) that work for narrower screens. + custom_css <- tags$head( + tags$style( + HTML( + gsub( + "_WIDTH_", titleWidth, fixed = TRUE, ' + @media (min-width: 768px) { + .main-header > .navbar { + margin-left: _WIDTH_; + } + .main-header .logo { + width: _WIDTH_; + } + }' + ) + ) + ) + ) + } + + tags$header( + class = "main-header", + id = 'header', + custom_css, + style = if (disable) "display: none;", + span(class = "logo", title), + tags$nav( + class = "navbar navbar-static-top", role = "navigation", + # Embed hidden icon so that we get the font-awesome dependency + span(shiny::icon("bars"), style = "display:none;"), + # Sidebar toggle button + a( + href="#", class="sidebar-toggle", `data-toggle`="offcanvas", + role="button", span(class="sr-only", "Toggle navigation") + ), + + # select inputs for dimension and function/problem ID + # HTML(' + #
+ #
+ # + # + # + # + # + #
+ # Dimension: + # + # + # Problem ID: + # + #
' + # ), + + div( + class = "navbar-custom-menu", + tags$ul(class = "nav navbar-nav", items) + ) + ) + ) +} + +header <- .dashboardHeader(title = HTML('
IOHanalyzer
')) + +# The side bar layout --------------------------------------------- +sidebar <- dashboardSidebar( + useShinyjs(), + sidebar_menu(), + hr(), + DIM_fID_panel() +) + +body <- dashboardBody( + tags$style( + HTML(' + .popover-title {color:black;} + .popover-content {color:black;} + .main-sidebar {z-index:auto;} + .fa-exclamation-triangle {color:#E87722} + .sticky { + position: fixed; + top: 0; + width: 100%; + } + + .sticky2 { + position: fixed; + } + + .table { + border-collapse: collapse; + width: 100%; + } + .table td, tr { + padding: 0px; + margin: 0px; + vertical-align: middle; + height: 45px; + } + .table th {height: 0px;}' + ) + ), + + # to show text on the header (heading banner) + tags$head( + tags$style( + HTML(' + .myClass { + font-size: 20px; + line-height: 50px; + text-align: left; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + padding: 0 15px; + overflow: hidden; + color: white; + }' + ) + ) + ), + + tags$head( + tags$style( + HTML( + '.box-title { + font-size: 20px; + line-height: 50px; + text-align: left; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + padding: 0 15px; + overflow: hidden; + color: white; + }' + ) + ) + ), + + tags$head( + tags$style( + HTML( + "label { font-size:120%; }" + ) + ) + ), + + tags$script( + HTML(' + $(document).ready(function() { + $("header").find("nav").append(\'Performance Evaluation for Iterative + Optimization Heuristics\'); + })' + ) + ), + + tags$script( + "Shiny.addCustomMessageHandler('background-color', + function(color) { + document.body.style.backgroundColor = color; + document.body.innerText = color; + });" + ), + + tags$script( + HTML(' + window.setInterval(function() { + var elem = document.getElementById("process_data_promt"); + if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; + }, 20);' + ) + ), + + tags$head( + tags$script( + HTML(" + Shiny.addCustomMessageHandler('manipulateMenuItem', function(message){ + var aNodeList = document.getElementsByTagName('a'); + + for (var i = 0; i < aNodeList.length; i++) { + if(aNodeList[i].getAttribute('data-value') == message.tabName || aNodeList[i].getAttribute('href') == message.tabName) { + if(message.action == 'hide'){ + aNodeList[i].setAttribute('style', 'display: none;'); + } else { + aNodeList[i].setAttribute('style', 'display: block;'); + }; + }; + } + }); + ") + ) + ), + + # make the data uploading prompt always scroll to the bottom + tags$script( + HTML(' + window.setInterval(function() { + var elem = document.getElementById("upload_data_promt"); + if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; + }, 20);' + ) + ), + + # render the header and the side bar 'sticky' + tags$script( + HTML( + '// When the user scrolls the page, execute myFunction + window.onscroll = function() {myFunction()}; + + // Get the header + var header = document.getElementById("header"); + + // Get the side bar + var sideBar = document.getElementById("sidebarCollapsed"); + sideBar.classList.add("sticky2"); + + // Get the offset position of the navbar + var sticky = header.offsetTop; + + // Add the sticky class to the header when you reach its scroll position. + // Remove "sticky" when you leave the scroll position + function myFunction() { + if (window.pageYOffset > sticky) { + header.classList.add("sticky"); + } else { + header.classList.remove("sticky"); + } + }' + ) + ), + + if (suppressWarnings(require("dashboardthemes", quietly = T))) { + shinyDashboardThemes( + theme = "grey_light" + ) + }, + + # load MathJax + # TODO: download MathJax and its license and include it in our package + HTML(""), + use_bs_tooltip(), + use_bs_popover(), + + # tabitems ---------------------- + tabItems( + tabItem(tabName = 'about', includeMarkdown('markdown/about.md')), + # tabItem(tabName = 'dataformat', includeMarkdown('markdown/dataformat.md')), + + # data uploading functionalities ----------------- + tabItem( + tabName = 'upload', + fluidRow( + welcome_bar(width = 12) + ), + + fluidRow( + column( + width = 6, + upload_box(collapsible = F) + ), + column( + width = 6, + repository_box(collapsible = F) + ) + ), + + fluidRow( + column( + width = 6, + upload_prompt_box(collapsible = F) + ), + column( + width = 6, + data_list_box(collapsible = F) + ) + ) + ), + + # General data overview ---------------------- + tabItem( + tabName = 'overview', + fluidRow( + column( + width = 12, + general_overview_box_all(collapsed = F), + general_overview_box_single(collapsed = F) + ) + ) + ), + + # RT (RunTime): Data Summary ----------------- + tabItem( + tabName = 'ERT_data', + fluidRow( + column( + width = 12, + rt_overview_box(collapsed = F), + rt_stats_box(collapsed = F), + rt_sample_box() + ) + ) + ), + + # RT: Expected Convergence Curve --------------------------------------------- + tabItem( + tabName = 'ERT_convergence_single', + fluidRow( + column( + width = 12, + ERT_box(collapsed = F), + ERT_comparison_box_dim() + ) + ) + ), + + tabItem( + tabName = 'ERT_convergence_aggr', + fluidRow( + column( + width = 12, + ERT_agg_box(collapsed = F), + ERT_comparison_box(collapsed = T) + ) + ) + ), + + # RT: histograms, violin plots ------------------------------------------ + tabItem( + tabName = 'RT_PMF', + fluidRow( + column( + width = 12, + rt_histogram_box(collapsed = F), + rt_pmf_box() + ) + ) + ), + + # RT ECDF ------------------------------------------ + tabItem( + tabName = 'RT_ECDF_single', + fluidRow( + column( + width = 12, + rt_ecdf_single_target_box(collapsed = F), + rt_ecdf_agg_targets_box() + # rt_ecdf_auc_box() + ) + ) + ), + + tabItem( + tabName = 'RT_ECDF_aggr', + fluidRow( + column( + width = 12, + rt_ecdf_agg_fct_box(collapsed = F) + ) + ) + ), + + # Parameter tab ------- + tabItem( + tabName = 'RT_PARAMETER', + fluidRow( + column( + width = 12, + rt_par_summary_box(collapsed = F), + rt_par_expected_value_box(), + rt_par_sample_box() + ) + ) + ), + + tabItem( + tabName = 'RT_Statistics_single', + fluidRow( + column( + width = 12, + rt_heatmap_box() + ) + ) + ), + tabItem(tabName = 'RT_DSC', + fluidRow( + column( + width = 12, + rt_dsc_box_rank(), + rt_dsc_box_omni(), + rt_dsc_box_posthoc() + ) + ) + ), + tabItem(tabName = 'RT_Statistics_aggr', + fluidRow( + column( + width = 12, + rt_glicko2_box(collapsed = F) + ) + ) + + ), + tabItem(tabName = 'RT_table_multi', + fluidRow( + column( + width = 12, + multi_function_ert_box(collapsed = F), + multi_function_sample_box(collapsed = T) + ) + ) + + ), + tabItem(tabName = 'RT_portfolio', + fluidRow( + column( + width = 12, + rt_shapleys_box(collapsed = F) + ) + ) + + ), + # FCE: Data Summary ----------------- + tabItem( + tabName = 'FCE_DATA', + fluidRow( + column( + width = 12, + fv_overview_box(collapsed = F), + fv_stats_box(collapsed = F), + fv_sample_box() + ) + ) + ), + + # FCE: Expected Convergence Curve --------------------------------------------- + tabItem( + tabName = 'FCE_convergence_single', + fluidRow( + column( + width = 12, + fv_per_fct_box(collapsed = F) + ) + ) + ), + + tabItem( + tabName = 'FCE_convergence_aggr', + fluidRow( + column( + width = 12, + fv_agg_box(collapsed = F), + fv_comparison_box() + ) + ) + ), + # FCE: historgrams, p.d.f. -------- + tabItem( + tabName = 'FCE_PDF', + fluidRow( + column( + width = 12, + fv_histgram_box(collapsed = F), + fv_pdf_box() + ) + ) + ), + + # FCE: empirical c.d.f. ------------------------------------------ + tabItem( + tabName = 'FCE_ECDF', + fluidRow( + column( + width = 12, + fv_ecdf_single_budget_box(collapsed = F), + fv_ecdf_agg_budgets_box(), + fv_ecdf_auc_box() + ) + ) + ), + + # Parameter tab ------- + tabItem( + tabName = 'FCE_PARAMETER', + fluidRow( + column( + width = 12, + fv_par_expected_value_box(collapsed = F), + fv_par_summary_box(), + fv_par_sample_box() + ) + ) + ), + + tabItem( + tabName = 'FCE_Statistics_single', + fluidRow( + column( + width = 12, + fv_heatmap_box(collapsed = F) + ) + ) + ), + tabItem(tabName = 'FV_table_multi', + fluidRow( + column( + width = 12, + multi_function_fv_box(collapsed = F), + multi_function_sample_box_fv(collapsed = T) + ) + ) + + ), + + tabItem(tabName = 'FCE_DSC', + fluidRow( + column( + width = 12, + fv_dsc_box_rank(), + fv_dsc_box_omni(), + fv_dsc_box_posthoc() + ) + ) + ), + + tabItem(tabName = 'FCE_Statistics_aggr', + fluidRow( + column( + width = 12, + fv_glicko2_box(collapsed = F) + ) + ) + ), + tabItem(tabName = 'Positions', + fluidRow( + column( + width = 12, + Par_coord_box() + ) + ) + ), + + tabItem( + tabName = 'Settings', + fluidRow( + column( + width = 12, + color_settings_box(), + general_settings_box() + ) + ) + ) + # tabItem(tabName = 'Report', + # fluidRow( + # column( + # width = 12, + # main_report_box() + # ) + # ) + # ) + ) +) + +# ----------------------------------------------------------- dashboardPage(title = 'IOHanalyzer', header, sidebar, body) \ No newline at end of file diff --git a/inst/shiny-server/ui/DIM_fID_panel.R b/inst/shiny-server/ui/DIM_fID_panel.R index 363f0bab..b9bbf9be 100644 --- a/inst/shiny-server/ui/DIM_fID_panel.R +++ b/inst/shiny-server/ui/DIM_fID_panel.R @@ -1,54 +1,54 @@ -DIM_fID_panel <- function() { - conditionalPanel( - "input.tabs!='Report' && input.tabs!='upload' && input.tabs!='readme' && input.tabs!='about' && input.tabs!='Settings'", - column(12, offset = 0, - div(id = 'overall_funcid_box', - # style = "padding: 0px 0px; margin-top:-5em; margin:0%", - fluidRow( - column( - width = 11, - selectInput( - "Overall.Funcid", - label = HTML('

Function ID:

'), - choices = NULL, selected = NULL) - ) - ) - ), - div(id = 'overall_funcname_box', - # style = "padding: 0px 0px; margin-top:-5em; margin:0%", - fluidRow( - column( - width = 11, - selectInput( - "Overall.Funcname", - label = HTML('

Function Name:

'), - choices = NULL, selected = NULL) - ) - ) - ), - div( - # style = "padding: 0px 0px; margin-top:-50em;", - fluidRow( - column( - width = 11, - selectInput( - "Overall.Dim", - label = HTML('

Dimension:

'), - choices = NULL, selected = NULL) - ) - ) - ), - div(id = 'overall_algid_box', - fluidRow( - column( - width = 11, - selectInput( - "Overall.ID", - label = HTML('

Algorithm:

'), - choices = NULL, selected = NULL) - ) - ) - )#, - ) - ) -} +DIM_fID_panel <- function() { + conditionalPanel( + "input.tabs!='Report' && input.tabs!='upload' && input.tabs!='readme' && input.tabs!='about' && input.tabs!='Settings'", + column(12, offset = 0, + div(id = 'overall_funcid_box', + # style = "padding: 0px 0px; margin-top:-5em; margin:0%", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Funcid", + label = HTML('

Function ID:

'), + choices = NULL, selected = NULL) + ) + ) + ), + div(id = 'overall_funcname_box', + # style = "padding: 0px 0px; margin-top:-5em; margin:0%", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Funcname", + label = HTML('

Function Name:

'), + choices = NULL, selected = NULL) + ) + ) + ), + div( + # style = "padding: 0px 0px; margin-top:-50em;", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Dim", + label = HTML('

Dimension:

'), + choices = NULL, selected = NULL) + ) + ) + ), + div(id = 'overall_algid_box', + fluidRow( + column( + width = 11, + selectInput( + "Overall.ID", + label = HTML('

Algorithm:

'), + choices = NULL, selected = NULL) + ) + ) + )#, + ) + ) +} diff --git a/inst/shiny-server/ui/DSC_interface.R b/inst/shiny-server/ui/DSC_interface.R index 8b0410fb..d5c21e5a 100644 --- a/inst/shiny-server/ui/DSC_interface.R +++ b/inst/shiny-server/ui/DSC_interface.R @@ -1,281 +1,281 @@ -fv_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Ranking per Function

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.DSC.ID', 'IDs to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.DSC.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - hr(), - numericInput('FV_Stats.DSC.Alpha_rank', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - numericInput('FV_Stats.DSC.Epsilon_rank', - label = "Threshold for practical significance (pDSC)", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Practical Significance", content = Pdsc_info, - placement = "auto" - ) - ), - numericInput('FV_Stats.DSC.MCsamples_rank', - label = "Number of monte carlo samples to use in the DSC procdure", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Monte Carlo Samples", content = Pdsc_mc_info, - placement = "auto" - ) - ), - selectInput("FV_Stats.DSC.Test_rank", "Test Type", - choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), - selected = "Anderson-Darling"), - actionButton('FV_Stats.DSC.Create_rank', 'Create Ranking'), - hr(), - selectInput('FV_Stats.DSC.Format_rank', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FV_Stats.DSC.Download_rank', label = 'Download the figure'), - selectInput('FV_Stats.DSC.TableFormat_rank', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') - - ), - - mainPanel( - width = 9, - HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based - framework for statistical comparison of stochastic optimization algorithms.' by - T. Eftimov et al. - This is the first of 3 parts of the process: the per-function ranking procedure. - The two other processes are the omnibus test and the post-hoc processing, - which are shown in the two boxes below this one. - Note that both of these are impacted by the settings selected for this - ranking procedure!"), - HTML_P("The chosen budget values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("FV_Stats.DSC.Targets"), - hr(), - HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques - as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and - Understand the Performance of Single-objective Optimization - Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, - from a single plot, which algorithms are most suited for a given problem, - the influence of each problem on the overall algorithm performance and - similarities among both algorithms and problems."), - plotOutput("FV_Stats.DSC.PerformViz", height = "800px") - ) - ) -} - -fv_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Omnibus Test

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.DSC.Omni_options', "Select which statistical test to use", - choices = NULL, selected = NULL), - numericInput('FV_Stats.DSC.Alpha_omni', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('FV_Stats.DSC.Create_omni', 'Perform Omnibus Test') - ), - - mainPanel( - width = 9, - HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. - Note that this test has to be performed before doing the post-hoc comparison!'), - hr(), - textOutput('FV_Stats.DSC.Output_omni') - ) - ) -} - -fv_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Posthoc comparison

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = - c('friedman', 'friedman-aligned-rank'), - selected = 'friedman', multiple = F), - selectInput('FV_Stats.DSC.Posthoc_method', 'Post-hoc P-value Correction Method', choices = - c('Holm', 'Hochberg', 'unadjusted P'), - selected = 'Holm', multiple = F), - numericInput('FV_Stats.DSC.Alpha_posthoc', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('FV_Stats.DSC.Create_posthoc', 'Create Comparison'), - hr(), - selectInput('FV_Stats.DSC.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FV_Stats.DSC.Download', label = 'Download the figure'), - hr(), - selectInput('FV_Stats.DSC.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.DSC.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P("The results of the post-hoc comparison are:"), - plotlyOutput.IOHanalyzer("FV_Stats.DSC.PosthocViz"), - DT::dataTableOutput('FV_Stats.DSC.PosthocTable') - ) - ) -} - -rt_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Ranking per Function

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.DSC.ID', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.DSC.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - hr(), - selectInput('RT_Stats.DSC.Value_type', "Select which type of hitting times to use", - choices = c('PAR-10', 'ERT', 'Remove-na', 'PAR-1')), - numericInput('RT_Stats.DSC.Alpha_rank', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - numericInput('RT_Stats.DSC.Epsilon_rank', - label = "Threshold for practical significance (pDSC)", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Practical Significance", content = Pdsc_info, - placement = "auto" - ) - ), - numericInput('RT_Stats.DSC.MCsamples_rank', - label = "Number of monte carlo samples to use in the DSC procdure", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Monte Carlo Samples", content = Pdsc_mc_info, - placement = "auto" - ) - ), - selectInput("RT_Stats.DSC.Test_rank", "Test Type", - choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), - selected = "Anderson-Darling"), - actionButton('RT_Stats.DSC.Create_rank', 'Create Ranking'), - hr(), - selectInput('RT_Stats.DSC.Format_rank', label = 'Select the figure format', - choices = c("pdf"), selected = 'pdf'), - - downloadButton('RT_Stats.DSC.Download_rank', label = 'Download the figure'), - selectInput('RT_Stats.DSC.TableFormat_rank', label = 'Select the table format', - choices = c('csv','tex'), selected = 'csv'), - downloadButton('RT_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') - - ), - - mainPanel( - width = 9, - HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based - framework for statistical comparison of stochastic optimization algorithms.' by - T. Eftimov et al. - This is the first of 3 parts of the process: the per-function ranking procedure. - The two other processes are the omnibus test and the post-hoc processing, - which are shown in the two boxes below this one. - Note that both of these are impacted by the settings selected for this - ranking procedure!"), - HTML_P("The chosen budget values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("RT_Stats.DSC.Targets"), - hr(), - HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques - as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and - Understand the Performance of Single-objective Optimization - Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, - from a single plot, which algorithms are most suited for a given problem, - the influence of each problem on the overall algorithm performance and - similarities among both algorithms and problems."), - plotOutput("RT_Stats.DSC.PerformViz", height = "800px") - ) - ) -} - -rt_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Omnibus Test

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.DSC.Omni_options', "Select which statistical test to use", - choices = NULL, selected = NULL), - numericInput('RT_Stats.DSC.Alpha_omni', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('RT_Stats.DSC.Create_omni', 'Perform Omnibus Test') - ), - - mainPanel( - width = 9, - HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. - Note that this test has to be performed before doing the post-hoc comparison!'), - hr(), - textOutput('RT_Stats.DSC.Output_omni') - ) - ) -} - -rt_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Posthoc comparison

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = - c('friedman', 'friedman-aligned-rank'), - selected = 'friedman', multiple = F), - selectInput('RT_Stats.DSC.Posthoc_method', 'Post-hoc Method', choices = - c('Holm', 'Hochberg', 'unadjusted P'), - selected = 'Holm', multiple = F), - numericInput('RT_Stats.DSC.Alpha_posthoc', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('RT_Stats.DSC.Create_posthoc', 'Create Comparison'), - hr(), - selectInput('RT_Stats.DSC.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RT_Stats.DSC.Download', label = 'Download the figure'), - hr(), - selectInput('RT_Stats.DSC.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('RT_Stats.DSC.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P("The results of the post-hoc comparison are:"), - plotlyOutput.IOHanalyzer("RT_Stats.DSC.PosthocViz"), - DT::dataTableOutput('RT_Stats.DSC.PosthocTable') - ) - ) +fv_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Ranking per Function

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.DSC.ID', 'IDs to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.DSC.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + hr(), + numericInput('FV_Stats.DSC.Alpha_rank', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + numericInput('FV_Stats.DSC.Epsilon_rank', + label = "Threshold for practical significance (pDSC)", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Practical Significance", content = Pdsc_info, + placement = "auto" + ) + ), + numericInput('FV_Stats.DSC.MCsamples_rank', + label = "Number of monte carlo samples to use in the DSC procdure", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Monte Carlo Samples", content = Pdsc_mc_info, + placement = "auto" + ) + ), + selectInput("FV_Stats.DSC.Test_rank", "Test Type", + choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), + selected = "Anderson-Darling"), + actionButton('FV_Stats.DSC.Create_rank', 'Create Ranking'), + hr(), + selectInput('FV_Stats.DSC.Format_rank', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FV_Stats.DSC.Download_rank', label = 'Download the figure'), + selectInput('FV_Stats.DSC.TableFormat_rank', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') + + ), + + mainPanel( + width = 9, + HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based + framework for statistical comparison of stochastic optimization algorithms.' by + T. Eftimov et al. + This is the first of 3 parts of the process: the per-function ranking procedure. + The two other processes are the omnibus test and the post-hoc processing, + which are shown in the two boxes below this one. + Note that both of these are impacted by the settings selected for this + ranking procedure!"), + HTML_P("The chosen budget values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("FV_Stats.DSC.Targets"), + hr(), + HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques + as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and + Understand the Performance of Single-objective Optimization + Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, + from a single plot, which algorithms are most suited for a given problem, + the influence of each problem on the overall algorithm performance and + similarities among both algorithms and problems."), + plotOutput("FV_Stats.DSC.PerformViz", height = "800px") + ) + ) +} + +fv_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Omnibus Test

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.DSC.Omni_options', "Select which statistical test to use", + choices = NULL, selected = NULL), + numericInput('FV_Stats.DSC.Alpha_omni', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('FV_Stats.DSC.Create_omni', 'Perform Omnibus Test') + ), + + mainPanel( + width = 9, + HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. + Note that this test has to be performed before doing the post-hoc comparison!'), + hr(), + textOutput('FV_Stats.DSC.Output_omni') + ) + ) +} + +fv_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Posthoc comparison

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = + c('friedman', 'friedman-aligned-rank'), + selected = 'friedman', multiple = F), + selectInput('FV_Stats.DSC.Posthoc_method', 'Post-hoc P-value Correction Method', choices = + c('Holm', 'Hochberg', 'unadjusted P'), + selected = 'Holm', multiple = F), + numericInput('FV_Stats.DSC.Alpha_posthoc', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('FV_Stats.DSC.Create_posthoc', 'Create Comparison'), + hr(), + selectInput('FV_Stats.DSC.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FV_Stats.DSC.Download', label = 'Download the figure'), + hr(), + selectInput('FV_Stats.DSC.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.DSC.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P("The results of the post-hoc comparison are:"), + plotlyOutput.IOHanalyzer("FV_Stats.DSC.PosthocViz"), + DT::dataTableOutput('FV_Stats.DSC.PosthocTable') + ) + ) +} + +rt_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Ranking per Function

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.DSC.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.DSC.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + hr(), + selectInput('RT_Stats.DSC.Value_type', "Select which type of hitting times to use", + choices = c('PAR-10', 'ERT', 'Remove-na', 'PAR-1')), + numericInput('RT_Stats.DSC.Alpha_rank', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + numericInput('RT_Stats.DSC.Epsilon_rank', + label = "Threshold for practical significance (pDSC)", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Practical Significance", content = Pdsc_info, + placement = "auto" + ) + ), + numericInput('RT_Stats.DSC.MCsamples_rank', + label = "Number of monte carlo samples to use in the DSC procdure", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Monte Carlo Samples", content = Pdsc_mc_info, + placement = "auto" + ) + ), + selectInput("RT_Stats.DSC.Test_rank", "Test Type", + choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), + selected = "Anderson-Darling"), + actionButton('RT_Stats.DSC.Create_rank', 'Create Ranking'), + hr(), + selectInput('RT_Stats.DSC.Format_rank', label = 'Select the figure format', + choices = c("pdf"), selected = 'pdf'), + + downloadButton('RT_Stats.DSC.Download_rank', label = 'Download the figure'), + selectInput('RT_Stats.DSC.TableFormat_rank', label = 'Select the table format', + choices = c('csv','tex'), selected = 'csv'), + downloadButton('RT_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') + + ), + + mainPanel( + width = 9, + HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based + framework for statistical comparison of stochastic optimization algorithms.' by + T. Eftimov et al. + This is the first of 3 parts of the process: the per-function ranking procedure. + The two other processes are the omnibus test and the post-hoc processing, + which are shown in the two boxes below this one. + Note that both of these are impacted by the settings selected for this + ranking procedure!"), + HTML_P("The chosen budget values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("RT_Stats.DSC.Targets"), + hr(), + HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques + as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and + Understand the Performance of Single-objective Optimization + Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, + from a single plot, which algorithms are most suited for a given problem, + the influence of each problem on the overall algorithm performance and + similarities among both algorithms and problems."), + plotOutput("RT_Stats.DSC.PerformViz", height = "800px") + ) + ) +} + +rt_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Omnibus Test

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.DSC.Omni_options', "Select which statistical test to use", + choices = NULL, selected = NULL), + numericInput('RT_Stats.DSC.Alpha_omni', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('RT_Stats.DSC.Create_omni', 'Perform Omnibus Test') + ), + + mainPanel( + width = 9, + HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. + Note that this test has to be performed before doing the post-hoc comparison!'), + hr(), + textOutput('RT_Stats.DSC.Output_omni') + ) + ) +} + +rt_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Posthoc comparison

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = + c('friedman', 'friedman-aligned-rank'), + selected = 'friedman', multiple = F), + selectInput('RT_Stats.DSC.Posthoc_method', 'Post-hoc Method', choices = + c('Holm', 'Hochberg', 'unadjusted P'), + selected = 'Holm', multiple = F), + numericInput('RT_Stats.DSC.Alpha_posthoc', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('RT_Stats.DSC.Create_posthoc', 'Create Comparison'), + hr(), + selectInput('RT_Stats.DSC.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RT_Stats.DSC.Download', label = 'Download the figure'), + hr(), + selectInput('RT_Stats.DSC.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('RT_Stats.DSC.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P("The results of the post-hoc comparison are:"), + plotlyOutput.IOHanalyzer("RT_Stats.DSC.PosthocViz"), + DT::dataTableOutput('RT_Stats.DSC.PosthocTable') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/ERT_agg_box.R b/inst/shiny-server/ui/ERT_agg_box.R index 02f5de4d..26e0d62e 100644 --- a/inst/shiny-server/ui/ERT_agg_box.R +++ b/inst/shiny-server/ui/ERT_agg_box.R @@ -1,46 +1,46 @@ -ERT_agg_box <- function(width = 12, height = '600px', collapsible = T, - collapsed = T) { - box( - title = HTML('

Expected Runtime (ERT): all functions

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('ERTPlot.Multi.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput('ERTPlot.Multi.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('ERTPlot.Multi.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - - actionButton('ERTPlot.Multi.PlotButton', label = 'Refresh the figure'), - hr(), - selectInput('ERTPlot.Multi.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Multi.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('The ERT is shown against the target - values for all functions in the selected dimension.'), - plotlyOutput.IOHanalyzer('ERTPlot.Multi.Plot', aspect_ratio = 1) - ) - ) - ) - ) -} +ERT_agg_box <- function(width = 12, height = '600px', collapsible = T, + collapsed = T) { + box( + title = HTML('

Expected Runtime (ERT): all functions

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('ERTPlot.Multi.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput('ERTPlot.Multi.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('ERTPlot.Multi.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + + actionButton('ERTPlot.Multi.PlotButton', label = 'Refresh the figure'), + hr(), + selectInput('ERTPlot.Multi.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Multi.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('The ERT is shown against the target + values for all functions in the selected dimension.'), + plotlyOutput.IOHanalyzer('ERTPlot.Multi.Plot', aspect_ratio = 1) + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/ERT_box.R b/inst/shiny-server/ui/ERT_box.R index 5c585e85..8c0ec0e2 100644 --- a/inst/shiny-server/ui/ERT_box.R +++ b/inst/shiny-server/ui/ERT_box.R @@ -1,133 +1,133 @@ -ERT_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Runtime (ERT): single function

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - - selectInput('ERTPlot.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - - HTML('

Range of the displayed target values

'), - - textInput('ERTPlot.Min', - label = F_MIN_LABEL, - value = '') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Interpolation", content = "The points on this plot are all interpollated where needed - to create solid lines. This means that for functions with limited domains (such as integers), - artifacts like step-wise behaviour can occur. Please carefully consider the domain of the function - when interpreting this plot.", - placement = "auto" - ) - ), - - textInput('ERTPlot.Max', - label = F_MAX_LABEL, - value = ''), - - checkboxInput('ERTPlot.show.ERT', - label = 'Show/hide ERT', - value = T), - - checkboxInput( - 'ERTPlot.show.mean', - label = 'Show/hide PAR-1', - value = F - ) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Penalized Average Runtime", content = "PAR-1 score is ther average of running time values, - where non-successful runs are counted as evaluation budget B.", - placement = "auto" - ) - ), - - checkboxInput('ERTPlot.show.CI', - label = 'Show/hide mean +/- sd', - value = F), - - checkboxInput('ERTPlot.show.Quantiles', - label = 'Show/hide outer quantiles', - value = F) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are - 2% and 98% by default. This can be changed in the settings-tab.", - placement = "auto" - ) - ), - - checkboxInput('ERTPlot.show.median', - label = 'Show/hide median', - value = F), - - checkboxInput('ERTPlot.semilogx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('ERTPlot.semilogy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - box(title = HTML('

Additional Options

'), collapsible = T, collapsed = T, width = width, solidHeader = T, status = 'info', - checkboxInput('ERTPlot.inclueOpts', - label = "Include optimal points found by each algorithm", - value = F), - checkboxInput('ERTPlot.show.runs', - label = 'Show individual runs', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_popover( - title = "Individual runs", content = "This procedure can be slow when many - runs are present in the data. Please use with caution.", - placement = "auto" - ) - ), - numericInput("ERTPlot.Additional.Budget", "Choose Custom Budget for ERT calculation", value = NA) %>% - shinyInput_label_embed( - custom_icon("info") %>% - bs_embed_popover( - title = "Custom Budget", content = "This will use this budget value instead of the ones in the - data for the ERT calculation. Any hitting times larger than this value are treated as unfinished runs. - Please use with caution.", - placement = "auto" - ) - ) - ), - hr(), - - selectInput('ERTPlot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Download', label = 'Download the figure') - - ), - - mainPanel( - width = 9, - column( - width = 12, - align = "center", - HTML_P('The mean, median, standard deviation and ERT of the runtime samples - are depicted against the best objective values. - The displayed elements (mean, median, standard deviations and ERT) - can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('ERT_PER_FUN') - ) - ) - ) - ) -} +ERT_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Runtime (ERT): single function

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + + selectInput('ERTPlot.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + HTML('

Range of the displayed target values

'), + + textInput('ERTPlot.Min', + label = F_MIN_LABEL, + value = '') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Interpolation", content = "The points on this plot are all interpollated where needed + to create solid lines. This means that for functions with limited domains (such as integers), + artifacts like step-wise behaviour can occur. Please carefully consider the domain of the function + when interpreting this plot.", + placement = "auto" + ) + ), + + textInput('ERTPlot.Max', + label = F_MAX_LABEL, + value = ''), + + checkboxInput('ERTPlot.show.ERT', + label = 'Show/hide ERT', + value = T), + + checkboxInput( + 'ERTPlot.show.mean', + label = 'Show/hide PAR-1', + value = F + ) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Penalized Average Runtime", content = "PAR-1 score is ther average of running time values, + where non-successful runs are counted as evaluation budget B.", + placement = "auto" + ) + ), + + checkboxInput('ERTPlot.show.CI', + label = 'Show/hide mean +/- sd', + value = F), + + checkboxInput('ERTPlot.show.Quantiles', + label = 'Show/hide outer quantiles', + value = F) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are + 2% and 98% by default. This can be changed in the settings-tab.", + placement = "auto" + ) + ), + + checkboxInput('ERTPlot.show.median', + label = 'Show/hide median', + value = F), + + checkboxInput('ERTPlot.semilogx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('ERTPlot.semilogy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + box(title = HTML('

Additional Options

'), collapsible = T, collapsed = T, width = width, solidHeader = T, status = 'info', + checkboxInput('ERTPlot.inclueOpts', + label = "Include optimal points found by each algorithm", + value = F), + checkboxInput('ERTPlot.show.runs', + label = 'Show individual runs', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_popover( + title = "Individual runs", content = "This procedure can be slow when many + runs are present in the data. Please use with caution.", + placement = "auto" + ) + ), + numericInput("ERTPlot.Additional.Budget", "Choose Custom Budget for ERT calculation", value = NA) %>% + shinyInput_label_embed( + custom_icon("info") %>% + bs_embed_popover( + title = "Custom Budget", content = "This will use this budget value instead of the ones in the + data for the ERT calculation. Any hitting times larger than this value are treated as unfinished runs. + Please use with caution.", + placement = "auto" + ) + ) + ), + hr(), + + selectInput('ERTPlot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Download', label = 'Download the figure') + + ), + + mainPanel( + width = 9, + column( + width = 12, + align = "center", + HTML_P('The mean, median, standard deviation and ERT of the runtime samples + are depicted against the best objective values. + The displayed elements (mean, median, standard deviations and ERT) + can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('ERT_PER_FUN') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/ERT_comparison_box.R b/inst/shiny-server/ui/ERT_comparison_box.R index 0aa2d4fc..9da19179 100644 --- a/inst/shiny-server/ui/ERT_comparison_box.R +++ b/inst/shiny-server/ui/ERT_comparison_box.R @@ -1,128 +1,128 @@ -ERT_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Runtime Comparisons (across functions on one dimension)

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('ERTPlot.Aggr.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - - selectInput('ERTPlot.Aggr.Funcs', label = 'Select which functions to aggregate over:', - multiple = T, selected = NULL, choices = NULL), - - selectInput('ERTPlot.Aggr.Mode', label = 'Select the plotting mode', - choices = c('radar', 'line'), selected = 'radar'), - - checkboxInput('ERTPlot.Aggr.Ranking', - label = 'Use ranking instead of ERT-values', - value = T), - - checkboxInput('ERTPlot.Aggr.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F), - actionButton("ERTPlot.Aggr.Refresh", "Refresh the figure and table"), - hr(), - - selectInput('ERTPlot.Aggr.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Aggr.Download', label = 'Download the figure'), - hr(), - selectInput('ERTPlot.Aggr.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('ERTPlot.Aggr.DownloadTable', label = 'Download the table') - - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('The ERT of the runtime samples across all functions. - ERT is decided based on the target values in the table below, - with the default being the best reached f(x) by any of the - selected algorithms. Infinite ERTS are shown as - seperate dots on the graph.'), - plotlyOutput.IOHanalyzer('ERTPlot.Aggr.Plot'), - hr(), - HTML_P("The chosen target values per function are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("ERTPlot.Aggr.Targets"), - hr(), - HTML_P("The raw ERT-values are:"), - DT::dataTableOutput("ERTPlot.Aggr.ERTTable") - ) - ) - ) - ) -} - -#TODO: combine with other function using proper namespacing and modularity -ERT_comparison_box_dim <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Runtime Comparisons (across dimensions)

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('ERTPlot.Aggr_Dim.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('ERTPlot.Aggr_Dim.Mode', label = 'Select the plotting mode', - choices = c('radar', 'line'), selected = 'line'), - - checkboxInput('ERTPlot.Aggr_Dim.Ranking', - label = 'Use ranking instead of ERT-values', - value = F), - - checkboxInput('ERTPlot.Aggr_Dim.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - actionButton("ERTPlot.Aggr_Dim.Refresh", "Refresh the figure and table"), - hr(), - - selectInput('ERTPlot.Aggr_Dim.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Aggr_Dim.Download', label = 'Download the figure'), - hr(), - selectInput('ERTPlot.Aggr_Dim.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('ERTPlot.Aggr_Dim.DownloadTable', label = 'Download the table') - - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('The ERT of the runtime samples across all functions. - ERT is decided based on the target values in the table below, - with the default being the best reached f(x) by any of the - selected algorithms. Infinite ERTS are shown as - seperate dots on the graph.'), - plotlyOutput.IOHanalyzer('ERTPlot.Aggr_Dim.Plot'), - hr(), - HTML_P("The chosen target values per dimension are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("ERTPlot.Aggr_Dim.Targets"), - hr(), - HTML_P("The raw ERT-values are:"), - DT::dataTableOutput("ERTPlot.Aggr_Dim.ERTTable") - ) - ) - ) - ) +ERT_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Runtime Comparisons (across functions on one dimension)

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('ERTPlot.Aggr.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + selectInput('ERTPlot.Aggr.Funcs', label = 'Select which functions to aggregate over:', + multiple = T, selected = NULL, choices = NULL), + + selectInput('ERTPlot.Aggr.Mode', label = 'Select the plotting mode', + choices = c('radar', 'line'), selected = 'radar'), + + checkboxInput('ERTPlot.Aggr.Ranking', + label = 'Use ranking instead of ERT-values', + value = T), + + checkboxInput('ERTPlot.Aggr.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F), + actionButton("ERTPlot.Aggr.Refresh", "Refresh the figure and table"), + hr(), + + selectInput('ERTPlot.Aggr.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Aggr.Download', label = 'Download the figure'), + hr(), + selectInput('ERTPlot.Aggr.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('ERTPlot.Aggr.DownloadTable', label = 'Download the table') + + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('The ERT of the runtime samples across all functions. + ERT is decided based on the target values in the table below, + with the default being the best reached f(x) by any of the + selected algorithms. Infinite ERTS are shown as + seperate dots on the graph.'), + plotlyOutput.IOHanalyzer('ERTPlot.Aggr.Plot'), + hr(), + HTML_P("The chosen target values per function are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("ERTPlot.Aggr.Targets"), + hr(), + HTML_P("The raw ERT-values are:"), + DT::dataTableOutput("ERTPlot.Aggr.ERTTable") + ) + ) + ) + ) +} + +#TODO: combine with other function using proper namespacing and modularity +ERT_comparison_box_dim <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Runtime Comparisons (across dimensions)

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('ERTPlot.Aggr_Dim.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('ERTPlot.Aggr_Dim.Mode', label = 'Select the plotting mode', + choices = c('radar', 'line'), selected = 'line'), + + checkboxInput('ERTPlot.Aggr_Dim.Ranking', + label = 'Use ranking instead of ERT-values', + value = F), + + checkboxInput('ERTPlot.Aggr_Dim.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + actionButton("ERTPlot.Aggr_Dim.Refresh", "Refresh the figure and table"), + hr(), + + selectInput('ERTPlot.Aggr_Dim.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Aggr_Dim.Download', label = 'Download the figure'), + hr(), + selectInput('ERTPlot.Aggr_Dim.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('ERTPlot.Aggr_Dim.DownloadTable', label = 'Download the table') + + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('The ERT of the runtime samples across all functions. + ERT is decided based on the target values in the table below, + with the default being the best reached f(x) by any of the + selected algorithms. Infinite ERTS are shown as + seperate dots on the graph.'), + plotlyOutput.IOHanalyzer('ERTPlot.Aggr_Dim.Plot'), + hr(), + HTML_P("The chosen target values per dimension are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("ERTPlot.Aggr_Dim.Targets"), + hr(), + HTML_P("The raw ERT-values are:"), + DT::dataTableOutput("ERTPlot.Aggr_Dim.ERTTable") + ) + ) + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/Parrallel_coord_box.R b/inst/shiny-server/ui/Parrallel_coord_box.R index c6bcffcf..44b993f0 100644 --- a/inst/shiny-server/ui/Parrallel_coord_box.R +++ b/inst/shiny-server/ui/Parrallel_coord_box.R @@ -1,33 +1,33 @@ -Par_coord_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Parallel coordinate plot of final position

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - - selectInput('ParCoordPlot.Algs', label = 'Select which algorithm to plot:', - multiple = F, selected = NULL, choices = NULL), - - hr(), - - selectInput('ParCoordPlot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ParCoordPlot.Download', label = 'Download the figure') - - ), - - mainPanel( - width = 9, - column( - width = 12, - align = "center", - HTML_P('The location of the best found solution by the algorithm, for each of the runs. - Each x-value corresponds to one coordinate of the solution.'), - plotlyOutput.IOHanalyzer('Parallel_Coord_Plot') - ) - ) - ) - ) -} +Par_coord_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Parallel coordinate plot of final position

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + + selectInput('ParCoordPlot.Algs', label = 'Select which algorithm to plot:', + multiple = F, selected = NULL, choices = NULL), + + hr(), + + selectInput('ParCoordPlot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ParCoordPlot.Download', label = 'Download the figure') + + ), + + mainPanel( + width = 9, + column( + width = 12, + align = "center", + HTML_P('The location of the best found solution by the algorithm, for each of the runs. + Each x-value corresponds to one coordinate of the solution.'), + plotlyOutput.IOHanalyzer('Parallel_Coord_Plot') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_box.R b/inst/shiny-server/ui/fv_box.R index ca6010d3..f4d593bc 100644 --- a/inst/shiny-server/ui/fv_box.R +++ b/inst/shiny-server/ui/fv_box.R @@ -1,175 +1,175 @@ -fv_per_fct_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Expected Target Value (per function)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarPanel( - width = 3, - HTML('

Range of the displayed budget values

'), - - textInput('FCEPlot.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCEPlot.Max', label = RT_MAX_LABEL, value = ''), - selectInput('FCEPlot.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - checkboxInput('FCEPlot.show.mean', - label = 'Show/hide mean', - value = T), - - checkboxInput('FCEPlot.show.median', - label = 'Show/hide median', - value = F), - - checkboxInput('FCEPlot.show.CI', - label = 'Show/hide mean +/- sd', - value = F), - - checkboxInput('FCEPlot.show.IQR', - label = 'Show/hide interquartile range', - value = F), - - checkboxInput('FCEPlot.semilogx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('FCEPlot.semilogy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - checkboxInput('FCEPlot.show.runs', - label = 'Show individual runs', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_popover( - title = "Individual runs", content = "This procedure can be slow when many - runs are present in the data. Please use with caution.", - placement = "auto" - ) - ), - - hr(), - selectInput('FCEPlot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPlot.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The mean, median and standard deviation of the best function values - found with a fixed-budget of evaluations are depicted against the budget. - The displayed elements can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('FCE_PER_FUN') - ) - ) - ) -} - -fv_agg_box <- function(width = 12, height = '600px', collapsible = T, collapsed = T) { - box(title = HTML('

Expected Function values: all functions

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('FCEPlot.Multi.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput('FCEPlot.Multi.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('FCEPlot.Multi.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - - actionButton('FCEPlot.Multi.PlotButton', label = 'Refresh the figure'), - hr(), - selectInput('FCEPlot.Multi.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPlot.Multi.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - plotlyOutput.IOHanalyzer('FCEPlot.Multi.Plot', aspect_ratio = 1) - ) - ) - ) - ) -} - -fv_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Function Value comparisons

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - selectInput('FCEPlot.Aggr.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('FCEPlot.Aggr.Mode', label = 'Select the plotting mode', - choices = c('radar','line'), selected = 'radar'), - - selectInput('FCEPlot.Aggr.Aggregator', label = 'Create plot for all?', - choices = c('Functions','Dimensions'), selected = 'Functions'), - - checkboxInput('FCEPlot.Aggr.Ranking', - label = 'Use ranking instead of ERT-values', - value = T), - - checkboxInput('FCEPlot.Aggr.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F), - actionButton("FCEPlot.Aggr.Refresh", "Refresh the figure and table"), - # textInput('FCEPlot.Aggr.Targets', label = 'Choose the runtimes (comma-separated)'), - hr(), - selectInput('FCEPlot.Aggr.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPlot.Aggr.Download', label = 'Download the figure'), - hr(), - selectInput('FCEPlot.Aggr.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FCEPlot.Aggr.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - plotlyOutput.IOHanalyzer('FCEPlot.Aggr.Plot'), - hr(), - HTML_P("The chosen budget values per function are as follows (double click an entry to edit it):"), - DT::dataTableOutput("FCEPlot.Aggr.Targets"), - hr(), - HTML_P("The raw function-values are:"), - DT::dataTableOutput("FCEPlot.Aggr.FCETable") - ) - ) - ) - ) -} +fv_per_fct_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Expected Target Value (per function)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarPanel( + width = 3, + HTML('

Range of the displayed budget values

'), + + textInput('FCEPlot.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCEPlot.Max', label = RT_MAX_LABEL, value = ''), + selectInput('FCEPlot.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + checkboxInput('FCEPlot.show.mean', + label = 'Show/hide mean', + value = T), + + checkboxInput('FCEPlot.show.median', + label = 'Show/hide median', + value = F), + + checkboxInput('FCEPlot.show.CI', + label = 'Show/hide mean +/- sd', + value = F), + + checkboxInput('FCEPlot.show.IQR', + label = 'Show/hide interquartile range', + value = F), + + checkboxInput('FCEPlot.semilogx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('FCEPlot.semilogy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + checkboxInput('FCEPlot.show.runs', + label = 'Show individual runs', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_popover( + title = "Individual runs", content = "This procedure can be slow when many + runs are present in the data. Please use with caution.", + placement = "auto" + ) + ), + + hr(), + selectInput('FCEPlot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPlot.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The mean, median and standard deviation of the best function values + found with a fixed-budget of evaluations are depicted against the budget. + The displayed elements can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('FCE_PER_FUN') + ) + ) + ) +} + +fv_agg_box <- function(width = 12, height = '600px', collapsible = T, collapsed = T) { + box(title = HTML('

Expected Function values: all functions

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('FCEPlot.Multi.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput('FCEPlot.Multi.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('FCEPlot.Multi.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + + actionButton('FCEPlot.Multi.PlotButton', label = 'Refresh the figure'), + hr(), + selectInput('FCEPlot.Multi.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPlot.Multi.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + plotlyOutput.IOHanalyzer('FCEPlot.Multi.Plot', aspect_ratio = 1) + ) + ) + ) + ) +} + +fv_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Function Value comparisons

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + selectInput('FCEPlot.Aggr.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('FCEPlot.Aggr.Mode', label = 'Select the plotting mode', + choices = c('radar','line'), selected = 'radar'), + + selectInput('FCEPlot.Aggr.Aggregator', label = 'Create plot for all?', + choices = c('Functions','Dimensions'), selected = 'Functions'), + + checkboxInput('FCEPlot.Aggr.Ranking', + label = 'Use ranking instead of ERT-values', + value = T), + + checkboxInput('FCEPlot.Aggr.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F), + actionButton("FCEPlot.Aggr.Refresh", "Refresh the figure and table"), + # textInput('FCEPlot.Aggr.Targets', label = 'Choose the runtimes (comma-separated)'), + hr(), + selectInput('FCEPlot.Aggr.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPlot.Aggr.Download', label = 'Download the figure'), + hr(), + selectInput('FCEPlot.Aggr.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FCEPlot.Aggr.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + plotlyOutput.IOHanalyzer('FCEPlot.Aggr.Plot'), + hr(), + HTML_P("The chosen budget values per function are as follows (double click an entry to edit it):"), + DT::dataTableOutput("FCEPlot.Aggr.Targets"), + hr(), + HTML_P("The raw function-values are:"), + DT::dataTableOutput("FCEPlot.Aggr.FCETable") + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_dist_box.R b/inst/shiny-server/ui/fv_dist_box.R index b9787b29..105e6fe9 100644 --- a/inst/shiny-server/ui/fv_dist_box.R +++ b/inst/shiny-server/ui/fv_dist_box.R @@ -1,96 +1,96 @@ -fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = 'Histogram of Fixed-Budget Targets', - width = width, collapsible = collapsible, - solidHeader = TRUE, collapsed = collapsed, - status = "primary", - sidebarPanel( - width = 2, - textInput('FCEPDF.Hist.Runtime', label = HTML('Select the budget value'), - value = ''), - selectInput('FCEPDF.Hist.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('Choose whether the histograms are overlaid in one plot - or separated in several subplots:'), - selectInput('FCEPDF.Hist.Mode', '', - choices = c("overlay", "subplot"), - selected = 'subplot'), - checkboxInput("FCEPDF.Hist.Equal", "Use equal bins for all IDs", F), - - hr(), - selectInput('FCEPDF.Hist.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPDF.Hist.Download', label = 'Download the figure') - - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML('

- This histogram counts the number of runs whose best-so-far function - values within the first \\(B\\) evaluations is between \\(v_i\\) and - \\(v_{i+1}\\). The buckets \\([v_i,v_{i+1})\\) are chosen automatically - according to the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= - 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) - and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. - The displayed IDs can be selected by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.

'), - plotlyOutput.IOHanalyzer('FCE_HIST') - ) - ) - ) -} - -fv_pdf_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = 'Empirical Probability Density Function of Fixed-Budget Function Values', - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - HTML('Select the budget for which the distribution of best-so-far function values is shown'), - - textInput('FCEPDF.Bar.Runtime', label = '', value = ''), - selectInput('FCEPDF.Bar.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - checkboxInput('FCEPDF.Bar.Samples', label = 'Show runtime samples', value = T), - checkboxInput('FCEPDF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = T), - - hr(), - selectInput('FCEPDF.Bar.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPDF.Bar.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML('

- The plot shows, for the budget selected on the left, the distribution - of the best-so-far function values of the individual runs (dots), and an estimated distribution of the probability mass function. - The displayed IDs can be selected by clicking on the legend on the right. A tooltip and toolbar - appear when hovering over the figure. A csv file with the runtime data can be downloaded from the - Data Summary tab.'), - plotlyOutput.IOHanalyzer('FCE_PDF') - ) - ) - ) - ) -} +fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = 'Histogram of Fixed-Budget Targets', + width = width, collapsible = collapsible, + solidHeader = TRUE, collapsed = collapsed, + status = "primary", + sidebarPanel( + width = 2, + textInput('FCEPDF.Hist.Runtime', label = HTML('Select the budget value'), + value = ''), + selectInput('FCEPDF.Hist.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('Choose whether the histograms are overlaid in one plot + or separated in several subplots:'), + selectInput('FCEPDF.Hist.Mode', '', + choices = c("overlay", "subplot"), + selected = 'subplot'), + checkboxInput("FCEPDF.Hist.Equal", "Use equal bins for all IDs", F), + + hr(), + selectInput('FCEPDF.Hist.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPDF.Hist.Download', label = 'Download the figure') + + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML('

+ This histogram counts the number of runs whose best-so-far function + values within the first \\(B\\) evaluations is between \\(v_i\\) and + \\(v_{i+1}\\). The buckets \\([v_i,v_{i+1})\\) are chosen automatically + according to the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= + 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) + and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. + The displayed IDs can be selected by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.

'), + plotlyOutput.IOHanalyzer('FCE_HIST') + ) + ) + ) +} + +fv_pdf_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = 'Empirical Probability Density Function of Fixed-Budget Function Values', + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + HTML('Select the budget for which the distribution of best-so-far function values is shown'), + + textInput('FCEPDF.Bar.Runtime', label = '', value = ''), + selectInput('FCEPDF.Bar.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + checkboxInput('FCEPDF.Bar.Samples', label = 'Show runtime samples', value = T), + checkboxInput('FCEPDF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = T), + + hr(), + selectInput('FCEPDF.Bar.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPDF.Bar.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML('

+ The plot shows, for the budget selected on the left, the distribution + of the best-so-far function values of the individual runs (dots), and an estimated distribution of the probability mass function. + The displayed IDs can be selected by clicking on the legend on the right. A tooltip and toolbar + appear when hovering over the figure. A csv file with the runtime data can be downloaded from the + Data Summary tab.'), + plotlyOutput.IOHanalyzer('FCE_PDF') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_ecdf_box.R b/inst/shiny-server/ui/fv_ecdf_box.R index 09042e32..d6570787 100644 --- a/inst/shiny-server/ui/fv_ecdf_box.R +++ b/inst/shiny-server/ui/fv_ecdf_box.R @@ -1,136 +1,136 @@ -fv_ecdf_single_budget_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Empirical Cumulative Distribution of - the Fixed-Budget Values: Single Budgets

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarLayout( - sidebarPanel( - width = 3, - selectInput('FCEECDF.Single.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('Select the budgets for which EDCF curves are displayed '), - textInput('FCEECDF.Single.Target', label = HTML('

\\(B_1\\)

'), value = ''), - checkboxInput('FCEECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('Each EDCF curve shows the proportion of the runs that have found - within the given budget B a solution of at least the required target - value given by the x-axis. The displayed curves can be selected - by clicking on the legend on the right. A tooltip and toolbar - appears when hovering over the figure.

'), - plotlyOutput.IOHanalyzer("FCE_ECDF_PER_TARGET") - ) - ) - ) - ) -} - -fv_ecdf_agg_budgets_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Empirical Cumulative Distribution - of the Fixed-Budget Values: Aggregation

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('FCEECDF.Mult.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('

Set the range and the granularity of the budgets - taken into account in the ECDF curve. The plot will show the ECDF curves - for evenly spaced budgets.

'), - textInput('FCEECDF.Mult.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCEECDF.Mult.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCEECDF.Mult.Step', label = RT_STEP_LABEL, value = ''), -# -# checkboxInput('FCEECDF.Mult.Targets', -# label = 'Show ECDF for each budget', -# value = F), - - checkboxInput('FCEECDF.Mult.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = F), - hr(), - selectInput('FCEECDF.Mult.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEECDF.Mult.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The evenly spaced budget values are:'), - verbatimTextOutput('FCE_RT_GRID'), - - HTML_P('The fraction of (run,budget) pairs \\((i,B)\\) satisfying that the best - solution that the algorithm has found in the \\(i\\)-th run within the - first \\(B\\) evaluations has quality at most \\(v\\) is plotted - against the target value \\(v\\). The displayed elements can be switched - on and off by clicking on the legend on the right. A tooltip and - toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('FCE_ECDF_AGGR') - ) - ) - ) -} - -fv_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Area Under the ECDF

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('FCEECDF.AUC.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('

Set the range and the granularity of the evenly spaced budgets.

'), - textInput('FCEECDF.AUC.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCEECDF.AUC.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCEECDF.AUC.Step', label = RT_STEP_LABEL, value = ''), - - hr(), - selectInput('FCEECDF.AUC.Format', label = 'select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEECDF.AUC.Download', label = 'download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The area under the ECDF is - caculated for the sequence of budget values specified on the left. The displayed - values are normalized against the maximal target value recorded for - each algorithm. Intuitively, the smaller the area, the better the algorithm. - The displayed IDs can be selected by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer("FCE_AUC") - ) - ) - ) -} +fv_ecdf_single_budget_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Empirical Cumulative Distribution of + the Fixed-Budget Values: Single Budgets

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarLayout( + sidebarPanel( + width = 3, + selectInput('FCEECDF.Single.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('Select the budgets for which EDCF curves are displayed '), + textInput('FCEECDF.Single.Target', label = HTML('

\\(B_1\\)

'), value = ''), + checkboxInput('FCEECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('Each EDCF curve shows the proportion of the runs that have found + within the given budget B a solution of at least the required target + value given by the x-axis. The displayed curves can be selected + by clicking on the legend on the right. A tooltip and toolbar + appears when hovering over the figure.

'), + plotlyOutput.IOHanalyzer("FCE_ECDF_PER_TARGET") + ) + ) + ) + ) +} + +fv_ecdf_agg_budgets_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Empirical Cumulative Distribution + of the Fixed-Budget Values: Aggregation

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('FCEECDF.Mult.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('

Set the range and the granularity of the budgets + taken into account in the ECDF curve. The plot will show the ECDF curves + for evenly spaced budgets.

'), + textInput('FCEECDF.Mult.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCEECDF.Mult.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCEECDF.Mult.Step', label = RT_STEP_LABEL, value = ''), +# +# checkboxInput('FCEECDF.Mult.Targets', +# label = 'Show ECDF for each budget', +# value = F), + + checkboxInput('FCEECDF.Mult.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = F), + hr(), + selectInput('FCEECDF.Mult.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEECDF.Mult.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The evenly spaced budget values are:'), + verbatimTextOutput('FCE_RT_GRID'), + + HTML_P('The fraction of (run,budget) pairs \\((i,B)\\) satisfying that the best + solution that the algorithm has found in the \\(i\\)-th run within the + first \\(B\\) evaluations has quality at most \\(v\\) is plotted + against the target value \\(v\\). The displayed elements can be switched + on and off by clicking on the legend on the right. A tooltip and + toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('FCE_ECDF_AGGR') + ) + ) + ) +} + +fv_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Area Under the ECDF

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('FCEECDF.AUC.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('

Set the range and the granularity of the evenly spaced budgets.

'), + textInput('FCEECDF.AUC.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCEECDF.AUC.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCEECDF.AUC.Step', label = RT_STEP_LABEL, value = ''), + + hr(), + selectInput('FCEECDF.AUC.Format', label = 'select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEECDF.AUC.Download', label = 'download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The area under the ECDF is + caculated for the sequence of budget values specified on the left. The displayed + values are normalized against the maximal target value recorded for + each algorithm. Intuitively, the smaller the area, the better the algorithm. + The displayed IDs can be selected by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer("FCE_AUC") + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_par_box.R b/inst/shiny-server/ui/fv_par_box.R index 7f26c8eb..763b5e8d 100644 --- a/inst/shiny-server/ui/fv_par_box.R +++ b/inst/shiny-server/ui/fv_par_box.R @@ -1,134 +1,134 @@ -fv_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Expected Parameter Value (per function)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarPanel( - width = 3, - HTML('

Range of the budget values (\\(x\\) axis)

'), - - textInput('FV_PAR.Plot.Min', label = RT_MIN_LABEL, value = ''), - textInput('FV_PAR.Plot.Max', label = RT_MAX_LABEL, value = ''), - selectInput('FV_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), - - selectInput('FV_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('FV_PAR.Plot.show.mean', label = 'Mean/median', - choices = c('mean', 'median'), - selected = 'mean'), - selectInput('FV_PAR.Plot.CI', "Show standard deviations or quantiles", - choices = c('Standard Deviation', 'Outer Quantiles', 'None'), - selected = 'None') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are - 2% and 98% by default. This can be changed in the settings-tab.", - placement = "auto" - ) - ), - checkboxInput('FV_PAR.Plot.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('FV_PAR.Plot.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_tooltip( - title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" - ) - ), - - hr(), - selectInput('FV_PAR.Plot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('FV_PAR.Plot.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The mean or median of internal parameters of the algorithm - found with a fixed-budget of evaluations are depicted against the budget. - The displayed elements can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('FV_PAR.Plot.Figure') - ) - ) - ) -} - -fv_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Statistics at Chosen Runtime Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-budget parameter values for evenly spaced budget values.'), - - textInput('FV_PAR.Summary.Min', label = RT_MIN_LABEL, value = ''), - textInput('FV_PAR.Summary.Max', label = RT_MAX_LABEL, value = ''), - textInput('FV_PAR.Summary.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FV_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - selectInput('FV_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('FV_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV_PAR.Summary.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', - includeMarkdown('markdown/FV_PAR_SUMMARY_TABLE.Rmd'), '
')), - DT::dataTableOutput('table_FV_PAR_summary') - ) - ) -} - -fv_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Sample at Chosen Runtime Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-budget parameter values for evenly spaced budget values.'), - - textInput('FV_PAR.Sample.Min', label = RT_MIN_LABEL, value = ''), - textInput('FV_PAR.Sample.Max', label = RT_MAX_LABEL, value = ''), - textInput('FV_PAR.Sample.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FV_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - selectInput('FV_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('FV_PAR.Sample.Format', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - selectInput('FV_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV_PAR.Sample.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML('

This table shows for each selected algorithm \\(A\\), - each selected runtime value \\(b\\), and each run \\(r\\) the parameter value - observed when the after a total budget of \\(b\\) function evaluations has been used.

'), - DT::dataTableOutput('table_FV_PAR_SAMPLE') - ) - ) -} +fv_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Expected Parameter Value (per function)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarPanel( + width = 3, + HTML('

Range of the budget values (\\(x\\) axis)

'), + + textInput('FV_PAR.Plot.Min', label = RT_MIN_LABEL, value = ''), + textInput('FV_PAR.Plot.Max', label = RT_MAX_LABEL, value = ''), + selectInput('FV_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), + + selectInput('FV_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('FV_PAR.Plot.show.mean', label = 'Mean/median', + choices = c('mean', 'median'), + selected = 'mean'), + selectInput('FV_PAR.Plot.CI', "Show standard deviations or quantiles", + choices = c('Standard Deviation', 'Outer Quantiles', 'None'), + selected = 'None') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are + 2% and 98% by default. This can be changed in the settings-tab.", + placement = "auto" + ) + ), + checkboxInput('FV_PAR.Plot.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('FV_PAR.Plot.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_tooltip( + title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" + ) + ), + + hr(), + selectInput('FV_PAR.Plot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('FV_PAR.Plot.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The mean or median of internal parameters of the algorithm + found with a fixed-budget of evaluations are depicted against the budget. + The displayed elements can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('FV_PAR.Plot.Figure') + ) + ) + ) +} + +fv_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Statistics at Chosen Runtime Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-budget parameter values for evenly spaced budget values.'), + + textInput('FV_PAR.Summary.Min', label = RT_MIN_LABEL, value = ''), + textInput('FV_PAR.Summary.Max', label = RT_MAX_LABEL, value = ''), + textInput('FV_PAR.Summary.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FV_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FV_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FV_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('FV_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV_PAR.Summary.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', + includeMarkdown('markdown/FV_PAR_SUMMARY_TABLE.Rmd'), '
')), + DT::dataTableOutput('table_FV_PAR_summary') + ) + ) +} + +fv_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Sample at Chosen Runtime Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-budget parameter values for evenly spaced budget values.'), + + textInput('FV_PAR.Sample.Min', label = RT_MIN_LABEL, value = ''), + textInput('FV_PAR.Sample.Max', label = RT_MAX_LABEL, value = ''), + textInput('FV_PAR.Sample.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FV_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FV_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FV_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('FV_PAR.Sample.Format', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + selectInput('FV_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV_PAR.Sample.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML('

This table shows for each selected algorithm \\(A\\), + each selected runtime value \\(b\\), and each run \\(r\\) the parameter value + observed when the after a total budget of \\(b\\) function evaluations has been used.

'), + DT::dataTableOutput('table_FV_PAR_SAMPLE') + ) + ) +} diff --git a/inst/shiny-server/ui/fv_stats_box.R b/inst/shiny-server/ui/fv_stats_box.R index 0463a387..892a502b 100644 --- a/inst/shiny-server/ui/fv_stats_box.R +++ b/inst/shiny-server/ui/fv_stats_box.R @@ -1,118 +1,118 @@ -fv_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Hypothesis Testing

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - textInput('FV_Stats.Overview.Target', label = RT_TAR_LABEL), - textInput('FV_Stats.Overview.Alpha', - label = HTML('

Significe level \\(\\alpha\\)

'), - value = 0.01), - # numericInput('FV_Stats.Overview.Samples', - # label = 'size of the bootstrap sample', - # min = 1, max = 1000, step = 1, value = 0), - hr(), - selectInput('FV_Stats.Overview.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.Overview.DownloadTable', label = 'Download the table'), - hr(), - selectInput('FV_Stats.Overview.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('FV_Stats.Overview.DownloadHeatmap', - label = 'Download the heatmap') - # downloadButton('FV_Stats.Overview.DownloadNetwork', - # label = 'Download the network-graph', status = F) - - ), - - mainPanel( - width = 9, - HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of - algorithms, in order to determine which algorithm gives a significantly - smaller running time distribution. The resulting p-values are arranged in a matrix, where - each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: - the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), - DT::dataTableOutput('FV_Stats.Overview.Pmatrix') - ), - - fluidRow( - column( - width = 12, - HTML('
'), - HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, - are visualized in a heatmap (left) and a network (right). - In each cell (row, column) of the heatmap, the alternative hypothesis is again: - the row algorithm has smaller (better) running time than the column. - The color indicates: -
    -
  • Red: A row is better than the column
    • -
  • Blue: A row is worse than the column
    • -
  • Gray: no significant distinction between the row and column
    • -
- On the right subplot, the partial order resulted from the test is visualized as a network, - where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than - \\(B\\) with the specified \\(\\alpha\\) value.') - ) - ), - - fluidRow( - column( - width = 6, align = 'center', - HTML('
'), - plotlyOutput.IOHanalyzer('FV_Stats.Overview.Heatmap', aspect_ratio = 1) - ), - column( - width = 6, align = 'center', - HTML('
'), - plotOutput("FV_Stats.Overview.Graph", height = '70vh') - ) - ) - ) -} - -fv_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Glicko2-based ranking

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - textInput('FV_Stats.Glicko.Nrgames', - label = "Number of games per (function,dimension) pair", - value = 25), - actionButton('FV_Stats.Glicko.Create', 'Create / Update Ranking'), - hr(), - selectInput('FV_Stats.Glicko.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FV_Stats.Glicko.Download', label = 'Download the figure'), - hr(), - selectInput('FV_Stats.Glicko.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.Glicko.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. - Every round, for every function and dimension of the datasetlist, - each pair of algorithms competes. This competition samples a random function value for the - provided runtime (maximum used among all algorithms). Whichever algorithm has the better - function value wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), - HTML_P("The chosen budget values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("FV_Stats.Glicko.Targets"), - hr(), - HTML_P("The results of the ranking are:"), - plotlyOutput.IOHanalyzer("FV_Stats.Glicko.Candlestick"), - DT::dataTableOutput('FV_Stats.Glicko.Dataframe') - ) - ) +fv_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Hypothesis Testing

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + textInput('FV_Stats.Overview.Target', label = RT_TAR_LABEL), + textInput('FV_Stats.Overview.Alpha', + label = HTML('

Significe level \\(\\alpha\\)

'), + value = 0.01), + # numericInput('FV_Stats.Overview.Samples', + # label = 'size of the bootstrap sample', + # min = 1, max = 1000, step = 1, value = 0), + hr(), + selectInput('FV_Stats.Overview.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.Overview.DownloadTable', label = 'Download the table'), + hr(), + selectInput('FV_Stats.Overview.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('FV_Stats.Overview.DownloadHeatmap', + label = 'Download the heatmap') + # downloadButton('FV_Stats.Overview.DownloadNetwork', + # label = 'Download the network-graph', status = F) + + ), + + mainPanel( + width = 9, + HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of + algorithms, in order to determine which algorithm gives a significantly + smaller running time distribution. The resulting p-values are arranged in a matrix, where + each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: + the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), + DT::dataTableOutput('FV_Stats.Overview.Pmatrix') + ), + + fluidRow( + column( + width = 12, + HTML('
'), + HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, + are visualized in a heatmap (left) and a network (right). + In each cell (row, column) of the heatmap, the alternative hypothesis is again: + the row algorithm has smaller (better) running time than the column. + The color indicates: +
    +
  • Red: A row is better than the column
    • +
  • Blue: A row is worse than the column
    • +
  • Gray: no significant distinction between the row and column
    • +
+ On the right subplot, the partial order resulted from the test is visualized as a network, + where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than + \\(B\\) with the specified \\(\\alpha\\) value.') + ) + ), + + fluidRow( + column( + width = 6, align = 'center', + HTML('
'), + plotlyOutput.IOHanalyzer('FV_Stats.Overview.Heatmap', aspect_ratio = 1) + ), + column( + width = 6, align = 'center', + HTML('
'), + plotOutput("FV_Stats.Overview.Graph", height = '70vh') + ) + ) + ) +} + +fv_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Glicko2-based ranking

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + textInput('FV_Stats.Glicko.Nrgames', + label = "Number of games per (function,dimension) pair", + value = 25), + actionButton('FV_Stats.Glicko.Create', 'Create / Update Ranking'), + hr(), + selectInput('FV_Stats.Glicko.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FV_Stats.Glicko.Download', label = 'Download the figure'), + hr(), + selectInput('FV_Stats.Glicko.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.Glicko.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. + Every round, for every function and dimension of the datasetlist, + each pair of algorithms competes. This competition samples a random function value for the + provided runtime (maximum used among all algorithms). Whichever algorithm has the better + function value wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), + HTML_P("The chosen budget values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("FV_Stats.Glicko.Targets"), + hr(), + HTML_P("The results of the ranking are:"), + plotlyOutput.IOHanalyzer("FV_Stats.Glicko.Candlestick"), + DT::dataTableOutput('FV_Stats.Glicko.Dataframe') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/fv_summary_box.R b/inst/shiny-server/ui/fv_summary_box.R index 5efd1cc7..f71f37ad 100644 --- a/inst/shiny-server/ui/fv_summary_box.R +++ b/inst/shiny-server/ui/fv_summary_box.R @@ -1,88 +1,88 @@ -fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Data Overview

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FCESummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - #TODO: implement this button - hr(), - selectInput('FCESummary.Overview.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FCESummary.Overview.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('table_FV_overview') - ) - ) -} - -fv_stats_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Target Statistics at Chosen Budget Values

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML(FCE_GRID_INPUT_TEXT), - - textInput('FCESummary.Statistics.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCESummary.Statistics.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCESummary.Statistics.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FCESummary.Statistics.Single', label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? - Once toggled, only \\(B_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - hr(), - selectInput('FCESummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FCESummary.Statistics.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', - includeMarkdown('markdown/TAR_SUMMARY_TABLE.Rmd'), - '
')), - DT::dataTableOutput('FCE_SUMMARY') - ) - ) -} - -fv_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Original Target Samples

'), - width = width, solidHeader = TRUE, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML(FCE_GRID_INPUT_TEXT), - - textInput('FCESummary.Sample.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCESummary.Sample.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCESummary.Sample.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FCESummary.Sample.Single', - label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? - Once toggled, only \\(B_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Sample.ID', 'Select which IDs to include:', - choices = NULL, selected = NULL, multiple = T), - - selectInput('FCESummary.Sample.Format', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - hr(), - selectInput('FCESummary.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FCESummary.Sample.Download", "Save the aligned runtime samples") - ), - - mainPanel( - width = 9, - HTML("
This table shows for each selected algorithm \\(A\\), - each selected target value \\(f(x)\\), and each run \\(r\\) - the number \\(T(A,f(x),r)\\) of evaluations performed by the - algorithm until it evaluated for the first time a solution of - quality at least \\(f(x)\\).
"), - br(), - DT::dataTableOutput('FCE_SAMPLE') - ) - ) -} +fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Data Overview

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FCESummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + #TODO: implement this button + hr(), + selectInput('FCESummary.Overview.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FCESummary.Overview.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('table_FV_overview') + ) + ) +} + +fv_stats_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Target Statistics at Chosen Budget Values

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML(FCE_GRID_INPUT_TEXT), + + textInput('FCESummary.Statistics.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCESummary.Statistics.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCESummary.Statistics.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FCESummary.Statistics.Single', label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? + Once toggled, only \\(B_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FCESummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('FCESummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FCESummary.Statistics.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', + includeMarkdown('markdown/TAR_SUMMARY_TABLE.Rmd'), + '
')), + DT::dataTableOutput('FCE_SUMMARY') + ) + ) +} + +fv_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Original Target Samples

'), + width = width, solidHeader = TRUE, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML(FCE_GRID_INPUT_TEXT), + + textInput('FCESummary.Sample.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCESummary.Sample.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCESummary.Sample.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FCESummary.Sample.Single', + label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? + Once toggled, only \\(B_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FCESummary.Sample.ID', 'Select which IDs to include:', + choices = NULL, selected = NULL, multiple = T), + + selectInput('FCESummary.Sample.Format', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + hr(), + selectInput('FCESummary.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FCESummary.Sample.Download", "Save the aligned runtime samples") + ), + + mainPanel( + width = 9, + HTML("
This table shows for each selected algorithm \\(A\\), + each selected target value \\(f(x)\\), and each run \\(r\\) + the number \\(T(A,f(x),r)\\) of evaluations performed by the + algorithm until it evaluated for the first time a solution of + quality at least \\(f(x)\\).
"), + br(), + DT::dataTableOutput('FCE_SAMPLE') + ) + ) +} diff --git a/inst/shiny-server/ui/overview_box.R b/inst/shiny-server/ui/overview_box.R index a8869c8d..5896306d 100644 --- a/inst/shiny-server/ui/overview_box.R +++ b/inst/shiny-server/ui/overview_box.R @@ -1,37 +1,37 @@ -general_overview_box_single <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Overview of Selected Function

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('Overview.Single.ID', 'IDs:', choices = NULL, selected = NULL, multiple = T), - hr(), - selectInput('Overview.Single.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("Overview.Single.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('Overview.Single.Table') - ) - ) -} - -general_overview_box_all <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Overview of All Available Functions

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('Overview.All.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("Overview.All.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('Overview.All.Table') - ) - ) +general_overview_box_single <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Overview of Selected Function

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('Overview.Single.ID', 'IDs:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('Overview.Single.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("Overview.Single.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('Overview.Single.Table') + ) + ) +} + +general_overview_box_all <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Overview of All Available Functions

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('Overview.All.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("Overview.All.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('Overview.All.Table') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/report_generation_box.R b/inst/shiny-server/ui/report_generation_box.R index 0150e150..5b71705a 100644 --- a/inst/shiny-server/ui/report_generation_box.R +++ b/inst/shiny-server/ui/report_generation_box.R @@ -1,119 +1,119 @@ -main_report_box <- function(width = 6, collapsible = T, collapsed = F) { - box(title = HTML('

Report Content

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - HTML("Select which components should be in the report. \n All relevant settings (bounds, targets, scaling...) - will be taken from the relevant tab if available, or set to a default value."), - checkboxInput("Report.Introduction", "Introduction", T), - fixed_target_report_box(), - fixed_budget_report_box(), - parameter_box(), - hr(), - downloadButton("Report.Generate", "Generate report pdf"), - downloadButton("Report.Generate.Tar", "Generate report tar", style="float:right") - ) -} - -fixed_target_report_box <- function(){ - box(title = "Fixed-Target", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "blue", - box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.Overview", "Data Overview", F), - conditionalSelection("Report.RT.Overview",'input["Report.RT.Overview"]'), - checkboxInput("Report.RT.Statistics", "Runtime Statistics", F), - conditionalSelection("Report.RT.Statistics",'input["Report.RT.Statistics"]'), - checkboxInput("Report.RT.Samples", "Runtime Samples", F), - conditionalSelection("Report.RT.Samples",'input["Report.RT.Samples"]') - ), - box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.Single_ERT", "Runtime plot (per function)", F), - conditionalSelection("Report.RT.Single_ERT",'input["Report.RT.Single_ERT"]'), - checkboxInput("Report.RT.Multi_ERT", "Runtime plot (all functions)", F), - conditionalSelection("Report.RT.Multi_ERT",'input["Report.RT.Multi_ERT"]', fid=F), - checkboxInput("Report.RT.Rank", "Runtime ranking", F), - conditionalSelection("Report.RT.Rank",'input["Report.RT.Rank"]', fid=F) #, plottype=c("radar", "line")) - ), - box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.Histogram", "Histogram of hitting times", F), - conditionalSelection("Report.RT.Histogram",'input["Report.RT.Histogram"]'), # plottype = c("subplot","overlay")), - checkboxInput("Report.RT.PMF", "Probability Mass Function", F), - conditionalSelection("Report.RT.PMF",'input["Report.RT.PMF"]') - ), - box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.ECDF_Single_Target", "Single-target ECDF", F), - conditionalSelection("Report.RT.ECDF_Single_Target",'input["Report.RT.ECDF_Single_Target"]'), - checkboxInput("Report.RT.ECDF_Single_Function", "Single-function ECDF", F), - conditionalSelection("Report.RT.ECDF_Single_Function",'input["Report.RT.ECDF_Single_Function"]'), - checkboxInput("Report.RT.ECDF_Aggregated", "Aggregated ECDF", F), - conditionalSelection("Report.RT.ECDF_Aggregated",'input["Report.RT.ECDF_Aggregated"]', F, F), - checkboxInput("Report.RT.ECDF_AUC", "AUC of ECDF", F), - conditionalSelection("Report.RT.ECDF_AUC",'input["Report.RT.ECDF_AUC"]') - ) - ) -} - -fixed_budget_report_box <- function(){ - box(title = "Fixed-Budget", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "blue", - box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.Overview", "Data Overview", F), - conditionalSelection("Report.FV.Overview",'input["Report.FV.Overview"]'), - checkboxInput("Report.FV.Statistics", "Runtime Statistics", F), - conditionalSelection("Report.FV.Statistics",'input["Report.FV.Statistics"]'), - checkboxInput("Report.FV.Samples", "Runtime Samples", F), - conditionalSelection("Report.FV.Samples",'input["Report.FV.Samples"]') - ), - box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.Single_FCE", "Runtime plot (per function)", F), - conditionalSelection("Report.FV.Single_FCE",'input["Report.FV.Single_FCE"]'), - checkboxInput("Report.FV.Multi_FCE", "Runtime plot (all functions)", F), - conditionalSelection("Report.FV.Multi_FCE",'input["Report.FV.Multi_FCE"]', fid=F), - checkboxInput("Report.FV.Rank", "Runtime ranking", F), - conditionalSelection("Report.FV.Rank",'input["Report.FV.Rank"]', fid=F) #, plottype=c("radar", "line")) - ), - box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.Histogram", "Histogram of hitting times", F), - conditionalSelection("Report.FV.Histogram",'input["Report.FV.Histogram"]'), # plottype = c("subplot","overlay")), - checkboxInput("Report.FV.PMF", "Probability Mass Function", F), - conditionalSelection("Report.FV.PMF",'input["Report.FV.PMF"]') - ), - box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.ECDF_Single_Target", "Single-target ECDF", F), - conditionalSelection("Report.FV.ECDF_Single_Target",'input["Report.FV.ECDF_Single_Target"]'), - checkboxInput("Report.FV.ECDF_Single_Function", "Single-function ECDF", F), - conditionalSelection("Report.FV.ECDF_Single_Function",'input["Report.FV.ECDF_Single_Function"]'), - checkboxInput("Report.FV.ECDF_AUC", "AUC of ECDF", F), - conditionalSelection("Report.FV.ECDF_AUC",'input["Report.FV.ECDF_AUC"]') - ) - ) -} - -parameter_box <- function(){ - box(title = "Parameters", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "blue", - checkboxInput("Report.Param.Plot", "Parameter plot", F), - conditionalSelection("Report.Param.Plot",'input["Report.Param.Plot"]'), - checkboxInput("Report.Param.Statistics", "Parameter statistics", F), - conditionalSelection("Report.Param.Statistics",'input["Report.Param.Statistics"]') - ) -} - -conditionalSelection <- function(id, condition, fid=T, dim=T, alg=T, scalex=F, scaley=F, plottype=NULL){ - ns <- NS(id) - conditionalPanel(condition = condition, - if (fid) selectInput(ns("FuncId"), label = "Function ids", choices = NULL, selected = NULL, multiple = T), - if (dim) selectInput(ns("DIM"), label = "Dimensions", choices = NULL, selected = NULL, multiple = T), - if (alg) selectInput(ns("Alg"), label = "Algorithms", choices = NULL, selected = NULL, multiple = T), - if (scalex) checkboxInput(ns("Scalex"), label = "Scale x-axis logarithmically", value = F), - if (scaley) checkboxInput(ns("Scaley"), label = "Scale y-axis logarithmically", value = F), - if (!is.null(plottype)) selectInput(ns("Type"), label = "Plot type", choices = plottype, selected = plottype[[1]]) - ) +main_report_box <- function(width = 6, collapsible = T, collapsed = F) { + box(title = HTML('

Report Content

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + HTML("Select which components should be in the report. \n All relevant settings (bounds, targets, scaling...) + will be taken from the relevant tab if available, or set to a default value."), + checkboxInput("Report.Introduction", "Introduction", T), + fixed_target_report_box(), + fixed_budget_report_box(), + parameter_box(), + hr(), + downloadButton("Report.Generate", "Generate report pdf"), + downloadButton("Report.Generate.Tar", "Generate report tar", style="float:right") + ) +} + +fixed_target_report_box <- function(){ + box(title = "Fixed-Target", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "blue", + box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.Overview", "Data Overview", F), + conditionalSelection("Report.RT.Overview",'input["Report.RT.Overview"]'), + checkboxInput("Report.RT.Statistics", "Runtime Statistics", F), + conditionalSelection("Report.RT.Statistics",'input["Report.RT.Statistics"]'), + checkboxInput("Report.RT.Samples", "Runtime Samples", F), + conditionalSelection("Report.RT.Samples",'input["Report.RT.Samples"]') + ), + box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.Single_ERT", "Runtime plot (per function)", F), + conditionalSelection("Report.RT.Single_ERT",'input["Report.RT.Single_ERT"]'), + checkboxInput("Report.RT.Multi_ERT", "Runtime plot (all functions)", F), + conditionalSelection("Report.RT.Multi_ERT",'input["Report.RT.Multi_ERT"]', fid=F), + checkboxInput("Report.RT.Rank", "Runtime ranking", F), + conditionalSelection("Report.RT.Rank",'input["Report.RT.Rank"]', fid=F) #, plottype=c("radar", "line")) + ), + box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.Histogram", "Histogram of hitting times", F), + conditionalSelection("Report.RT.Histogram",'input["Report.RT.Histogram"]'), # plottype = c("subplot","overlay")), + checkboxInput("Report.RT.PMF", "Probability Mass Function", F), + conditionalSelection("Report.RT.PMF",'input["Report.RT.PMF"]') + ), + box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.ECDF_Single_Target", "Single-target ECDF", F), + conditionalSelection("Report.RT.ECDF_Single_Target",'input["Report.RT.ECDF_Single_Target"]'), + checkboxInput("Report.RT.ECDF_Single_Function", "Single-function ECDF", F), + conditionalSelection("Report.RT.ECDF_Single_Function",'input["Report.RT.ECDF_Single_Function"]'), + checkboxInput("Report.RT.ECDF_Aggregated", "Aggregated ECDF", F), + conditionalSelection("Report.RT.ECDF_Aggregated",'input["Report.RT.ECDF_Aggregated"]', F, F), + checkboxInput("Report.RT.ECDF_AUC", "AUC of ECDF", F), + conditionalSelection("Report.RT.ECDF_AUC",'input["Report.RT.ECDF_AUC"]') + ) + ) +} + +fixed_budget_report_box <- function(){ + box(title = "Fixed-Budget", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "blue", + box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.Overview", "Data Overview", F), + conditionalSelection("Report.FV.Overview",'input["Report.FV.Overview"]'), + checkboxInput("Report.FV.Statistics", "Runtime Statistics", F), + conditionalSelection("Report.FV.Statistics",'input["Report.FV.Statistics"]'), + checkboxInput("Report.FV.Samples", "Runtime Samples", F), + conditionalSelection("Report.FV.Samples",'input["Report.FV.Samples"]') + ), + box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.Single_FCE", "Runtime plot (per function)", F), + conditionalSelection("Report.FV.Single_FCE",'input["Report.FV.Single_FCE"]'), + checkboxInput("Report.FV.Multi_FCE", "Runtime plot (all functions)", F), + conditionalSelection("Report.FV.Multi_FCE",'input["Report.FV.Multi_FCE"]', fid=F), + checkboxInput("Report.FV.Rank", "Runtime ranking", F), + conditionalSelection("Report.FV.Rank",'input["Report.FV.Rank"]', fid=F) #, plottype=c("radar", "line")) + ), + box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.Histogram", "Histogram of hitting times", F), + conditionalSelection("Report.FV.Histogram",'input["Report.FV.Histogram"]'), # plottype = c("subplot","overlay")), + checkboxInput("Report.FV.PMF", "Probability Mass Function", F), + conditionalSelection("Report.FV.PMF",'input["Report.FV.PMF"]') + ), + box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.ECDF_Single_Target", "Single-target ECDF", F), + conditionalSelection("Report.FV.ECDF_Single_Target",'input["Report.FV.ECDF_Single_Target"]'), + checkboxInput("Report.FV.ECDF_Single_Function", "Single-function ECDF", F), + conditionalSelection("Report.FV.ECDF_Single_Function",'input["Report.FV.ECDF_Single_Function"]'), + checkboxInput("Report.FV.ECDF_AUC", "AUC of ECDF", F), + conditionalSelection("Report.FV.ECDF_AUC",'input["Report.FV.ECDF_AUC"]') + ) + ) +} + +parameter_box <- function(){ + box(title = "Parameters", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "blue", + checkboxInput("Report.Param.Plot", "Parameter plot", F), + conditionalSelection("Report.Param.Plot",'input["Report.Param.Plot"]'), + checkboxInput("Report.Param.Statistics", "Parameter statistics", F), + conditionalSelection("Report.Param.Statistics",'input["Report.Param.Statistics"]') + ) +} + +conditionalSelection <- function(id, condition, fid=T, dim=T, alg=T, scalex=F, scaley=F, plottype=NULL){ + ns <- NS(id) + conditionalPanel(condition = condition, + if (fid) selectInput(ns("FuncId"), label = "Function ids", choices = NULL, selected = NULL, multiple = T), + if (dim) selectInput(ns("DIM"), label = "Dimensions", choices = NULL, selected = NULL, multiple = T), + if (alg) selectInput(ns("Alg"), label = "Algorithms", choices = NULL, selected = NULL, multiple = T), + if (scalex) checkboxInput(ns("Scalex"), label = "Scale x-axis logarithmically", value = F), + if (scaley) checkboxInput(ns("Scaley"), label = "Scale y-axis logarithmically", value = F), + if (!is.null(plottype)) selectInput(ns("Type"), label = "Plot type", choices = plottype, selected = plottype[[1]]) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/rt_dist_box.R b/inst/shiny-server/ui/rt_dist_box.R index f7efbc39..1631b4ef 100644 --- a/inst/shiny-server/ui/rt_dist_box.R +++ b/inst/shiny-server/ui/rt_dist_box.R @@ -1,114 +1,114 @@ -rt_histogram_box <- function(width = 12, collapsed = T, collapsible = T) { - box(title = 'Histogram of Fixed-Target Runtimes', - width = width, collapsible = collapsible, solidHeader = TRUE, collapsed = collapsed, - status = "primary", - sidebarPanel( - width = 2, - textInput('RTPMF.Hist.Target', label = HTML('Select the target value'), - value = ''), - selectInput( - 'RTPMF.Hist.Algs', - label = 'Select which IDs to include:', - multiple = T, - selected = NULL, - choices = NULL - ) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", - content = alg_select_info, - placement = "auto" - ) - ), - - HTML_P('Choose whether the histograms are overlaid in one plot - or separated in several subplots:'), - selectInput('RTPMF.Hist.Mode', '', - choices = c("overlay", "subplot"), - selected = 'subplot'), - checkboxInput("RTPMF.Hist.Equal", "Use equal bins for all algorithms", F), - hr(), - selectInput('RTPMF.Hist.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTPMF.Hist.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('This histogram counts how many runs needed between - \\(t\\) and \\(t+1\\) function evaluations. The bins - \\([t,t+1)\\) are chosen automatically. The bin size is determined - by the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= - 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) - and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. - The displayed algorithms can be selected by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('RT_HIST', aspect_ratio = 16/14) - ) - ) - ) -} - -rt_pmf_box <- function(width = 12, collapsed = T, collapsible = T) { - box(title = 'Empirical Probability Mass Function of the Runtime', - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - textInput('RTPMF.Bar.Target', label = 'Select the target value', value = ''), - - selectInput( - 'RTPMF.Bar.Algs', - label = 'Select which IDs to include:', - multiple = T, - selected = NULL, - choices = NULL - ) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", - content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput('RTPMF.Bar.Sample', label = 'Show runtime for each run', value = T), - checkboxInput('RTPMF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = F), - - hr(), - selectInput('RTPMF.Bar.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTPMF.Bar.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML('

Warning! The - probability mass function of the runtime is approximated by the - treating the runtime as a continuous random variable and - applying the kernel estimation (KDE):

'), - HTML('

- The plot shows the distribution of the first hitting - times of the individual runs (dots), and an estimated - distribution of the probability mass function. - The displayed algorithms can be selected by clicking on - the legend on the right. A tooltip and toolbar - appear when hovering over the figure. This also includes the - option to download the plot as png file. A csv file with the runtime - data can be downlowaded from the - Data Summary tab.'), - plotlyOutput.IOHanalyzer('RT_PMF') - ) - ) - ) - ) -} +rt_histogram_box <- function(width = 12, collapsed = T, collapsible = T) { + box(title = 'Histogram of Fixed-Target Runtimes', + width = width, collapsible = collapsible, solidHeader = TRUE, collapsed = collapsed, + status = "primary", + sidebarPanel( + width = 2, + textInput('RTPMF.Hist.Target', label = HTML('Select the target value'), + value = ''), + selectInput( + 'RTPMF.Hist.Algs', + label = 'Select which IDs to include:', + multiple = T, + selected = NULL, + choices = NULL + ) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", + content = alg_select_info, + placement = "auto" + ) + ), + + HTML_P('Choose whether the histograms are overlaid in one plot + or separated in several subplots:'), + selectInput('RTPMF.Hist.Mode', '', + choices = c("overlay", "subplot"), + selected = 'subplot'), + checkboxInput("RTPMF.Hist.Equal", "Use equal bins for all algorithms", F), + hr(), + selectInput('RTPMF.Hist.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTPMF.Hist.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('This histogram counts how many runs needed between + \\(t\\) and \\(t+1\\) function evaluations. The bins + \\([t,t+1)\\) are chosen automatically. The bin size is determined + by the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= + 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) + and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. + The displayed algorithms can be selected by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('RT_HIST', aspect_ratio = 16/14) + ) + ) + ) +} + +rt_pmf_box <- function(width = 12, collapsed = T, collapsible = T) { + box(title = 'Empirical Probability Mass Function of the Runtime', + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + textInput('RTPMF.Bar.Target', label = 'Select the target value', value = ''), + + selectInput( + 'RTPMF.Bar.Algs', + label = 'Select which IDs to include:', + multiple = T, + selected = NULL, + choices = NULL + ) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", + content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput('RTPMF.Bar.Sample', label = 'Show runtime for each run', value = T), + checkboxInput('RTPMF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = F), + + hr(), + selectInput('RTPMF.Bar.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTPMF.Bar.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML('

Warning! The + probability mass function of the runtime is approximated by the + treating the runtime as a continuous random variable and + applying the kernel estimation (KDE):

'), + HTML('

+ The plot shows the distribution of the first hitting + times of the individual runs (dots), and an estimated + distribution of the probability mass function. + The displayed algorithms can be selected by clicking on + the legend on the right. A tooltip and toolbar + appear when hovering over the figure. This also includes the + option to download the plot as png file. A csv file with the runtime + data can be downlowaded from the + Data Summary tab.'), + plotlyOutput.IOHanalyzer('RT_PMF') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/rt_ecdf_box.R b/inst/shiny-server/ui/rt_ecdf_box.R index 2e7eb064..5ccffbd6 100644 --- a/inst/shiny-server/ui/rt_ecdf_box.R +++ b/inst/shiny-server/ui/rt_ecdf_box.R @@ -1,256 +1,256 @@ -rt_ecdf_single_target_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Empirical Cumulative Distribution: Single target

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - selectInput('RTECDF.Single.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('Select the target values for which EDCF curves are displayed'), - textInput('RTECDF.Single.Target', label = HTML('

\\(f_{target}\\)

'), - value = ''), - - checkboxInput('RTECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) - ), - - mainPanel( - width = 9, - column( - width = 12, - align = "center", - HTML_P('Each EDCF curve shows the proportion of the runs - that have found a solution of at least the required - target value within the budget given by the \\(x\\)-axis. - The displayed curves can be selected by clicking on the legend on the right. A tooltip - and toolbar appears when hovering over the figure. - This also includes the option to download the plot as png file.'), - plotlyOutput.IOHanalyzer("RT_ECDF"), - HTML_P('The approximated Area Under the Curve of this displayed single-target ECDF is:'), - DT::dataTableOutput('AUC_GRID_GENERATED_SINGLE') - ) - ) - ) - ) -} - -rt_ecdf_agg_targets_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Aggregated Empirical Cumulative - Distribution: Single function

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('RTECDF.Multi.Algs', label = 'Select which IDs to include:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('

Set the range and the granularity - of the quality targets taken into account in the ECDF curve. - The plot will show the ECDF curves for evenly spaced target values.

'), - textInput('RTECDF.Multi.Min', label = F_MIN_LABEL, value = ''), - textInput('RTECDF.Multi.Max', label = F_MAX_LABEL, value = ''), - textInput('RTECDF.Multi.Step', label = F_STEP_LABEL, value = ''), - # checkboxInput('RTECDF.Multi.Targets', - # label = 'Show ECDFs for each target', - # value = F), - checkboxInput('RTECDF.Multi.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - hr(), - selectInput('RTECDF.Multi.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTECDF.Multi.Download', label = 'download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The evenly spaced target values are:'), - verbatimTextOutput('RT_GRID'), - HTML_P('The fraction of (run,target value) - pairs \\((i,v)\\) satisfying that the best solution that the algorithm has - found in the \\(i\\)-th run within the given time budget \\(t\\) has quality at least - \\(v\\) is plotted against the available budget \\(t\\). The displayed elements can be switched - on and off by clicking on the legend on the right. A tooltip - and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('RT_ECDF_AGGR'), - HTML_P('The approximated Area Under the Curve of this displayed single-function ECDF is:'), - DT::dataTableOutput('AUC_GRID_GENERATED_FUNC') - ) - ) - ) -} - -rt_ecdf_agg_fct_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Aggregated Empirical Cumulative - Distribution: All functions

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput( - 'RTECDF.Aggr.Algs', - label = 'Select which IDs to include:', - multiple = T, - selected = NULL, - choices = NULL - ) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput("RTECDF.Aggr.Func", "Aggregate functions", value = T), - checkboxInput("RTECDF.Aggr.Dim", "Aggregate dimensions", value = F), - checkboxInput("RTECDF.Aggr.Logx", "Scale x axis \\(\\log_{10}\\)", value = T), - checkboxInput("RTECDF.Aggr.Logy", "Scale y axis \\(\\log_{10}\\)", value = F) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Scaling", content = "The logorithmic scaling might cause some visual issues - when the smallest y-values are (very close to) 0. - Please be mindful of this fact when using this option.", - placement = "auto" - ) - ), - - br(), - actionButton( - "RTECDF.Aggr.Refresh", - label = HTML('

Refresh the figure

') - ), - - hr(), - selectInput("RTECDF.Aggr.Target_type", label = "Select the spacing for the - automatically generated ECDF-targets:", - choices = c('linear', 'log-linear', 'bbob'), - selected = 'linear') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Default targets", content = "The log-linear spacing only works correctly - when no negative target values are present in the data. The BBOB-spacing is pre-defined - to 51 log-linear targets between 10^2 and 10^-8.", - placement = "auto" - ) - ), - numericInput("RTECDF.Aggr.Target_number", label = "Select the number of ECDF-targets to - generate for each function/dimension", value = 10, min = 1, max = 100), - - HTML_P('Alternatively, you can download the table containing the target values for each - (function, dimension)-pair and edit the table as you want. Please keep - the file format when modifying it.'), - downloadButton('RTECDF.Aggr.Table.Download', label = 'Download the table of targets'), - br(), - br(), - br(), - - HTML_P('Upload the table you just downloaded and edited'), - fileInput( - "RTECDF.Aggr.Table.Upload", - label = NULL, - multiple = FALSE, - accept = c( - "text/csv", - "text/comma-separated-values,text/plain", - ".csv" - ) - ), - - hr(), - selectInput('RTECDF.Aggr.Format', label = 'figure format to download', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTECDF.Aggr.Download', label = 'Download the figure'), - hr(), - selectInput('RTECDF.AUC.Table.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('RTECDF.AUC.Table.Download', label = 'Download the AUC table') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - hr(), - - HTML_P('The fraction of (run,target value, ...) - pairs \\((i,v, ...)\\) satisfying that the best solution that the algorithm has - found in the \\(i\\)-th (run of function \\(f\\) in dimension \\(d\\)) within - the given time budget \\(t\\) has quality at least \\(v\\) is plotted against - the available budget \\(t\\). The displayed elements can be switched - on and off by clicking on the legend on the right. A tooltip - and toolbar appears when hovering over the figure. Aggregation over - functions and dimension can be switched on or off using the checkboxes on - the left; when aggregation is off the selected function / dimension - is chosen according the the value in the bottom-left selection-box.'), - plotlyOutput.IOHanalyzer('RT_ECDF_MULT'), - HTML_P('The approximated Area Under the Curve of this displayed ECDF is:'), - DT::dataTableOutput('AUC_GRID_GENERATED'), - HTML_P('The selected targets are:'), - DT::dataTableOutput('RT_GRID_GENERATED') - ) - ) - ) -} - -# rt_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { -# box( -# title = HTML('

Area Under the ECDF

'), -# width = width, collapsible = collapsible, collapsed = collapsed, -# solidHeader = T, status = "primary", -# sidebarPanel( -# width = 3, -# selectInput('RTECDF.AUC.Algs', label = 'Select which IDs to include:', -# multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( -# custom_icon() %>% -# bs_embed_popover( -# title = "ID selection", content = alg_select_info, -# placement = "auto" -# ) -# ), -# HTML('

Set the range and the granularity of -# the evenly spaced quality targets taken into account in the plot.

'), -# textInput('RTECDF.AUC.Min', label = F_MIN_LABEL, value = ''), -# textInput('RTECDF.AUC.Max', label = F_MAX_LABEL, value = ''), -# textInput('RTECDF.AUC.Step', label = F_STEP_LABEL, value = ''), -# -# hr(), -# selectInput('RTECDF.AUC.Format', label = 'Select the figure format', -# choices = supported_fig_format, selected = 'pdf'), -# downloadButton('RTECDF.AUC.Download', label = 'Download the figure') -# ), -# -# mainPanel( -# width = 9, -# column( -# width = 12, align = "center", -# HTML_P('The area under the ECDF is -# caculated for the sequence of target values specified on the left. The displayed -# values are normalized against the maximal number of function evaluations for -# each algorithm. Intuitively, the larger the area, the better the algorithm. -# The displayed algorithms can be selected by clicking on the legend on the right. -# A tooltip and toolbar appears when hovering over the figure. -# This also includes the option to download the plot as png file.'), -# plotlyOutput.IOHanalyzer("RT_AUC") -# ) -# ) -# ) -# } +rt_ecdf_single_target_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Empirical Cumulative Distribution: Single target

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + selectInput('RTECDF.Single.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('Select the target values for which EDCF curves are displayed'), + textInput('RTECDF.Single.Target', label = HTML('

\\(f_{target}\\)

'), + value = ''), + + checkboxInput('RTECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) + ), + + mainPanel( + width = 9, + column( + width = 12, + align = "center", + HTML_P('Each EDCF curve shows the proportion of the runs + that have found a solution of at least the required + target value within the budget given by the \\(x\\)-axis. + The displayed curves can be selected by clicking on the legend on the right. A tooltip + and toolbar appears when hovering over the figure. + This also includes the option to download the plot as png file.'), + plotlyOutput.IOHanalyzer("RT_ECDF"), + HTML_P('The approximated Area Under the Curve of this displayed single-target ECDF is:'), + DT::dataTableOutput('AUC_GRID_GENERATED_SINGLE') + ) + ) + ) + ) +} + +rt_ecdf_agg_targets_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Aggregated Empirical Cumulative + Distribution: Single function

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('RTECDF.Multi.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('

Set the range and the granularity + of the quality targets taken into account in the ECDF curve. + The plot will show the ECDF curves for evenly spaced target values.

'), + textInput('RTECDF.Multi.Min', label = F_MIN_LABEL, value = ''), + textInput('RTECDF.Multi.Max', label = F_MAX_LABEL, value = ''), + textInput('RTECDF.Multi.Step', label = F_STEP_LABEL, value = ''), + # checkboxInput('RTECDF.Multi.Targets', + # label = 'Show ECDFs for each target', + # value = F), + checkboxInput('RTECDF.Multi.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + hr(), + selectInput('RTECDF.Multi.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTECDF.Multi.Download', label = 'download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The evenly spaced target values are:'), + verbatimTextOutput('RT_GRID'), + HTML_P('The fraction of (run,target value) + pairs \\((i,v)\\) satisfying that the best solution that the algorithm has + found in the \\(i\\)-th run within the given time budget \\(t\\) has quality at least + \\(v\\) is plotted against the available budget \\(t\\). The displayed elements can be switched + on and off by clicking on the legend on the right. A tooltip + and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('RT_ECDF_AGGR'), + HTML_P('The approximated Area Under the Curve of this displayed single-function ECDF is:'), + DT::dataTableOutput('AUC_GRID_GENERATED_FUNC') + ) + ) + ) +} + +rt_ecdf_agg_fct_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Aggregated Empirical Cumulative + Distribution: All functions

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput( + 'RTECDF.Aggr.Algs', + label = 'Select which IDs to include:', + multiple = T, + selected = NULL, + choices = NULL + ) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput("RTECDF.Aggr.Func", "Aggregate functions", value = T), + checkboxInput("RTECDF.Aggr.Dim", "Aggregate dimensions", value = F), + checkboxInput("RTECDF.Aggr.Logx", "Scale x axis \\(\\log_{10}\\)", value = T), + checkboxInput("RTECDF.Aggr.Logy", "Scale y axis \\(\\log_{10}\\)", value = F) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Scaling", content = "The logorithmic scaling might cause some visual issues + when the smallest y-values are (very close to) 0. + Please be mindful of this fact when using this option.", + placement = "auto" + ) + ), + + br(), + actionButton( + "RTECDF.Aggr.Refresh", + label = HTML('

Refresh the figure

') + ), + + hr(), + selectInput("RTECDF.Aggr.Target_type", label = "Select the spacing for the + automatically generated ECDF-targets:", + choices = c('linear', 'log-linear', 'bbob'), + selected = 'linear') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Default targets", content = "The log-linear spacing only works correctly + when no negative target values are present in the data. The BBOB-spacing is pre-defined + to 51 log-linear targets between 10^2 and 10^-8.", + placement = "auto" + ) + ), + numericInput("RTECDF.Aggr.Target_number", label = "Select the number of ECDF-targets to + generate for each function/dimension", value = 10, min = 1, max = 100), + + HTML_P('Alternatively, you can download the table containing the target values for each + (function, dimension)-pair and edit the table as you want. Please keep + the file format when modifying it.'), + downloadButton('RTECDF.Aggr.Table.Download', label = 'Download the table of targets'), + br(), + br(), + br(), + + HTML_P('Upload the table you just downloaded and edited'), + fileInput( + "RTECDF.Aggr.Table.Upload", + label = NULL, + multiple = FALSE, + accept = c( + "text/csv", + "text/comma-separated-values,text/plain", + ".csv" + ) + ), + + hr(), + selectInput('RTECDF.Aggr.Format', label = 'figure format to download', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTECDF.Aggr.Download', label = 'Download the figure'), + hr(), + selectInput('RTECDF.AUC.Table.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('RTECDF.AUC.Table.Download', label = 'Download the AUC table') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + hr(), + + HTML_P('The fraction of (run,target value, ...) + pairs \\((i,v, ...)\\) satisfying that the best solution that the algorithm has + found in the \\(i\\)-th (run of function \\(f\\) in dimension \\(d\\)) within + the given time budget \\(t\\) has quality at least \\(v\\) is plotted against + the available budget \\(t\\). The displayed elements can be switched + on and off by clicking on the legend on the right. A tooltip + and toolbar appears when hovering over the figure. Aggregation over + functions and dimension can be switched on or off using the checkboxes on + the left; when aggregation is off the selected function / dimension + is chosen according the the value in the bottom-left selection-box.'), + plotlyOutput.IOHanalyzer('RT_ECDF_MULT'), + HTML_P('The approximated Area Under the Curve of this displayed ECDF is:'), + DT::dataTableOutput('AUC_GRID_GENERATED'), + HTML_P('The selected targets are:'), + DT::dataTableOutput('RT_GRID_GENERATED') + ) + ) + ) +} + +# rt_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { +# box( +# title = HTML('

Area Under the ECDF

'), +# width = width, collapsible = collapsible, collapsed = collapsed, +# solidHeader = T, status = "primary", +# sidebarPanel( +# width = 3, +# selectInput('RTECDF.AUC.Algs', label = 'Select which IDs to include:', +# multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( +# custom_icon() %>% +# bs_embed_popover( +# title = "ID selection", content = alg_select_info, +# placement = "auto" +# ) +# ), +# HTML('

Set the range and the granularity of +# the evenly spaced quality targets taken into account in the plot.

'), +# textInput('RTECDF.AUC.Min', label = F_MIN_LABEL, value = ''), +# textInput('RTECDF.AUC.Max', label = F_MAX_LABEL, value = ''), +# textInput('RTECDF.AUC.Step', label = F_STEP_LABEL, value = ''), +# +# hr(), +# selectInput('RTECDF.AUC.Format', label = 'Select the figure format', +# choices = supported_fig_format, selected = 'pdf'), +# downloadButton('RTECDF.AUC.Download', label = 'Download the figure') +# ), +# +# mainPanel( +# width = 9, +# column( +# width = 12, align = "center", +# HTML_P('The area under the ECDF is +# caculated for the sequence of target values specified on the left. The displayed +# values are normalized against the maximal number of function evaluations for +# each algorithm. Intuitively, the larger the area, the better the algorithm. +# The displayed algorithms can be selected by clicking on the legend on the right. +# A tooltip and toolbar appears when hovering over the figure. +# This also includes the option to download the plot as png file.'), +# plotlyOutput.IOHanalyzer("RT_AUC") +# ) +# ) +# ) +# } diff --git a/inst/shiny-server/ui/rt_par_box.R b/inst/shiny-server/ui/rt_par_box.R index 21b964a7..2feef3ad 100644 --- a/inst/shiny-server/ui/rt_par_box.R +++ b/inst/shiny-server/ui/rt_par_box.R @@ -1,134 +1,134 @@ -rt_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Expected Parameter Value (per function)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarPanel( - width = 3, - HTML('

Range of the function values (\\(x\\) axis)

'), - - textInput('RT_PAR.Plot.Min', label = F_MIN_LABEL, value = ''), - textInput('RT_PAR.Plot.Max', label = F_MAX_LABEL, value = ''), - selectInput('RT_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), - - selectInput('RT_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "ID selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('RT_PAR.Plot.show.mean', label = 'Mean/median', - choices = c('mean', 'median'), - selected = 'mean'), - selectInput('RT_PAR.Plot.CI', "Show standard deviations or quantiles", - choices = c('Standard Deviation', 'Outer Quantiles', 'None'), - selected = 'None') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are - 2% and 98% by default. This can be changed in the settings-tab.", - placement = "auto" - ) - ), - checkboxInput('RT_PAR.Plot.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('RT_PAR.Plot.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_tooltip( - title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" - ) - ), - - hr(), - selectInput('RT_PAR.Plot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('RT_PAR.Plot.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The mean or median of internal parameters of the algorithm - found with a fixed-budget of evaluations are depicted against the budget. - The displayed elements can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('RT_PAR.Plot.Figure') - ) - ) - ) -} - -rt_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Statistics at Chosen Target Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-target parameter values for evenly spaced target values.'), - - textInput('RT_PAR.Summary.Min', label = F_MIN_LABEL, value = ''), - textInput('RT_PAR.Summary.Max', label = F_MAX_LABEL, value = ''), - textInput('RT_PAR.Summary.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RT_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - selectInput('RT_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('RT_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT_PAR.Summary.Download", "Save this table as csv") - ), - - mainPanel( - width = 9, - HTML(paste0('
', - includeMarkdown('markdown/PAR_SUMMARY_TABLE.Rmd'), '
')), - DT::dataTableOutput('table_RT_PAR_summary') - ) - ) -} - -rt_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Sample at Chosen Target Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-target parameter values for evenly spaced target values.'), - - textInput('RT_PAR.Sample.Min', label = F_MIN_LABEL, value = ''), - textInput('RT_PAR.Sample.Max', label = F_MAX_LABEL, value = ''), - textInput('RT_PAR.Sample.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RT_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - selectInput('RT_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('RT_PAR.Sample.Format', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - selectInput('RT_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT_PAR.Sample.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML('

This table shows for each selected algorithm \\(A\\), - each selected target value \\(f(x)\\), and each run \\(r\\) the parameter value - observed when the target value \\(f(x)\\) is reached for the first time.

'), - DT::dataTableOutput('table_RT_PAR_SAMPLE') - ) - ) -} +rt_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Expected Parameter Value (per function)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarPanel( + width = 3, + HTML('

Range of the function values (\\(x\\) axis)

'), + + textInput('RT_PAR.Plot.Min', label = F_MIN_LABEL, value = ''), + textInput('RT_PAR.Plot.Max', label = F_MAX_LABEL, value = ''), + selectInput('RT_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), + + selectInput('RT_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('RT_PAR.Plot.show.mean', label = 'Mean/median', + choices = c('mean', 'median'), + selected = 'mean'), + selectInput('RT_PAR.Plot.CI', "Show standard deviations or quantiles", + choices = c('Standard Deviation', 'Outer Quantiles', 'None'), + selected = 'None') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are + 2% and 98% by default. This can be changed in the settings-tab.", + placement = "auto" + ) + ), + checkboxInput('RT_PAR.Plot.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('RT_PAR.Plot.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_tooltip( + title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" + ) + ), + + hr(), + selectInput('RT_PAR.Plot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('RT_PAR.Plot.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The mean or median of internal parameters of the algorithm + found with a fixed-budget of evaluations are depicted against the budget. + The displayed elements can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('RT_PAR.Plot.Figure') + ) + ) + ) +} + +rt_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Statistics at Chosen Target Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-target parameter values for evenly spaced target values.'), + + textInput('RT_PAR.Summary.Min', label = F_MIN_LABEL, value = ''), + textInput('RT_PAR.Summary.Max', label = F_MAX_LABEL, value = ''), + textInput('RT_PAR.Summary.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RT_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('RT_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RT_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('RT_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT_PAR.Summary.Download", "Save this table as csv") + ), + + mainPanel( + width = 9, + HTML(paste0('
', + includeMarkdown('markdown/PAR_SUMMARY_TABLE.Rmd'), '
')), + DT::dataTableOutput('table_RT_PAR_summary') + ) + ) +} + +rt_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Sample at Chosen Target Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-target parameter values for evenly spaced target values.'), + + textInput('RT_PAR.Sample.Min', label = F_MIN_LABEL, value = ''), + textInput('RT_PAR.Sample.Max', label = F_MAX_LABEL, value = ''), + textInput('RT_PAR.Sample.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RT_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('RT_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RT_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('RT_PAR.Sample.Format', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + selectInput('RT_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT_PAR.Sample.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML('

This table shows for each selected algorithm \\(A\\), + each selected target value \\(f(x)\\), and each run \\(r\\) the parameter value + observed when the target value \\(f(x)\\) is reached for the first time.

'), + DT::dataTableOutput('table_RT_PAR_SAMPLE') + ) + ) +} diff --git a/inst/shiny-server/ui/rt_portfolio_box.R b/inst/shiny-server/ui/rt_portfolio_box.R index 0807394c..da955139 100644 --- a/inst/shiny-server/ui/rt_portfolio_box.R +++ b/inst/shiny-server/ui/rt_portfolio_box.R @@ -1,97 +1,97 @@ -rt_shapleys_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Contribution to portfolio (Shapley-values)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('RTportfolio.Shapley.Algs', 'Algorithms to include:', multiple = T, selected = NULL, choices = NULL), - selectInput("RTportfolio.Shapley.Funcs", "Functions to include:", multiple = T, selected = NULL, choices = NULL), - selectInput("RTportfolio.Shapley.Dims", "Dimensions to include:", multiple = T, selected = NULL, choices = NULL), - hr(), - checkboxInput("RTportfolio.Shapley.Logx", "Scale runtimes to sample at \\(\\log_{10}\\)", value = T), - numericInput("RTportfolio.Shapley.Groupsize", "Maximum permutation size:", value = 5, min = 1, max = 100) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Groupsize", content = "This parameter controls how many groups of permutations are used to - calculate the Shapley-values. Larger values give more accurate estimates, but take longer to compute.", - placement = "auto" - ) - ), - numericInput("RTportfolio.Shapley.Permsize", "Maximum permutation size", value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Scaling", content = "This parameter controls the maximum size of permutations to include - in the calculation of the marginal contribution. So, if this parameter is low, only small portfolios - will be considered. Higher values lead to more accurate Shapley-values (relative to eachother) but take - longer to compute.", - placement = "auto" - ) - ), - - br(), - actionButton( - "RTportfolio.Shapley.Refresh", - label = HTML('

Refresh the figure

') - ), - - hr(), - selectInput("RTportfolio.Shapley.Target_type", label = "Select the spacing for the - automatically generated ECDF-targets:", - choices = c('linear', 'log-linear', 'bbob'), - selected = 'linear') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Default targets", content = "The log-linear spacing only works correctly - when no negative target values are present in the data. The BBOB-spacing is pre-defined - to 51 log-linear targets between 10^2 and 10^-8.", - placement = "auto" - ) - ), - numericInput("RTportfolio.Shapley.Target_number", label = "Select the number of ECDF-targets to - generate for each function/dimension", value = 10, min = 1, max = 100), - - HTML_P('Alternatively, you can download the table containing the target values for each - (function, dimension)-pair and edit the table as you want. Please keep - the file format when modifying it.'), - downloadButton('RTportfolio.Shapley.Table.Download', label = 'Download the table of targets'), - br(), - br(), - br(), - - HTML_P('Upload the table you just downloaded and edited'), - fileInput( - "RTportfolio.Shapley.Table.Upload", - label = NULL, - multiple = FALSE, - accept = c( - "text/csv", - "text/comma-separated-values,text/plain", - ".csv" - ) - ), - - hr(), - selectInput('RTportfolio.Shapley.Format', label = 'figure format to download', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTportfolio.Shapley.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - hr(), - - HTML_P('DESCRIPTION OF SHAPLEY VALUES HERE'), - plotlyOutput.IOHanalyzer('RT_SHAPLEY'), - HTML_P('The selected targets are:'), - DT::dataTableOutput('RT_SHAPLEY_TARGETS_GENERATED') - ) - ) - ) +rt_shapleys_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Contribution to portfolio (Shapley-values)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('RTportfolio.Shapley.Algs', 'Algorithms to include:', multiple = T, selected = NULL, choices = NULL), + selectInput("RTportfolio.Shapley.Funcs", "Functions to include:", multiple = T, selected = NULL, choices = NULL), + selectInput("RTportfolio.Shapley.Dims", "Dimensions to include:", multiple = T, selected = NULL, choices = NULL), + hr(), + checkboxInput("RTportfolio.Shapley.Logx", "Scale runtimes to sample at \\(\\log_{10}\\)", value = T), + numericInput("RTportfolio.Shapley.Groupsize", "Maximum permutation size:", value = 5, min = 1, max = 100) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Groupsize", content = "This parameter controls how many groups of permutations are used to + calculate the Shapley-values. Larger values give more accurate estimates, but take longer to compute.", + placement = "auto" + ) + ), + numericInput("RTportfolio.Shapley.Permsize", "Maximum permutation size", value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Scaling", content = "This parameter controls the maximum size of permutations to include + in the calculation of the marginal contribution. So, if this parameter is low, only small portfolios + will be considered. Higher values lead to more accurate Shapley-values (relative to eachother) but take + longer to compute.", + placement = "auto" + ) + ), + + br(), + actionButton( + "RTportfolio.Shapley.Refresh", + label = HTML('

Refresh the figure

') + ), + + hr(), + selectInput("RTportfolio.Shapley.Target_type", label = "Select the spacing for the + automatically generated ECDF-targets:", + choices = c('linear', 'log-linear', 'bbob'), + selected = 'linear') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Default targets", content = "The log-linear spacing only works correctly + when no negative target values are present in the data. The BBOB-spacing is pre-defined + to 51 log-linear targets between 10^2 and 10^-8.", + placement = "auto" + ) + ), + numericInput("RTportfolio.Shapley.Target_number", label = "Select the number of ECDF-targets to + generate for each function/dimension", value = 10, min = 1, max = 100), + + HTML_P('Alternatively, you can download the table containing the target values for each + (function, dimension)-pair and edit the table as you want. Please keep + the file format when modifying it.'), + downloadButton('RTportfolio.Shapley.Table.Download', label = 'Download the table of targets'), + br(), + br(), + br(), + + HTML_P('Upload the table you just downloaded and edited'), + fileInput( + "RTportfolio.Shapley.Table.Upload", + label = NULL, + multiple = FALSE, + accept = c( + "text/csv", + "text/comma-separated-values,text/plain", + ".csv" + ) + ), + + hr(), + selectInput('RTportfolio.Shapley.Format', label = 'figure format to download', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTportfolio.Shapley.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + hr(), + + HTML_P('DESCRIPTION OF SHAPLEY VALUES HERE'), + plotlyOutput.IOHanalyzer('RT_SHAPLEY'), + HTML_P('The selected targets are:'), + DT::dataTableOutput('RT_SHAPLEY_TARGETS_GENERATED') + ) + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/rt_summary_box.R b/inst/shiny-server/ui/rt_summary_box.R index 75d8e671..9a3d35f2 100644 --- a/inst/shiny-server/ui/rt_summary_box.R +++ b/inst/shiny-server/ui/rt_summary_box.R @@ -1,92 +1,92 @@ -rt_stats_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Runtime Statistics at Chosen Target Values

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Set the range and the granularity of the results. - The table will show fixed-target runtimes for evenly spaced target values.

'), - - textInput('RTSummary.Statistics.Min', label = F_MIN_LABEL, value = ''), - textInput('RTSummary.Statistics.Max', label = F_MAX_LABEL, value = ''), - textInput('RTSummary.Statistics.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RTSummary.Statistics.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('RTSummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - hr(), - selectInput('RTSummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RTSummary.Statistics.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', includeMarkdown('markdown/RT_SUMMARY_TABLE.Rmd'),'
')), - DT::dataTableOutput('table_RT_summary') - ) - ) -} - -rt_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Original Runtime Samples

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Set the range and the granularity of the results. - The table will show fixed-target runtimes for evenly spaced target values.

'), - - textInput('RTSummary.Sample.Min', label = F_MIN_LABEL, value = ''), - textInput('RTSummary.Sample.Max', label = F_MAX_LABEL, value = ''), - textInput('RTSummary.Sample.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RTSummary.Sample.Single', - label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - - # TODO: do we need this log scaling? - # checkboxInput('F_LOGSPACE_DATA_SUMMARY', - # label = HTML('Evenly space target values in \\(log_{10}\\) space')), - selectInput('RTSummary.Sample.ID', 'Select which IDs to include:', - choices = NULL, selected = NULL, multiple = T), - - hr(), - selectInput('RTSummary.Sample.DownloadFormat', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - selectInput('RTSummary.Sample.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), - - downloadButton("RTSummary.Sample.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML('

This table shows for each selected algorithm \\(A\\), - each selected target value \\(f(x)\\), and each run \\(r\\) - the number \\(T(A,f(x),r)\\) of evaluations performed by the - algorithm until it evaluated for the first time a solution of - quality at least \\(f(x)\\).

'), - DT::dataTableOutput('table_RT_sample') - ) - ) -} - -rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Data Overview

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RTSummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), - hr(), - selectInput('RTSummary.Overview.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RTSummary.Overview.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', includeMarkdown('markdown/RT_OVERVIEW_TABLE.Rmd'), '
')), - DT::dataTableOutput('table_RT_overview') - ) - ) -} - +rt_stats_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Runtime Statistics at Chosen Target Values

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Set the range and the granularity of the results. + The table will show fixed-target runtimes for evenly spaced target values.

'), + + textInput('RTSummary.Statistics.Min', label = F_MIN_LABEL, value = ''), + textInput('RTSummary.Statistics.Max', label = F_MAX_LABEL, value = ''), + textInput('RTSummary.Statistics.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RTSummary.Statistics.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('RTSummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('RTSummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RTSummary.Statistics.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', includeMarkdown('markdown/RT_SUMMARY_TABLE.Rmd'),'
')), + DT::dataTableOutput('table_RT_summary') + ) + ) +} + +rt_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Original Runtime Samples

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Set the range and the granularity of the results. + The table will show fixed-target runtimes for evenly spaced target values.

'), + + textInput('RTSummary.Sample.Min', label = F_MIN_LABEL, value = ''), + textInput('RTSummary.Sample.Max', label = F_MAX_LABEL, value = ''), + textInput('RTSummary.Sample.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RTSummary.Sample.Single', + label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + + # TODO: do we need this log scaling? + # checkboxInput('F_LOGSPACE_DATA_SUMMARY', + # label = HTML('Evenly space target values in \\(log_{10}\\) space')), + selectInput('RTSummary.Sample.ID', 'Select which IDs to include:', + choices = NULL, selected = NULL, multiple = T), + + hr(), + selectInput('RTSummary.Sample.DownloadFormat', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + selectInput('RTSummary.Sample.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), + + downloadButton("RTSummary.Sample.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML('

This table shows for each selected algorithm \\(A\\), + each selected target value \\(f(x)\\), and each run \\(r\\) + the number \\(T(A,f(x),r)\\) of evaluations performed by the + algorithm until it evaluated for the first time a solution of + quality at least \\(f(x)\\).

'), + DT::dataTableOutput('table_RT_sample') + ) + ) +} + +rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Data Overview

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RTSummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('RTSummary.Overview.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RTSummary.Overview.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', includeMarkdown('markdown/RT_OVERVIEW_TABLE.Rmd'), '
')), + DT::dataTableOutput('table_RT_overview') + ) + ) +} + diff --git a/inst/shiny-server/ui/settings.R b/inst/shiny-server/ui/settings.R index 1d0dadf0..28da0f74 100644 --- a/inst/shiny-server/ui/settings.R +++ b/inst/shiny-server/ui/settings.R @@ -1,92 +1,92 @@ -color_settings_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Color Settings

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - selectInput(inputId = "Settings.Color.Scheme", label = "Color schemes", - choices = c("Default", "Variant 1", "Variant 2", "Variant 3", "Custom")), - conditionalPanel(condition = 'input["Settings.Color.Scheme"] == "Custom"', - downloadButton("Settings.Color.Example","Download an example color settings file"), - fileInput("Settings.Color.Upload","Upload a color settings file") - ), - colourInput("Settings.Color.Bg", "Plot background colour", value = "#E6E6E6"), - colourInput("Settings.Color.Grid", "Plot gridline colour", value = "#FFFFFF"), - colourInput("Settings.Color.Tick", "Plot ticks colour", value = "#333333"), - numericInput("Settings.Color.Linewidth", "Line Width", value = 2), - numericInput("Settings.Color.Markersize", "Marker Size", value = 4), - selectInput("Settings.Legend.Location", "Legend location", - c("Outside, right", "Inside, right", "Inside, left", "Below", "Custom"), "Below"), - conditionalPanel(condition = 'input["Settings.Legend.Location"] == "Custom"', - numericInput("Settings.Legend.LocationX", "X-position of Legend", value = 0.5, step = 0.05), - numericInput("Settings.Legend.LocationY", "Y-position of Legend", value = -0.2, step = 0.05) - ), - downloadButton("Settings.Plot.Download", label = "Download sample plot") - ), - - mainPanel( - width = 9, - column( - width = 12, - align = 'center', - HTML_P('Example of the current colorscheme.'), - plotlyOutput.IOHanalyzer('Settings.Color.Plot') - ) - ) - ) - ) -} - -general_settings_box <- function(width=12, collapsible = T, collapsed = F) { - box(title = HTML('

General settings

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - mainPanel( - width = 12, - column( - width = 3, - align = "Left", - HTML_P('Set the general properties'), - #TODO: get probabilities from get_probability and put them as default - textInput("Settings.General.Probs", label = "Probability-quantiles", - value = "0.02,0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.98"), - numericInput("Settings.General.Max_samples", - label = "Maximum samples shown per algorithm", - value = 100), - selectInput("Settings.General.Backend", "Plotting backend", - c('plotly', 'ggplot2'), 'plotly'), - numericInput("Settings.General.Precision", label = "Function value precision (digits)", - value = 2), - hr(), - HTML_P("ID variables"), - selectInput("Settings.ID.Variables", label = "Attributes used to create IDs", - multiple = T, selected = NULL, choices = NULL), - checkboxInput("Settings.Use_Funcname", "Use function names instead of IDs"), - hr(), - downloadButton("Settings.Download", "Download current general settings file"), - fileInput("Settings.Upload", "Upload a settings file", accept = "rds") - ), - column( - width = 3, - align = "Left", - HTML_P('Set the figure download properties'), - selectInput("Settings.Download.Preset", label = "Choose preset for download and font sizes", - choices = c("Default", "Paper-1col", "Paper-2col"), selected = "Default"), - hr(), - numericInput("Settings.Download.Width", label = "Image width (px)", value = 1000, min = 100, max = 4096), - numericInput("Settings.Download.Height", label = "Image height (px)", value = 1000, min = 100, max = 4096) - ), - column( - width = 3, - align = "Left", - HTML_P('Set the figure fontsizes'), - numericInput("Settings.Font.Title", label = "Title", value = 16, min = 8, max = 100), - numericInput("Settings.Font.Label", label = "Axis labels", value = 16, min = 8, max = 100), - numericInput("Settings.Font.Legend", label = "Legend", value = 13, min = 8, max = 100), - numericInput("Settings.Font.Tick", label = "Ticks", value = 12, min = 8, max = 100) - ) - ) - - ) +color_settings_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Color Settings

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + selectInput(inputId = "Settings.Color.Scheme", label = "Color schemes", + choices = c("Default", "Variant 1", "Variant 2", "Variant 3", "Custom")), + conditionalPanel(condition = 'input["Settings.Color.Scheme"] == "Custom"', + downloadButton("Settings.Color.Example","Download an example color settings file"), + fileInput("Settings.Color.Upload","Upload a color settings file") + ), + colourInput("Settings.Color.Bg", "Plot background colour", value = "#E6E6E6"), + colourInput("Settings.Color.Grid", "Plot gridline colour", value = "#FFFFFF"), + colourInput("Settings.Color.Tick", "Plot ticks colour", value = "#333333"), + numericInput("Settings.Color.Linewidth", "Line Width", value = 2), + numericInput("Settings.Color.Markersize", "Marker Size", value = 4), + selectInput("Settings.Legend.Location", "Legend location", + c("Outside, right", "Inside, right", "Inside, left", "Below", "Custom"), "Below"), + conditionalPanel(condition = 'input["Settings.Legend.Location"] == "Custom"', + numericInput("Settings.Legend.LocationX", "X-position of Legend", value = 0.5, step = 0.05), + numericInput("Settings.Legend.LocationY", "Y-position of Legend", value = -0.2, step = 0.05) + ), + downloadButton("Settings.Plot.Download", label = "Download sample plot") + ), + + mainPanel( + width = 9, + column( + width = 12, + align = 'center', + HTML_P('Example of the current colorscheme.'), + plotlyOutput.IOHanalyzer('Settings.Color.Plot') + ) + ) + ) + ) +} + +general_settings_box <- function(width=12, collapsible = T, collapsed = F) { + box(title = HTML('

General settings

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + mainPanel( + width = 12, + column( + width = 3, + align = "Left", + HTML_P('Set the general properties'), + #TODO: get probabilities from get_probability and put them as default + textInput("Settings.General.Probs", label = "Probability-quantiles", + value = "0.02,0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.98"), + numericInput("Settings.General.Max_samples", + label = "Maximum samples shown per algorithm", + value = 100), + selectInput("Settings.General.Backend", "Plotting backend", + c('plotly', 'ggplot2'), 'plotly'), + numericInput("Settings.General.Precision", label = "Function value precision (digits)", + value = 2), + hr(), + HTML_P("ID variables"), + selectInput("Settings.ID.Variables", label = "Attributes used to create IDs", + multiple = T, selected = NULL, choices = NULL), + checkboxInput("Settings.Use_Funcname", "Use function names instead of IDs"), + hr(), + downloadButton("Settings.Download", "Download current general settings file"), + fileInput("Settings.Upload", "Upload a settings file", accept = "rds") + ), + column( + width = 3, + align = "Left", + HTML_P('Set the figure download properties'), + selectInput("Settings.Download.Preset", label = "Choose preset for download and font sizes", + choices = c("Default", "Paper-1col", "Paper-2col"), selected = "Default"), + hr(), + numericInput("Settings.Download.Width", label = "Image width (px)", value = 1000, min = 100, max = 4096), + numericInput("Settings.Download.Height", label = "Image height (px)", value = 1000, min = 100, max = 4096) + ), + column( + width = 3, + align = "Left", + HTML_P('Set the figure fontsizes'), + numericInput("Settings.Font.Title", label = "Title", value = 16, min = 8, max = 100), + numericInput("Settings.Font.Label", label = "Axis labels", value = 16, min = 8, max = 100), + numericInput("Settings.Font.Legend", label = "Legend", value = 13, min = 8, max = 100), + numericInput("Settings.Font.Tick", label = "Ticks", value = 12, min = 8, max = 100) + ) + ) + + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/sidebar_menu.R b/inst/shiny-server/ui/sidebar_menu.R index 761e8e7c..3294a140 100644 --- a/inst/shiny-server/ui/sidebar_menu.R +++ b/inst/shiny-server/ui/sidebar_menu.R @@ -1,49 +1,49 @@ -sidebar_menu <- function() { - sidebarMenu(id = "tabs", - menuItem("Upload Data", tabName = "upload", icon = icon('upload', lib = 'glyphicon'), - selected = T), - menuItem("General Overview", tabName = "overview", icon = icon("table")), - - menuItem("Fixed-Target Results", tabName = "ERT", icon = icon("file-text-o"), - menuItem("Single Function", tabName = "RT_single", icon = icon("line-chart"), selected = F, - menuSubItem("Data Summary", tabName = "ERT_data", icon = icon("table")), - menuSubItem("Expected Runtime", tabName = "ERT_convergence_single", icon = icon("line-chart"), selected = F), - menuSubItem("Probability Mass Function", tabName = "RT_PMF", icon = icon("bar-chart"), selected = F), - menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_single", icon = icon("line-chart"), selected = F), - menuSubItem("Algorithm Parameters", tabName = "RT_PARAMETER", icon = icon('file-text-o'), selected = F), - menuSubItem("Statistics", tabName = "RT_Statistics_single", icon = icon("file-text-o"), selected = F) - ), - menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, - menuSubItem("Data Summary", tabName = "RT_table_multi", icon = icon("table"), selected = F), - menuSubItem("Expected Runtime", tabName = "ERT_convergence_aggr", icon = icon("line-chart"), selected = F), - menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_aggr", icon = icon("line-chart"), selected = F), - menuSubItem("Deep Statistics", tabName = "RT_DSC", icon = icon("not-equal"), selected = F), - menuSubItem("Ranking", tabName = "RT_Statistics_aggr", icon = icon("file-text-o"), selected = F), - menuSubItem("Portfolio", tabName = "RT_portfolio", icon = icon("bar-chart"), selected = F) - ) - ), - - menuItem("Fixed-Budget Results", tabName = "FCE", icon = icon("file-text-o"), - menuItem("Single Function", tabName = "FCE_single", icon = icon("line-chart"), selected = F, - menuSubItem("Data Summary", tabName = "FCE_DATA", icon = icon("table")), - menuSubItem("Expected Target Value", tabName = "FCE_convergence_single", icon = icon("bar-chart")), - menuSubItem("Probability Density Function", tabName = "FCE_PDF", icon = icon("bar-chart"), selected = F), - menuSubItem("Cumulative Distribution", tabName = "FCE_ECDF", icon = icon("line-chart")), - menuSubItem("Algorithm Parameters", tabName = "FCE_PARAMETER", icon = icon('file-text-o'), selected = F), - menuSubItem('Statistics', tabName = "FCE_Statistics_single", icon = icon('file-text-o')) - ), - menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, - menuSubItem("Data Summary", tabName = "FV_table_multi", icon = icon("table"), selected = F), - menuSubItem("Expected Target Value", tabName = "FCE_convergence_aggr", icon = icon("bar-chart")), - menuSubItem('Deep Statistics', tabName = "FCE_DSC", icon = icon('not-equal')), - menuSubItem('Ranking', tabName = "FCE_Statistics_aggr", icon = icon('file-text-o')) - ) - ), - menuItem("Position Information", tabName = "Positions", icon = icon("arrows-alt")), - # menuItem("Data Format", tabName = "dataformat", icon = icon("fas fa-database")), - menuItem("About", tabName = "about", icon = icon("question")), - menuItem("Settings", tabName = "Settings", icon = icon("cog")) - # , - # menuItem("Report", tabName = "Report", icon = icon("file-pdf")) - ) -} +sidebar_menu <- function() { + sidebarMenu(id = "tabs", + menuItem("Upload Data", tabName = "upload", icon = icon('upload', lib = 'glyphicon'), + selected = T), + menuItem("General Overview", tabName = "overview", icon = icon("table")), + + menuItem("Fixed-Target Results", tabName = "ERT", icon = icon("file-text-o"), + menuItem("Single Function", tabName = "RT_single", icon = icon("line-chart"), selected = F, + menuSubItem("Data Summary", tabName = "ERT_data", icon = icon("table")), + menuSubItem("Expected Runtime", tabName = "ERT_convergence_single", icon = icon("line-chart"), selected = F), + menuSubItem("Probability Mass Function", tabName = "RT_PMF", icon = icon("bar-chart"), selected = F), + menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_single", icon = icon("line-chart"), selected = F), + menuSubItem("Algorithm Parameters", tabName = "RT_PARAMETER", icon = icon('file-text-o'), selected = F), + menuSubItem("Statistics", tabName = "RT_Statistics_single", icon = icon("file-text-o"), selected = F) + ), + menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, + menuSubItem("Data Summary", tabName = "RT_table_multi", icon = icon("table"), selected = F), + menuSubItem("Expected Runtime", tabName = "ERT_convergence_aggr", icon = icon("line-chart"), selected = F), + menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_aggr", icon = icon("line-chart"), selected = F), + menuSubItem("Deep Statistics", tabName = "RT_DSC", icon = icon("not-equal"), selected = F), + menuSubItem("Ranking", tabName = "RT_Statistics_aggr", icon = icon("file-text-o"), selected = F), + menuSubItem("Portfolio", tabName = "RT_portfolio", icon = icon("bar-chart"), selected = F) + ) + ), + + menuItem("Fixed-Budget Results", tabName = "FCE", icon = icon("file-text-o"), + menuItem("Single Function", tabName = "FCE_single", icon = icon("line-chart"), selected = F, + menuSubItem("Data Summary", tabName = "FCE_DATA", icon = icon("table")), + menuSubItem("Expected Target Value", tabName = "FCE_convergence_single", icon = icon("bar-chart")), + menuSubItem("Probability Density Function", tabName = "FCE_PDF", icon = icon("bar-chart"), selected = F), + menuSubItem("Cumulative Distribution", tabName = "FCE_ECDF", icon = icon("line-chart")), + menuSubItem("Algorithm Parameters", tabName = "FCE_PARAMETER", icon = icon('file-text-o'), selected = F), + menuSubItem('Statistics', tabName = "FCE_Statistics_single", icon = icon('file-text-o')) + ), + menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, + menuSubItem("Data Summary", tabName = "FV_table_multi", icon = icon("table"), selected = F), + menuSubItem("Expected Target Value", tabName = "FCE_convergence_aggr", icon = icon("bar-chart")), + menuSubItem('Deep Statistics', tabName = "FCE_DSC", icon = icon('not-equal')), + menuSubItem('Ranking', tabName = "FCE_Statistics_aggr", icon = icon('file-text-o')) + ) + ), + menuItem("Position Information", tabName = "Positions", icon = icon("arrows-alt")), + # menuItem("Data Format", tabName = "dataformat", icon = icon("fas fa-database")), + menuItem("About", tabName = "about", icon = icon("question")), + menuItem("Settings", tabName = "Settings", icon = icon("cog")) + # , + # menuItem("Report", tabName = "Report", icon = icon("file-pdf")) + ) +} diff --git a/inst/shiny-server/ui/stats_box.R b/inst/shiny-server/ui/stats_box.R index 3190bbab..b93782a1 100644 --- a/inst/shiny-server/ui/stats_box.R +++ b/inst/shiny-server/ui/stats_box.R @@ -1,125 +1,125 @@ -rt_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Hypothesis Testing

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - textInput('RT_Stats.Overview.Target', label = F_TAR_LABEL), - textInput('RT_Stats.Overview.Alpha', - label = HTML('

Significe level \\(\\alpha\\)

'), - value = 0.01), - numericInput('RT_Stats.Overview.Samples', - label = 'Size of the bootstrap sample', - min = 1, max = 1000, step = 1, value = 0) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_tooltip( - title = "Using bootstrapped running times might result in misleading significance - levels. Use only when absolutely sure bootstrapping is applicable." - ) - ), - hr(), - selectInput('RT_Stats.Overview.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('RT_Stats.Overview.DownloadTable', label = 'Download the table'), - hr(), - selectInput('RT_Stats.Overview.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('RT_Stats.Overview.DownloadHeatmap', - label = 'Download the heatmap') - # downloadButton('RT_Stats.Overview.DownloadNetwork', - # label = 'Download the network-graph', status = F) - - ), - - mainPanel( - width = 9, - HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of - algorithms, in order to determine which algorithm gives a significantly - smaller running time distribution. The resulting p-values are arranged in a matrix, where - each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: - the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), - DT::dataTableOutput('RT_Stats.Overview.Pmatrix') - ), - - fluidRow( - column( - width = 12, - HTML('
'), - HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, - are visualized in a heatmap (left) and a network (right). - In each cell (row, column) of the heatmap, the alternative hypothesis is again: - the row algorithm has smaller (better) running time than the column. - The color indicates: -
    -
  • Red: A row is better than the column
    • -
  • Blue: A row is worse than the column
    • -
  • Gray: no significant distinction between the row and column
    • -
- On the right subplot, the partial order resulted from the test is visualized as a network, - where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than - \\(B\\) with the specified \\(\\alpha\\) value.') - ) - ), - - fluidRow( - column( - width = 6, align = 'center', - HTML('
'), - plotlyOutput.IOHanalyzer('RT_Stats.Overview.Heatmap', aspect_ratio = 1) - ), - column( - width = 6, align = 'center', - HTML('
'), - plotOutput("RT_Stats.Overview.Graph", height = '70vh') - ) - ) - ) -} - -rt_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Glicko2-based ranking

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - textInput('RT_Stats.Glicko.Nrgames', - label = "Number of games per (function,dimension) pair", - value = 25), - actionButton('RT_Stats.Glicko.Create', 'Create Ranking'), - hr(), - selectInput('RT_Stats.Glicko.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format), - - downloadButton('RT_Stats.Glicko.Download', label = 'Download the figure'), - hr(), - selectInput('RT_Stats.Glicko.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format), - downloadButton('RT_Stats.Glicko.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. - Every round, for every function and dimension of the datasetlist, - each pair of algorithms competes. This competition samples a random runtime for the - provided target (best achieved target among all algorithms). Whichever algorithm has the lower - runtime wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), - HTML_P("The chosen target function values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("RT_Stats.Glicko.Targets"), - hr(), - HTML_P("The results of the ranking are:"), - plotlyOutput.IOHanalyzer("RT_Stats.Glicko.Candlestick"), - DT::dataTableOutput('RT_Stats.Glicko.Dataframe') - ) - ) -} +rt_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Hypothesis Testing

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + textInput('RT_Stats.Overview.Target', label = F_TAR_LABEL), + textInput('RT_Stats.Overview.Alpha', + label = HTML('

Significe level \\(\\alpha\\)

'), + value = 0.01), + numericInput('RT_Stats.Overview.Samples', + label = 'Size of the bootstrap sample', + min = 1, max = 1000, step = 1, value = 0) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_tooltip( + title = "Using bootstrapped running times might result in misleading significance + levels. Use only when absolutely sure bootstrapping is applicable." + ) + ), + hr(), + selectInput('RT_Stats.Overview.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('RT_Stats.Overview.DownloadTable', label = 'Download the table'), + hr(), + selectInput('RT_Stats.Overview.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('RT_Stats.Overview.DownloadHeatmap', + label = 'Download the heatmap') + # downloadButton('RT_Stats.Overview.DownloadNetwork', + # label = 'Download the network-graph', status = F) + + ), + + mainPanel( + width = 9, + HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of + algorithms, in order to determine which algorithm gives a significantly + smaller running time distribution. The resulting p-values are arranged in a matrix, where + each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: + the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), + DT::dataTableOutput('RT_Stats.Overview.Pmatrix') + ), + + fluidRow( + column( + width = 12, + HTML('
'), + HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, + are visualized in a heatmap (left) and a network (right). + In each cell (row, column) of the heatmap, the alternative hypothesis is again: + the row algorithm has smaller (better) running time than the column. + The color indicates: +
    +
  • Red: A row is better than the column
    • +
  • Blue: A row is worse than the column
    • +
  • Gray: no significant distinction between the row and column
    • +
+ On the right subplot, the partial order resulted from the test is visualized as a network, + where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than + \\(B\\) with the specified \\(\\alpha\\) value.') + ) + ), + + fluidRow( + column( + width = 6, align = 'center', + HTML('
'), + plotlyOutput.IOHanalyzer('RT_Stats.Overview.Heatmap', aspect_ratio = 1) + ), + column( + width = 6, align = 'center', + HTML('
'), + plotOutput("RT_Stats.Overview.Graph", height = '70vh') + ) + ) + ) +} + +rt_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Glicko2-based ranking

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + textInput('RT_Stats.Glicko.Nrgames', + label = "Number of games per (function,dimension) pair", + value = 25), + actionButton('RT_Stats.Glicko.Create', 'Create Ranking'), + hr(), + selectInput('RT_Stats.Glicko.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format), + + downloadButton('RT_Stats.Glicko.Download', label = 'Download the figure'), + hr(), + selectInput('RT_Stats.Glicko.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format), + downloadButton('RT_Stats.Glicko.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. + Every round, for every function and dimension of the datasetlist, + each pair of algorithms competes. This competition samples a random runtime for the + provided target (best achieved target among all algorithms). Whichever algorithm has the lower + runtime wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), + HTML_P("The chosen target function values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("RT_Stats.Glicko.Targets"), + hr(), + HTML_P("The results of the ranking are:"), + plotlyOutput.IOHanalyzer("RT_Stats.Glicko.Candlestick"), + DT::dataTableOutput('RT_Stats.Glicko.Dataframe') + ) + ) +} diff --git a/inst/shiny-server/ui/tables_multi_func_box.R b/inst/shiny-server/ui/tables_multi_func_box.R index cbb24c1b..dd986317 100644 --- a/inst/shiny-server/ui/tables_multi_func_box.R +++ b/inst/shiny-server/ui/tables_multi_func_box.R @@ -1,104 +1,104 @@ -multi_function_ert_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Statistics

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('RT.MultiERT.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.MultiERT.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.MultiERT.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('RT.MultiERT.Target', 'Target Value', value = ''), - hr(), - selectInput('RT.MultiERT.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT.MultiERT.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('RT.MultiERT.Table') - ) - ) -} - - -multi_function_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Hitting Times

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('RT.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('RT.Multisample.Target', 'Target Value', value = ''), - hr(), - selectInput('RT.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), - selectInput('RT.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT.Multisample.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('RT.Multisample.Table') - ) - ) -} - - -multi_function_fv_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Statistics

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('FV.MultiFV.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.MultiFV.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.MultiFV.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('FV.MultiFV.Target', 'Budget Value', value = ''), - hr(), - selectInput('FV.MultiFV.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV.MultiFV.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('FV.MultiFV.Table') - ) - ) -} - - -multi_function_sample_box_fv <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Hitting Times

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which IDs to include:

'), - - selectInput('FV.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('FV.Multisample.Target', 'Budget Value', value = ''), - hr(), - selectInput('FV.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), - selectInput('FV.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV.Multisample.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('FV.Multisample.Table') - ) - ) +multi_function_ert_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Statistics

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('RT.MultiERT.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.MultiERT.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.MultiERT.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('RT.MultiERT.Target', 'Target Value', value = ''), + hr(), + selectInput('RT.MultiERT.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT.MultiERT.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('RT.MultiERT.Table') + ) + ) +} + + +multi_function_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Hitting Times

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('RT.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('RT.Multisample.Target', 'Target Value', value = ''), + hr(), + selectInput('RT.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), + selectInput('RT.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT.Multisample.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('RT.Multisample.Table') + ) + ) +} + + +multi_function_fv_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Statistics

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('FV.MultiFV.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.MultiFV.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.MultiFV.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('FV.MultiFV.Target', 'Budget Value', value = ''), + hr(), + selectInput('FV.MultiFV.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV.MultiFV.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('FV.MultiFV.Table') + ) + ) +} + + +multi_function_sample_box_fv <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Hitting Times

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('FV.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('FV.Multisample.Target', 'Budget Value', value = ''), + hr(), + selectInput('FV.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), + selectInput('FV.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV.Multisample.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('FV.Multisample.Table') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/upload_box.R b/inst/shiny-server/ui/upload_box.R index 59a621b7..0b30367e 100644 --- a/inst/shiny-server/ui/upload_box.R +++ b/inst/shiny-server/ui/upload_box.R @@ -1,149 +1,149 @@ -upload_box <- function(width = 12, collapsible = T, collapsed = T, # TODO: find a way to include all potential arguments - height = '75vh') { - box( - title = HTML('

Upload Data

'), - width = width, height = height, collapsed = collapsed, collapsible = collapsible, - solidHeader = T, status = "primary", - sidebarPanel( - width = 12, - - HTML_P('IOHexperimenter, Nevergrad, and BBOB/COCO data files are accepted. - For alternative data files, please convert them to the format described here.'), - - selectInput( - 'upload.data_format', - label = "Please choose the format of your datasets", - choices = c(AUTOMATIC, TWO_COL), - selected = AUTOMATIC, width = '60%' - ) %>% - shinyInput_label_embed( - custom_icon("info") %>% - bs_embed_tooltip( - title = "The IOHprofiler, COCO and Nevergrad formats can be automatically detected." - ) - ), - - selectInput('upload.maximization', - label = "Maximization or minimization?", - choices = c(AUTOMATIC,"MAXIMIZE", "MINIMIZE"), - selected = AUTOMATIC, width = '60%') , - - HTML('

When the dataset is huge, - the alignment can take a very long time. In this case, you could toggle - the efficient mode to subsample the dataset. However, - the precision of data will be compromised.

'), - - checkboxInput('upload.subsampling', - label = HTML('

- Efficient mode

'), - value = F), - - fileInput("upload.add_zip", - label = HTML('

- Please choose a zip file containing the - benchmark data

'), - multiple = TRUE, accept = c("Application/zip", ".zip", - ".csv", 'bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')), - - actionButton('upload.remove_data', - label = HTML('

- Remove all the data

')) - ) - ) -} - -upload_prompt_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Data Processing Prompt

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - - verbatimTextOutput('process_data_promt'), - # modifying CSS for 'process_data_promt' - tags$head(tags$style("#process_data_promt{ - color:black; font-size:12px; font-style:italic; - max-height: 500px; - overflow-y:visible; overflow-x: auto; - white-space: pre-wrap; - white-space: -moz-pre-wrap; - white-space: -pre-wrap; - white-space: -o-pre-wrap; - word-wrap: normal; - background: ghostwhite;}")) - ) -} - -data_list_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

List of Processed Data

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - dataTableOutput('data_info') - ) -} - -repository_box <- function(width = 12, collapsible = F, collapsed = T, - height = '75vh') { - box( - title = HTML('

Load Data from Repository

'), - width = width, height = height, collapsed = collapsed, collapsible = collapsible, - solidHeader = T, status = "primary", - sidebarPanel( - width = 12, - - HTML_P("Load the data from the available repositories. There are currently three available sources: -
    -
  • - Data generated with the PBO-suite, implemented in the IOHexperimenter -
    • -
  • - All data generated by the nevergrad benchmarking framework -
    • -
  • - The majority of the publicly available benchmark data on the single-objective BBOB framework -
    • "), - - selectInput('repository.type', label = "Select the dataset source", - choices = NULL, selected = NULL, width = '50%'), - - selectInput('repository.dataset', - label = "Select the dataset", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - selectInput('repository.funcId', - label = "Please choose the function", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - selectInput('repository.dim', - label = "Please choose the dimension", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - selectInput('repository.ID', - label = "Please choose the algorithm", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - shinyjs::disabled( - actionButton('repository.load_button', 'load data') - ) - ) - ) -} - -welcome_bar <- function(width = 12, collapsible = T, collapsed = F) { - box( - title = HTML('

    Welcome to IOHanalyzer!

    '), - width = width, collapsed = collapsed, collapsible = collapsible, - solidHeader = T, status = "primary", - mainPanel( - width = 12, - HTML(paste0( - '', - includeMarkdown('markdown/welcome.md'), - '') - )), - fluidRow(valueBoxOutput("VersionBox"), - valueBoxOutput("WikiBox"), - valueBoxOutput("ContactBox") - ) - ) -} - +upload_box <- function(width = 12, collapsible = T, collapsed = T, # TODO: find a way to include all potential arguments + height = '75vh') { + box( + title = HTML('

    Upload Data

    '), + width = width, height = height, collapsed = collapsed, collapsible = collapsible, + solidHeader = T, status = "primary", + sidebarPanel( + width = 12, + + HTML_P('IOHexperimenter, Nevergrad, and BBOB/COCO data files are accepted. + For alternative data files, please convert them to the format described here.'), + + selectInput( + 'upload.data_format', + label = "Please choose the format of your datasets", + choices = c(AUTOMATIC, TWO_COL), + selected = AUTOMATIC, width = '60%' + ) %>% + shinyInput_label_embed( + custom_icon("info") %>% + bs_embed_tooltip( + title = "The IOHprofiler, COCO and Nevergrad formats can be automatically detected." + ) + ), + + selectInput('upload.maximization', + label = "Maximization or minimization?", + choices = c(AUTOMATIC,"MAXIMIZE", "MINIMIZE"), + selected = AUTOMATIC, width = '60%') , + + HTML('

    When the dataset is huge, + the alignment can take a very long time. In this case, you could toggle + the efficient mode to subsample the dataset. However, + the precision of data will be compromised.

    '), + + checkboxInput('upload.subsampling', + label = HTML('

    + Efficient mode

    '), + value = F), + + fileInput("upload.add_zip", + label = HTML('

    + Please choose a zip file containing the + benchmark data

    '), + multiple = TRUE, accept = c("Application/zip", ".zip", + ".csv", 'bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')), + + actionButton('upload.remove_data', + label = HTML('

    + Remove all the data

    ')) + ) + ) +} + +upload_prompt_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

    Data Processing Prompt

    '), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + + verbatimTextOutput('process_data_promt'), + # modifying CSS for 'process_data_promt' + tags$head(tags$style("#process_data_promt{ + color:black; font-size:12px; font-style:italic; + max-height: 500px; + overflow-y:visible; overflow-x: auto; + white-space: pre-wrap; + white-space: -moz-pre-wrap; + white-space: -pre-wrap; + white-space: -o-pre-wrap; + word-wrap: normal; + background: ghostwhite;}")) + ) +} + +data_list_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

    List of Processed Data

    '), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + dataTableOutput('data_info') + ) +} + +repository_box <- function(width = 12, collapsible = F, collapsed = T, + height = '75vh') { + box( + title = HTML('

    Load Data from Repository

    '), + width = width, height = height, collapsed = collapsed, collapsible = collapsible, + solidHeader = T, status = "primary", + sidebarPanel( + width = 12, + + HTML_P("Load the data from the available repositories. There are currently three available sources: +
      +
    • + Data generated with the PBO-suite, implemented in the IOHexperimenter +
      • +
    • + All data generated by the nevergrad benchmarking framework +
      • +
    • + The majority of the publicly available benchmark data on the single-objective BBOB framework +
      • "), + + selectInput('repository.type', label = "Select the dataset source", + choices = NULL, selected = NULL, width = '50%'), + + selectInput('repository.dataset', + label = "Select the dataset", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + selectInput('repository.funcId', + label = "Please choose the function", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + selectInput('repository.dim', + label = "Please choose the dimension", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + selectInput('repository.ID', + label = "Please choose the algorithm", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + shinyjs::disabled( + actionButton('repository.load_button', 'load data') + ) + ) + ) +} + +welcome_bar <- function(width = 12, collapsible = T, collapsed = F) { + box( + title = HTML('

      Welcome to IOHanalyzer!

      '), + width = width, collapsed = collapsed, collapsible = collapsible, + solidHeader = T, status = "primary", + mainPanel( + width = 12, + HTML(paste0( + '', + includeMarkdown('markdown/welcome.md'), + '') + )), + fluidRow(valueBoxOutput("VersionBox"), + valueBoxOutput("WikiBox"), + valueBoxOutput("ContactBox") + ) + ) +} + diff --git a/inst/shiny-server/ui/variable.R b/inst/shiny-server/ui/variable.R index f69f6439..5a25071d 100644 --- a/inst/shiny-server/ui/variable.R +++ b/inst/shiny-server/ui/variable.R @@ -1,38 +1,38 @@ -FCE_GRID_INPUT_TEXT <- '

      Set the range and the granularity of the results. - The table will show function values that have been reached within evenly - spaced evaluation budgets.

      ' - -RT_TAR_LABEL <- HTML('

      \\(B:\\) Budget value

      ') -RT_MIN_LABEL <- HTML('

      \\(B_{\\text{min}}:\\) Smallest budget value

      ') -RT_MAX_LABEL <- HTML('

      \\(B_{\\text{max}}:\\) Largest budget value

      ') -RT_STEP_LABEL <- HTML('

      \\(\\Delta B:\\) Granularity (step size)

      ') - -F_TAR_LABEL <- HTML('Target function value

      \\(f_{\\text{target}}:\\)

      ') -F_MIN_LABEL <- HTML('

      \\(f_{\\text{min}}:\\) Smallest target value

      ') -F_MAX_LABEL <- HTML('

      \\(f_{\\text{max}}:\\) Largest target value

      ') -F_STEP_LABEL <- HTML('

      \\(\\Delta f:\\) Granularity (step size)

      ') - -Pdsc_info <- "Practical Deep Comparison uses this - as a threshold parameter to determine when two performance measure values are - equal or different from a practical point of view. - Since the practical significance - is handled with preprocessing using the predefined threshold in sequential order of - the obtained runs, to make more robust analysis a Monte-Carlo simulation is required." - -Pdsc_mc_info <- "Practical Deep Comparison uses this - to determine the number of monte-carlo simulations to perform. - The Monte-Carlo simulation involves preprocessing done with different orders of the - independent runs before comparing the distribution of the data. - This only has effect if threshold for practical significance > 0" - -HTML_P <- function(s) HTML(paste0('

      ', s, '

      ')) - -alg_select_info <- "Use this option to select which IDs to plot. - This will have an effect the dowloaded plot, - as opposed to using the legend-entries to show or hide algorithms. - The attributes which define the ID can be changed in the settings." - -custom_icon <- function(name = NULL){ - if (is.null(name)) name <- "info-circle" - htmltools::tags$a(shiny::icon(name = name), href = "javascript:void(0);") +FCE_GRID_INPUT_TEXT <- '

      Set the range and the granularity of the results. + The table will show function values that have been reached within evenly + spaced evaluation budgets.

      ' + +RT_TAR_LABEL <- HTML('

      \\(B:\\) Budget value

      ') +RT_MIN_LABEL <- HTML('

      \\(B_{\\text{min}}:\\) Smallest budget value

      ') +RT_MAX_LABEL <- HTML('

      \\(B_{\\text{max}}:\\) Largest budget value

      ') +RT_STEP_LABEL <- HTML('

      \\(\\Delta B:\\) Granularity (step size)

      ') + +F_TAR_LABEL <- HTML('Target function value

      \\(f_{\\text{target}}:\\)

      ') +F_MIN_LABEL <- HTML('

      \\(f_{\\text{min}}:\\) Smallest target value

      ') +F_MAX_LABEL <- HTML('

      \\(f_{\\text{max}}:\\) Largest target value

      ') +F_STEP_LABEL <- HTML('

      \\(\\Delta f:\\) Granularity (step size)

      ') + +Pdsc_info <- "Practical Deep Comparison uses this + as a threshold parameter to determine when two performance measure values are + equal or different from a practical point of view. + Since the practical significance + is handled with preprocessing using the predefined threshold in sequential order of + the obtained runs, to make more robust analysis a Monte-Carlo simulation is required." + +Pdsc_mc_info <- "Practical Deep Comparison uses this + to determine the number of monte-carlo simulations to perform. + The Monte-Carlo simulation involves preprocessing done with different orders of the + independent runs before comparing the distribution of the data. + This only has effect if threshold for practical significance > 0" + +HTML_P <- function(s) HTML(paste0('

      ', s, '

      ')) + +alg_select_info <- "Use this option to select which IDs to plot. + This will have an effect the dowloaded plot, + as opposed to using the legend-entries to show or hide algorithms. + The attributes which define the ID can be changed in the settings." + +custom_icon <- function(name = NULL){ + if (is.null(name)) name <- "info-circle" + htmltools::tags$a(shiny::icon(name = name), href = "javascript:void(0);") } \ No newline at end of file diff --git a/man/AUC.Rd b/man/AUC.Rd index e0bbe3ce..30a2df1a 100644 --- a/man/AUC.Rd +++ b/man/AUC.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{AUC} -\alias{AUC} -\alias{AUC.ECDF} -\title{Area Under Curve (Empirical Cumulative Dsitribution Function)} -\usage{ -AUC(fun, from = NULL, to = NULL) - -\method{AUC}{ECDF}(fun, from = NULL, to = NULL) -} -\arguments{ -\item{fun}{A ECDF object.} - -\item{from}{double. Starting point of the area on x-axis} - -\item{to}{double. Ending point of the area on x-axis} -} -\value{ -a object of type 'ECDF' -} -\description{ -Area Under Curve (Empirical Cumulative Dsitribution Function) -} -\examples{ -ecdf <- ECDF(dsl,c(12,14)) -AUC(ecdf, 0, 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{AUC} +\alias{AUC} +\alias{AUC.ECDF} +\title{Area Under Curve (Empirical Cumulative Dsitribution Function)} +\usage{ +AUC(fun, from = NULL, to = NULL) + +\method{AUC}{ECDF}(fun, from = NULL, to = NULL) +} +\arguments{ +\item{fun}{A ECDF object.} + +\item{from}{double. Starting point of the area on x-axis} + +\item{to}{double. Ending point of the area on x-axis} +} +\value{ +a object of type 'ECDF' +} +\description{ +Area Under Curve (Empirical Cumulative Dsitribution Function) +} +\examples{ +ecdf <- ECDF(dsl,c(12,14)) +AUC(ecdf, 0, 100) +} diff --git a/man/DataSet.Rd b/man/DataSet.Rd index 6d377f4c..3306e293 100644 --- a/man/DataSet.Rd +++ b/man/DataSet.Rd @@ -1,41 +1,41 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{DataSet} -\alias{DataSet} -\title{Constructor of S3 class 'DataSet'} -\usage{ -DataSet(info, verbose = F, maximization = NULL, format = IOHprofiler, - subsampling = FALSE) -} -\arguments{ -\item{info}{A List. Contains a set of in a *.info file.} - -\item{verbose}{Logical.} - -\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization? -Set to NULL to determine automatically based on format} - -\item{format}{A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL"} - -\item{subsampling}{Logical. Whether *.cdat files are subsampled?} -} -\value{ -A S3 object 'DataSet' -} -\description{ -DataSet contains the following attributes - * funId - * DIM - * algId - * datafile - * instance - * maxEvals - * finalFunEvals - * comment - * Additional attributes based on the original format -} -\examples{ -path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') -info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) -DataSet(info[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{DataSet} +\alias{DataSet} +\title{Constructor of S3 class 'DataSet'} +\usage{ +DataSet(info, verbose = F, maximization = NULL, format = IOHprofiler, + subsampling = FALSE) +} +\arguments{ +\item{info}{A List. Contains a set of in a *.info file.} + +\item{verbose}{Logical.} + +\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization? +Set to NULL to determine automatically based on format} + +\item{format}{A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL"} + +\item{subsampling}{Logical. Whether *.cdat files are subsampled?} +} +\value{ +A S3 object 'DataSet' +} +\description{ +DataSet contains the following attributes + * funId + * DIM + * algId + * datafile + * instance + * maxEvals + * finalFunEvals + * comment + * Additional attributes based on the original format +} +\examples{ +path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') +info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) +DataSet(info[[1]]) +} diff --git a/man/DataSetList.Rd b/man/DataSetList.Rd index a8bc05d5..61230cc9 100644 --- a/man/DataSetList.Rd +++ b/man/DataSetList.Rd @@ -1,50 +1,50 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{DataSetList} -\alias{DataSetList} -\title{S3 constructor of the 'DataSetList'} -\usage{ -DataSetList(path = NULL, verbose = T, print_fun = NULL, - maximization = NULL, format = IOHprofiler, subsampling = FALSE, - full_aggregation = TRUE) -} -\arguments{ -\item{path}{Path to the data files. Will look for all .info-files in this directory and use -the corresponding datafiles to create the DataSetList} - -\item{verbose}{Logical.} - -\item{print_fun}{Function used to print output when in verbose mode} - -\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization?} - -\item{format}{A character. The format of data source, options are: -\itemize{ -\item'IOHProfiler' -\item'COCO' -\item'TWO_COL' -\item'COCO_BIOBJ' -\item'NEVERGRAD' -\item'SOS' -} -These formats are specified in more detail in our github wiki.} - -\item{subsampling}{Logical. Whether *.cdat files are subsampled?} - -\item{full_aggregation}{If True, individual DataSets are aggregated as much as possible: all DataSets -with the same algorithmname, function id and dimension are combined together. This leads to information loss -related to static variables, so only use if that information is not required.} -} -\value{ -A DataSetList object -} -\description{ -Attributes - funId - DIM - algId -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -DataSetList(path) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{DataSetList} +\alias{DataSetList} +\title{S3 constructor of the 'DataSetList'} +\usage{ +DataSetList(path = NULL, verbose = T, print_fun = NULL, + maximization = NULL, format = IOHprofiler, subsampling = FALSE, + full_aggregation = TRUE) +} +\arguments{ +\item{path}{Path to the data files. Will look for all .info-files in this directory and use +the corresponding datafiles to create the DataSetList} + +\item{verbose}{Logical.} + +\item{print_fun}{Function used to print output when in verbose mode} + +\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization?} + +\item{format}{A character. The format of data source, options are: +\itemize{ +\item'IOHProfiler' +\item'COCO' +\item'TWO_COL' +\item'COCO_BIOBJ' +\item'NEVERGRAD' +\item'SOS' +} +These formats are specified in more detail in our github wiki.} + +\item{subsampling}{Logical. Whether *.cdat files are subsampled?} + +\item{full_aggregation}{If True, individual DataSets are aggregated as much as possible: all DataSets +with the same algorithmname, function id and dimension are combined together. This leads to information loss +related to static variables, so only use if that information is not required.} +} +\value{ +A DataSetList object +} +\description{ +Attributes + funId + DIM + algId +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +DataSetList(path) +} diff --git a/man/ECDF.Rd b/man/ECDF.Rd index 674c6fdf..28077296 100644 --- a/man/ECDF.Rd +++ b/man/ECDF.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{ECDF} -\alias{ECDF} -\alias{ECDF.DataSet} -\alias{ECDF.DataSetList} -\title{Empirical Cumulative Dsitribution Function of Runtime of a single data set} -\usage{ -ECDF(ds, ftarget, ...) - -\method{ECDF}{DataSet}(ds, ftarget, ...) - -\method{ECDF}{DataSetList}(ds, ftarget, ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object.} - -\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} - -\item{...}{Arguments passed to other methods} -} -\value{ -a object of type 'ECDF' -} -\description{ -Empirical Cumulative Dsitribution Function of Runtime of a single data set -} -\examples{ -ECDF(dsl,c(12,14)) -ECDF(dsl[[1]],c(12,14)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{ECDF} +\alias{ECDF} +\alias{ECDF.DataSet} +\alias{ECDF.DataSetList} +\title{Empirical Cumulative Dsitribution Function of Runtime of a single data set} +\usage{ +ECDF(ds, ftarget, ...) + +\method{ECDF}{DataSet}(ds, ftarget, ...) + +\method{ECDF}{DataSetList}(ds, ftarget, ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object.} + +\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} + +\item{...}{Arguments passed to other methods} +} +\value{ +a object of type 'ECDF' +} +\description{ +Empirical Cumulative Dsitribution Function of Runtime of a single data set +} +\examples{ +ECDF(dsl,c(12,14)) +ECDF(dsl[[1]],c(12,14)) +} diff --git a/man/IOH_plot_ly_default.Rd b/man/IOH_plot_ly_default.Rd index 9b8c9553..d44af531 100644 --- a/man/IOH_plot_ly_default.Rd +++ b/man/IOH_plot_ly_default.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{IOH_plot_ly_default} -\alias{IOH_plot_ly_default} -\title{Template for creating plots in the IOHanalyzer-style} -\usage{ -IOH_plot_ly_default(title = NULL, x.title = NULL, y.title = NULL) -} -\arguments{ -\item{title}{Title for the plot} - -\item{x.title}{X-axis label} - -\item{y.title}{Y-axis label} -} -\description{ -Template for creating plots in the IOHanalyzer-style -} -\examples{ -IOH_plot_ly_default("Example plot","x-axis","y-axis") -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{IOH_plot_ly_default} +\alias{IOH_plot_ly_default} +\title{Template for creating plots in the IOHanalyzer-style} +\usage{ +IOH_plot_ly_default(title = NULL, x.title = NULL, y.title = NULL) +} +\arguments{ +\item{title}{Title for the plot} + +\item{x.title}{X-axis label} + +\item{y.title}{Y-axis label} +} +\description{ +Template for creating plots in the IOHanalyzer-style +} +\examples{ +IOH_plot_ly_default("Example plot","x-axis","y-axis") +} diff --git a/man/IOHanalyzer.Rd b/man/IOHanalyzer.Rd index 41ca129d..650ff685 100644 --- a/man/IOHanalyzer.Rd +++ b/man/IOHanalyzer.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer.R -\docType{package} -\name{IOHanalyzer} -\alias{IOHanalyzer} -\alias{IOHanalyzer-package} -\title{IOHanalyzer: Data Analysis Part of IOHprofiler} -\description{ -The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). -This module provides statistical analysis methods for the benchmark data generated by -optimization heuristics, which can be visualized through a -web-based interface. The benchmark data is usually generated by the -experimentation module, called IOHexperimenter. IOHanalyzer also supports -the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. -} -\section{Functions}{ - -The IOHanalyzer consists of 3 main functionalities: -\itemize{ -\item Reading and alligning data from different heuristics, such as IOHExperimenter. -This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions -\item Processing and summarizing this data -\item Creating various plots -} -} - -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -dsList <- DataSetList(path) -summary(dsList) -Plot.RT.Single_Func(dsList[1]) - -\dontrun{ -runServer() -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer.R +\docType{package} +\name{IOHanalyzer} +\alias{IOHanalyzer} +\alias{IOHanalyzer-package} +\title{IOHanalyzer: Data Analysis Part of IOHprofiler} +\description{ +The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). +This module provides statistical analysis methods for the benchmark data generated by +optimization heuristics, which can be visualized through a +web-based interface. The benchmark data is usually generated by the +experimentation module, called IOHexperimenter. IOHanalyzer also supports +the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. +} +\section{Functions}{ + +The IOHanalyzer consists of 3 main functionalities: +\itemize{ +\item Reading and alligning data from different heuristics, such as IOHExperimenter. +This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions +\item Processing and summarizing this data +\item Creating various plots +} +} + +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +dsList <- DataSetList(path) +summary(dsList) +Plot.RT.Single_Func(dsList[1]) + +\dontrun{ +runServer() +} +} diff --git a/man/IOHanlyzer-deprecated.Rd b/man/IOHanlyzer-deprecated.Rd index 38b68118..b932c49b 100644 --- a/man/IOHanlyzer-deprecated.Rd +++ b/man/IOHanlyzer-deprecated.Rd @@ -1,11 +1,11 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{IOHanlyzer-deprecated} -\alias{IOHanlyzer-deprecated} -\title{Deprecated function in package \pkg{IOHanalyzer}} -\description{ -The functions listed below are deprecated and will be defunct in - the near future. When possible, alternative functions with similar - functionality are also mentioned. -} -\keyword{internal} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{IOHanlyzer-deprecated} +\alias{IOHanlyzer-deprecated} +\title{Deprecated function in package \pkg{IOHanalyzer}} +\description{ +The functions listed below are deprecated and will be defunct in + the near future. When possible, alternative functions with similar + functionality are also mentioned. +} +\keyword{internal} diff --git a/man/Plot.FV.Aggregated.Rd b/man/Plot.FV.Aggregated.Rd index ef91b3a2..a11be39d 100644 --- a/man/Plot.FV.Aggregated.Rd +++ b/man/Plot.FV.Aggregated.Rd @@ -1,42 +1,42 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Aggregated} -\alias{Plot.FV.Aggregated} -\alias{Plot.FV.Aggregated.DataSetList} -\title{Plot expected function value-based comparison over multiple functions or dimensions} -\usage{ -Plot.FV.Aggregated(dsList, aggr_on = "funcId", runtimes = NULL, - plot_mode = "radar", use_rank = F, scale.ylog = T, fvs = NULL) - -\method{Plot.FV.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", - runtimes = NULL, plot_mode = "radar", use_rank = F, - scale.ylog = T, fvs = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} - -\item{runtimes}{Custom list of function-value targets, one for each function or dimension.} - -\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} - -\item{use_rank}{Wheter to use a ranking system. If False, the actual expected function- -values will be used.} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{fvs}{Pre-calculated expected function-values for the provided runtimes Created by the -max_ERTs function of DataSetList. Can be provided to prevent needless computation -in recalculating ERTs when recreating this plot.} -} -\value{ -A plot of expected function value-based comparison on the provided functions - or dimensions of the DataSetList -} -\description{ -Plot expected function value-based comparison over multiple functions or dimensions -} -\examples{ -Plot.FV.Aggregated(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Aggregated} +\alias{Plot.FV.Aggregated} +\alias{Plot.FV.Aggregated.DataSetList} +\title{Plot expected function value-based comparison over multiple functions or dimensions} +\usage{ +Plot.FV.Aggregated(dsList, aggr_on = "funcId", runtimes = NULL, + plot_mode = "radar", use_rank = F, scale.ylog = T, fvs = NULL) + +\method{Plot.FV.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", + runtimes = NULL, plot_mode = "radar", use_rank = F, + scale.ylog = T, fvs = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} + +\item{runtimes}{Custom list of function-value targets, one for each function or dimension.} + +\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} + +\item{use_rank}{Wheter to use a ranking system. If False, the actual expected function- +values will be used.} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{fvs}{Pre-calculated expected function-values for the provided runtimes Created by the +max_ERTs function of DataSetList. Can be provided to prevent needless computation +in recalculating ERTs when recreating this plot.} +} +\value{ +A plot of expected function value-based comparison on the provided functions + or dimensions of the DataSetList +} +\description{ +Plot expected function value-based comparison over multiple functions or dimensions +} +\examples{ +Plot.FV.Aggregated(dsl) +} diff --git a/man/Plot.FV.ECDF_AUC.Rd b/man/Plot.FV.ECDF_AUC.Rd index d2c3c427..8e430fef 100644 --- a/man/Plot.FV.ECDF_AUC.Rd +++ b/man/Plot.FV.ECDF_AUC.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.ECDF_AUC} -\alias{Plot.FV.ECDF_AUC} -\alias{Plot.FV.ECDF_AUC.DataSetList} -\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} -\usage{ -Plot.FV.ECDF_AUC(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL) - -\method{Plot.FV.ECDF_AUC}{DataSetList}(dsList, rt_min = NULL, - rt_max = NULL, rt_step = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{rt_min}{The starting runtime} - -\item{rt_max}{The final runtime} - -\item{rt_step}{The spacing between starting and final runtimes} -} -\value{ -A radarplot of the area under the aggregated ECDF-curve of the DataSetList -} -\description{ -Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -} -\examples{ -Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.ECDF_AUC} +\alias{Plot.FV.ECDF_AUC} +\alias{Plot.FV.ECDF_AUC.DataSetList} +\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} +\usage{ +Plot.FV.ECDF_AUC(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL) + +\method{Plot.FV.ECDF_AUC}{DataSetList}(dsList, rt_min = NULL, + rt_max = NULL, rt_step = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{rt_min}{The starting runtime} + +\item{rt_max}{The final runtime} + +\item{rt_step}{The spacing between starting and final runtimes} +} +\value{ +A radarplot of the area under the aggregated ECDF-curve of the DataSetList +} +\description{ +Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +} +\examples{ +Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.FV.ECDF_Per_Target.Rd b/man/Plot.FV.ECDF_Per_Target.Rd index a1c4c81c..9b15c256 100644 --- a/man/Plot.FV.ECDF_Per_Target.Rd +++ b/man/Plot.FV.ECDF_Per_Target.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.ECDF_Per_Target} -\alias{Plot.FV.ECDF_Per_Target} -\alias{Plot.FV.ECDF_Per_Target.DataSetList} -\title{Plot the empirical cumulative distriburtion as a function of the target values of -a DataSetList at certain target runtimes} -\usage{ -Plot.FV.ECDF_Per_Target(dsList, runtimes, scale.xlog = F, - scale.reverse = F) - -\method{Plot.FV.ECDF_Per_Target}{DataSetList}(dsList, runtimes, - scale.xlog = F, scale.reverse = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{runtimes}{The target runtimes} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the fucntion values of the DataSetList at the target runtimes -} -\description{ -Plot the empirical cumulative distriburtion as a function of the target values of -a DataSetList at certain target runtimes -} -\examples{ -Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.ECDF_Per_Target} +\alias{Plot.FV.ECDF_Per_Target} +\alias{Plot.FV.ECDF_Per_Target.DataSetList} +\title{Plot the empirical cumulative distriburtion as a function of the target values of +a DataSetList at certain target runtimes} +\usage{ +Plot.FV.ECDF_Per_Target(dsList, runtimes, scale.xlog = F, + scale.reverse = F) + +\method{Plot.FV.ECDF_Per_Target}{DataSetList}(dsList, runtimes, + scale.xlog = F, scale.reverse = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{runtimes}{The target runtimes} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the fucntion values of the DataSetList at the target runtimes +} +\description{ +Plot the empirical cumulative distriburtion as a function of the target values of +a DataSetList at certain target runtimes +} +\examples{ +Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) +} diff --git a/man/Plot.FV.ECDF_Single_Func.Rd b/man/Plot.FV.ECDF_Single_Func.Rd index 39ec35a5..03a0c7ae 100644 --- a/man/Plot.FV.ECDF_Single_Func.Rd +++ b/man/Plot.FV.ECDF_Single_Func.Rd @@ -1,42 +1,42 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.ECDF_Single_Func} -\alias{Plot.FV.ECDF_Single_Func} -\alias{Plot.FV.ECDF_Single_Func.DataSetList} -\title{Plot the aggregated empirical cumulative distriburtion as a function of the function values of -a DataSetList.} -\usage{ -Plot.FV.ECDF_Single_Func(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL, scale.xlog = F, show.per_target = F, - scale.reverse = F) - -\method{Plot.FV.ECDF_Single_Func}{DataSetList}(dsList, rt_min = NULL, - rt_max = NULL, rt_step = NULL, scale.xlog = F, - show.per_target = F, scale.reverse = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{rt_min}{The starting runtime} - -\item{rt_max}{The final runtime} - -\item{rt_step}{The spacing between starting and final runtimes} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{show.per_target}{Whether or not to show the individual ECDF-curves for each runtime} - -\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the function values of the DataSetList -} -\description{ -Plot the aggregated empirical cumulative distriburtion as a function of the function values of -a DataSetList. -} -\examples{ -Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.ECDF_Single_Func} +\alias{Plot.FV.ECDF_Single_Func} +\alias{Plot.FV.ECDF_Single_Func.DataSetList} +\title{Plot the aggregated empirical cumulative distriburtion as a function of the function values of +a DataSetList.} +\usage{ +Plot.FV.ECDF_Single_Func(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL, scale.xlog = F, show.per_target = F, + scale.reverse = F) + +\method{Plot.FV.ECDF_Single_Func}{DataSetList}(dsList, rt_min = NULL, + rt_max = NULL, rt_step = NULL, scale.xlog = F, + show.per_target = F, scale.reverse = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{rt_min}{The starting runtime} + +\item{rt_max}{The final runtime} + +\item{rt_step}{The spacing between starting and final runtimes} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{show.per_target}{Whether or not to show the individual ECDF-curves for each runtime} + +\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the function values of the DataSetList +} +\description{ +Plot the aggregated empirical cumulative distriburtion as a function of the function values of +a DataSetList. +} +\examples{ +Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.FV.Histogram.Rd b/man/Plot.FV.Histogram.Rd index 15e9d762..dacf141f 100644 --- a/man/Plot.FV.Histogram.Rd +++ b/man/Plot.FV.Histogram.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Histogram} -\alias{Plot.FV.Histogram} -\alias{Plot.FV.Histogram.DataSetList} -\title{Plot histograms of the function values of a DataSetList at a certain target runtime} -\usage{ -Plot.FV.Histogram(dsList, runtime, plot_mode = "overlay", - use.equal.bins = F) - -\method{Plot.FV.Histogram}{DataSetList}(dsList, runtime, - plot_mode = "overlay", use.equal.bins = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{runtime}{The target runtime} - -\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either -'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} - -\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual -bin sizes for each algorithm} -} -\value{ -A plot of the histograms of the function values at a the - target runtime of the DataSetList -} -\description{ -Plot histograms of the function values of a DataSetList at a certain target runtime -} -\examples{ -Plot.FV.Histogram(subset(dsl, funcId == 1), 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Histogram} +\alias{Plot.FV.Histogram} +\alias{Plot.FV.Histogram.DataSetList} +\title{Plot histograms of the function values of a DataSetList at a certain target runtime} +\usage{ +Plot.FV.Histogram(dsList, runtime, plot_mode = "overlay", + use.equal.bins = F) + +\method{Plot.FV.Histogram}{DataSetList}(dsList, runtime, + plot_mode = "overlay", use.equal.bins = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{runtime}{The target runtime} + +\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either +'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} + +\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual +bin sizes for each algorithm} +} +\value{ +A plot of the histograms of the function values at a the + target runtime of the DataSetList +} +\description{ +Plot histograms of the function values of a DataSetList at a certain target runtime +} +\examples{ +Plot.FV.Histogram(subset(dsl, funcId == 1), 100) +} diff --git a/man/Plot.FV.Multi_Func.Rd b/man/Plot.FV.Multi_Func.Rd index e3ac971e..464f7157 100644 --- a/man/Plot.FV.Multi_Func.Rd +++ b/man/Plot.FV.Multi_Func.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Multi_Func} -\alias{Plot.FV.Multi_Func} -\alias{Plot.FV.Multi_Func.DataSetList} -\title{Plot FV-plots for multiple functions or dimensions} -\usage{ -Plot.FV.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, - backend = NULL) - -\method{Plot.FV.Multi_Func}{DataSetList}(dsList, scale.xlog = F, - scale.ylog = F, backend = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} -} -\value{ -A plot of Function-values of the DataSetList -} -\description{ -Plot FV-plots for multiple functions or dimensions -} -\examples{ -Plot.FV.Multi_Func(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Multi_Func} +\alias{Plot.FV.Multi_Func} +\alias{Plot.FV.Multi_Func.DataSetList} +\title{Plot FV-plots for multiple functions or dimensions} +\usage{ +Plot.FV.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, + backend = NULL) + +\method{Plot.FV.Multi_Func}{DataSetList}(dsList, scale.xlog = F, + scale.ylog = F, backend = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} +} +\value{ +A plot of Function-values of the DataSetList +} +\description{ +Plot FV-plots for multiple functions or dimensions +} +\examples{ +Plot.FV.Multi_Func(dsl) +} diff --git a/man/Plot.FV.PDF.Rd b/man/Plot.FV.PDF.Rd index 2d879670..02ed6ad9 100644 --- a/man/Plot.FV.PDF.Rd +++ b/man/Plot.FV.PDF.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.PDF} -\alias{Plot.FV.PDF} -\alias{Plot.FV.PDF.DataSetList} -\title{Plot probablity density function of the function values of a DataSetList at -a certain target runtime} -\usage{ -Plot.FV.PDF(dsList, runtime, show.sample = F, scale.ylog = F) - -\method{Plot.FV.PDF}{DataSetList}(dsList, runtime, show.sample = F, - scale.ylog = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{runtime}{The target runtime} - -\item{show.sample}{Whether or not to show the individual function value samples} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} -} -\value{ -A plot of the probablity density function of the runtimes at a the - target function value of the DataSetList -} -\description{ -Plot probablity density function of the function values of a DataSetList at -a certain target runtime -} -\examples{ -Plot.FV.PDF(subset(dsl, funcId == 1), 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.PDF} +\alias{Plot.FV.PDF} +\alias{Plot.FV.PDF.DataSetList} +\title{Plot probablity density function of the function values of a DataSetList at +a certain target runtime} +\usage{ +Plot.FV.PDF(dsList, runtime, show.sample = F, scale.ylog = F) + +\method{Plot.FV.PDF}{DataSetList}(dsList, runtime, show.sample = F, + scale.ylog = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{runtime}{The target runtime} + +\item{show.sample}{Whether or not to show the individual function value samples} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} +} +\value{ +A plot of the probablity density function of the runtimes at a the + target function value of the DataSetList +} +\description{ +Plot probablity density function of the function values of a DataSetList at +a certain target runtime +} +\examples{ +Plot.FV.PDF(subset(dsl, funcId == 1), 100) +} diff --git a/man/Plot.FV.Parameters.Rd b/man/Plot.FV.Parameters.Rd index 65ebb0c5..893cd455 100644 --- a/man/Plot.FV.Parameters.Rd +++ b/man/Plot.FV.Parameters.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Parameters} -\alias{Plot.FV.Parameters} -\alias{Plot.FV.Parameters.DataSetList} -\title{Plot the parameter values recorded in a DataSetList (aligned by budget)} -\usage{ -Plot.FV.Parameters(dsList, rt_min = NULL, rt_max = NULL, - algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, show.CI = F) - -\method{Plot.FV.Parameters}{DataSetList}(dsList, rt_min = NULL, - rt_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, - scale.ylog = F, show.mean = T, show.median = F, show.CI = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{rt_min}{The starting budget value.} - -\item{rt_max}{The final budget value.} - -\item{algids}{Which algorithms from dsList to use} - -\item{par_name}{Which parameters to create plots for; set to NULL to use all -parameters found in dsList.} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{show.mean}{Whether or not to show the mean parameter values} - -\item{show.median}{Whether or not to show the median parameter values} - -\item{show.CI}{Whether or not to show the standard deviation} -} -\value{ -A plot of for every recorded parameter in the DataSetList -} -\description{ -Plot the parameter values recorded in a DataSetList (aligned by budget) -} -\examples{ -Plot.FV.Parameters(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Parameters} +\alias{Plot.FV.Parameters} +\alias{Plot.FV.Parameters.DataSetList} +\title{Plot the parameter values recorded in a DataSetList (aligned by budget)} +\usage{ +Plot.FV.Parameters(dsList, rt_min = NULL, rt_max = NULL, + algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, show.CI = F) + +\method{Plot.FV.Parameters}{DataSetList}(dsList, rt_min = NULL, + rt_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, + scale.ylog = F, show.mean = T, show.median = F, show.CI = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{rt_min}{The starting budget value.} + +\item{rt_max}{The final budget value.} + +\item{algids}{Which algorithms from dsList to use} + +\item{par_name}{Which parameters to create plots for; set to NULL to use all +parameters found in dsList.} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{show.mean}{Whether or not to show the mean parameter values} + +\item{show.median}{Whether or not to show the median parameter values} + +\item{show.CI}{Whether or not to show the standard deviation} +} +\value{ +A plot of for every recorded parameter in the DataSetList +} +\description{ +Plot the parameter values recorded in a DataSetList (aligned by budget) +} +\examples{ +Plot.FV.Parameters(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.FV.Single_Func.Rd b/man/Plot.FV.Single_Func.Rd index ed014fa7..74fc741b 100644 --- a/man/Plot.FV.Single_Func.Rd +++ b/man/Plot.FV.Single_Func.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Single_Func} -\alias{Plot.FV.Single_Func} -\alias{Plot.FV.Single_Func.DataSetList} -\title{Plot lineplot of the expected function values of a DataSetList} -\usage{ -Plot.FV.Single_Func(dsList, RTstart = NULL, RTstop = NULL, - show.CI = F, show.mean = T, show.median = F, backend = NULL, - scale.xlog = F, scale.ylog = F, scale.reverse = F) - -\method{Plot.FV.Single_Func}{DataSetList}(dsList, RTstart = NULL, - RTstop = NULL, show.CI = F, show.mean = T, show.median = F, - backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{RTstart}{The starting runtime value.} - -\item{RTstop}{The final runtime value.} - -\item{show.CI}{Whether or not to show the standard deviations} - -\item{show.mean}{Whether or not to show the mean runtimes} - -\item{show.median}{Whether or not to show the median runtimes} - -\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} -} -\value{ -A plot of ERT-values of the DataSetList -} -\description{ -Plot lineplot of the expected function values of a DataSetList -} -\examples{ -Plot.FV.Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Single_Func} +\alias{Plot.FV.Single_Func} +\alias{Plot.FV.Single_Func.DataSetList} +\title{Plot lineplot of the expected function values of a DataSetList} +\usage{ +Plot.FV.Single_Func(dsList, RTstart = NULL, RTstop = NULL, + show.CI = F, show.mean = T, show.median = F, backend = NULL, + scale.xlog = F, scale.ylog = F, scale.reverse = F) + +\method{Plot.FV.Single_Func}{DataSetList}(dsList, RTstart = NULL, + RTstop = NULL, show.CI = F, show.mean = T, show.median = F, + backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{RTstart}{The starting runtime value.} + +\item{RTstop}{The final runtime value.} + +\item{show.CI}{Whether or not to show the standard deviations} + +\item{show.mean}{Whether or not to show the mean runtimes} + +\item{show.median}{Whether or not to show the median runtimes} + +\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} +} +\value{ +A plot of ERT-values of the DataSetList +} +\description{ +Plot lineplot of the expected function values of a DataSetList +} +\examples{ +Plot.FV.Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.Performviz.Rd b/man/Plot.Performviz.Rd index 16ef829d..bfe78c58 100644 --- a/man/Plot.Performviz.Rd +++ b/man/Plot.Performviz.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Performviz} -\alias{Plot.Performviz} -\title{Create the PerformViz plot} -\usage{ -Plot.Performviz(DSC_rank_result) -} -\arguments{ -\item{DSC_rank_result}{The result from a call to DSCtool rank service (`get_dsc_rank`)} -} -\value{ -A performviz plot -} -\description{ -From the paper: -} -\examples{ -\dontrun{ -Plot.Performviz(get_dsc_rank(dsl)) -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Performviz} +\alias{Plot.Performviz} +\title{Create the PerformViz plot} +\usage{ +Plot.Performviz(DSC_rank_result) +} +\arguments{ +\item{DSC_rank_result}{The result from a call to DSCtool rank service (`get_dsc_rank`)} +} +\value{ +A performviz plot +} +\description{ +From the paper: +} +\examples{ +\dontrun{ +Plot.Performviz(get_dsc_rank(dsl)) +} +} diff --git a/man/Plot.RT.Aggregated.Rd b/man/Plot.RT.Aggregated.Rd index 0a2e62c5..98aa73f3 100644 --- a/man/Plot.RT.Aggregated.Rd +++ b/man/Plot.RT.Aggregated.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Aggregated} -\alias{Plot.RT.Aggregated} -\alias{Plot.RT.Aggregated.DataSetList} -\title{Plot ERT-based comparison over multiple functions or dimensions} -\usage{ -Plot.RT.Aggregated(dsList, aggr_on = "funcId", targets = NULL, - plot_mode = "radar", use_rank = F, scale.ylog = T, maximize = T, - erts = NULL, inf.action = "overlap") - -\method{Plot.RT.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", - targets = NULL, plot_mode = "radar", use_rank = F, - scale.ylog = T, maximize = T, erts = NULL, - inf.action = "overlap") -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} - -\item{targets}{Custom list of function-value targets, one for each function or dimension.} - -\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} - -\item{use_rank}{Wheter to use a ranking system. If False, the actual ERT-values will be used.} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{maximize}{Wheter or not to the data is of a maximization problem} - -\item{erts}{Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function -of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating -this plot.} - -\item{inf.action}{How to handle infinite ERTs ('overlap' or 'jitter')} -} -\value{ -A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList -} -\description{ -Plot ERT-based comparison over multiple functions or dimensions -} -\examples{ -Plot.RT.Aggregated(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Aggregated} +\alias{Plot.RT.Aggregated} +\alias{Plot.RT.Aggregated.DataSetList} +\title{Plot ERT-based comparison over multiple functions or dimensions} +\usage{ +Plot.RT.Aggregated(dsList, aggr_on = "funcId", targets = NULL, + plot_mode = "radar", use_rank = F, scale.ylog = T, maximize = T, + erts = NULL, inf.action = "overlap") + +\method{Plot.RT.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", + targets = NULL, plot_mode = "radar", use_rank = F, + scale.ylog = T, maximize = T, erts = NULL, + inf.action = "overlap") +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} + +\item{targets}{Custom list of function-value targets, one for each function or dimension.} + +\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} + +\item{use_rank}{Wheter to use a ranking system. If False, the actual ERT-values will be used.} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{maximize}{Wheter or not to the data is of a maximization problem} + +\item{erts}{Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function +of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating +this plot.} + +\item{inf.action}{How to handle infinite ERTs ('overlap' or 'jitter')} +} +\value{ +A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList +} +\description{ +Plot ERT-based comparison over multiple functions or dimensions +} +\examples{ +Plot.RT.Aggregated(dsl) +} diff --git a/man/Plot.RT.ECDF_AUC.Rd b/man/Plot.RT.ECDF_AUC.Rd index d047fd5d..767f1cc6 100644 --- a/man/Plot.RT.ECDF_AUC.Rd +++ b/man/Plot.RT.ECDF_AUC.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_AUC} -\alias{Plot.RT.ECDF_AUC} -\alias{Plot.RT.ECDF_AUC.DataSetList} -\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} -\usage{ -Plot.RT.ECDF_AUC(dsList, fstart = NULL, fstop = NULL, fstep = NULL, - fval_formatter = as.integer) - -\method{Plot.RT.ECDF_AUC}{DataSetList}(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, fval_formatter = as.integer) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{fstart}{The starting function value} - -\item{fstop}{The final function value} - -\item{fstep}{The spacing between starting and final function values} - -\item{fval_formatter}{Function to format the function-value labels} -} -\value{ -A radarplot of the area under the aggregated ECDF-curve of the DataSetList -} -\description{ -Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -} -\examples{ -Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_AUC} +\alias{Plot.RT.ECDF_AUC} +\alias{Plot.RT.ECDF_AUC.DataSetList} +\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} +\usage{ +Plot.RT.ECDF_AUC(dsList, fstart = NULL, fstop = NULL, fstep = NULL, + fval_formatter = as.integer) + +\method{Plot.RT.ECDF_AUC}{DataSetList}(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, fval_formatter = as.integer) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{fstart}{The starting function value} + +\item{fstop}{The final function value} + +\item{fstep}{The spacing between starting and final function values} + +\item{fval_formatter}{Function to format the function-value labels} +} +\value{ +A radarplot of the area under the aggregated ECDF-curve of the DataSetList +} +\description{ +Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +} +\examples{ +Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.RT.ECDF_Multi_Func.Rd b/man/Plot.RT.ECDF_Multi_Func.Rd index 27c19799..86442f78 100644 --- a/man/Plot.RT.ECDF_Multi_Func.Rd +++ b/man/Plot.RT.ECDF_Multi_Func.Rd @@ -1,32 +1,32 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_Multi_Func} -\alias{Plot.RT.ECDF_Multi_Func} -\alias{Plot.RT.ECDF_Multi_Func.DataSetList} -\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList. Aggregated over multiple functions or dimensions.} -\usage{ -Plot.RT.ECDF_Multi_Func(dsList, targets = NULL, scale.xlog = F) - -\method{Plot.RT.ECDF_Multi_Func}{DataSetList}(dsList, targets = NULL, - scale.xlog = F) -} -\arguments{ -\item{dsList}{A DataSetList.} - -\item{targets}{The target function values. Specified in a data.frame, as can be generated} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically -by the function 'get_ECDF_targets'} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the running times of the DataSetList -} -\description{ -Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList. Aggregated over multiple functions or dimensions. -} -\examples{ -Plot.RT.ECDF_Multi_Func(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_Multi_Func} +\alias{Plot.RT.ECDF_Multi_Func} +\alias{Plot.RT.ECDF_Multi_Func.DataSetList} +\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList. Aggregated over multiple functions or dimensions.} +\usage{ +Plot.RT.ECDF_Multi_Func(dsList, targets = NULL, scale.xlog = F) + +\method{Plot.RT.ECDF_Multi_Func}{DataSetList}(dsList, targets = NULL, + scale.xlog = F) +} +\arguments{ +\item{dsList}{A DataSetList.} + +\item{targets}{The target function values. Specified in a data.frame, as can be generated} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically +by the function 'get_ECDF_targets'} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the running times of the DataSetList +} +\description{ +Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList. Aggregated over multiple functions or dimensions. +} +\examples{ +Plot.RT.ECDF_Multi_Func(dsl) +} diff --git a/man/Plot.RT.ECDF_Per_Target.Rd b/man/Plot.RT.ECDF_Per_Target.Rd index e2668271..0d652fac 100644 --- a/man/Plot.RT.ECDF_Per_Target.Rd +++ b/man/Plot.RT.ECDF_Per_Target.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_Per_Target} -\alias{Plot.RT.ECDF_Per_Target} -\alias{Plot.RT.ECDF_Per_Target.DataSetList} -\title{Plot the empirical cumulative distriburtion as a function of the running times of -a DataSetList at certain target function values} -\usage{ -Plot.RT.ECDF_Per_Target(dsList, ftargets, scale.xlog = F) - -\method{Plot.RT.ECDF_Per_Target}{DataSetList}(dsList, ftargets, - scale.xlog = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftargets}{The target function values} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the running times of the DataSetList at the target function values -} -\description{ -Plot the empirical cumulative distriburtion as a function of the running times of -a DataSetList at certain target function values -} -\examples{ -Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_Per_Target} +\alias{Plot.RT.ECDF_Per_Target} +\alias{Plot.RT.ECDF_Per_Target.DataSetList} +\title{Plot the empirical cumulative distriburtion as a function of the running times of +a DataSetList at certain target function values} +\usage{ +Plot.RT.ECDF_Per_Target(dsList, ftargets, scale.xlog = F) + +\method{Plot.RT.ECDF_Per_Target}{DataSetList}(dsList, ftargets, + scale.xlog = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftargets}{The target function values} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the running times of the DataSetList at the target function values +} +\description{ +Plot the empirical cumulative distriburtion as a function of the running times of +a DataSetList at certain target function values +} +\examples{ +Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) +} diff --git a/man/Plot.RT.ECDF_Single_Func.Rd b/man/Plot.RT.ECDF_Single_Func.Rd index 4c9bac6c..5174e043 100644 --- a/man/Plot.RT.ECDF_Single_Func.Rd +++ b/man/Plot.RT.ECDF_Single_Func.Rd @@ -1,38 +1,38 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_Single_Func} -\alias{Plot.RT.ECDF_Single_Func} -\alias{Plot.RT.ECDF_Single_Func.DataSetList} -\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList.} -\usage{ -Plot.RT.ECDF_Single_Func(dsList, fstart = NULL, fstop = NULL, - fstep = NULL, show.per_target = F, scale.xlog = F) - -\method{Plot.RT.ECDF_Single_Func}{DataSetList}(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, show.per_target = F, scale.xlog = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{fstart}{The starting function value} - -\item{fstop}{The final function value} - -\item{fstep}{The spacing between starting and final function values} - -\item{show.per_target}{Whether or not to show the individual ECDF-curves for each target} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the running times of the DataSetList -} -\description{ -Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList. -} -\examples{ -Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_Single_Func} +\alias{Plot.RT.ECDF_Single_Func} +\alias{Plot.RT.ECDF_Single_Func.DataSetList} +\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList.} +\usage{ +Plot.RT.ECDF_Single_Func(dsList, fstart = NULL, fstop = NULL, + fstep = NULL, show.per_target = F, scale.xlog = F) + +\method{Plot.RT.ECDF_Single_Func}{DataSetList}(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, show.per_target = F, scale.xlog = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{fstart}{The starting function value} + +\item{fstop}{The final function value} + +\item{fstep}{The spacing between starting and final function values} + +\item{show.per_target}{Whether or not to show the individual ECDF-curves for each target} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the running times of the DataSetList +} +\description{ +Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList. +} +\examples{ +Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.RT.Histogram.Rd b/man/Plot.RT.Histogram.Rd index 5a839b84..6d733180 100644 --- a/man/Plot.RT.Histogram.Rd +++ b/man/Plot.RT.Histogram.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Histogram} -\alias{Plot.RT.Histogram} -\alias{Plot.RT.Histogram.DataSetList} -\title{Plot histograms of the runtimes of a DataSetList at a certain target function value} -\usage{ -Plot.RT.Histogram(dsList, ftarget, plot_mode = "overlay", - use.equal.bins = F) - -\method{Plot.RT.Histogram}{DataSetList}(dsList, ftarget, - plot_mode = "overlay", use.equal.bins = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value.} - -\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either -'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} - -\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual -bin sizes for each algorithm} -} -\value{ -A plot of the histograms of the runtimes at a the - target function value of the DataSetList -} -\description{ -Plot histograms of the runtimes of a DataSetList at a certain target function value -} -\examples{ -Plot.RT.Histogram(subset(dsl, funcId == 1), 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Histogram} +\alias{Plot.RT.Histogram} +\alias{Plot.RT.Histogram.DataSetList} +\title{Plot histograms of the runtimes of a DataSetList at a certain target function value} +\usage{ +Plot.RT.Histogram(dsList, ftarget, plot_mode = "overlay", + use.equal.bins = F) + +\method{Plot.RT.Histogram}{DataSetList}(dsList, ftarget, + plot_mode = "overlay", use.equal.bins = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value.} + +\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either +'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} + +\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual +bin sizes for each algorithm} +} +\value{ +A plot of the histograms of the runtimes at a the + target function value of the DataSetList +} +\description{ +Plot histograms of the runtimes of a DataSetList at a certain target function value +} +\examples{ +Plot.RT.Histogram(subset(dsl, funcId == 1), 14) +} diff --git a/man/Plot.RT.Multi_Func.Rd b/man/Plot.RT.Multi_Func.Rd index 1121a3ae..9dd528fb 100644 --- a/man/Plot.RT.Multi_Func.Rd +++ b/man/Plot.RT.Multi_Func.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Multi_Func} -\alias{Plot.RT.Multi_Func} -\alias{Plot.RT.Multi_Func.DataSetList} -\title{Plot ERT-plots for multiple functions or dimensions} -\usage{ -Plot.RT.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, - scale.reverse = F, backend = NULL) - -\method{Plot.RT.Multi_Func}{DataSetList}(dsList, scale.xlog = F, - scale.ylog = F, scale.reverse = F, backend = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} - -\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} -} -\value{ -A plot of ERT-values of the DataSetList -} -\description{ -Plot ERT-plots for multiple functions or dimensions -} -\examples{ -Plot.RT.Multi_Func(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Multi_Func} +\alias{Plot.RT.Multi_Func} +\alias{Plot.RT.Multi_Func.DataSetList} +\title{Plot ERT-plots for multiple functions or dimensions} +\usage{ +Plot.RT.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, + scale.reverse = F, backend = NULL) + +\method{Plot.RT.Multi_Func}{DataSetList}(dsList, scale.xlog = F, + scale.ylog = F, scale.reverse = F, backend = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} + +\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} +} +\value{ +A plot of ERT-values of the DataSetList +} +\description{ +Plot ERT-plots for multiple functions or dimensions +} +\examples{ +Plot.RT.Multi_Func(dsl) +} diff --git a/man/Plot.RT.PMF.Rd b/man/Plot.RT.PMF.Rd index 8d270f5d..aa6c9db6 100644 --- a/man/Plot.RT.PMF.Rd +++ b/man/Plot.RT.PMF.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.PMF} -\alias{Plot.RT.PMF} -\alias{Plot.RT.PMF.DataSetList} -\title{Plot probablity mass function of the runtimes of a DataSetList at a certain target function value} -\usage{ -Plot.RT.PMF(dsList, ftarget, show.sample = F, scale.ylog = F, - backend = NULL) - -\method{Plot.RT.PMF}{DataSetList}(dsList, ftarget, show.sample = F, - scale.ylog = F, backend = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value.} - -\item{show.sample}{Whether or not to show the individual runtime samples} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} -} -\value{ -A plot of the probablity mass function of the runtimes at a the - target function value of the DataSetList -} -\description{ -Plot probablity mass function of the runtimes of a DataSetList at a certain target function value -} -\examples{ -Plot.RT.PMF(subset(dsl, funcId == 1), 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.PMF} +\alias{Plot.RT.PMF} +\alias{Plot.RT.PMF.DataSetList} +\title{Plot probablity mass function of the runtimes of a DataSetList at a certain target function value} +\usage{ +Plot.RT.PMF(dsList, ftarget, show.sample = F, scale.ylog = F, + backend = NULL) + +\method{Plot.RT.PMF}{DataSetList}(dsList, ftarget, show.sample = F, + scale.ylog = F, backend = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value.} + +\item{show.sample}{Whether or not to show the individual runtime samples} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} +} +\value{ +A plot of the probablity mass function of the runtimes at a the + target function value of the DataSetList +} +\description{ +Plot probablity mass function of the runtimes of a DataSetList at a certain target function value +} +\examples{ +Plot.RT.PMF(subset(dsl, funcId == 1), 14) +} diff --git a/man/Plot.RT.Parameters.Rd b/man/Plot.RT.Parameters.Rd index 1ed81362..9fc96bba 100644 --- a/man/Plot.RT.Parameters.Rd +++ b/man/Plot.RT.Parameters.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Parameters} -\alias{Plot.RT.Parameters} -\alias{Plot.RT.Parameters.DataSetList} -\title{Plot the parameter values recorded in a DataSetList (aligned by funcion value)} -\usage{ -Plot.RT.Parameters(dsList, f_min = NULL, f_max = NULL, - algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, show.CI = F) - -\method{Plot.RT.Parameters}{DataSetList}(dsList, f_min = NULL, - f_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, - scale.ylog = F, show.mean = T, show.median = F, show.CI = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{f_min}{The starting function value.} - -\item{f_max}{The final function value.} - -\item{algids}{Which algorithms from dsList to use} - -\item{par_name}{Which parameters to create plots for; set to NULL to use all -parameters found in dsList.} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{show.mean}{Whether or not to show the mean parameter values} - -\item{show.median}{Whether or not to show the median parameter values} - -\item{show.CI}{Whether or not to show the standard deviation} -} -\value{ -A plot of for every recorded parameter in the DataSetList -} -\description{ -Plot the parameter values recorded in a DataSetList (aligned by funcion value) -} -\examples{ -Plot.RT.Parameters(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Parameters} +\alias{Plot.RT.Parameters} +\alias{Plot.RT.Parameters.DataSetList} +\title{Plot the parameter values recorded in a DataSetList (aligned by funcion value)} +\usage{ +Plot.RT.Parameters(dsList, f_min = NULL, f_max = NULL, + algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, show.CI = F) + +\method{Plot.RT.Parameters}{DataSetList}(dsList, f_min = NULL, + f_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, + scale.ylog = F, show.mean = T, show.median = F, show.CI = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{f_min}{The starting function value.} + +\item{f_max}{The final function value.} + +\item{algids}{Which algorithms from dsList to use} + +\item{par_name}{Which parameters to create plots for; set to NULL to use all +parameters found in dsList.} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{show.mean}{Whether or not to show the mean parameter values} + +\item{show.median}{Whether or not to show the median parameter values} + +\item{show.CI}{Whether or not to show the standard deviation} +} +\value{ +A plot of for every recorded parameter in the DataSetList +} +\description{ +Plot the parameter values recorded in a DataSetList (aligned by funcion value) +} +\examples{ +Plot.RT.Parameters(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.RT.Single_Func.Rd b/man/Plot.RT.Single_Func.Rd index 2d5bc0b2..9577a353 100644 --- a/man/Plot.RT.Single_Func.Rd +++ b/man/Plot.RT.Single_Func.Rd @@ -1,53 +1,53 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Single_Func} -\alias{Plot.RT.Single_Func} -\alias{Plot.RT.Single_Func.DataSetList} -\title{Plot lineplot of the ERTs of a DataSetList} -\usage{ -Plot.RT.Single_Func(dsList, Fstart = NULL, Fstop = NULL, - show.ERT = T, show.CI = F, show.mean = F, show.median = F, - backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F, includeOpts = F, p = NULL) - -\method{Plot.RT.Single_Func}{DataSetList}(dsList, Fstart = NULL, - Fstop = NULL, show.ERT = T, show.CI = T, show.mean = F, - show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F, includeOpts = F, p = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{Fstart}{The starting function value.} - -\item{Fstop}{The final function value.} - -\item{show.ERT}{Whether or not to show the ERT-values} - -\item{show.CI}{Whether or not to show the standard deviations} - -\item{show.mean}{Whether or not to show the mean hitting times} - -\item{show.median}{Whether or not to show the median hitting times} - -\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} - -\item{includeOpts}{Whether or not to include all best points reached by each algorithm} - -\item{p}{Existing plot to which to add the current data} -} -\value{ -A plot of ERT-values of the DataSetList -} -\description{ -Plot lineplot of the ERTs of a DataSetList -} -\examples{ -Plot.RT.Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Single_Func} +\alias{Plot.RT.Single_Func} +\alias{Plot.RT.Single_Func.DataSetList} +\title{Plot lineplot of the ERTs of a DataSetList} +\usage{ +Plot.RT.Single_Func(dsList, Fstart = NULL, Fstop = NULL, + show.ERT = T, show.CI = F, show.mean = F, show.median = F, + backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F, includeOpts = F, p = NULL) + +\method{Plot.RT.Single_Func}{DataSetList}(dsList, Fstart = NULL, + Fstop = NULL, show.ERT = T, show.CI = T, show.mean = F, + show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F, includeOpts = F, p = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{Fstart}{The starting function value.} + +\item{Fstop}{The final function value.} + +\item{show.ERT}{Whether or not to show the ERT-values} + +\item{show.CI}{Whether or not to show the standard deviations} + +\item{show.mean}{Whether or not to show the mean hitting times} + +\item{show.median}{Whether or not to show the median hitting times} + +\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} + +\item{includeOpts}{Whether or not to include all best points reached by each algorithm} + +\item{p}{Existing plot to which to add the current data} +} +\value{ +A plot of ERT-values of the DataSetList +} +\description{ +Plot lineplot of the ERTs of a DataSetList +} +\examples{ +Plot.RT.Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.Stats.Glicko2_Candlestick.Rd b/man/Plot.Stats.Glicko2_Candlestick.Rd index 25f6d212..cf54c63e 100644 --- a/man/Plot.Stats.Glicko2_Candlestick.Rd +++ b/man/Plot.Stats.Glicko2_Candlestick.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Stats.Glicko2_Candlestick} -\alias{Plot.Stats.Glicko2_Candlestick} -\alias{Plot.Stats.Glicko2_Candlestick.DataSetList} -\title{Create a candlestick plot of Glicko2-rankings} -\usage{ -Plot.Stats.Glicko2_Candlestick(dsList, nr_rounds = 100, - glicko2_rank_df = NULL, which = "by_FV", target_dt = NULL) - -\method{Plot.Stats.Glicko2_Candlestick}{DataSetList}(dsList, - nr_rounds = 100, glicko2_rank_df = NULL, which = "by_FV", - target_dt = NULL) -} -\arguments{ -\item{dsList}{A DataSetList} - -\item{nr_rounds}{The number of rounds in the tournament} - -\item{glicko2_rank_df}{Optional. Dataframe containing the glicko2 rating to avoid needless recalculation.} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} - -\item{target_dt}{Optional: data table containing the targets for each function and dimension} -} -\description{ -Create a candlestick plot of Glicko2-rankings -} -\examples{ -Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Stats.Glicko2_Candlestick} +\alias{Plot.Stats.Glicko2_Candlestick} +\alias{Plot.Stats.Glicko2_Candlestick.DataSetList} +\title{Create a candlestick plot of Glicko2-rankings} +\usage{ +Plot.Stats.Glicko2_Candlestick(dsList, nr_rounds = 100, + glicko2_rank_df = NULL, which = "by_FV", target_dt = NULL) + +\method{Plot.Stats.Glicko2_Candlestick}{DataSetList}(dsList, + nr_rounds = 100, glicko2_rank_df = NULL, which = "by_FV", + target_dt = NULL) +} +\arguments{ +\item{dsList}{A DataSetList} + +\item{nr_rounds}{The number of rounds in the tournament} + +\item{glicko2_rank_df}{Optional. Dataframe containing the glicko2 rating to avoid needless recalculation.} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} + +\item{target_dt}{Optional: data table containing the targets for each function and dimension} +} +\description{ +Create a candlestick plot of Glicko2-rankings +} +\examples{ +Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) +} diff --git a/man/Plot.Stats.Significance_Graph.Rd b/man/Plot.Stats.Significance_Graph.Rd index 6894fce3..5c77551c 100644 --- a/man/Plot.Stats.Significance_Graph.Rd +++ b/man/Plot.Stats.Significance_Graph.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Stats.Significance_Graph} -\alias{Plot.Stats.Significance_Graph} -\alias{Plot.Stats.Significance_Graph.DataSetList} -\title{Plot a network graph showing the statistically different algorithms} -\usage{ -Plot.Stats.Significance_Graph(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = "by_FV") - -\method{Plot.Stats.Significance_Graph}{DataSetList}(dsList, ftarget, - alpha = 0.01, bootstrap.size = 30, which = "by_FV") -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value to use} - -\item{alpha}{The cutoff for statistical significance} - -\item{bootstrap.size}{The amound of bootstrapped samples used} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} -} -\value{ -A graph showing the statistical significance between algorithms -} -\description{ -Plot a network graph showing the statistically different algorithms -} -\examples{ -Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Stats.Significance_Graph} +\alias{Plot.Stats.Significance_Graph} +\alias{Plot.Stats.Significance_Graph.DataSetList} +\title{Plot a network graph showing the statistically different algorithms} +\usage{ +Plot.Stats.Significance_Graph(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = "by_FV") + +\method{Plot.Stats.Significance_Graph}{DataSetList}(dsList, ftarget, + alpha = 0.01, bootstrap.size = 30, which = "by_FV") +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value to use} + +\item{alpha}{The cutoff for statistical significance} + +\item{bootstrap.size}{The amound of bootstrapped samples used} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} +} +\value{ +A graph showing the statistical significance between algorithms +} +\description{ +Plot a network graph showing the statistically different algorithms +} +\examples{ +Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) +} diff --git a/man/Plot.Stats.Significance_Heatmap.Rd b/man/Plot.Stats.Significance_Heatmap.Rd index 9c3693e0..e50c1880 100644 --- a/man/Plot.Stats.Significance_Heatmap.Rd +++ b/man/Plot.Stats.Significance_Heatmap.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Stats.Significance_Heatmap} -\alias{Plot.Stats.Significance_Heatmap} -\alias{Plot.Stats.Significance_Heatmap.DataSetList} -\title{Plot a heatmap showing the statistically different algorithms} -\usage{ -Plot.Stats.Significance_Heatmap(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = "by_FV") - -\method{Plot.Stats.Significance_Heatmap}{DataSetList}(dsList, ftarget, - alpha = 0.01, bootstrap.size = 30, which = "by_FV") -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value to use} - -\item{alpha}{The cutoff for statistical significance} - -\item{bootstrap.size}{The amound of bootstrapped samples used} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} -} -\value{ -A heatmap showing the statistical significance between algorithms -} -\description{ -Plot a heatmap showing the statistically different algorithms -} -\examples{ -Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Stats.Significance_Heatmap} +\alias{Plot.Stats.Significance_Heatmap} +\alias{Plot.Stats.Significance_Heatmap.DataSetList} +\title{Plot a heatmap showing the statistically different algorithms} +\usage{ +Plot.Stats.Significance_Heatmap(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = "by_FV") + +\method{Plot.Stats.Significance_Heatmap}{DataSetList}(dsList, ftarget, + alpha = 0.01, bootstrap.size = 30, which = "by_FV") +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value to use} + +\item{alpha}{The cutoff for statistical significance} + +\item{bootstrap.size}{The amound of bootstrapped samples used} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} +} +\value{ +A heatmap showing the statistical significance between algorithms +} +\description{ +Plot a heatmap showing the statistically different algorithms +} +\examples{ +Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) +} diff --git a/man/SP.Rd b/man/SP.Rd index c202d82e..cb6552d0 100644 --- a/man/SP.Rd +++ b/man/SP.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{SP} -\alias{SP} -\title{Estimator 'SP' for the Expected Running Time (ERT)} -\usage{ -SP(data, max_runtime) -} -\arguments{ -\item{data}{A dataframe or matrix. Each row stores the runtime sample points from -several runs} - -\item{max_runtime}{The budget to use for calculating ERT. If this is a vector, the largest value is taken. -Using this as a vector is being deprecated, and will be removed in a future update} -} -\value{ -A list containing ERTs, number of succesfull runs and the succes rate -} -\description{ -Estimator 'SP' for the Expected Running Time (ERT) -} -\examples{ -SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{SP} +\alias{SP} +\title{Estimator 'SP' for the Expected Running Time (ERT)} +\usage{ +SP(data, max_runtime) +} +\arguments{ +\item{data}{A dataframe or matrix. Each row stores the runtime sample points from +several runs} + +\item{max_runtime}{The budget to use for calculating ERT. If this is a vector, the largest value is taken. +Using this as a vector is being deprecated, and will be removed in a future update} +} +\value{ +A list containing ERTs, number of succesfull runs and the succes rate +} +\description{ +Estimator 'SP' for the Expected Running Time (ERT) +} +\examples{ +SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) +} diff --git a/man/arrange.Rd b/man/arrange.Rd index 2f4ae67c..c36ce771 100644 --- a/man/arrange.Rd +++ b/man/arrange.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{arrange} -\alias{arrange} -\alias{arrange.DataSetList} -\title{S3 sort function for DataSetList} -\usage{ -arrange(dsl, ...) - -\method{arrange}{DataSetList}(dsl, ...) -} -\arguments{ -\item{dsl}{The DataSetList to sort} - -\item{...}{attribute by which `dsl` is sorted. Multiple attributes can be specified.} -} -\description{ -Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). -Default is as ascending, can be made descending by adding a - in front of the attribute. -Sorting accross multiple attributes is supported, in the order they are specified. -} -\examples{ -arrange(dsl, DIM, -funcId, algId) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{arrange} +\alias{arrange} +\alias{arrange.DataSetList} +\title{S3 sort function for DataSetList} +\usage{ +arrange(dsl, ...) + +\method{arrange}{DataSetList}(dsl, ...) +} +\arguments{ +\item{dsl}{The DataSetList to sort} + +\item{...}{attribute by which `dsl` is sorted. Multiple attributes can be specified.} +} +\description{ +Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). +Default is as ascending, can be made descending by adding a - in front of the attribute. +Sorting accross multiple attributes is supported, in the order they are specified. +} +\examples{ +arrange(dsl, DIM, -funcId, algId) +} diff --git a/man/as.character.DataSet.Rd b/man/as.character.DataSet.Rd index 9329de82..84a75c24 100644 --- a/man/as.character.DataSet.Rd +++ b/man/as.character.DataSet.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{as.character.DataSet} -\alias{as.character.DataSet} -\title{S3 generic as.character operator for DataSet} -\usage{ -\method{as.character}{DataSet}(x, verbose = F, ...) -} -\arguments{ -\item{x}{A DataSet object} - -\item{verbose}{Verbose mode, currently not implemented} - -\item{...}{Arguments passed to other methods} -} -\value{ -A short description of the DataSet -} -\description{ -S3 generic as.character operator for DataSet -} -\examples{ -as.character(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{as.character.DataSet} +\alias{as.character.DataSet} +\title{S3 generic as.character operator for DataSet} +\usage{ +\method{as.character}{DataSet}(x, verbose = F, ...) +} +\arguments{ +\item{x}{A DataSet object} + +\item{verbose}{Verbose mode, currently not implemented} + +\item{...}{Arguments passed to other methods} +} +\value{ +A short description of the DataSet +} +\description{ +S3 generic as.character operator for DataSet +} +\examples{ +as.character(dsl[[1]]) +} diff --git a/man/bootstrap_RT.Rd b/man/bootstrap_RT.Rd index 416a7da5..4ff38ccf 100644 --- a/man/bootstrap_RT.Rd +++ b/man/bootstrap_RT.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{bootstrap_RT} -\alias{bootstrap_RT} -\title{Bootstrapping for running time samples} -\usage{ -bootstrap_RT(x, max_eval, bootstrap.size) -} -\arguments{ -\item{x}{A numeric vector. A sample of the running time.} - -\item{max_eval}{A numeric vector, containing the maximal running time in -each run. It should have the same size as x} - -\item{bootstrap.size}{integer, the size of the bootstrapped sample} -} -\value{ -A numeric vector of the bootstrapped running time sample -} -\description{ -Bootstrapping for running time samples -} -\examples{ -ds <- dsl[[1]] -x <- get_RT_sample(ds, ftarget = 16, output = 'long') -max_eval <- get_maxRT(dsl, output = 'long') -bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{bootstrap_RT} +\alias{bootstrap_RT} +\title{Bootstrapping for running time samples} +\usage{ +bootstrap_RT(x, max_eval, bootstrap.size) +} +\arguments{ +\item{x}{A numeric vector. A sample of the running time.} + +\item{max_eval}{A numeric vector, containing the maximal running time in +each run. It should have the same size as x} + +\item{bootstrap.size}{integer, the size of the bootstrapped sample} +} +\value{ +A numeric vector of the bootstrapped running time sample +} +\description{ +Bootstrapping for running time samples +} +\examples{ +ds <- dsl[[1]] +x <- get_RT_sample(ds, ftarget = 16, output = 'long') +max_eval <- get_maxRT(dsl, output = 'long') +bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) +} diff --git a/man/c.DataSet.Rd b/man/c.DataSet.Rd index 2c395bc7..af05b5a7 100644 --- a/man/c.DataSet.Rd +++ b/man/c.DataSet.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{c.DataSet} -\alias{c.DataSet} -\title{S3 concatenation function for DataSet} -\usage{ -\method{c}{DataSet}(...) -} -\arguments{ -\item{...}{The DataSets to concatenate} -} -\value{ -A new DataSet -} -\description{ -Concatenation for DataSets. Combines multiple runs from separate DataSets -into a single DataSet object if all provided arguments have the same dimension, function ID and -algorithm ID, and each contains only a single run. Currently does not support parameter tracking -} -\examples{ -c(dsl[[1]], dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{c.DataSet} +\alias{c.DataSet} +\title{S3 concatenation function for DataSet} +\usage{ +\method{c}{DataSet}(...) +} +\arguments{ +\item{...}{The DataSets to concatenate} +} +\value{ +A new DataSet +} +\description{ +Concatenation for DataSets. Combines multiple runs from separate DataSets +into a single DataSet object if all provided arguments have the same dimension, function ID and +algorithm ID, and each contains only a single run. Currently does not support parameter tracking +} +\examples{ +c(dsl[[1]], dsl[[1]]) +} diff --git a/man/c.DataSetList.Rd b/man/c.DataSetList.Rd index 246d46b4..14571fa2 100644 --- a/man/c.DataSetList.Rd +++ b/man/c.DataSetList.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{c.DataSetList} -\alias{c.DataSetList} -\title{S3 concatenation function for DataSetList} -\usage{ -\method{c}{DataSetList}(...) -} -\arguments{ -\item{...}{The DataSetLists to concatenate} -} -\value{ -A new DataSetList -} -\description{ -S3 concatenation function for DataSetList -} -\examples{ -c(dsl[1], dsl[3]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{c.DataSetList} +\alias{c.DataSetList} +\title{S3 concatenation function for DataSetList} +\usage{ +\method{c}{DataSetList}(...) +} +\arguments{ +\item{...}{The DataSetLists to concatenate} +} +\value{ +A new DataSetList +} +\description{ +S3 concatenation function for DataSetList +} +\examples{ +c(dsl[1], dsl[3]) +} diff --git a/man/cat.DataSet.Rd b/man/cat.DataSet.Rd index d4b4ce7c..6fa1905e 100644 --- a/man/cat.DataSet.Rd +++ b/man/cat.DataSet.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{cat.DataSet} -\alias{cat.DataSet} -\title{S3 generic cat operator for DataSet} -\usage{ -cat.DataSet(x) -} -\arguments{ -\item{x}{A DataSet object} -} -\value{ -A short description of the DataSet -} -\description{ -S3 generic cat operator for DataSet -} -\examples{ -cat.DataSet(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{cat.DataSet} +\alias{cat.DataSet} +\title{S3 generic cat operator for DataSet} +\usage{ +cat.DataSet(x) +} +\arguments{ +\item{x}{A DataSet object} +} +\value{ +A short description of the DataSet +} +\description{ +S3 generic cat operator for DataSet +} +\examples{ +cat.DataSet(dsl[[1]]) +} diff --git a/man/change_id.Rd b/man/change_id.Rd index 96b99795..6b4033a2 100644 --- a/man/change_id.Rd +++ b/man/change_id.Rd @@ -1,25 +1,25 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{change_id} -\alias{change_id} -\title{Add unique identifiers to each DataSet in the provided DataSetList based on static attributes} -\usage{ -change_id(dsl, attrs) -} -\arguments{ -\item{dsl}{The DataSetList} - -\item{attrs}{The list of attributes to combine into a unique identifier} -} -\value{ -A new DataSetList object where the split has been done based on the provided attributes, and the unique -identifier has been added. -} -\description{ -Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to -ensure each dataset has exactly one unique identifier. -Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. -} -\examples{ -change_id(dsl, c('instance')) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{change_id} +\alias{change_id} +\title{Add unique identifiers to each DataSet in the provided DataSetList based on static attributes} +\usage{ +change_id(dsl, attrs) +} +\arguments{ +\item{dsl}{The DataSetList} + +\item{attrs}{The list of attributes to combine into a unique identifier} +} +\value{ +A new DataSetList object where the split has been done based on the provided attributes, and the unique +identifier has been added. +} +\description{ +Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to +ensure each dataset has exactly one unique identifier. +Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. +} +\examples{ +change_id(dsl, c('instance')) +} diff --git a/man/check_dsc_configured.Rd b/man/check_dsc_configured.Rd index e685f58f..1405e520 100644 --- a/man/check_dsc_configured.Rd +++ b/man/check_dsc_configured.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{check_dsc_configured} -\alias{check_dsc_configured} -\title{Verify that the credentials for DSCtool have been set} -\usage{ -check_dsc_configured() -} -\description{ -This uses the keyring package to store and load credentials. -If the keyring package does not exists, it will default to look for -a config-file in the 'repository'-folder, under your home directory. -This can be changed by setting the option IOHprofiler.config_dir -If you already have an account, please call `set_DSC_credentials` -with the corresponding username and password. -If you don't have an account, you can register for one using `register_DSC` -} -\examples{ -check_dsc_configured() -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{check_dsc_configured} +\alias{check_dsc_configured} +\title{Verify that the credentials for DSCtool have been set} +\usage{ +check_dsc_configured() +} +\description{ +This uses the keyring package to store and load credentials. +If the keyring package does not exists, it will default to look for +a config-file in the 'repository'-folder, under your home directory. +This can be changed by setting the option IOHprofiler.config_dir +If you already have an account, please call `set_DSC_credentials` +with the corresponding username and password. +If you don't have an account, you can register for one using `register_DSC` +} +\examples{ +check_dsc_configured() +} diff --git a/man/check_format.Rd b/man/check_format.Rd index 8e11463f..dccedf7e 100644 --- a/man/check_format.Rd +++ b/man/check_format.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{check_format} -\alias{check_format} -\title{Check the format of data} -\usage{ -check_format(path) -} -\arguments{ -\item{path}{The path to the folder to check} -} -\value{ -The format of the data in the given folder. Either 'COCO', 'IOHprofiler', -'NEVERGRAD' or 'SOS'. -} -\description{ -Throws a warning when multiple formats are found in the same folder. -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -check_format(path) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{check_format} +\alias{check_format} +\title{Check the format of data} +\usage{ +check_format(path) +} +\arguments{ +\item{path}{The path to the folder to check} +} +\value{ +The format of the data in the given folder. Either 'COCO', 'IOHprofiler', +'NEVERGRAD' or 'SOS'. +} +\description{ +Throws a warning when multiple formats are found in the same folder. +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +check_format(path) +} diff --git a/man/clean_DataSetList.Rd b/man/clean_DataSetList.Rd index 05af812f..8a8caf1b 100644 --- a/man/clean_DataSetList.Rd +++ b/man/clean_DataSetList.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{clean_DataSetList} -\alias{clean_DataSetList} -\title{Clean DataSetList object by concatenating DataSets} -\usage{ -clean_DataSetList(dsList) -} -\arguments{ -\item{dsList}{The DataSetList object to clean} -} -\description{ -Concatenates all DataSets with the same ID, function id and dimension -} -\examples{ -clean_DataSetList(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{clean_DataSetList} +\alias{clean_DataSetList} +\title{Clean DataSetList object by concatenating DataSets} +\usage{ +clean_DataSetList(dsList) +} +\arguments{ +\item{dsList}{The DataSetList object to clean} +} +\description{ +Concatenates all DataSets with the same ID, function id and dimension +} +\examples{ +clean_DataSetList(dsl) +} diff --git a/man/dsl.Rd b/man/dsl.Rd index 2358e21a..d2729899 100644 --- a/man/dsl.Rd +++ b/man/dsl.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/data.R -\docType{data} -\name{dsl} -\alias{dsl} -\title{Example DataSetList used in tests / examples} -\format{DataSetList} -\usage{ -dsl -} -\description{ -A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D -} -\examples{ -summary(dsl) -} -\keyword{datasets} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data.R +\docType{data} +\name{dsl} +\alias{dsl} +\title{Example DataSetList used in tests / examples} +\format{DataSetList} +\usage{ +dsl +} +\description{ +A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D +} +\examples{ +summary(dsl) +} +\keyword{datasets} diff --git a/man/dsl_large.Rd b/man/dsl_large.Rd index 88df6756..36637559 100644 --- a/man/dsl_large.Rd +++ b/man/dsl_large.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/data.R -\docType{data} -\name{dsl_large} -\alias{dsl_large} -\title{Larger example DataSetList used in tests / examples} -\format{DataSetList} -\usage{ -dsl_large -} -\description{ -A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D -} -\examples{ -summary(dsl_large) -} -\keyword{datasets} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data.R +\docType{data} +\name{dsl_large} +\alias{dsl_large} +\title{Larger example DataSetList used in tests / examples} +\format{DataSetList} +\usage{ +dsl_large +} +\description{ +A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D +} +\examples{ +summary(dsl_large) +} +\keyword{datasets} diff --git a/man/equals-.DataSet.Rd b/man/equals-.DataSet.Rd index 417eb711..a3b9513a 100644 --- a/man/equals-.DataSet.Rd +++ b/man/equals-.DataSet.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{==.DataSet} -\alias{==.DataSet} -\title{S3 generic == operator for DataSets} -\usage{ -\method{==}{DataSet}(dsL, dsR) -} -\arguments{ -\item{dsL}{A `DataSet` object} - -\item{dsR}{A `DataSet` object} -} -\value{ -True if the DataSets contain the same function, dimension and algorithm, -and have the exact same attributes -} -\description{ -S3 generic == operator for DataSets -} -\examples{ -dsl[[1]] == dsl[[2]] -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{==.DataSet} +\alias{==.DataSet} +\title{S3 generic == operator for DataSets} +\usage{ +\method{==}{DataSet}(dsL, dsR) +} +\arguments{ +\item{dsL}{A `DataSet` object} + +\item{dsR}{A `DataSet` object} +} +\value{ +True if the DataSets contain the same function, dimension and algorithm, +and have the exact same attributes +} +\description{ +S3 generic == operator for DataSets +} +\examples{ +dsl[[1]] == dsl[[2]] +} diff --git a/man/fast_RT_samples.Rd b/man/fast_RT_samples.Rd index 45c2cd4c..fa3c8a14 100644 --- a/man/fast_RT_samples.Rd +++ b/man/fast_RT_samples.Rd @@ -1,18 +1,18 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{fast_RT_samples} -\alias{fast_RT_samples} -\title{Function to get just the RT samples needed, without any formatting to improve speed} -\usage{ -fast_RT_samples(RT_mat, target, maximization = F) -} -\arguments{ -\item{RT_mat}{A matrix containing the RT-values of a dataset} - -\item{target}{Which target-value to use} - -\item{maximization}{Whether maximization is needed or not} -} -\description{ -Function to get just the RT samples needed, without any formatting to improve speed -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{fast_RT_samples} +\alias{fast_RT_samples} +\title{Function to get just the RT samples needed, without any formatting to improve speed} +\usage{ +fast_RT_samples(RT_mat, target, maximization = F) +} +\arguments{ +\item{RT_mat}{A matrix containing the RT-values of a dataset} + +\item{target}{Which target-value to use} + +\item{maximization}{Whether maximization is needed or not} +} +\description{ +Function to get just the RT samples needed, without any formatting to improve speed +} diff --git a/man/generate_data.AUC.Rd b/man/generate_data.AUC.Rd index 4dd79451..31956512 100644 --- a/man/generate_data.AUC.Rd +++ b/man/generate_data.AUC.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.AUC} -\alias{generate_data.AUC} -\title{Generate dataframe containing the AUC for any ECDF-curves} -\usage{ -generate_data.AUC(dsList, targets, scale_log = F, which = "by_RT", - dt_ecdf = NULL, multiple_x = FALSE) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale_log}{Whether to use logarithmic scaling or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{dt_ecdf}{A data table of the ECDF to avoid needless recomputations. Will take preference if it -is provided together with dsList and targets} - -\item{multiple_x}{Boolean, whether to get only the total AUC or get stepwise AUC values} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.AUC(dsl, get_ECDF_targets(dsl)) -generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.AUC} +\alias{generate_data.AUC} +\title{Generate dataframe containing the AUC for any ECDF-curves} +\usage{ +generate_data.AUC(dsList, targets, scale_log = F, which = "by_RT", + dt_ecdf = NULL, multiple_x = FALSE) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale_log}{Whether to use logarithmic scaling or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{dt_ecdf}{A data table of the ECDF to avoid needless recomputations. Will take preference if it +is provided together with dsList and targets} + +\item{multiple_x}{Boolean, whether to get only the total AUC or get stepwise AUC values} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.AUC(dsl, get_ECDF_targets(dsl)) +generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) +} diff --git a/man/generate_data.Aggr.Rd b/man/generate_data.Aggr.Rd index afb0f11e..c7d23e05 100644 --- a/man/generate_data.Aggr.Rd +++ b/man/generate_data.Aggr.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.Aggr} -\alias{generate_data.Aggr} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.Aggr(dsList, aggr_on = "funcId", targets = NULL, - which = "by_RT") -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{aggr_on}{Which attribute to use for aggregation. Either 'funcId' or 'DIM'} - -\item{targets}{Optional list of target values (Runtime or target value)} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.Aggr(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.Aggr} +\alias{generate_data.Aggr} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.Aggr(dsList, aggr_on = "funcId", targets = NULL, + which = "by_RT") +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{aggr_on}{Which attribute to use for aggregation. Either 'funcId' or 'DIM'} + +\item{targets}{Optional list of target values (Runtime or target value)} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.Aggr(dsl) +} diff --git a/man/generate_data.ECDF.Rd b/man/generate_data.ECDF.Rd index dcf2df3a..fc1e5365 100644 --- a/man/generate_data.ECDF.Rd +++ b/man/generate_data.ECDF.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.ECDF} -\alias{generate_data.ECDF} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.ECDF(dsList, targets, scale_log = F, which = "by_RT", - use_full_range = TRUE) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{use_full_range}{Whether or not to use the full range of the x-axis or cut it off as soon as -all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.ECDF} +\alias{generate_data.ECDF} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.ECDF(dsList, targets, scale_log = F, which = "by_RT", + use_full_range = TRUE) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{use_full_range}{Whether or not to use the full range of the x-axis or cut it off as soon as +all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) +} diff --git a/man/generate_data.ECDF_raw.Rd b/man/generate_data.ECDF_raw.Rd index 8ddbbc49..0c3ea4de 100644 --- a/man/generate_data.ECDF_raw.Rd +++ b/man/generate_data.ECDF_raw.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.ECDF_raw} -\alias{generate_data.ECDF_raw} -\title{Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of} -\usage{ -generate_data.ECDF_raw(dsList, targets, scale_log = F) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} -} -\description{ -This provides an unaggregated version of the function `generate_data.ECDF`. -} -\examples{ -generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.ECDF_raw} +\alias{generate_data.ECDF_raw} +\title{Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of} +\usage{ +generate_data.ECDF_raw(dsList, targets, scale_log = F) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} +} +\description{ +This provides an unaggregated version of the function `generate_data.ECDF`. +} +\examples{ +generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) +} diff --git a/man/generate_data.PMF.Rd b/man/generate_data.PMF.Rd index 5922249a..4d0d9127 100644 --- a/man/generate_data.PMF.Rd +++ b/man/generate_data.PMF.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.PMF} -\alias{generate_data.PMF} -\title{Generate dataframe of a single function/dimension pair for creating PDF or PMF plots} -\usage{ -generate_data.PMF(dsList, target, which = "by_RT") -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{target}{The target value (Runtime or target value)} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.PMF} +\alias{generate_data.PMF} +\title{Generate dataframe of a single function/dimension pair for creating PDF or PMF plots} +\usage{ +generate_data.PMF(dsList, target, which = "by_RT") +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{target}{The target value (Runtime or target value)} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +} diff --git a/man/generate_data.Parameters.Rd b/man/generate_data.Parameters.Rd index 85edb3a6..492b4c55 100644 --- a/man/generate_data.Parameters.Rd +++ b/man/generate_data.Parameters.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.Parameters} -\alias{generate_data.Parameters} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.Parameters(dsList, which = "by_RT", scale_log = F) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.Parameters(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.Parameters} +\alias{generate_data.Parameters} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.Parameters(dsList, which = "by_RT", scale_log = F) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.Parameters(subset(dsl, funcId == 1)) +} diff --git a/man/generate_data.Single_Function.Rd b/man/generate_data.Single_Function.Rd index 6c10b5bd..c71f6478 100644 --- a/man/generate_data.Single_Function.Rd +++ b/man/generate_data.Single_Function.Rd @@ -1,32 +1,32 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.Single_Function} -\alias{generate_data.Single_Function} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.Single_Function(dsList, start = NULL, stop = NULL, - scale_log = F, which = "by_RT", include_opts = F, budget = NULL) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{start}{Optional start value (Runtime or target value)} - -\item{stop}{Optional end value (Runtime or target value)} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{include_opts}{Whether or not to also include the best value hit by each algorithm to -the generated datapoints} - -\item{budget}{Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works -in fixed_target mode.} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.Single_Function} +\alias{generate_data.Single_Function} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.Single_Function(dsList, start = NULL, stop = NULL, + scale_log = F, which = "by_RT", include_opts = F, budget = NULL) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{start}{Optional start value (Runtime or target value)} + +\item{stop}{Optional end value (Runtime or target value)} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{include_opts}{Whether or not to also include the best value hit by each algorithm to +the generated datapoints} + +\item{budget}{Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works +in fixed_target mode.} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +} diff --git a/man/generate_data.hist.Rd b/man/generate_data.hist.Rd index 93f215eb..7cd46dd8 100644 --- a/man/generate_data.hist.Rd +++ b/man/generate_data.hist.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.hist} -\alias{generate_data.hist} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.hist(dsList, target, use.equal.bins = F, which = "by_RT") -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{target}{The target value (Runtime or target value)} - -\item{use.equal.bins}{Whether all bins should be equal size for each algorithm or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.hist} +\alias{generate_data.hist} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.hist(dsList, target, use.equal.bins = F, which = "by_RT") +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{target}{The target value (Runtime or target value)} + +\item{use.equal.bins}{Whether all bins should be equal size for each algorithm or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +} diff --git a/man/get_ECDF_targets.Rd b/man/get_ECDF_targets.Rd index 56fe681d..5d82a5d0 100644 --- a/man/get_ECDF_targets.Rd +++ b/man/get_ECDF_targets.Rd @@ -1,25 +1,25 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_ECDF_targets} -\alias{get_ECDF_targets} -\title{Generation of default ECDF-targets} -\usage{ -get_ECDF_targets(dsList, type = "log-linear", number_targets = 10) -} -\arguments{ -\item{dsList}{The DataSetList object for which to generate the targets} - -\item{type}{The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, -equal for all functions / dimensions)} - -\item{number_targets}{The amount of targets to generate} -} -\value{ -A data.table with 3 columns: funcId, DIM and target -} -\description{ -Generation of default ECDF-targets -} -\examples{ -get_ECDF_targets(dsl, 'linear', 10) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_ECDF_targets} +\alias{get_ECDF_targets} +\title{Generation of default ECDF-targets} +\usage{ +get_ECDF_targets(dsList, type = "log-linear", number_targets = 10) +} +\arguments{ +\item{dsList}{The DataSetList object for which to generate the targets} + +\item{type}{The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, +equal for all functions / dimensions)} + +\item{number_targets}{The amount of targets to generate} +} +\value{ +A data.table with 3 columns: funcId, DIM and target +} +\description{ +Generation of default ECDF-targets +} +\examples{ +get_ECDF_targets(dsl, 'linear', 10) +} diff --git a/man/get_ERT.Rd b/man/get_ERT.Rd index 1b2b333f..9e3ced27 100644 --- a/man/get_ERT.Rd +++ b/man/get_ERT.Rd @@ -1,37 +1,37 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_ERT} -\alias{get_ERT} -\alias{get_ERT.DataSet} -\alias{get_ERT.DataSetList} -\title{Get Expected RunTime} -\usage{ -get_ERT(ds, ftarget, budget, ...) - -\method{get_ERT}{DataSet}(ds, ftarget, budget = NULL, ...) - -\method{get_ERT}{DataSetList}(ds, ftarget, budget = NULL, - algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{ftarget}{The function target(s) for which to get the ERT} - -\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the runtime samples for each provided target -function value -} -\description{ -Get Expected RunTime -} -\examples{ -get_ERT(dsl, 14) -get_ERT(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_ERT} +\alias{get_ERT} +\alias{get_ERT.DataSet} +\alias{get_ERT.DataSetList} +\title{Get Expected RunTime} +\usage{ +get_ERT(ds, ftarget, budget, ...) + +\method{get_ERT}{DataSet}(ds, ftarget, budget = NULL, ...) + +\method{get_ERT}{DataSetList}(ds, ftarget, budget = NULL, + algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{ftarget}{The function target(s) for which to get the ERT} + +\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} + +\item{...}{Arguments passed to other methods} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the runtime samples for each provided target +function value +} +\description{ +Get Expected RunTime +} +\examples{ +get_ERT(dsl, 14) +get_ERT(dsl[[1]], 14) +} diff --git a/man/get_FV_overview.Rd b/man/get_FV_overview.Rd index a031cbf4..6ec007fe 100644 --- a/man/get_FV_overview.Rd +++ b/man/get_FV_overview.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_FV_overview} -\alias{get_FV_overview} -\alias{get_FV_overview.DataSet} -\alias{get_FV_overview.DataSetList} -\title{Get Function Value condensed overview} -\usage{ -get_FV_overview(ds, ...) - -\method{get_FV_overview}{DataSet}(ds, ...) - -\method{get_FV_overview}{DataSetList}(ds, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A `DataSet` or `DataSetList` object} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the algorithm ID, best, worst and mean reached function -values, the number of runs and available budget for the DataSet -} -\description{ -Get Function Value condensed overview -} -\examples{ -get_FV_overview(dsl) -get_FV_overview(dsl[[1]]) -get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_FV_overview} +\alias{get_FV_overview} +\alias{get_FV_overview.DataSet} +\alias{get_FV_overview.DataSetList} +\title{Get Function Value condensed overview} +\usage{ +get_FV_overview(ds, ...) + +\method{get_FV_overview}{DataSet}(ds, ...) + +\method{get_FV_overview}{DataSetList}(ds, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A `DataSet` or `DataSetList` object} + +\item{...}{Arguments passed to other methods} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the algorithm ID, best, worst and mean reached function +values, the number of runs and available budget for the DataSet +} +\description{ +Get Function Value condensed overview +} +\examples{ +get_FV_overview(dsl) +get_FV_overview(dsl[[1]]) +get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') +} diff --git a/man/get_FV_sample.Rd b/man/get_FV_sample.Rd index 209eaa53..c75c0870 100644 --- a/man/get_FV_sample.Rd +++ b/man/get_FV_sample.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_FV_sample} -\alias{get_FV_sample} -\alias{get_FV_sample.DataSet} -\alias{get_FV_sample.DataSetList} -\title{Get Funtion Value Samples} -\usage{ -get_FV_sample(ds, ...) - -\method{get_FV_sample}{DataSet}(ds, runtime, output = "wide", ...) - -\method{get_FV_sample}{DataSetList}(ds, runtime, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{runtime}{A Numerical vector. Runtimes at which function values are reached} - -\item{output}{A String. The format of the output data: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the function value samples for each provided -target runtime -} -\description{ -Get Funtion Value Samples -} -\examples{ -get_FV_sample(dsl, 100) -get_FV_sample(dsl[[1]], 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_FV_sample} +\alias{get_FV_sample} +\alias{get_FV_sample.DataSet} +\alias{get_FV_sample.DataSetList} +\title{Get Funtion Value Samples} +\usage{ +get_FV_sample(ds, ...) + +\method{get_FV_sample}{DataSet}(ds, runtime, output = "wide", ...) + +\method{get_FV_sample}{DataSetList}(ds, runtime, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{runtime}{A Numerical vector. Runtimes at which function values are reached} + +\item{output}{A String. The format of the output data: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the function value samples for each provided +target runtime +} +\description{ +Get Funtion Value Samples +} +\examples{ +get_FV_sample(dsl, 100) +get_FV_sample(dsl[[1]], 100) +} diff --git a/man/get_FV_summary.Rd b/man/get_FV_summary.Rd index ec94fcdd..941b6408 100644 --- a/man/get_FV_summary.Rd +++ b/man/get_FV_summary.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_FV_summary} -\alias{get_FV_summary} -\alias{get_FV_summary.DataSet} -\alias{get_FV_summary.DataSetList} -\title{Get Function Value Summary} -\usage{ -get_FV_summary(ds, ...) - -\method{get_FV_summary}{DataSet}(ds, runtime, ...) - -\method{get_FV_summary}{DataSetList}(ds, runtime, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{runtime}{A Numerical vector. Runtimes at which function values are reached} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the function value statistics for each provided -target runtime value -} -\description{ -Get Function Value Summary -} -\examples{ -get_FV_summary(dsl, 100) -get_FV_summary(dsl[[1]], 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_FV_summary} +\alias{get_FV_summary} +\alias{get_FV_summary.DataSet} +\alias{get_FV_summary.DataSetList} +\title{Get Function Value Summary} +\usage{ +get_FV_summary(ds, ...) + +\method{get_FV_summary}{DataSet}(ds, runtime, ...) + +\method{get_FV_summary}{DataSetList}(ds, runtime, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{runtime}{A Numerical vector. Runtimes at which function values are reached} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the function value statistics for each provided +target runtime value +} +\description{ +Get Function Value Summary +} +\examples{ +get_FV_summary(dsl, 100) +get_FV_summary(dsl[[1]], 100) +} diff --git a/man/get_PAR_name.Rd b/man/get_PAR_name.Rd index 937c7f95..e618aa2e 100644 --- a/man/get_PAR_name.Rd +++ b/man/get_PAR_name.Rd @@ -1,27 +1,27 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{get_PAR_name} -\alias{get_PAR_name} -\alias{get_PAR_name.DataSet} -\title{Get the parameter names of the algorithm} -\usage{ -get_PAR_name(ds, which) - -\method{get_PAR_name}{DataSet}(ds, which = "by_FV") -} -\arguments{ -\item{ds}{A DataSet object} - -\item{which}{a string takes it value in `c('by_FV', 'by_RT')`, indicating the -parameters aligned against the running time (RT) or function value (FV). `'by_FV'` -is the default value.} -} -\value{ -a character list of paramter names, if recorded in the data set -} -\description{ -Get the parameter names of the algorithm -} -\examples{ -get_PAR_name(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{get_PAR_name} +\alias{get_PAR_name} +\alias{get_PAR_name.DataSet} +\title{Get the parameter names of the algorithm} +\usage{ +get_PAR_name(ds, which) + +\method{get_PAR_name}{DataSet}(ds, which = "by_FV") +} +\arguments{ +\item{ds}{A DataSet object} + +\item{which}{a string takes it value in `c('by_FV', 'by_RT')`, indicating the +parameters aligned against the running time (RT) or function value (FV). `'by_FV'` +is the default value.} +} +\value{ +a character list of paramter names, if recorded in the data set +} +\description{ +Get the parameter names of the algorithm +} +\examples{ +get_PAR_name(dsl[[1]]) +} diff --git a/man/get_PAR_sample.Rd b/man/get_PAR_sample.Rd index 204344d5..cdd586c2 100644 --- a/man/get_PAR_sample.Rd +++ b/man/get_PAR_sample.Rd @@ -1,45 +1,45 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_PAR_sample} -\alias{get_PAR_sample} -\alias{get_PAR_sample.DataSet} -\alias{get_PAR_sample.DataSetList} -\title{Get Parameter Value Samples} -\usage{ -get_PAR_sample(ds, idxValue, ...) - -\method{get_PAR_sample}{DataSet}(ds, idxValue, parId = "all", - which = "by_FV", output = "wide", ...) - -\method{get_PAR_sample}{DataSetList}(ds, idxValue, algorithm = "all", - ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. -The index value can either take its value in the range of running times, or function values. -Such a value type is signified by `which` parameter.} - -\item{...}{Arguments passed to other methods} - -\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} - -\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -is the default value.} - -\item{output}{A character. The format of the output data: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table object containing parameter values aligned at each given target value -} -\description{ -Get Parameter Value Samples -} -\examples{ -get_PAR_sample(dsl, 14) -get_PAR_sample(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_PAR_sample} +\alias{get_PAR_sample} +\alias{get_PAR_sample.DataSet} +\alias{get_PAR_sample.DataSetList} +\title{Get Parameter Value Samples} +\usage{ +get_PAR_sample(ds, idxValue, ...) + +\method{get_PAR_sample}{DataSet}(ds, idxValue, parId = "all", + which = "by_FV", output = "wide", ...) + +\method{get_PAR_sample}{DataSetList}(ds, idxValue, algorithm = "all", + ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. +The index value can either take its value in the range of running times, or function values. +Such a value type is signified by `which` parameter.} + +\item{...}{Arguments passed to other methods} + +\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} + +\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +is the default value.} + +\item{output}{A character. The format of the output data: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table object containing parameter values aligned at each given target value +} +\description{ +Get Parameter Value Samples +} +\examples{ +get_PAR_sample(dsl, 14) +get_PAR_sample(dsl[[1]], 14) +} diff --git a/man/get_PAR_summary.Rd b/man/get_PAR_summary.Rd index a96470b4..a234173c 100644 --- a/man/get_PAR_summary.Rd +++ b/man/get_PAR_summary.Rd @@ -1,43 +1,43 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_PAR_summary} -\alias{get_PAR_summary} -\alias{get_PAR_summary.DataSet} -\alias{get_PAR_summary.DataSetList} -\title{Get Parameter Value Summary} -\usage{ -get_PAR_summary(ds, idxValue, ...) - -\method{get_PAR_summary}{DataSet}(ds, idxValue, parId = "all", - which = "by_FV", ...) - -\method{get_PAR_summary}{DataSetList}(ds, idxValue, algorithm = "all", - ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. -The index value can either take its value in the range of running times, or function values. -Such a value type is signified by `which` parameter.} - -\item{...}{Arguments passed to other methods} - -\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} - -\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -is the default value.} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table object containing basic statistics of parameter values aligned at each given target value -} -\description{ -Get Parameter Value Summary -} -\examples{ -get_PAR_summary(dsl, 14) -get_PAR_summary(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_PAR_summary} +\alias{get_PAR_summary} +\alias{get_PAR_summary.DataSet} +\alias{get_PAR_summary.DataSetList} +\title{Get Parameter Value Summary} +\usage{ +get_PAR_summary(ds, idxValue, ...) + +\method{get_PAR_summary}{DataSet}(ds, idxValue, parId = "all", + which = "by_FV", ...) + +\method{get_PAR_summary}{DataSetList}(ds, idxValue, algorithm = "all", + ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. +The index value can either take its value in the range of running times, or function values. +Such a value type is signified by `which` parameter.} + +\item{...}{Arguments passed to other methods} + +\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} + +\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +is the default value.} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table object containing basic statistics of parameter values aligned at each given target value +} +\description{ +Get Parameter Value Summary +} +\examples{ +get_PAR_summary(dsl, 14) +get_PAR_summary(dsl[[1]], 14) +} diff --git a/man/get_RT_overview.Rd b/man/get_RT_overview.Rd index 7f02a0aa..b454c27e 100644 --- a/man/get_RT_overview.Rd +++ b/man/get_RT_overview.Rd @@ -1,32 +1,32 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_RT_overview} -\alias{get_RT_overview} -\alias{get_RT_overview.DataSet} -\alias{get_RT_overview.DataSetList} -\title{Get Runtime Value condensed overview} -\usage{ -get_RT_overview(ds, ...) - -\method{get_RT_overview}{DataSet}(ds, ...) - -\method{get_RT_overview}{DataSetList}(ds, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the algorithm ID, minimum and maximum used evaluations, -number of runs and available budget for the DataSet -} -\description{ -Get Runtime Value condensed overview -} -\examples{ -get_RT_overview(dsl) -get_RT_overview(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_RT_overview} +\alias{get_RT_overview} +\alias{get_RT_overview.DataSet} +\alias{get_RT_overview.DataSetList} +\title{Get Runtime Value condensed overview} +\usage{ +get_RT_overview(ds, ...) + +\method{get_RT_overview}{DataSet}(ds, ...) + +\method{get_RT_overview}{DataSetList}(ds, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the algorithm ID, minimum and maximum used evaluations, +number of runs and available budget for the DataSet +} +\description{ +Get Runtime Value condensed overview +} +\examples{ +get_RT_overview(dsl) +get_RT_overview(dsl[[1]]) +} diff --git a/man/get_RT_sample.Rd b/man/get_RT_sample.Rd index 4aa676ed..d996bb97 100644 --- a/man/get_RT_sample.Rd +++ b/man/get_RT_sample.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_RT_sample} -\alias{get_RT_sample} -\alias{get_RT_sample.DataSet} -\alias{get_RT_sample.DataSetList} -\title{Get RunTime Sample} -\usage{ -get_RT_sample(ds, ftarget, ...) - -\method{get_RT_sample}{DataSet}(ds, ftarget, output = "wide", ...) - -\method{get_RT_sample}{DataSetList}(ds, ftarget, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} - -\item{...}{Arguments passed to other methods} - -\item{output}{A character determining the format of output data.table: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the runtime samples for each provided target -function value -} -\description{ -Get RunTime Sample -} -\examples{ -get_RT_sample(dsl, 14) -get_RT_sample(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_RT_sample} +\alias{get_RT_sample} +\alias{get_RT_sample.DataSet} +\alias{get_RT_sample.DataSetList} +\title{Get RunTime Sample} +\usage{ +get_RT_sample(ds, ftarget, ...) + +\method{get_RT_sample}{DataSet}(ds, ftarget, output = "wide", ...) + +\method{get_RT_sample}{DataSetList}(ds, ftarget, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} + +\item{...}{Arguments passed to other methods} + +\item{output}{A character determining the format of output data.table: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the runtime samples for each provided target +function value +} +\description{ +Get RunTime Sample +} +\examples{ +get_RT_sample(dsl, 14) +get_RT_sample(dsl[[1]], 14) +} diff --git a/man/get_RT_summary.Rd b/man/get_RT_summary.Rd index 32e5d85b..44c511d7 100644 --- a/man/get_RT_summary.Rd +++ b/man/get_RT_summary.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_RT_summary} -\alias{get_RT_summary} -\alias{get_RT_summary.DataSet} -\alias{get_RT_summary.DataSetList} -\title{Get RunTime Summary} -\usage{ -get_RT_summary(ds, ftarget, budget, ...) - -\method{get_RT_summary}{DataSet}(ds, ftarget, budget = NULL, ...) - -\method{get_RT_summary}{DataSetList}(ds, ftarget, budget = NULL, ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{ftarget}{The function target(s) for which to get the runtime summary} - -\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} - -\item{...}{Arguments passed to other methods} -} -\value{ -A data.table containing the runtime statistics for each provided target -function value -} -\description{ -Get RunTime Summary -} -\examples{ -get_RT_summary(dsl, 14) -get_RT_summary(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_RT_summary} +\alias{get_RT_summary} +\alias{get_RT_summary.DataSet} +\alias{get_RT_summary.DataSetList} +\title{Get RunTime Summary} +\usage{ +get_RT_summary(ds, ftarget, budget, ...) + +\method{get_RT_summary}{DataSet}(ds, ftarget, budget = NULL, ...) + +\method{get_RT_summary}{DataSetList}(ds, ftarget, budget = NULL, ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{ftarget}{The function target(s) for which to get the runtime summary} + +\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} + +\item{...}{Arguments passed to other methods} +} +\value{ +A data.table containing the runtime statistics for each provided target +function value +} +\description{ +Get RunTime Summary +} +\examples{ +get_RT_summary(dsl, 14) +get_RT_summary(dsl[[1]], 14) +} diff --git a/man/get_algId.Rd b/man/get_algId.Rd index ca7ec9fc..d99a56cd 100644 --- a/man/get_algId.Rd +++ b/man/get_algId.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_algId} -\alias{get_algId} -\title{Get all algorithm ids present in a DataSetList} -\usage{ -get_algId(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A sorted list of all unique algorithm ids which occur in the DataSetList -} -\description{ -Get all algorithm ids present in a DataSetList -} -\examples{ -get_algId(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_algId} +\alias{get_algId} +\title{Get all algorithm ids present in a DataSetList} +\usage{ +get_algId(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A sorted list of all unique algorithm ids which occur in the DataSetList +} +\description{ +Get all algorithm ids present in a DataSetList +} +\examples{ +get_algId(dsl) +} diff --git a/man/get_color_scheme.Rd b/man/get_color_scheme.Rd index dadf3e87..4e8c29e2 100644 --- a/man/get_color_scheme.Rd +++ b/man/get_color_scheme.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{get_color_scheme} -\alias{get_color_scheme} -\title{Get colors according to the current colorScheme of the IOHanalyzer} -\usage{ -get_color_scheme(ids_in) -} -\arguments{ -\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get colors} -} -\description{ -Get colors according to the current colorScheme of the IOHanalyzer -} -\examples{ -get_color_scheme(get_algId(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{get_color_scheme} +\alias{get_color_scheme} +\title{Get colors according to the current colorScheme of the IOHanalyzer} +\usage{ +get_color_scheme(ids_in) +} +\arguments{ +\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get colors} +} +\description{ +Get colors according to the current colorScheme of the IOHanalyzer +} +\examples{ +get_color_scheme(get_algId(dsl)) +} diff --git a/man/get_color_scheme_dt.Rd b/man/get_color_scheme_dt.Rd index 6fb2c779..c0ac0e4b 100644 --- a/man/get_color_scheme_dt.Rd +++ b/man/get_color_scheme_dt.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{get_color_scheme_dt} -\alias{get_color_scheme_dt} -\title{Get datatable of current color (and linestyle) scheme to file} -\usage{ -get_color_scheme_dt() -} -\value{ -data.table object with 3 columns: ids, colors, linestyles -} -\description{ -Get datatable of current color (and linestyle) scheme to file -} -\examples{ -get_color_scheme_dt() -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{get_color_scheme_dt} +\alias{get_color_scheme_dt} +\title{Get datatable of current color (and linestyle) scheme to file} +\usage{ +get_color_scheme_dt() +} +\value{ +data.table object with 3 columns: ids, colors, linestyles +} +\description{ +Get datatable of current color (and linestyle) scheme to file +} +\examples{ +get_color_scheme_dt() +} diff --git a/man/get_default_ECDF_targets.Rd b/man/get_default_ECDF_targets.Rd index e2823a7f..79a9e313 100644 --- a/man/get_default_ECDF_targets.Rd +++ b/man/get_default_ECDF_targets.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{get_default_ECDF_targets} -\alias{get_default_ECDF_targets} -\title{Generate ECDF targets for a DataSetList} -\usage{ -get_default_ECDF_targets(data, format_func = as.integer) -} -\arguments{ -\item{data}{A DataSetList} - -\item{format_func}{function to format the targets} -} -\value{ -a vector of targets -} -\description{ -Generate ECDF targets for a DataSetList -} -\examples{ -get_default_ECDF_targets(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{get_default_ECDF_targets} +\alias{get_default_ECDF_targets} +\title{Generate ECDF targets for a DataSetList} +\usage{ +get_default_ECDF_targets(data, format_func = as.integer) +} +\arguments{ +\item{data}{A DataSetList} + +\item{format_func}{function to format the targets} +} +\value{ +a vector of targets +} +\description{ +Generate ECDF targets for a DataSetList +} +\examples{ +get_default_ECDF_targets(dsl) +} diff --git a/man/get_dim.Rd b/man/get_dim.Rd index f31bb411..ce0fe941 100644 --- a/man/get_dim.Rd +++ b/man/get_dim.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_dim} -\alias{get_dim} -\title{Get all dimensions present in a DataSetList} -\usage{ -get_dim(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A sorted list of all unique dimensions which occur in the DataSetList -} -\description{ -Get all dimensions present in a DataSetList -} -\examples{ -get_dim(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_dim} +\alias{get_dim} +\title{Get all dimensions present in a DataSetList} +\usage{ +get_dim(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A sorted list of all unique dimensions which occur in the DataSetList +} +\description{ +Get all dimensions present in a DataSetList +} +\examples{ +get_dim(dsl) +} diff --git a/man/get_dsc_omnibus.Rd b/man/get_dsc_omnibus.Rd index 75cf4fa9..be9e0e5e 100644 --- a/man/get_dsc_omnibus.Rd +++ b/man/get_dsc_omnibus.Rd @@ -1,26 +1,26 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_dsc_omnibus} -\alias{get_dsc_omnibus} -\title{Perform omnibus statistical tests on the matrix of rankings from the DSCtool api} -\usage{ -get_dsc_omnibus(res, method = NULL, alpha = 0.05) -} -\arguments{ -\item{res}{The result of a call to the `get_dsc_rank`} - -\item{method}{Which method to use to do the tests. -Has be be one of the allowed ones in `res$valid_methods`. -When NULL, the first valid option is chosen by default} - -\item{alpha}{Threshold value for statistical significance} -} -\value{ -A named list containing the algorithm means -} -\description{ -Perform omnibus statistical tests on the matrix of rankings from the DSCtool api -} -\examples{ -get_dsc_omnibus(get_dsc_rank(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_dsc_omnibus} +\alias{get_dsc_omnibus} +\title{Perform omnibus statistical tests on the matrix of rankings from the DSCtool api} +\usage{ +get_dsc_omnibus(res, method = NULL, alpha = 0.05) +} +\arguments{ +\item{res}{The result of a call to the `get_dsc_rank`} + +\item{method}{Which method to use to do the tests. +Has be be one of the allowed ones in `res$valid_methods`. +When NULL, the first valid option is chosen by default} + +\item{alpha}{Threshold value for statistical significance} +} +\value{ +A named list containing the algorithm means +} +\description{ +Perform omnibus statistical tests on the matrix of rankings from the DSCtool api +} +\examples{ +get_dsc_omnibus(get_dsc_rank(dsl)) +} diff --git a/man/get_dsc_posthoc.Rd b/man/get_dsc_posthoc.Rd index acfc6e9d..44356ee7 100644 --- a/man/get_dsc_posthoc.Rd +++ b/man/get_dsc_posthoc.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_dsc_posthoc} -\alias{get_dsc_posthoc} -\title{Perform post-hoc processing on data from DSCtool} -\usage{ -get_dsc_posthoc(omni_res, nr_algs, nr_problems, base_algorithm = NULL, - method = "friedman", alpha = 0.05) -} -\arguments{ -\item{omni_res}{The result from a call to `get_dsc_omnibus`} - -\item{nr_algs}{The number of algorithms present in `omni_res`} - -\item{nr_problems}{The number of problems present in `omni_res`} - -\item{base_algorithm}{The base algorithm to which the other are compared. -This has to be present in `omni_res$algorithm_means` as an `algorithm` property} - -\item{method}{Either 'friedman' or 'friedman-aligned-rank'} - -\item{alpha}{Threshold value for statistical significance} -} -\value{ -A named list containing 4 types of analyses: -* Zvalue -* UnadjustedPValue -* Holm -* Hochberg -} -\description{ -Perform post-hoc processing on data from DSCtool -} -\examples{ -get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_dsc_posthoc} +\alias{get_dsc_posthoc} +\title{Perform post-hoc processing on data from DSCtool} +\usage{ +get_dsc_posthoc(omni_res, nr_algs, nr_problems, base_algorithm = NULL, + method = "friedman", alpha = 0.05) +} +\arguments{ +\item{omni_res}{The result from a call to `get_dsc_omnibus`} + +\item{nr_algs}{The number of algorithms present in `omni_res`} + +\item{nr_problems}{The number of problems present in `omni_res`} + +\item{base_algorithm}{The base algorithm to which the other are compared. +This has to be present in `omni_res$algorithm_means` as an `algorithm` property} + +\item{method}{Either 'friedman' or 'friedman-aligned-rank'} + +\item{alpha}{Threshold value for statistical significance} +} +\value{ +A named list containing 4 types of analyses: +* Zvalue +* UnadjustedPValue +* Holm +* Hochberg +} +\description{ +Perform post-hoc processing on data from DSCtool +} +\examples{ +get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) +} diff --git a/man/get_dsc_rank.Rd b/man/get_dsc_rank.Rd index c6073f6f..b9aa00b3 100644 --- a/man/get_dsc_rank.Rd +++ b/man/get_dsc_rank.Rd @@ -1,47 +1,47 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_dsc_rank} -\alias{get_dsc_rank} -\title{Get the matrix of rankings using the DSCtool api for a DataSetList} -\usage{ -get_dsc_rank(dsList, targets = NULL, which = "by_RT", - test_type = "AD", alpha = 0.05, epsilon = 0, - monte_carlo_iterations = 0, na.correction = NULL) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{Optional list of target values (Runtime or target value)} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{test_type}{Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests} - -\item{alpha}{Threshold value for statistical significance} - -\item{epsilon}{Minimum threshold to have practical difference between algorithms (eDSC)} - -\item{monte_carlo_iterations}{How many monte-carlo-simulations to perform -(set to 0 to use regular DSC)} - -\item{na.correction}{How to deal with missing values. Only used in fixed-target perspective. -Options are: -- 'NULL': No correction is done. This will likely result in an error, as the DSCtool -does not allow for na values -- 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) -- 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) -- 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this -reverts to nr_runs * budget -- 'Remove-na' Removes all NA values} -} -\value{ -A named list containing a ranked-matrix which has the rankin of each algorithm -on each problem, as well as a list of which omnibus tests can be used to further process -this data. This can be further analyzed using `get_dsc_omnibus` -} -\description{ -Get the matrix of rankings using the DSCtool api for a DataSetList -} -\examples{ -get_dsc_rank(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_dsc_rank} +\alias{get_dsc_rank} +\title{Get the matrix of rankings using the DSCtool api for a DataSetList} +\usage{ +get_dsc_rank(dsList, targets = NULL, which = "by_RT", + test_type = "AD", alpha = 0.05, epsilon = 0, + monte_carlo_iterations = 0, na.correction = NULL) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{Optional list of target values (Runtime or target value)} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{test_type}{Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests} + +\item{alpha}{Threshold value for statistical significance} + +\item{epsilon}{Minimum threshold to have practical difference between algorithms (eDSC)} + +\item{monte_carlo_iterations}{How many monte-carlo-simulations to perform +(set to 0 to use regular DSC)} + +\item{na.correction}{How to deal with missing values. Only used in fixed-target perspective. +Options are: +- 'NULL': No correction is done. This will likely result in an error, as the DSCtool +does not allow for na values +- 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) +- 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) +- 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this +reverts to nr_runs * budget +- 'Remove-na' Removes all NA values} +} +\value{ +A named list containing a ranked-matrix which has the rankin of each algorithm +on each problem, as well as a list of which omnibus tests can be used to further process +this data. This can be further analyzed using `get_dsc_omnibus` +} +\description{ +Get the matrix of rankings using the DSCtool api for a DataSetList +} +\examples{ +get_dsc_rank(dsl) +} diff --git a/man/get_funcId.Rd b/man/get_funcId.Rd index e570ea41..847b6f08 100644 --- a/man/get_funcId.Rd +++ b/man/get_funcId.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_funcId} -\alias{get_funcId} -\title{Get all function ids present in a DataSetList} -\usage{ -get_funcId(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A sorted list of all unique function ids which occur in the DataSetList -} -\description{ -Get all function ids present in a DataSetList -} -\examples{ -get_funcId(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funcId} +\alias{get_funcId} +\title{Get all function ids present in a DataSetList} +\usage{ +get_funcId(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A sorted list of all unique function ids which occur in the DataSetList +} +\description{ +Get all function ids present in a DataSetList +} +\examples{ +get_funcId(dsl) +} diff --git a/man/get_funcName.Rd b/man/get_funcName.Rd index 415d15cd..fc2e5519 100644 --- a/man/get_funcName.Rd +++ b/man/get_funcName.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_funcName} -\alias{get_funcName} -\title{Get all function names present in a DataSetList} -\usage{ -get_funcName(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A list of all unique function names which occur in the DataSetList -} -\description{ -Get all function names present in a DataSetList -} -\examples{ -get_funcName(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funcName} +\alias{get_funcName} +\title{Get all function names present in a DataSetList} +\usage{ +get_funcName(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list of all unique function names which occur in the DataSetList +} +\description{ +Get all function names present in a DataSetList +} +\examples{ +get_funcName(dsl) +} diff --git a/man/get_funvals.Rd b/man/get_funvals.Rd index 4152144a..9cbcca3f 100644 --- a/man/get_funvals.Rd +++ b/man/get_funvals.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_funvals} -\alias{get_funvals} -\title{Get all function values present in a DataSetList} -\usage{ -get_funvals(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A list matrices of all function values which occur in the DataSetList -} -\description{ -Get all function values present in a DataSetList -} -\examples{ -get_funvals(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funvals} +\alias{get_funvals} +\title{Get all function values present in a DataSetList} +\usage{ +get_funvals(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list matrices of all function values which occur in the DataSetList +} +\description{ +Get all function values present in a DataSetList +} +\examples{ +get_funvals(dsl) +} diff --git a/man/get_id.Rd b/man/get_id.Rd index bff5e0be..74b5ad5a 100644 --- a/man/get_id.Rd +++ b/man/get_id.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_id} -\alias{get_id} -\alias{get_id.DataSet} -\alias{get_id.DataSetList} -\title{Get condensed overview of datasets} -\usage{ -get_id(ds, ...) - -\method{get_id}{DataSet}(ds, ...) - -\method{get_id}{DataSetList}(ds, ...) -} -\arguments{ -\item{ds}{The DataSetList} - -\item{...}{Arguments passed to other methods} -} -\value{ -The list of unique identiefiers present in dsl -} -\description{ -Get the unique identifiers for each DataSet in the provided DataSetList -} -\details{ -If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), -this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility -} -\examples{ -get_id(dsl) -get_id(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_id} +\alias{get_id} +\alias{get_id.DataSet} +\alias{get_id.DataSetList} +\title{Get condensed overview of datasets} +\usage{ +get_id(ds, ...) + +\method{get_id}{DataSet}(ds, ...) + +\method{get_id}{DataSetList}(ds, ...) +} +\arguments{ +\item{ds}{The DataSetList} + +\item{...}{Arguments passed to other methods} +} +\value{ +The list of unique identiefiers present in dsl +} +\description{ +Get the unique identifiers for each DataSet in the provided DataSetList +} +\details{ +If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility +} +\examples{ +get_id(dsl) +get_id(dsl[[1]]) +} diff --git a/man/get_line_style.Rd b/man/get_line_style.Rd index beb5adbd..36d46b9d 100644 --- a/man/get_line_style.Rd +++ b/man/get_line_style.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{get_line_style} -\alias{get_line_style} -\title{Get line styles according to the current styleScheme of the IOHanalyzer} -\usage{ -get_line_style(ids_in) -} -\arguments{ -\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get linestyles} -} -\description{ -Get line styles according to the current styleScheme of the IOHanalyzer -} -\examples{ -get_line_style(get_algId(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{get_line_style} +\alias{get_line_style} +\title{Get line styles according to the current styleScheme of the IOHanalyzer} +\usage{ +get_line_style(ids_in) +} +\arguments{ +\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get linestyles} +} +\description{ +Get line styles according to the current styleScheme of the IOHanalyzer +} +\examples{ +get_line_style(get_algId(dsl)) +} diff --git a/man/get_marg_contrib_ecdf.Rd b/man/get_marg_contrib_ecdf.Rd index 55b91023..2200786a 100644 --- a/man/get_marg_contrib_ecdf.Rd +++ b/man/get_marg_contrib_ecdf.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_marg_contrib_ecdf} -\alias{get_marg_contrib_ecdf} -\title{Get the marginal contribution of an algorithm to a portfolio} -\usage{ -get_marg_contrib_ecdf(id, perm, j, dt) -} -\arguments{ -\item{id}{The id for which to get the contribution} - -\item{perm}{The permutation of algorithms to which is being contributed} - -\item{j}{At which point in the permutation the contribution should be measured} - -\item{dt}{The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`)} -} -\description{ -Based on the contribution to the ECDF-curve of the VBS of the portfolio -} -\examples{ -dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) -get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_marg_contrib_ecdf} +\alias{get_marg_contrib_ecdf} +\title{Get the marginal contribution of an algorithm to a portfolio} +\usage{ +get_marg_contrib_ecdf(id, perm, j, dt) +} +\arguments{ +\item{id}{The id for which to get the contribution} + +\item{perm}{The permutation of algorithms to which is being contributed} + +\item{j}{At which point in the permutation the contribution should be measured} + +\item{dt}{The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`)} +} +\description{ +Based on the contribution to the ECDF-curve of the VBS of the portfolio +} +\examples{ +dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) +get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) +} diff --git a/man/get_maxRT.Rd b/man/get_maxRT.Rd index 3d7191ae..f762b834 100644 --- a/man/get_maxRT.Rd +++ b/man/get_maxRT.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_maxRT} -\alias{get_maxRT} -\alias{get_maxRT.DataSet} -\alias{get_maxRT.DataSetList} -\title{Get the maximal running time} -\usage{ -get_maxRT(ds, ...) - -\method{get_maxRT}{DataSet}(ds, output = "wide", ...) - -\method{get_maxRT}{DataSetList}(ds, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{output}{The format of the outputted table: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table object containing the algorithm ID and the running time -when the algorithm terminates in each run -} -\description{ -Get the maximal running time -} -\examples{ -get_maxRT(dsl) -get_maxRT(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_maxRT} +\alias{get_maxRT} +\alias{get_maxRT.DataSet} +\alias{get_maxRT.DataSetList} +\title{Get the maximal running time} +\usage{ +get_maxRT(ds, ...) + +\method{get_maxRT}{DataSet}(ds, output = "wide", ...) + +\method{get_maxRT}{DataSetList}(ds, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{output}{The format of the outputted table: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table object containing the algorithm ID and the running time +when the algorithm terminates in each run +} +\description{ +Get the maximal running time +} +\examples{ +get_maxRT(dsl) +get_maxRT(dsl[[1]]) +} diff --git a/man/get_overview.Rd b/man/get_overview.Rd index d32eeefd..18697ec8 100644 --- a/man/get_overview.Rd +++ b/man/get_overview.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_overview} -\alias{get_overview} -\alias{get_overview.DataSet} -\alias{get_overview.DataSetList} -\title{Get condensed overview of datasets} -\usage{ -get_overview(ds, ...) - -\method{get_overview}{DataSet}(ds, ...) - -\method{get_overview}{DataSetList}(ds, ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} -} -\value{ -A data.table containing some basic information about the provided DataSet(List) -} -\description{ -Get condensed overview of datasets -} -\examples{ -get_overview(dsl) -get_overview(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_overview} +\alias{get_overview} +\alias{get_overview.DataSet} +\alias{get_overview.DataSetList} +\title{Get condensed overview of datasets} +\usage{ +get_overview(ds, ...) + +\method{get_overview}{DataSet}(ds, ...) + +\method{get_overview}{DataSetList}(ds, ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} +} +\value{ +A data.table containing some basic information about the provided DataSet(List) +} +\description{ +Get condensed overview of datasets +} +\examples{ +get_overview(dsl) +get_overview(dsl[[1]]) +} diff --git a/man/get_parId.Rd b/man/get_parId.Rd index 95ddad93..b64e87fd 100644 --- a/man/get_parId.Rd +++ b/man/get_parId.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_parId} -\alias{get_parId} -\title{Get all parameter ids present in a DataSetList} -\usage{ -get_parId(dsList, which = "by_FV") -} -\arguments{ -\item{dsList}{The DataSetList} - -\item{which}{A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned -by the running time (RT) or the function value (FV). Note that parameters in each case are -not necessary the same.} -} -\value{ -A sorted list of all unique parameter ids which occur in the DataSetList -} -\description{ -Get all parameter ids present in a DataSetList -} -\examples{ -get_parId(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_parId} +\alias{get_parId} +\title{Get all parameter ids present in a DataSetList} +\usage{ +get_parId(dsList, which = "by_FV") +} +\arguments{ +\item{dsList}{The DataSetList} + +\item{which}{A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned +by the running time (RT) or the function value (FV). Note that parameters in each case are +not necessary the same.} +} +\value{ +A sorted list of all unique parameter ids which occur in the DataSetList +} +\description{ +Get all parameter ids present in a DataSetList +} +\examples{ +get_parId(dsl) +} diff --git a/man/get_runtimes.Rd b/man/get_runtimes.Rd index dd910f07..7a75d6f7 100644 --- a/man/get_runtimes.Rd +++ b/man/get_runtimes.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_runtimes} -\alias{get_runtimes} -\title{Get all runtime values present in a DataSetList} -\usage{ -get_runtimes(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A list matrices of all runtime values which occur in the DataSetList -} -\description{ -Get all runtime values present in a DataSetList -} -\examples{ -get_runtimes(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_runtimes} +\alias{get_runtimes} +\title{Get all runtime values present in a DataSetList} +\usage{ +get_runtimes(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list matrices of all runtime values which occur in the DataSetList +} +\description{ +Get all runtime values present in a DataSetList +} +\examples{ +get_runtimes(dsl) +} diff --git a/man/get_shapley_values.Rd b/man/get_shapley_values.Rd index 28ddc9e0..cbeb9855 100644 --- a/man/get_shapley_values.Rd +++ b/man/get_shapley_values.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_shapley_values} -\alias{get_shapley_values} -\title{Get the shapley-values of a portfolio of algorithms} -\usage{ -get_shapley_values(dsList, targets, scale.log = T, group_size = 5, - max_perm_size = 10, normalize = T) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale.log}{Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not} - -\item{group_size}{How many permutation groups will be considered} - -\item{max_perm_size}{The maximum limit for permutations to be considered} - -\item{normalize}{Whether or not to ensure the resulting values will be in [0,1]} -} -\description{ -Based on the contribution to the ECDF-curve of the VBS of the portfolio -} -\examples{ -get_shapley_values(dsl, get_ECDF_targets(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_shapley_values} +\alias{get_shapley_values} +\title{Get the shapley-values of a portfolio of algorithms} +\usage{ +get_shapley_values(dsList, targets, scale.log = T, group_size = 5, + max_perm_size = 10, normalize = T) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale.log}{Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not} + +\item{group_size}{How many permutation groups will be considered} + +\item{max_perm_size}{The maximum limit for permutations to be considered} + +\item{normalize}{Whether or not to ensure the resulting values will be in [0,1]} +} +\description{ +Based on the contribution to the ECDF-curve of the VBS of the portfolio +} +\examples{ +get_shapley_values(dsl, get_ECDF_targets(dsl)) +} diff --git a/man/get_static_attribute_values.Rd b/man/get_static_attribute_values.Rd index fae27df0..33a34eea 100644 --- a/man/get_static_attribute_values.Rd +++ b/man/get_static_attribute_values.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_static_attribute_values} -\alias{get_static_attribute_values} -\title{Get all options for a specific attribute which can be used to subset a DataSetList} -\usage{ -get_static_attribute_values(dsl, attribute) -} -\arguments{ -\item{dsl}{The DataSetList} - -\item{attribute}{the name of the attribute for which to get the available options in dsl} -} -\value{ -The list of options for the specified attribute -} -\description{ -This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. -Note the only attributes returned by `get_static_attributes` are supported in this funcion -} -\examples{ -get_static_attribute_values(dsl, 'funcId') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_static_attribute_values} +\alias{get_static_attribute_values} +\title{Get all options for a specific attribute which can be used to subset a DataSetList} +\usage{ +get_static_attribute_values(dsl, attribute) +} +\arguments{ +\item{dsl}{The DataSetList} + +\item{attribute}{the name of the attribute for which to get the available options in dsl} +} +\value{ +The list of options for the specified attribute +} +\description{ +This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. +Note the only attributes returned by `get_static_attributes` are supported in this funcion +} +\examples{ +get_static_attribute_values(dsl, 'funcId') +} diff --git a/man/get_static_attributes.Rd b/man/get_static_attributes.Rd index ea7489d5..aad011bc 100644 --- a/man/get_static_attributes.Rd +++ b/man/get_static_attributes.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_static_attributes} -\alias{get_static_attributes} -\title{Get all attributes which can be used to subset a DataSetList} -\usage{ -get_static_attributes(dsl) -} -\arguments{ -\item{dsl}{The DataSetList} -} -\value{ -The list of available attributes -} -\description{ -Get all attributes which can be used to subset a DataSetList -} -\examples{ -get_static_attributes(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_static_attributes} +\alias{get_static_attributes} +\title{Get all attributes which can be used to subset a DataSetList} +\usage{ +get_static_attributes(dsl) +} +\arguments{ +\item{dsl}{The DataSetList} +} +\value{ +The list of available attributes +} +\description{ +Get all attributes which can be used to subset a DataSetList +} +\examples{ +get_static_attributes(dsl) +} diff --git a/man/get_target_dt.Rd b/man/get_target_dt.Rd index df418bec..4305a7cd 100644 --- a/man/get_target_dt.Rd +++ b/man/get_target_dt.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_target_dt} -\alias{get_target_dt} -\title{Generate datatables of runtime or function value targets for a DataSetList} -\usage{ -get_target_dt(dsList, which = "by_RT") -} -\arguments{ -\item{dsList}{A DataSetList} - -\item{which}{Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets} -} -\value{ -a data.table of targets -} -\description{ -Only one target is generated per (function, dimension)-pair, as opposed to the -function `get_default_ECDF_targets`, which generates multiple targets. -} -\examples{ -get_target_dt(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_target_dt} +\alias{get_target_dt} +\title{Generate datatables of runtime or function value targets for a DataSetList} +\usage{ +get_target_dt(dsList, which = "by_RT") +} +\arguments{ +\item{dsList}{A DataSetList} + +\item{which}{Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets} +} +\value{ +a data.table of targets +} +\description{ +Only one target is generated per (function, dimension)-pair, as opposed to the +function `get_default_ECDF_targets`, which generates multiple targets. +} +\examples{ +get_target_dt(dsl) +} diff --git a/man/glicko2_ranking.Rd b/man/glicko2_ranking.Rd index 085167a0..013ceccb 100644 --- a/man/glicko2_ranking.Rd +++ b/man/glicko2_ranking.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{glicko2_ranking} -\alias{glicko2_ranking} -\title{Glicko2 raning of algorithms} -\usage{ -glicko2_ranking(dsl, nr_rounds = 100, which = "by_FV", - target_dt = NULL) -} -\arguments{ -\item{dsl}{The DataSetList, can contain multiple functions and dimensions, but should have the -same algorithms for all of them} - -\item{nr_rounds}{The number of rounds to run. More rounds leads to a more accurate ranking.} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} - -\item{target_dt}{Custom data.table target value to use. When NULL, this is selected automatically.} -} -\value{ -A dataframe containing the glicko2-ratings and some additional info -} -\description{ -This procedure ranks algorithms based on a glicko2-procedure. -Every round (total nr_rounds), for every function and dimension of the datasetlist, -each pair of algorithms competes. This competition samples a random runtime for the -provided target (defaults to best achieved target). Whichever algorithm has the lower -runtime wins the game. Then, from these games, the glicko2-rating is determined. -} -\examples{ -glicko2_ranking(dsl, nr_round = 25) -glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{glicko2_ranking} +\alias{glicko2_ranking} +\title{Glicko2 raning of algorithms} +\usage{ +glicko2_ranking(dsl, nr_rounds = 100, which = "by_FV", + target_dt = NULL) +} +\arguments{ +\item{dsl}{The DataSetList, can contain multiple functions and dimensions, but should have the +same algorithms for all of them} + +\item{nr_rounds}{The number of rounds to run. More rounds leads to a more accurate ranking.} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} + +\item{target_dt}{Custom data.table target value to use. When NULL, this is selected automatically.} +} +\value{ +A dataframe containing the glicko2-ratings and some additional info +} +\description{ +This procedure ranks algorithms based on a glicko2-procedure. +Every round (total nr_rounds), for every function and dimension of the datasetlist, +each pair of algorithms competes. This competition samples a random runtime for the +provided target (defaults to best achieved target). Whichever algorithm has the lower +runtime wins the game. Then, from these games, the glicko2-rating is determined. +} +\examples{ +glicko2_ranking(dsl, nr_round = 25) +glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') +} diff --git a/man/limit.data.Rd b/man/limit.data.Rd index acfd9ebb..cb2175cd 100644 --- a/man/limit.data.Rd +++ b/man/limit.data.Rd @@ -1,19 +1,19 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{limit.data} -\alias{limit.data} -\title{Reduce the size of the data set by evenly subsampling the records} -\usage{ -limit.data(df, n) -} -\arguments{ -\item{df}{The data to subsample} - -\item{n}{The amount of samples} -} -\value{ -A smaller data.frame -} -\description{ -Reduce the size of the data set by evenly subsampling the records -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{limit.data} +\alias{limit.data} +\title{Reduce the size of the data set by evenly subsampling the records} +\usage{ +limit.data(df, n) +} +\arguments{ +\item{df}{The data to subsample} + +\item{n}{The amount of samples} +} +\value{ +A smaller data.frame +} +\description{ +Reduce the size of the data set by evenly subsampling the records +} diff --git a/man/max_ERTs.Rd b/man/max_ERTs.Rd index b370ac57..e18371c4 100644 --- a/man/max_ERTs.Rd +++ b/man/max_ERTs.Rd @@ -1,30 +1,30 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{max_ERTs} -\alias{max_ERTs} -\alias{max_ERTs.DataSetList} -\title{Get the ERT-values for all DataSets in a DataSetList at certain targets} -\usage{ -max_ERTs(dsList, aggr_on = "funcId", targets = NULL, maximize = T) - -\method{max_ERTs}{DataSetList}(dsList, aggr_on = "funcId", - targets = NULL, maximize = T) -} -\arguments{ -\item{dsList}{The DataSetLsit} - -\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} - -\item{targets}{Predifined target function-values. Should be one for each function/dimension} - -\item{maximize}{Whether the DataSetList is from a maximization or minimization problem} -} -\value{ -A data.table containing ERT-values -} -\description{ -Get the ERT-values for all DataSets in a DataSetList at certain targets -} -\examples{ -max_ERTs(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{max_ERTs} +\alias{max_ERTs} +\alias{max_ERTs.DataSetList} +\title{Get the ERT-values for all DataSets in a DataSetList at certain targets} +\usage{ +max_ERTs(dsList, aggr_on = "funcId", targets = NULL, maximize = T) + +\method{max_ERTs}{DataSetList}(dsList, aggr_on = "funcId", + targets = NULL, maximize = T) +} +\arguments{ +\item{dsList}{The DataSetLsit} + +\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} + +\item{targets}{Predifined target function-values. Should be one for each function/dimension} + +\item{maximize}{Whether the DataSetList is from a maximization or minimization problem} +} +\value{ +A data.table containing ERT-values +} +\description{ +Get the ERT-values for all DataSets in a DataSetList at certain targets +} +\examples{ +max_ERTs(dsl) +} diff --git a/man/mean_FVs.Rd b/man/mean_FVs.Rd index 37a89e2f..3094936d 100644 --- a/man/mean_FVs.Rd +++ b/man/mean_FVs.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{mean_FVs} -\alias{mean_FVs} -\alias{mean_FVs.DataSetList} -\title{Get the expected function-values for all DataSets in a DataSetList at certain runtimes} -\usage{ -mean_FVs(dsList, aggr_on = "funcId", runtimes = NULL) - -\method{mean_FVs}{DataSetList}(dsList, aggr_on = "funcId", - runtimes = NULL) -} -\arguments{ -\item{dsList}{The DataSetLsit} - -\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} - -\item{runtimes}{Predifined target runtimes-values. Should be one for each function/dimension} -} -\value{ -A data.table containing expected fucntion-values -} -\description{ -Get the expected function-values for all DataSets in a DataSetList at certain runtimes -} -\examples{ -mean_FVs(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{mean_FVs} +\alias{mean_FVs} +\alias{mean_FVs.DataSetList} +\title{Get the expected function-values for all DataSets in a DataSetList at certain runtimes} +\usage{ +mean_FVs(dsList, aggr_on = "funcId", runtimes = NULL) + +\method{mean_FVs}{DataSetList}(dsList, aggr_on = "funcId", + runtimes = NULL) +} +\arguments{ +\item{dsList}{The DataSetLsit} + +\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} + +\item{runtimes}{Predifined target runtimes-values. Should be one for each function/dimension} +} +\value{ +A data.table containing expected fucntion-values +} +\description{ +Get the expected function-values for all DataSets in a DataSetList at certain runtimes +} +\examples{ +mean_FVs(dsl) +} diff --git a/man/pairwise.test.Rd b/man/pairwise.test.Rd index 914c322f..963594b5 100644 --- a/man/pairwise.test.Rd +++ b/man/pairwise.test.Rd @@ -1,45 +1,45 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{pairwise.test} -\alias{pairwise.test} -\alias{pairwise.test.list} -\alias{pairwise.test.DataSetList} -\title{Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times -among a data set} -\usage{ -pairwise.test(x, ...) - -\method{pairwise.test}{list}(x, max_eval, bootstrap.size = 30, ...) - -\method{pairwise.test}{DataSetList}(x, ftarget, bootstrap.size = 0, - which = "by_FV", ...) -} -\arguments{ -\item{x}{either a list that contains running time sample for each algorithm as -sub-lists, or a DataSetList object} - -\item{...}{all other options} - -\item{max_eval}{list that contains the maximal running time for each algorithm -as sub-lists} - -\item{bootstrap.size}{integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping} - -\item{ftarget}{float, the target value used to determine the running / hitting} - -\item{which}{wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison -time} -} -\value{ -A matrix containing p-values of the test -} -\description{ -This function performs a Kolmogorov-Smirnov test on each pair of -algorithms in the input x to determine which algorithm gives a significantly -smaller running time. The resulting p-values are arranged in a matrix, where -each cell (i, j) contains a p-value from the test with alternative hypothesis: -the running time of algorithm i is smaller (thus better) than that of j. -} -\examples{ -pairwise.test(subset(dsl, funcId == 1), 16) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{pairwise.test} +\alias{pairwise.test} +\alias{pairwise.test.list} +\alias{pairwise.test.DataSetList} +\title{Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times +among a data set} +\usage{ +pairwise.test(x, ...) + +\method{pairwise.test}{list}(x, max_eval, bootstrap.size = 30, ...) + +\method{pairwise.test}{DataSetList}(x, ftarget, bootstrap.size = 0, + which = "by_FV", ...) +} +\arguments{ +\item{x}{either a list that contains running time sample for each algorithm as +sub-lists, or a DataSetList object} + +\item{...}{all other options} + +\item{max_eval}{list that contains the maximal running time for each algorithm +as sub-lists} + +\item{bootstrap.size}{integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping} + +\item{ftarget}{float, the target value used to determine the running / hitting} + +\item{which}{wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison +time} +} +\value{ +A matrix containing p-values of the test +} +\description{ +This function performs a Kolmogorov-Smirnov test on each pair of +algorithms in the input x to determine which algorithm gives a significantly +smaller running time. The resulting p-values are arranged in a matrix, where +each cell (i, j) contains a p-value from the test with alternative hypothesis: +the running time of algorithm i is smaller (thus better) than that of j. +} +\examples{ +pairwise.test(subset(dsl, funcId == 1), 16) +} diff --git a/man/plot_general_data.Rd b/man/plot_general_data.Rd index 5a2d2b19..b9aeb184 100644 --- a/man/plot_general_data.Rd +++ b/man/plot_general_data.Rd @@ -1,57 +1,57 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{plot_general_data} -\alias{plot_general_data} -\title{General function for plotting within IOHanalyzer} -\usage{ -plot_general_data(df, x_attr = "ID", y_attr = "vals", - type = "violin", legend_attr = "ID", scale.xlog = F, - scale.ylog = F, scale.reverse = F, p = NULL, x_title = NULL, - y_title = NULL, plot_title = NULL, upper_attr = NULL, - lower_attr = NULL, subplot_attr = NULL, show.legend = F, - inf.action = "none", ...) -} -\arguments{ -\item{df}{The dataframe containing the data to plot. It should contain at least two columns: -'x_attr' and 'y_attr'} - -\item{x_attr}{The column to specify the x_axis. Default is 'algId'} - -\item{y_attr}{The column to specify the y_axis} - -\item{type}{The type of plot to use. Currently available: 'violin', 'line', 'radar', -'bar', hist' and 'ribbon'} - -\item{legend_attr}{Default is 'algId' This is also used for the selection of colorschemes} - -\item{scale.xlog}{Logarithmic scaling of x-axis} - -\item{scale.ylog}{Logarithmic scaling of y-axis} - -\item{scale.reverse}{Decreasing or increasing x-axis} - -\item{p}{A previously existing plot on which to add traces. If NULL, a new canvas is created} - -\item{x_title}{Title of x-axis. Defaults to x_attr} - -\item{y_title}{Title of x-axis. Defaults to x_attr} - -\item{plot_title}{Title of x-axis. Defaults to no title} - -\item{upper_attr}{When using ribbon-plot, this can be used to create a shaded area. -Only works in combination with`lower_attr` and `type` == 'ribbon'} - -\item{lower_attr}{When using ribbon-plot, this can be used to create a shaded area. -Only works in combination with`upper_attr` and `type` == 'ribbon'} - -\item{subplot_attr}{Which attribute of the dataframe to use for creating subplots} - -\item{show.legend}{Whether or not to include a legend} - -\item{inf.action}{How to deal with infinite values. Can be 'none', 'overlap' or 'jitter'} - -\item{...}{Additional parameters for the add_trace function} -} -\description{ -General function for plotting within IOHanalyzer -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{plot_general_data} +\alias{plot_general_data} +\title{General function for plotting within IOHanalyzer} +\usage{ +plot_general_data(df, x_attr = "ID", y_attr = "vals", + type = "violin", legend_attr = "ID", scale.xlog = F, + scale.ylog = F, scale.reverse = F, p = NULL, x_title = NULL, + y_title = NULL, plot_title = NULL, upper_attr = NULL, + lower_attr = NULL, subplot_attr = NULL, show.legend = F, + inf.action = "none", ...) +} +\arguments{ +\item{df}{The dataframe containing the data to plot. It should contain at least two columns: +'x_attr' and 'y_attr'} + +\item{x_attr}{The column to specify the x_axis. Default is 'algId'} + +\item{y_attr}{The column to specify the y_axis} + +\item{type}{The type of plot to use. Currently available: 'violin', 'line', 'radar', +'bar', hist' and 'ribbon'} + +\item{legend_attr}{Default is 'algId' This is also used for the selection of colorschemes} + +\item{scale.xlog}{Logarithmic scaling of x-axis} + +\item{scale.ylog}{Logarithmic scaling of y-axis} + +\item{scale.reverse}{Decreasing or increasing x-axis} + +\item{p}{A previously existing plot on which to add traces. If NULL, a new canvas is created} + +\item{x_title}{Title of x-axis. Defaults to x_attr} + +\item{y_title}{Title of x-axis. Defaults to x_attr} + +\item{plot_title}{Title of x-axis. Defaults to no title} + +\item{upper_attr}{When using ribbon-plot, this can be used to create a shaded area. +Only works in combination with`lower_attr` and `type` == 'ribbon'} + +\item{lower_attr}{When using ribbon-plot, this can be used to create a shaded area. +Only works in combination with`upper_attr` and `type` == 'ribbon'} + +\item{subplot_attr}{Which attribute of the dataframe to use for creating subplots} + +\item{show.legend}{Whether or not to include a legend} + +\item{inf.action}{How to deal with infinite values. Can be 'none', 'overlap' or 'jitter'} + +\item{...}{Additional parameters for the add_trace function} +} +\description{ +General function for plotting within IOHanalyzer +} diff --git a/man/print.DataSet.Rd b/man/print.DataSet.Rd index 2654e253..72d10ac5 100644 --- a/man/print.DataSet.Rd +++ b/man/print.DataSet.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{print.DataSet} -\alias{print.DataSet} -\title{S3 generic print operator for DataSet} -\usage{ -\method{print}{DataSet}(x, ...) -} -\arguments{ -\item{x}{A DataSet object} - -\item{...}{Arguments passed to other methods} -} -\value{ -A short description of the DataSet -} -\description{ -S3 generic print operator for DataSet -} -\examples{ -print(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{print.DataSet} +\alias{print.DataSet} +\title{S3 generic print operator for DataSet} +\usage{ +\method{print}{DataSet}(x, ...) +} +\arguments{ +\item{x}{A DataSet object} + +\item{...}{Arguments passed to other methods} +} +\value{ +A short description of the DataSet +} +\description{ +S3 generic print operator for DataSet +} +\examples{ +print(dsl[[1]]) +} diff --git a/man/print.DataSetList.Rd b/man/print.DataSetList.Rd index 24c714bc..8547c7f7 100644 --- a/man/print.DataSetList.Rd +++ b/man/print.DataSetList.Rd @@ -1,19 +1,19 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{print.DataSetList} -\alias{print.DataSetList} -\title{S3 print function for DataSetList} -\usage{ -\method{print}{DataSetList}(x, ...) -} -\arguments{ -\item{x}{The DataSetList to print} - -\item{...}{Arguments for underlying print function?} -} -\description{ -S3 print function for DataSetList -} -\examples{ -print(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{print.DataSetList} +\alias{print.DataSetList} +\title{S3 print function for DataSetList} +\usage{ +\method{print}{DataSetList}(x, ...) +} +\arguments{ +\item{x}{The DataSetList to print} + +\item{...}{Arguments for underlying print function?} +} +\description{ +S3 print function for DataSetList +} +\examples{ +print(dsl) +} diff --git a/man/read_index_file.Rd b/man/read_index_file.Rd index 0e5398d7..e5e9cb28 100644 --- a/man/read_index_file.Rd +++ b/man/read_index_file.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{read_index_file} -\alias{read_index_file} -\title{Read .info files and extract information} -\usage{ -read_index_file(fname) -} -\arguments{ -\item{fname}{The path to the .info file} -} -\value{ -The data contained in the .info file -} -\description{ -Read .info files and extract information -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{read_index_file} +\alias{read_index_file} +\title{Read .info files and extract information} +\usage{ +read_index_file(fname) +} +\arguments{ +\item{fname}{The path to the .info file} +} +\value{ +The data contained in the .info file +} +\description{ +Read .info files and extract information +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) +} diff --git a/man/register_DSC.Rd b/man/register_DSC.Rd index 541c2207..fe3d3961 100644 --- a/man/register_DSC.Rd +++ b/man/register_DSC.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{register_DSC} -\alias{register_DSC} -\title{Register an account to the DSCtool API} -\usage{ -register_DSC(name, username, affiliation, email, password = NULL) -} -\arguments{ -\item{name}{Your name} - -\item{username}{A usename to be identified with. Will be stored on keyring under 'DSCtool_name'} - -\item{affiliation}{Your affiliation (university / company)} - -\item{email}{Your email adress} - -\item{password}{The password to use. If NULL, this will be generated at random. -Will be stored on keyring under 'DSCtool'} -} -\description{ -This uses the keyring package to store and load credentials. -If you already have an account, please call `set_DSC_credentials` instead -} -\examples{ -\dontrun{ -register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{register_DSC} +\alias{register_DSC} +\title{Register an account to the DSCtool API} +\usage{ +register_DSC(name, username, affiliation, email, password = NULL) +} +\arguments{ +\item{name}{Your name} + +\item{username}{A usename to be identified with. Will be stored on keyring under 'DSCtool_name'} + +\item{affiliation}{Your affiliation (university / company)} + +\item{email}{Your email adress} + +\item{password}{The password to use. If NULL, this will be generated at random. +Will be stored on keyring under 'DSCtool'} +} +\description{ +This uses the keyring package to store and load credentials. +If you already have an account, please call `set_DSC_credentials` instead +} +\examples{ +\dontrun{ +register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") +} +} diff --git a/man/runServer.Rd b/man/runServer.Rd index 3c6a21c4..07b9212f 100644 --- a/man/runServer.Rd +++ b/man/runServer.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/runServer.R -\name{runServer} -\alias{runServer} -\title{Create a shiny-server GUI to interactively use the IOHanalyzer} -\usage{ -runServer(port = getOption("shiny.port"), open_browser = TRUE) -} -\arguments{ -\item{port}{Optional; which port the server should be opened at. Defaults -to the option set for 'shiny.port'} - -\item{open_browser}{Whether or not to open a browser tab with the -IOHanalyzer GUI. Defaults to TRUE.} -} -\description{ -Create a shiny-server GUI to interactively use the IOHanalyzer -} -\examples{ -\dontrun{ -runServer(6563, TRUE) -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/runServer.R +\name{runServer} +\alias{runServer} +\title{Create a shiny-server GUI to interactively use the IOHanalyzer} +\usage{ +runServer(port = getOption("shiny.port"), open_browser = TRUE) +} +\arguments{ +\item{port}{Optional; which port the server should be opened at. Defaults +to the option set for 'shiny.port'} + +\item{open_browser}{Whether or not to open a browser tab with the +IOHanalyzer GUI. Defaults to TRUE.} +} +\description{ +Create a shiny-server GUI to interactively use the IOHanalyzer +} +\examples{ +\dontrun{ +runServer(6563, TRUE) +} +} diff --git a/man/save_plotly.Rd b/man/save_plotly.Rd index a8d8b80f..dd574553 100644 --- a/man/save_plotly.Rd +++ b/man/save_plotly.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{save_plotly} -\alias{save_plotly} -\title{Save plotly figure in multiple format} -\usage{ -save_plotly(p, file, width = NULL, height = NULL, ...) -} -\arguments{ -\item{p}{plotly object. The plot to be saved} - -\item{file}{String. The name of the figure file, with the extension of the required file-format} - -\item{width}{Optional. Width of the figure} - -\item{height}{Optional. Height of the figure} - -\item{...}{Additional arguments for orca} -} -\description{ -NOTE: This function requires orca to be installed -} -\examples{ -\dontrun{ -p <- Plot.RT.Single_Func(dsl[1]) -save_plotly(p, 'example_file.png') -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{save_plotly} +\alias{save_plotly} +\title{Save plotly figure in multiple format} +\usage{ +save_plotly(p, file, width = NULL, height = NULL, ...) +} +\arguments{ +\item{p}{plotly object. The plot to be saved} + +\item{file}{String. The name of the figure file, with the extension of the required file-format} + +\item{width}{Optional. Width of the figure} + +\item{height}{Optional. Height of the figure} + +\item{...}{Additional arguments for orca} +} +\description{ +NOTE: This function requires orca to be installed +} +\examples{ +\dontrun{ +p <- Plot.RT.Single_Func(dsl[1]) +save_plotly(p, 'example_file.png') +} +} diff --git a/man/save_table.Rd b/man/save_table.Rd index 531a6bdf..09e4e3a3 100644 --- a/man/save_table.Rd +++ b/man/save_table.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{save_table} -\alias{save_table} -\title{Save DataTable in multiple formats} -\usage{ -save_table(df, file, format = NULL) -} -\arguments{ -\item{df}{The DataTable to store} - -\item{file}{String. The name of the figure file, with the extension of the required file-format} - -\item{format}{Optional, string. Overwrites the extension of the `file` parameter. If not specified while -file does not have an extension, it defaults to csv} -} -\description{ -Save DataTable in multiple formats -} -\examples{ -df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -save_table(df, tempfile(fileext = ".md")) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{save_table} +\alias{save_table} +\title{Save DataTable in multiple formats} +\usage{ +save_table(df, file, format = NULL) +} +\arguments{ +\item{df}{The DataTable to store} + +\item{file}{String. The name of the figure file, with the extension of the required file-format} + +\item{format}{Optional, string. Overwrites the extension of the `file` parameter. If not specified while +file does not have an extension, it defaults to csv} +} +\description{ +Save DataTable in multiple formats +} +\examples{ +df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +save_table(df, tempfile(fileext = ".md")) +} diff --git a/man/scan_index_file.Rd b/man/scan_index_file.Rd index 33d04866..448f2611 100644 --- a/man/scan_index_file.Rd +++ b/man/scan_index_file.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{scan_index_file} -\alias{scan_index_file} -\title{Scan *.info files for IOHProfiler or COCO} -\usage{ -scan_index_file(folder) -} -\arguments{ -\item{folder}{The folder containing the .info and .json files} -} -\value{ -The paths to all found .info and .json-files -} -\description{ -Scan *.info files for IOHProfiler or COCO -} -\note{ -This automatically filetrs our files of size 0 -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -scan_index_file(path) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{scan_index_file} +\alias{scan_index_file} +\title{Scan *.info files for IOHProfiler or COCO} +\usage{ +scan_index_file(folder) +} +\arguments{ +\item{folder}{The folder containing the .info and .json files} +} +\value{ +The paths to all found .info and .json-files +} +\description{ +Scan *.info files for IOHProfiler or COCO +} +\note{ +This automatically filetrs our files of size 0 +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +scan_index_file(path) +} diff --git a/man/seq_FV.Rd b/man/seq_FV.Rd index ea063685..811042cb 100644 --- a/man/seq_FV.Rd +++ b/man/seq_FV.Rd @@ -1,35 +1,35 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{seq_FV} -\alias{seq_FV} -\title{Function for generating sequences of function values} -\usage{ -seq_FV(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, - scale = NULL) -} -\arguments{ -\item{FV}{A list of function values} - -\item{from}{Starting function value. Will be replaced by min(FV) if it is NULL or too small} - -\item{to}{Stopping function value. Will be replaced by max(FV) if it is NULL or too large} - -\item{by}{Stepsize of the sequence. Will be replaced if it is too small} - -\item{length.out}{Number of values in the sequence. -'by' takes preference if both it and length.out are provided.} - -\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a -linear or log-linear spacing respectively. If NULL, the scale will be predicted -based on FV} -} -\value{ -A sequence of function values -} -\description{ -Function for generating sequences of function values -} -\examples{ -FVall <- get_runtimes(dsl) -seq_FV(FVall, 10, 16, 1, scale='linear') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{seq_FV} +\alias{seq_FV} +\title{Function for generating sequences of function values} +\usage{ +seq_FV(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, + scale = NULL) +} +\arguments{ +\item{FV}{A list of function values} + +\item{from}{Starting function value. Will be replaced by min(FV) if it is NULL or too small} + +\item{to}{Stopping function value. Will be replaced by max(FV) if it is NULL or too large} + +\item{by}{Stepsize of the sequence. Will be replaced if it is too small} + +\item{length.out}{Number of values in the sequence. +'by' takes preference if both it and length.out are provided.} + +\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a +linear or log-linear spacing respectively. If NULL, the scale will be predicted +based on FV} +} +\value{ +A sequence of function values +} +\description{ +Function for generating sequences of function values +} +\examples{ +FVall <- get_runtimes(dsl) +seq_FV(FVall, 10, 16, 1, scale='linear') +} diff --git a/man/seq_RT.Rd b/man/seq_RT.Rd index a242c0d0..74ebe44c 100644 --- a/man/seq_RT.Rd +++ b/man/seq_RT.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{seq_RT} -\alias{seq_RT} -\title{Function for generating sequences of runtime values} -\usage{ -seq_RT(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, - scale = "linear") -} -\arguments{ -\item{RT}{A list of runtime values} - -\item{from}{Starting runtime value. Will be replaced by min(RT) if it is NULL or too small} - -\item{to}{Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large} - -\item{by}{Stepsize of the sequence. Will be replaced if it is too small} - -\item{length.out}{Number of values in the sequence. -'by' takes preference if both it and length.out are provided.} - -\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a -linear or log-linear spacing respectively.} -} -\value{ -A sequence of runtime values -} -\description{ -Function for generating sequences of runtime values -} -\examples{ -RTall <- get_runtimes(dsl) -seq_RT(RTall, 0, 500, length.out=10, scale='log') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{seq_RT} +\alias{seq_RT} +\title{Function for generating sequences of runtime values} +\usage{ +seq_RT(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, + scale = "linear") +} +\arguments{ +\item{RT}{A list of runtime values} + +\item{from}{Starting runtime value. Will be replaced by min(RT) if it is NULL or too small} + +\item{to}{Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large} + +\item{by}{Stepsize of the sequence. Will be replaced if it is too small} + +\item{length.out}{Number of values in the sequence. +'by' takes preference if both it and length.out are provided.} + +\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a +linear or log-linear spacing respectively.} +} +\value{ +A sequence of runtime values +} +\description{ +Function for generating sequences of runtime values +} +\examples{ +RTall <- get_runtimes(dsl) +seq_RT(RTall, 0, 500, length.out=10, scale='log') +} diff --git a/man/set_DSC_credentials.Rd b/man/set_DSC_credentials.Rd index 5366817d..bd6b51ab 100644 --- a/man/set_DSC_credentials.Rd +++ b/man/set_DSC_credentials.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{set_DSC_credentials} -\alias{set_DSC_credentials} -\title{Register an account to the DSCtool API} -\usage{ -set_DSC_credentials(username, password) -} -\arguments{ -\item{username}{The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name'} - -\item{password}{The password you use on DSCtool. Will be stored on keyring under 'DSCtool'} -} -\description{ -This uses the keyring package to store and load credentials. -If you already have an account, please call `add_DSC_credentials` instead -} -\examples{ -\dontrun{set_DSC_credentials('jdoe', 'monkey123')} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{set_DSC_credentials} +\alias{set_DSC_credentials} +\title{Register an account to the DSCtool API} +\usage{ +set_DSC_credentials(username, password) +} +\arguments{ +\item{username}{The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name'} + +\item{password}{The password you use on DSCtool. Will be stored on keyring under 'DSCtool'} +} +\description{ +This uses the keyring package to store and load credentials. +If you already have an account, please call `add_DSC_credentials` instead +} +\examples{ +\dontrun{set_DSC_credentials('jdoe', 'monkey123')} +} diff --git a/man/set_color_scheme.Rd b/man/set_color_scheme.Rd index 51c1da0d..af55f1d2 100644 --- a/man/set_color_scheme.Rd +++ b/man/set_color_scheme.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{set_color_scheme} -\alias{set_color_scheme} -\title{Set the colorScheme of the IOHanalyzer plots} -\usage{ -set_color_scheme(schemename, ids, path = NULL) -} -\arguments{ -\item{schemename}{Three default colorschemes are implemented: -\itemize{ -\item Default -\item Variant 1 -\item Variant 2 -\item Variant 3 -} -And it is also possible to select "Custom", which allows uploading of a custom set of colors} - -\item{ids}{The names of the algorithms (or custom ids, see `change_id`) for which to set the colors} - -\item{path}{The path to the file containing the colors to use. Only used if -schemename is "Custom"} -} -\description{ -Set the colorScheme of the IOHanalyzer plots -} -\examples{ -set_color_scheme("Default", get_algId(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{set_color_scheme} +\alias{set_color_scheme} +\title{Set the colorScheme of the IOHanalyzer plots} +\usage{ +set_color_scheme(schemename, ids, path = NULL) +} +\arguments{ +\item{schemename}{Three default colorschemes are implemented: +\itemize{ +\item Default +\item Variant 1 +\item Variant 2 +\item Variant 3 +} +And it is also possible to select "Custom", which allows uploading of a custom set of colors} + +\item{ids}{The names of the algorithms (or custom ids, see `change_id`) for which to set the colors} + +\item{path}{The path to the file containing the colors to use. Only used if +schemename is "Custom"} +} +\description{ +Set the colorScheme of the IOHanalyzer plots +} +\examples{ +set_color_scheme("Default", get_algId(dsl)) +} diff --git a/man/sub-.DataSetList.Rd b/man/sub-.DataSetList.Rd index 2c79a2a1..227f217f 100644 --- a/man/sub-.DataSetList.Rd +++ b/man/sub-.DataSetList.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{[.DataSetList} -\alias{[.DataSetList} -\title{S3 extraction function for DataSetList} -\usage{ -\method{[}{DataSetList}(x, i, drop = FALSE) -} -\arguments{ -\item{x}{The DataSetList to use} - -\item{i}{The indices to extract} - -\item{drop}{Currently unused parameter} -} -\value{ -The DataSetList of the DataSets at indices i of DataSetList x -} -\description{ -S3 extraction function for DataSetList -} -\examples{ -dsl[c(1, 3)] -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{[.DataSetList} +\alias{[.DataSetList} +\title{S3 extraction function for DataSetList} +\usage{ +\method{[}{DataSetList}(x, i, drop = FALSE) +} +\arguments{ +\item{x}{The DataSetList to use} + +\item{i}{The indices to extract} + +\item{drop}{Currently unused parameter} +} +\value{ +The DataSetList of the DataSets at indices i of DataSetList x +} +\description{ +S3 extraction function for DataSetList +} +\examples{ +dsl[c(1, 3)] +} diff --git a/man/subset.DataSet.Rd b/man/subset.DataSet.Rd index 1d225720..0997c696 100644 --- a/man/subset.DataSet.Rd +++ b/man/subset.DataSet.Rd @@ -1,26 +1,26 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{subset.DataSet} -\alias{subset.DataSet} -\title{S3 subset function for DataSet} -\usage{ -\method{subset}{DataSet}(x, mask, ...) -} -\arguments{ -\item{x}{The DataSet from which to get a subset} - -\item{mask}{The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs -present in the provided dataset object x.} - -\item{...}{Arguments passed to underlying subset method (not yet supported)} -} -\value{ -A new DataSet -} -\description{ -Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet -and turned into a new DataSet object. -} -\examples{ -subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{subset.DataSet} +\alias{subset.DataSet} +\title{S3 subset function for DataSet} +\usage{ +\method{subset}{DataSet}(x, mask, ...) +} +\arguments{ +\item{x}{The DataSet from which to get a subset} + +\item{mask}{The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs +present in the provided dataset object x.} + +\item{...}{Arguments passed to underlying subset method (not yet supported)} +} +\value{ +A new DataSet +} +\description{ +Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet +and turned into a new DataSet object. +} +\examples{ +subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) +} diff --git a/man/subset.DataSetList.Rd b/man/subset.DataSetList.Rd index e9d36a79..ab325b8a 100644 --- a/man/subset.DataSetList.Rd +++ b/man/subset.DataSetList.Rd @@ -1,30 +1,30 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{subset.DataSetList} -\alias{subset.DataSetList} -\title{Filter a DataSetList by some criteria} -\usage{ -\method{subset}{DataSetList}(x, ...) -} -\arguments{ -\item{x}{The DataSetList} - -\item{...}{The conditions to filter on. Can be any expression which assigns True or False -to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes -(funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should -be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a -string to be parsed. This allows exectution of subset commands on arbitrary variables in code.} -} -\value{ -The filtered DataSetList -} -\description{ -Filter a DataSetList by some criteria -} -\examples{ -subset(dsl, funcId == 1) -subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes -subset(dsl, instance == 1) -subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes -subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{subset.DataSetList} +\alias{subset.DataSetList} +\title{Filter a DataSetList by some criteria} +\usage{ +\method{subset}{DataSetList}(x, ...) +} +\arguments{ +\item{x}{The DataSetList} + +\item{...}{The conditions to filter on. Can be any expression which assigns True or False +to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes +(funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should +be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a +string to be parsed. This allows exectution of subset commands on arbitrary variables in code.} +} +\value{ +The filtered DataSetList +} +\description{ +Filter a DataSetList by some criteria +} +\examples{ +subset(dsl, funcId == 1) +subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes +subset(dsl, instance == 1) +subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes +subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND +} diff --git a/man/summary.DataSet.Rd b/man/summary.DataSet.Rd index 65688097..f52f92e7 100644 --- a/man/summary.DataSet.Rd +++ b/man/summary.DataSet.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{summary.DataSet} -\alias{summary.DataSet} -\title{S3 generic summary operator for DataSet} -\usage{ -\method{summary}{DataSet}(object, ...) -} -\arguments{ -\item{object}{A DataSet object} - -\item{...}{Arguments passed to other methods} -} -\value{ -A summary of the DataSet containing both function-value and runtime based statistics. -} -\description{ -S3 generic summary operator for DataSet -} -\examples{ -summary(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{summary.DataSet} +\alias{summary.DataSet} +\title{S3 generic summary operator for DataSet} +\usage{ +\method{summary}{DataSet}(object, ...) +} +\arguments{ +\item{object}{A DataSet object} + +\item{...}{Arguments passed to other methods} +} +\value{ +A summary of the DataSet containing both function-value and runtime based statistics. +} +\description{ +S3 generic summary operator for DataSet +} +\examples{ +summary(dsl[[1]]) +} diff --git a/man/summary.DataSetList.Rd b/man/summary.DataSetList.Rd index a6a26e5d..401bec81 100644 --- a/man/summary.DataSetList.Rd +++ b/man/summary.DataSetList.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{summary.DataSetList} -\alias{summary.DataSetList} -\title{S3 summary function for DataSetList} -\usage{ -\method{summary}{DataSetList}(object, ...) -} -\arguments{ -\item{object}{The DataSetList to print} - -\item{...}{Arguments for underlying summary function?} -} -\description{ -Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every -DataSet in the DataSetList -} -\examples{ -summary(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{summary.DataSetList} +\alias{summary.DataSetList} +\title{S3 summary function for DataSetList} +\usage{ +\method{summary}{DataSetList}(object, ...) +} +\arguments{ +\item{object}{The DataSetList to print} + +\item{...}{Arguments for underlying summary function?} +} +\description{ +Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every +DataSet in the DataSetList +} +\examples{ +summary(dsl) +} diff --git a/src/align.cc b/src/align.cc index 4531c32e..0f1be787 100644 --- a/src/align.cc +++ b/src/align.cc @@ -1,146 +1,146 @@ -#include -#include -using namespace Rcpp; - -// [[Rcpp::export]] -NumericVector align_by_target_inner_loop(double t, int idxEvals, int idxTarget, -List data, NumericVector index, NumericMatrix next_lines, NumericVector curr_eval, bool maximization) { - - bool condition; - int n_row, n_col, iter, N; - N = data.size(); - NumericVector out = clone(curr_eval); - - for (int k = 0; k < N; k++) { - NumericMatrix d = as(data[k]); - n_row = d.nrow(); - n_col = d.ncol(); - iter = index[k]; - - while (true) { - if (maximization) { - condition = next_lines(k, idxTarget) >= t; - } else { - condition = next_lines(k, idxTarget) <= t; - } - - if (condition) { - out[k] = next_lines(k, idxEvals); - break; - } - - if (iter < (n_row - 1)) { - iter++; - for (int j = 0; j < n_col; j++) { - next_lines(k, j) = d(iter, j); - } - } else { - break; - } - } - index[k] = iter; - } - return out; -} - -// [[Rcpp::export]] -NumericVector c_impute(NumericVector x, NumericVector y, NumericVector rowname) { - int N = rowname.size(); - int L = x.size(); - NumericVector res(N, NA_REAL); - - int i = 0; - int j = 0; - while (i < N) { - if (j < (L - 1) && rowname[i] >= y[j]) { - if (rowname[i] < y[j + 1]) { - res[i] = x[j]; - ++j; - } else { - ++j; - continue; - } - // if the query value is larger than the last runtime value `y[L - 1]` - } else if (j == L - 1 && rowname[i] >= y[j]) { - res[i] = x[j]; - // take the previous function value if the next runtime in y[j] is not reached - } else if (rowname[i] < y[j] && j > 0) { - res[i] = x[j - 1]; - } - ++i; - } - return res; -} - -// [[Rcpp::export]] -NumericMatrix c_impute_running_time(NumericVector index, // index value recorded - NumericMatrix value, // value to align: runtime + parameters - NumericVector FV, // function values to match - bool maximization // the data are collected from a maximization problem? - ) { - int N = FV.size(); - int L = index.size(); - int M = value.ncol(); - NumericMatrix res(N, M); - std::fill(res.begin(), res.end(), NA_REAL); - - bool condition; - int j = 0; - for (int i = 0; i < N; ++i) { - while (j < L) { - if (maximization) { - condition = FV[i] > index[j]; - } else { - condition = FV[i] < index[j]; - } - - if (condition) { - ++j; - } else { - res(i, _) = value(j, _); - break; - } - } - } - return res; -} - -// TODO: Better comments -//' Align a list of data set by function values -//' -//' @param data the data -//' @param FV Function values -//' @param idxValue index of the function values -//' @param maximization Boolean -//' @param idxTarget index of the target -//' @noRd -// [[Rcpp::export]] -List c_align_running_time(List data, NumericVector FV, NumericVector idxValue, bool maximization, int idxTarget) { - int NC = data.size(); - int NR = FV.size(); - int M = idxValue.size(); - - List res(M); - for (int i = 0; i < M; i++) { - NumericMatrix aux(NR, NC); - rownames(aux) = FV; - res[i] = clone(aux); - } - - for (int i = 0; i < NC; i++) { - NumericMatrix d = data[i]; - NumericMatrix value(d.nrow(), M); - - for (int j = 0; j < M; j++) { - value(_, j) = d(_, idxValue[j]); - } - - NumericMatrix tmp = c_impute_running_time(d(_, idxTarget), value, FV, maximization); - - for (int k = 0; k < M; k++) { - NumericMatrix aux = res[k]; - aux(_, i) = tmp(_, k); - } - } - return res; -} +#include +#include +using namespace Rcpp; + +// [[Rcpp::export]] +NumericVector align_by_target_inner_loop(double t, int idxEvals, int idxTarget, +List data, NumericVector index, NumericMatrix next_lines, NumericVector curr_eval, bool maximization) { + + bool condition; + int n_row, n_col, iter, N; + N = data.size(); + NumericVector out = clone(curr_eval); + + for (int k = 0; k < N; k++) { + NumericMatrix d = as(data[k]); + n_row = d.nrow(); + n_col = d.ncol(); + iter = index[k]; + + while (true) { + if (maximization) { + condition = next_lines(k, idxTarget) >= t; + } else { + condition = next_lines(k, idxTarget) <= t; + } + + if (condition) { + out[k] = next_lines(k, idxEvals); + break; + } + + if (iter < (n_row - 1)) { + iter++; + for (int j = 0; j < n_col; j++) { + next_lines(k, j) = d(iter, j); + } + } else { + break; + } + } + index[k] = iter; + } + return out; +} + +// [[Rcpp::export]] +NumericVector c_impute(NumericVector x, NumericVector y, NumericVector rowname) { + int N = rowname.size(); + int L = x.size(); + NumericVector res(N, NA_REAL); + + int i = 0; + int j = 0; + while (i < N) { + if (j < (L - 1) && rowname[i] >= y[j]) { + if (rowname[i] < y[j + 1]) { + res[i] = x[j]; + ++j; + } else { + ++j; + continue; + } + // if the query value is larger than the last runtime value `y[L - 1]` + } else if (j == L - 1 && rowname[i] >= y[j]) { + res[i] = x[j]; + // take the previous function value if the next runtime in y[j] is not reached + } else if (rowname[i] < y[j] && j > 0) { + res[i] = x[j - 1]; + } + ++i; + } + return res; +} + +// [[Rcpp::export]] +NumericMatrix c_impute_running_time(NumericVector index, // index value recorded + NumericMatrix value, // value to align: runtime + parameters + NumericVector FV, // function values to match + bool maximization // the data are collected from a maximization problem? + ) { + int N = FV.size(); + int L = index.size(); + int M = value.ncol(); + NumericMatrix res(N, M); + std::fill(res.begin(), res.end(), NA_REAL); + + bool condition; + int j = 0; + for (int i = 0; i < N; ++i) { + while (j < L) { + if (maximization) { + condition = FV[i] > index[j]; + } else { + condition = FV[i] < index[j]; + } + + if (condition) { + ++j; + } else { + res(i, _) = value(j, _); + break; + } + } + } + return res; +} + +// TODO: Better comments +//' Align a list of data set by function values +//' +//' @param data the data +//' @param FV Function values +//' @param idxValue index of the function values +//' @param maximization Boolean +//' @param idxTarget index of the target +//' @noRd +// [[Rcpp::export]] +List c_align_running_time(List data, NumericVector FV, NumericVector idxValue, bool maximization, int idxTarget) { + int NC = data.size(); + int NR = FV.size(); + int M = idxValue.size(); + + List res(M); + for (int i = 0; i < M; i++) { + NumericMatrix aux(NR, NC); + rownames(aux) = FV; + res[i] = clone(aux); + } + + for (int i = 0; i < NC; i++) { + NumericMatrix d = data[i]; + NumericMatrix value(d.nrow(), M); + + for (int j = 0; j < M; j++) { + value(_, j) = d(_, idxValue[j]); + } + + NumericMatrix tmp = c_impute_running_time(d(_, idxTarget), value, FV, maximization); + + for (int k = 0; k < M; k++) { + NumericMatrix aux = res[k]; + aux(_, i) = tmp(_, k); + } + } + return res; +} diff --git a/src/read.cc b/src/read.cc index 9ff229e3..7eb4483d 100644 --- a/src/read.cc +++ b/src/read.cc @@ -1,68 +1,68 @@ -// [[Rcpp::plugins(cpp11)]] -#include -#include -#include -#include -#include -#include - -using namespace Rcpp; -using namespace std; - -Rcpp::NumericMatrix make_mat(Rcpp::List input_list){ - - unsigned int n = input_list.length(); - - if(n == 0) { - Rcpp::stop("Must supply a list with more than 1 element."); - } - - Rcpp::NumericVector testvals = input_list[0]; - unsigned int elems = testvals.length(); - - Rcpp::NumericMatrix result_mat = Rcpp::no_init(n, elems); - - for(unsigned int i = 0; i < n; i++) { - Rcpp::NumericVector row_val = input_list[i]; - - if(elems != (unsigned int)row_val.length()) { - Rcpp::stop("Length of row does not match matrix requirements"); - } - - result_mat(i, Rcpp::_) = row_val; - } - - return result_mat; -} - -// [[Rcpp::export]] -List c_read_dat(std::string dat, int NC, std::string leading) { - int i; - std::string line, val, replacement (" "); - std::ifstream fp(dat.c_str()); - std::regex re("\\s+|\t+"); // in case of multiple whitespaces or tab - - List dataSets, one_run; - NumericVector row(NC); - - while (std::getline(fp, line)) { - if (line.find(leading) == 0) { - if (one_run.size() != 0) { - dataSets.push_back(make_mat(one_run)); - } - one_run = Rcpp::List::create(); - continue; - } - - std::stringstream ss(line); - - for (i = 0; i < NC; ++i) { - std::getline(ss, val, ' '); - row[i] = std::stod(val); - } - - one_run.push_back(Rcpp::clone(row)); - } - dataSets.push_back(make_mat(one_run)); - return dataSets; -} +// [[Rcpp::plugins(cpp11)]] +#include +#include +#include +#include +#include +#include + +using namespace Rcpp; +using namespace std; + +Rcpp::NumericMatrix make_mat(Rcpp::List input_list){ + + unsigned int n = input_list.length(); + + if(n == 0) { + Rcpp::stop("Must supply a list with more than 1 element."); + } + + Rcpp::NumericVector testvals = input_list[0]; + unsigned int elems = testvals.length(); + + Rcpp::NumericMatrix result_mat = Rcpp::no_init(n, elems); + + for(unsigned int i = 0; i < n; i++) { + Rcpp::NumericVector row_val = input_list[i]; + + if(elems != (unsigned int)row_val.length()) { + Rcpp::stop("Length of row does not match matrix requirements"); + } + + result_mat(i, Rcpp::_) = row_val; + } + + return result_mat; +} + +// [[Rcpp::export]] +List c_read_dat(std::string dat, int NC, std::string leading) { + int i; + std::string line, val, replacement (" "); + std::ifstream fp(dat.c_str()); + std::regex re("\\s+|\t+"); // in case of multiple whitespaces or tab + + List dataSets, one_run; + NumericVector row(NC); + + while (std::getline(fp, line)) { + if (line.find(leading) == 0) { + if (one_run.size() != 0) { + dataSets.push_back(make_mat(one_run)); + } + one_run = Rcpp::List::create(); + continue; + } + + std::stringstream ss(line); + + for (i = 0; i < NC; ++i) { + std::getline(ss, val, ' '); + row[i] = std::stod(val); + } + + one_run.push_back(Rcpp::clone(row)); + } + dataSets.push_back(make_mat(one_run)); + return dataSets; +} diff --git a/tests/testthat.R b/tests/testthat.R index 121c9107..cc4f02e3 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -1,4 +1,4 @@ -library(testthat) -library(IOHanalyzer) - -test_check("IOHanalyzer") +library(testthat) +library(IOHanalyzer) + +test_check("IOHanalyzer") diff --git a/tests/testthat/test_DataSetList.R b/tests/testthat/test_DataSetList.R index e8863565..c93d42be 100644 --- a/tests/testthat/test_DataSetList.R +++ b/tests/testthat/test_DataSetList.R @@ -1,47 +1,47 @@ -library(testthat) -library(IOHanalyzer) - -context("Basic DataSetList functionality") - -test_that("Can DataSetLists be loaded?",{ - expect_true(any(match(class(dsl), "DataSetList"))) - expect_true(any(match(class(dsl[[1]]), "DataSet"))) -}) - -test_that("Validate reading of files",{ - path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") - expect_equal(check_format(path), "IOHprofiler") - dsl1 <- DataSetList(path) - - expect_true(any(match(class(dsl1), "DataSetList"))) - expect_equal(get_dim(dsl1), 100) - expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") - expect_equal(get_funcId(dsl1), 1) - - ds1 <- dsl1[[1]] - expect_true(any(match(class(ds1), "DataSet"))) - expect_equal(attr(ds1,'DIM'), 100) - expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") - expect_equal(attr(ds1,'suite'), "PBO") - expect_equal(attr(ds1,'funcId'), 1) - expect_true(all(attr(ds1,'instance') == 1)) - expect_equal(attr(ds1,'format'), "IOHprofiler") - expect_equal(attr(ds1,'maximization'), TRUE) -}) - -test_that("Validate overview, summary and sample functions",{ - expect_equal(get_FV_summary(dsl[1], 12)$"98%", 14) - expect_equal(get_FV_summary(dsl[[1]], 12)$"98%", 14) - expect_equal(get_RT_summary(dsl[1], 12)$"98%", 21) - expect_equal(get_RT_summary(dsl[[1]], 12)$"98%", 21) - expect_equal(get_FV_overview(dsl[1])$"mean reached", 16) - expect_equal(get_FV_overview(dsl[[1]])$"mean reached", 16) - expect_equal(get_RT_overview(dsl[[1]])$"runs", 11) - expect_equal(get_RT_overview(dsl[1])$"runs", 11) - expect_equal(get_FV_sample(dsl[[1]],12)$"run.5",13) - expect_equal(get_FV_sample(dsl[1],12)$"run.5",13) - expect_equal(get_RT_sample(dsl[[1]],12)$"run.5",10) - expect_equal(get_RT_sample(dsl[1],12)$"run.5",10) - expect_equal(min(get_funvals(dsl[1])),5) - expect_equal(min(get_runtimes(dsl[1])),1) -}) +library(testthat) +library(IOHanalyzer) + +context("Basic DataSetList functionality") + +test_that("Can DataSetLists be loaded?",{ + expect_true(any(match(class(dsl), "DataSetList"))) + expect_true(any(match(class(dsl[[1]]), "DataSet"))) +}) + +test_that("Validate reading of files",{ + path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") + expect_equal(check_format(path), "IOHprofiler") + dsl1 <- DataSetList(path) + + expect_true(any(match(class(dsl1), "DataSetList"))) + expect_equal(get_dim(dsl1), 100) + expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") + expect_equal(get_funcId(dsl1), 1) + + ds1 <- dsl1[[1]] + expect_true(any(match(class(ds1), "DataSet"))) + expect_equal(attr(ds1,'DIM'), 100) + expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") + expect_equal(attr(ds1,'suite'), "PBO") + expect_equal(attr(ds1,'funcId'), 1) + expect_true(all(attr(ds1,'instance') == 1)) + expect_equal(attr(ds1,'format'), "IOHprofiler") + expect_equal(attr(ds1,'maximization'), TRUE) +}) + +test_that("Validate overview, summary and sample functions",{ + expect_equal(get_FV_summary(dsl[1], 12)$"98%", 14) + expect_equal(get_FV_summary(dsl[[1]], 12)$"98%", 14) + expect_equal(get_RT_summary(dsl[1], 12)$"98%", 21) + expect_equal(get_RT_summary(dsl[[1]], 12)$"98%", 21) + expect_equal(get_FV_overview(dsl[1])$"mean reached", 16) + expect_equal(get_FV_overview(dsl[[1]])$"mean reached", 16) + expect_equal(get_RT_overview(dsl[[1]])$"runs", 11) + expect_equal(get_RT_overview(dsl[1])$"runs", 11) + expect_equal(get_FV_sample(dsl[[1]],12)$"run.5",13) + expect_equal(get_FV_sample(dsl[1],12)$"run.5",13) + expect_equal(get_RT_sample(dsl[[1]],12)$"run.5",10) + expect_equal(get_RT_sample(dsl[1],12)$"run.5",10) + expect_equal(min(get_funvals(dsl[1])),5) + expect_equal(min(get_runtimes(dsl[1])),1) +}) diff --git a/tests/testthat/test_DataSetList_named_with_strings.R b/tests/testthat/test_DataSetList_named_with_strings.R index bca267fe..a6aab6a8 100644 --- a/tests/testthat/test_DataSetList_named_with_strings.R +++ b/tests/testthat/test_DataSetList_named_with_strings.R @@ -1,62 +1,62 @@ -library(testthat) -library(IOHanalyzer) - -context("Basic DataSetList functionality, but with String-Named Functions") - - - -test_that("Validate reading of files",{ - path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") - expect_equal(check_format(path1), "IOHprofiler") - dsl1 <- DataSetList(path1) - - expect_true(any(match(class(dsl1), "DataSetList"))) - expect_equal(get_dim(dsl1), 100) - expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") - expect_equal(get_funcId(dsl1), c('bla', 'blubb')) - - ds1 <- dsl1[[1]] - expect_true(any(match(class(ds1), "DataSet"))) - expect_equal(attr(ds1,'DIM'), 100) - expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") - #expect_equal(attr(ds1,'suite'), "PBO") - expect_equal(attr(ds1,'funcId'), 'bla') - expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) - expect_equal(attr(ds1,'format'), "IOHprofiler") - expect_equal(attr(ds1,'maximization'), TRUE) - - ds1 <- dsl1[[2]] - expect_true(any(match(class(ds1), "DataSet"))) - expect_equal(attr(ds1,'DIM'), 100) - expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") - #expect_equal(attr(ds1,'suite'), "PBO") - expect_equal(attr(ds1,'funcId'), 'blubb') - expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) - expect_equal(attr(ds1,'format'), "IOHprofiler") - expect_equal(attr(ds1,'maximization'), TRUE) -}) - -test_that("Validate overview, summary and sample functions",{ - path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") - expect_equal(check_format(path1), "IOHprofiler") - dsl1 <- DataSetList(path1) - - path2 <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") - expect_equal(check_format(path2), "IOHprofiler") - dsl2 <- DataSetList(path2) - - expect_equal(get_FV_summary(dsl1[1], 12)$"98%", get_FV_summary(dsl2[1], 12)$"98%") - expect_equal(get_FV_summary(dsl1[[1]], 12)$"98%", get_FV_summary(dsl2[[1]], 12)$"98%") - expect_equal(get_RT_summary(dsl1[1], 12)$"98%", get_RT_summary(dsl2[1], 12)$"98%") - expect_equal(get_RT_summary(dsl1[[1]], 12)$"98%", get_RT_summary(dsl2[[1]], 12)$"98%") - expect_equal(get_FV_overview(dsl1[1])$"mean reached", get_FV_overview(dsl2[1])$"mean reached") - expect_equal(get_FV_overview(dsl1[[1]])$"mean reached", get_FV_overview(dsl2[[1]])$"mean reached") - expect_equal(get_RT_overview(dsl1[[1]])$"runs", get_RT_overview(dsl2[[1]])$"runs") - expect_equal(get_RT_overview(dsl1[1])$"runs", get_RT_overview(dsl2[1])$"runs") - expect_equal(get_FV_sample(dsl1[[1]],12)$"run.5",get_FV_sample(dsl2[[1]],12)$"run.5") - expect_equal(get_FV_sample(dsl1[1],12)$"run.5",get_FV_sample(dsl2[1],12)$"run.5") - expect_equal(get_RT_sample(dsl1[[1]],12)$"run.5",get_RT_sample(dsl2[[1]],12)$"run.5") - expect_equal(get_RT_sample(dsl1[1],12)$"run.5",get_RT_sample(dsl2[1],12)$"run.5") - expect_equal(min(get_funvals(dsl1[1])),min(get_funvals(dsl2[1]))) - expect_equal(min(get_runtimes(dsl1[1])),min(get_runtimes(dsl2[1]))) -}) +library(testthat) +library(IOHanalyzer) + +context("Basic DataSetList functionality, but with String-Named Functions") + + + +test_that("Validate reading of files",{ + path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") + expect_equal(check_format(path1), "IOHprofiler") + dsl1 <- DataSetList(path1) + + expect_true(any(match(class(dsl1), "DataSetList"))) + expect_equal(get_dim(dsl1), 100) + expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") + expect_equal(get_funcId(dsl1), c('bla', 'blubb')) + + ds1 <- dsl1[[1]] + expect_true(any(match(class(ds1), "DataSet"))) + expect_equal(attr(ds1,'DIM'), 100) + expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") + #expect_equal(attr(ds1,'suite'), "PBO") + expect_equal(attr(ds1,'funcId'), 'bla') + expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) + expect_equal(attr(ds1,'format'), "IOHprofiler") + expect_equal(attr(ds1,'maximization'), TRUE) + + ds1 <- dsl1[[2]] + expect_true(any(match(class(ds1), "DataSet"))) + expect_equal(attr(ds1,'DIM'), 100) + expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") + #expect_equal(attr(ds1,'suite'), "PBO") + expect_equal(attr(ds1,'funcId'), 'blubb') + expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) + expect_equal(attr(ds1,'format'), "IOHprofiler") + expect_equal(attr(ds1,'maximization'), TRUE) +}) + +test_that("Validate overview, summary and sample functions",{ + path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") + expect_equal(check_format(path1), "IOHprofiler") + dsl1 <- DataSetList(path1) + + path2 <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") + expect_equal(check_format(path2), "IOHprofiler") + dsl2 <- DataSetList(path2) + + expect_equal(get_FV_summary(dsl1[1], 12)$"98%", get_FV_summary(dsl2[1], 12)$"98%") + expect_equal(get_FV_summary(dsl1[[1]], 12)$"98%", get_FV_summary(dsl2[[1]], 12)$"98%") + expect_equal(get_RT_summary(dsl1[1], 12)$"98%", get_RT_summary(dsl2[1], 12)$"98%") + expect_equal(get_RT_summary(dsl1[[1]], 12)$"98%", get_RT_summary(dsl2[[1]], 12)$"98%") + expect_equal(get_FV_overview(dsl1[1])$"mean reached", get_FV_overview(dsl2[1])$"mean reached") + expect_equal(get_FV_overview(dsl1[[1]])$"mean reached", get_FV_overview(dsl2[[1]])$"mean reached") + expect_equal(get_RT_overview(dsl1[[1]])$"runs", get_RT_overview(dsl2[[1]])$"runs") + expect_equal(get_RT_overview(dsl1[1])$"runs", get_RT_overview(dsl2[1])$"runs") + expect_equal(get_FV_sample(dsl1[[1]],12)$"run.5",get_FV_sample(dsl2[[1]],12)$"run.5") + expect_equal(get_FV_sample(dsl1[1],12)$"run.5",get_FV_sample(dsl2[1],12)$"run.5") + expect_equal(get_RT_sample(dsl1[[1]],12)$"run.5",get_RT_sample(dsl2[[1]],12)$"run.5") + expect_equal(get_RT_sample(dsl1[1],12)$"run.5",get_RT_sample(dsl2[1],12)$"run.5") + expect_equal(min(get_funvals(dsl1[1])),min(get_funvals(dsl2[1]))) + expect_equal(min(get_runtimes(dsl1[1])),min(get_runtimes(dsl2[1]))) +}) diff --git a/tests/testthat/test_data_generation.R b/tests/testthat/test_data_generation.R index 66a98b67..221182f4 100644 --- a/tests/testthat/test_data_generation.R +++ b/tests/testthat/test_data_generation.R @@ -1,116 +1,116 @@ -context("Data generation for plotting") - -test_that("Single-function info",{ - dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "ID", "target", "ERT", - "lower", "upper", "mean", "median") %in% colnames(dt))) - - dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "ID", "runtime", - "lower", "upper", "mean", "median") %in% colnames(dt))) - expect_false("ERT" %in% colnames(dt)) - expect_error(generate_data.Single_Function(dsl)) -}) - -test_that("PMF-data", { - dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "ID", "target", "RT") %in% colnames(dt))) - expect_false("f(x)" %in% colnames(dt)) - - dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 100, which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "ID", "runtime", "f(x)") %in% colnames(dt))) - expect_false("RT" %in% colnames(dt)) - expect_error(generate_data.PMF(dsl)) -}) - -test_that("Histogram-data", { - dt <- generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) - - dt <- generate_data.hist(subset(dsl, funcId == 1), target = 100, which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) - expect_error(generate_data.hist(dsl)) -}) - -test_that("ECDF-data (single function)", { - dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) - - dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) - expect_error(generate_data.ECDF(dsl, c(10, 15, 16))) -}) - -test_that("ECDF-data (multiple functions, auto-generated targets)", { - targets <- get_ECDF_targets(dsl, 'linear', 3) - expect_true(is.data.table(targets)) - - dt <- generate_data.ECDF(dsl, targets) - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) - - dt <- generate_data.ECDF(dsl, c(1, 10, 100), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) -}) - -test_that("AUC-data", { - dt <- generate_data.AUC(subset(dsl, funcId == 1), c(10, 13, 16)) - expect_true(is.data.table(dt)) - expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) - expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) - - dt <- generate_data.AUC(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) - expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) - - dt <- generate_data.AUC(dsl, get_ECDF_targets(dsl)) - expect_true(is.data.table(dt)) - subset(dsl, funcId == 1) - expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) -}) - -test_that("Parameter-data", { - dt <- generate_data.Parameters(subset(dsl, funcId == 1)) - expect_true(is.data.table(dt)) - expect_true(all(c("ID", "runtime", "parId", - "lower", "upper", "mean", "median") %in% colnames(dt))) - - dt <- generate_data.Parameters(subset(dsl, funcId == 1), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("ID", "target", "parId", - "lower", "upper", "mean", "median") %in% colnames(dt))) - - expect_error(generate_data.Parameters(dsl)) -}) - -test_that("Aggregated data for multiple functions / dimensions", { - targets <- get_target_dt(dsl) - expect_true(is.data.table(targets)) - expect_true(all(c("funcId", "DIM", "target") %in% colnames(targets))) - - dt <- generate_data.Aggr(dsl, targets = targets) - expect_true(is.data.table(dt)) - expect_true(all(c("ID", "target", "rank", "DIM", "funcId", "value", - "median") %in% colnames(dt))) - - dt <- generate_data.Aggr(dsl, which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("ID", "runtime", "rank", "DIM", "funcId", "value", - "median") %in% colnames(dt))) - - expect_error(generate_data.Aggr(dsl, targets = c(12, 16))) -}) +context("Data generation for plotting") + +test_that("Single-function info",{ + dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "target", "ERT", + "lower", "upper", "mean", "median") %in% colnames(dt))) + + dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "runtime", + "lower", "upper", "mean", "median") %in% colnames(dt))) + expect_false("ERT" %in% colnames(dt)) + expect_error(generate_data.Single_Function(dsl)) +}) + +test_that("PMF-data", { + dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "target", "RT") %in% colnames(dt))) + expect_false("f(x)" %in% colnames(dt)) + + dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 100, which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "runtime", "f(x)") %in% colnames(dt))) + expect_false("RT" %in% colnames(dt)) + expect_error(generate_data.PMF(dsl)) +}) + +test_that("Histogram-data", { + dt <- generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) + + dt <- generate_data.hist(subset(dsl, funcId == 1), target = 100, which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) + expect_error(generate_data.hist(dsl)) +}) + +test_that("ECDF-data (single function)", { + dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) + + dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) + expect_error(generate_data.ECDF(dsl, c(10, 15, 16))) +}) + +test_that("ECDF-data (multiple functions, auto-generated targets)", { + targets <- get_ECDF_targets(dsl, 'linear', 3) + expect_true(is.data.table(targets)) + + dt <- generate_data.ECDF(dsl, targets) + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) + + dt <- generate_data.ECDF(dsl, c(1, 10, 100), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) +}) + +test_that("AUC-data", { + dt <- generate_data.AUC(subset(dsl, funcId == 1), c(10, 13, 16)) + expect_true(is.data.table(dt)) + expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) + expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) + + dt <- generate_data.AUC(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) + expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) + + dt <- generate_data.AUC(dsl, get_ECDF_targets(dsl)) + expect_true(is.data.table(dt)) + subset(dsl, funcId == 1) + expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) +}) + +test_that("Parameter-data", { + dt <- generate_data.Parameters(subset(dsl, funcId == 1)) + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "runtime", "parId", + "lower", "upper", "mean", "median") %in% colnames(dt))) + + dt <- generate_data.Parameters(subset(dsl, funcId == 1), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "target", "parId", + "lower", "upper", "mean", "median") %in% colnames(dt))) + + expect_error(generate_data.Parameters(dsl)) +}) + +test_that("Aggregated data for multiple functions / dimensions", { + targets <- get_target_dt(dsl) + expect_true(is.data.table(targets)) + expect_true(all(c("funcId", "DIM", "target") %in% colnames(targets))) + + dt <- generate_data.Aggr(dsl, targets = targets) + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "target", "rank", "DIM", "funcId", "value", + "median") %in% colnames(dt))) + + dt <- generate_data.Aggr(dsl, which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "runtime", "rank", "DIM", "funcId", "value", + "median") %in% colnames(dt))) + + expect_error(generate_data.Aggr(dsl, targets = c(12, 16))) +}) diff --git a/tests/testthat/test_diagram_examples_with_string_names.R b/tests/testthat/test_diagram_examples_with_string_names.R index e554cfa7..c0a203e3 100644 --- a/tests/testthat/test_diagram_examples_with_string_names.R +++ b/tests/testthat/test_diagram_examples_with_string_names.R @@ -1,36 +1,36 @@ -library(testthat) -library(IOHanalyzer) - -context("Test the Diagram Examples with String Names") - - - -test_that("Test the diagram examples with string names",{ - path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") - expect_equal(check_format(path1), "IOHprofiler") - dsl1 <- DataSetList(path1) - - Plot.RT.Single_Func(subset(dsl1, funcId == 'bla')) - Plot.FV.Single_Func(subset(dsl1, funcId == 'bla')) - Plot.RT.PMF(subset(dsl1, funcId == 'bla'), 14) - Plot.RT.Histogram(subset(dsl1, funcId == 'bla'), 14) - Plot.RT.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 14) - Plot.RT.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) - suppressWarnings(Plot.RT.ECDF_AUC(subset(dsl1, funcId == 'bla'))) - Plot.FV.PDF(subset(dsl1, funcId == 'bla'), 100) - Plot.FV.Histogram(subset(dsl1, funcId == 'bla'), 100) - Plot.FV.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 10) - Plot.FV.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) - suppressWarnings(Plot.FV.ECDF_AUC(subset(dsl1, funcId == 'bla'))) - Plot.RT.Parameters(subset(dsl1, funcId == 'bla')) - Plot.FV.Parameters(subset(dsl1, funcId == 'bla')) - Plot.RT.ECDF_Multi_Func(dsl1) - Plot.RT.Multi_Func(dsl1) - Plot.RT.Aggregated(dsl1) - Plot.FV.Aggregated(dsl1) - Plot.FV.Multi_Func(dsl1) - Plot.Stats.Significance_Heatmap(subset(dsl1, funcId == 'bla'), 16) - Plot.Stats.Significance_Graph(subset(dsl1, funcId == 'bla'), 16) - #this will fail also with the original data, since only one alg present - #Plot.Stats.Glicko2_Candlestick(dsl1, nr_rounds=2) -}) +library(testthat) +library(IOHanalyzer) + +context("Test the Diagram Examples with String Names") + + + +test_that("Test the diagram examples with string names",{ + path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") + expect_equal(check_format(path1), "IOHprofiler") + dsl1 <- DataSetList(path1) + + Plot.RT.Single_Func(subset(dsl1, funcId == 'bla')) + Plot.FV.Single_Func(subset(dsl1, funcId == 'bla')) + Plot.RT.PMF(subset(dsl1, funcId == 'bla'), 14) + Plot.RT.Histogram(subset(dsl1, funcId == 'bla'), 14) + Plot.RT.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 14) + Plot.RT.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) + suppressWarnings(Plot.RT.ECDF_AUC(subset(dsl1, funcId == 'bla'))) + Plot.FV.PDF(subset(dsl1, funcId == 'bla'), 100) + Plot.FV.Histogram(subset(dsl1, funcId == 'bla'), 100) + Plot.FV.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 10) + Plot.FV.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) + suppressWarnings(Plot.FV.ECDF_AUC(subset(dsl1, funcId == 'bla'))) + Plot.RT.Parameters(subset(dsl1, funcId == 'bla')) + Plot.FV.Parameters(subset(dsl1, funcId == 'bla')) + Plot.RT.ECDF_Multi_Func(dsl1) + Plot.RT.Multi_Func(dsl1) + Plot.RT.Aggregated(dsl1) + Plot.FV.Aggregated(dsl1) + Plot.FV.Multi_Func(dsl1) + Plot.Stats.Significance_Heatmap(subset(dsl1, funcId == 'bla'), 16) + Plot.Stats.Significance_Graph(subset(dsl1, funcId == 'bla'), 16) + #this will fail also with the original data, since only one alg present + #Plot.Stats.Glicko2_Candlestick(dsl1, nr_rounds=2) +}) From 7f202da749685f35ffb9590151824ff4d2addb2c Mon Sep 17 00:00:00 2001 From: Dvermetten Date: Wed, 18 Aug 2021 14:17:30 +0200 Subject: [PATCH 27/28] Minor updates to json-processing --- R/readFiles.R | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/R/readFiles.R b/R/readFiles.R index 550d1138..ef00ca06 100644 --- a/R/readFiles.R +++ b/R/readFiles.R @@ -60,12 +60,9 @@ read_index_file <- function(fname) { read_index_file__json <- function(fname) { json_data <- fromJSON(file = fname) - attribute_names <- attributes(json_data) - - exp_attrs <- json_data$experiment_attributes + exp_attrs <- sapply(json_data$experiment_attributes, function(x) {x}) data <- list() - tryCatch({ fid <- json_data$function_id fname <- json_data$function_name @@ -74,12 +71,14 @@ read_index_file__json <- function(fname) { algid <- json_data$algorithm$name }, error = function(e) {return(NULL)}) - data <- lapply(json_data$dimensions, function(scenario) { + data <- lapply(json_data$scenarios, function(scenario) { + run_attrs <- list() for (run_attr in json_data$run_attributes) { attr(run_attrs, run_attr) <- sapply(scenario$runs, function(x) x$run_attr) } + temp <- c(list( funcId = fid, @@ -93,8 +92,8 @@ read_index_file__json <- function(fname) { maxRT = sapply(scenario$runs, function(x) x$evals), finalFV = sapply(scenario$runs, function(x) x$best$y), final_pos = sapply(scenario$runs, function(x) x$best$x) - ), exp_attrs, - run_attrs) + ), run_attrs, + exp_attrs) }) data From ee77413e9c0eb693cc877addba047bfdd36a1772 Mon Sep 17 00:00:00 2001 From: DVermetten Date: Wed, 17 Aug 2022 13:27:56 +0200 Subject: [PATCH 28/28] Update datapath attribute when reading json file --- R/readFiles.R | 110 +++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/R/readFiles.R b/R/readFiles.R index ef00ca06..78348df6 100644 --- a/R/readFiles.R +++ b/R/readFiles.R @@ -40,7 +40,7 @@ scan_index_file <- function(folder) { #' info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) read_index_file <- function(fname) { format <- tools::file_ext(fname) - if (format == 'json') + if (format == 'json') read_index_file__json(fname) else { tryCatch( @@ -58,10 +58,11 @@ read_index_file <- function(fname) { #' @return The data contained in the json info-file #' @noRd read_index_file__json <- function(fname) { - + json_data <- fromJSON(file = fname) + base_dir <- dirname(fname) exp_attrs <- sapply(json_data$experiment_attributes, function(x) {x}) - + data <- list() tryCatch({ fid <- json_data$function_id @@ -70,16 +71,17 @@ read_index_file__json <- function(fname) { maximization <- json_data$maximization algid <- json_data$algorithm$name }, error = function(e) {return(NULL)}) - + data <- lapply(json_data$scenarios, function(scenario) { - + run_attrs <- list() - + for (run_attr in json_data$run_attributes) { attr(run_attrs, run_attr) <- sapply(scenario$runs, function(x) x$run_attr) } - + datafile <- file.path(base_dir, scenario$path) + temp <- c(list( funcId = fid, funcName = fname, @@ -87,14 +89,14 @@ read_index_file__json <- function(fname) { maximization = maximization, algId = algid, DIM = scenario$dimension, - datafile = scenario$path, + datafile = datafile, instance = sapply(scenario$runs, function(x) x$instance), maxRT = sapply(scenario$runs, function(x) x$evals), finalFV = sapply(scenario$runs, function(x) x$best$y), final_pos = sapply(scenario$runs, function(x) x$best$x) - ), run_attrs, + ), run_attrs, exp_attrs) - + }) data } @@ -135,12 +137,12 @@ read_index_file__IOH <- function(fname) { for (name in .[1, ]) { value <- ans[[name]] ans[[name]] <- gsub("'", '', value) - + if (name == 'maximization') value <- as.logical(value) else value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - + if (!is.na(value)) ans[[name]] <- value } ans @@ -156,7 +158,7 @@ read_index_file__IOH <- function(fname) { maxRTs <- NULL } else { res <- matrix(unlist(strsplit(record[-1], ':')), nrow = 2) - info <- matrix(unlist(strsplit(res[2, ], '\\|')), nrow = 2) + info <- matrix(unlist(strsplit(res[2, ], '\\|')), nrow = 2) #Check for incorrect usages of reset_problem and remove them maxRTs <- as.numeric(info[1,]) idx_correct <- which(maxRTs > 0) @@ -164,7 +166,7 @@ read_index_file__IOH <- function(fname) { instances <- as.numeric(res[1,])[idx_correct] maxRTs <- maxRTs[idx_correct] } - + record[1] <- gsub("\\\\", "/", record[1]) datafile <- file.path(path, record[1]) @@ -196,14 +198,14 @@ read_index_file__COCO <- function(fname) { data <- list() i <- 1 while (TRUE) { - + lines <- suppressWarnings(readLines(f, n = 3)) # read header and comments if (length(lines) < 3) { break } comment <- lines[2] name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) - + header <- trimws(name_value) %>% { regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' } %>% @@ -221,11 +223,11 @@ read_index_file__COCO <- function(fname) { } ans } - + names(header) <- gsub('algorithm', 'algId', names(header)) - + record <- strsplit(lines[3], ',')[[1]] %>% trimws - + if (length(record) < 2) { warning(sprintf('File %s is incomplete!', fname)) res <- NULL @@ -234,14 +236,14 @@ read_index_file__COCO <- function(fname) { res <- matrix(unlist(strsplit(record[-c(1)], ':')), nrow = 2) info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) } - + record[1] <- gsub("\\\\", "/", record[1]) if ('folder' %in% names(header)) datafile <- file.path(path, header$folder, record[1]) else datafile <- file.path(path, record[1]) - - + + # TODO: check the name of the attributes and fix them! data[[i]] <- c( header, @@ -360,14 +362,14 @@ read_index_file__BIOBJ_COCO <- function(fname) { check_format <- function(path) { if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") return(NEVERGRAD) - + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "rds") return("RDS") - + index_files <- scan_index_file(path) - if (length(index_files) == 0) + if (length(index_files) == 0) return(SOS) - + info <- unlist(lapply(index_files, read_index_file), recursive = F) datafile <- sapply(info, function(item) item$datafile) @@ -408,7 +410,7 @@ check_format <- function(path) { csv_files <- file.path(path, list.files(path, pattern = '.csv', recursive = T)) if (length(csv_files) > 0) format <- c(format, NEVERGRAD) - + txt_files <- file.path(path, list.files(path, pattern = '.txt', recursive = T)) if (length(txt_files) > 0) format <- c(format, SOS) @@ -693,7 +695,7 @@ read_nevergrad <- function(path){ if (!'name' %in% colnames(dt)) { dt[, name := function_class] } - + triplets <- unique(dt[, .(optimizer_name, dimension, name)]) algIds <- unique(triplets$optimizer_name) DIMs <- unique(triplets$dimension) @@ -770,7 +772,7 @@ read_nevergrad <- function(path){ } #' Read single DataSet of SOS-based data -#' +#' #' Read single .txt files in SOS format and extract information as a DataSet #' #' @param file The path to the .txt file @@ -778,9 +780,9 @@ read_nevergrad <- function(path){ #' @noRd read_single_file_SOS <- function(file) { V1 <- NULL #Local binding to remove CRAN warnings - + algId <- substr(basename(file), 1, stringi::stri_locate_last(basename(file), fixed = 'D')[[1]] - 1) - + dt <- fread(file, header = F) header <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) splitted <- header %>% trimws %>% strsplit("\\s+") %>% .[[1]] %>% .[2:length(.)] @@ -793,7 +795,7 @@ read_single_file_SOS <- function(file) { names(temp) <- name info <- c(info, temp) } - + dim <- as.numeric(info$DIM) #Hardcoded fix for SB-related data if (is.null(dim) || length(dim) == 0) { @@ -801,28 +803,28 @@ read_single_file_SOS <- function(file) { dim <- 30 info$DIM <- dim } - + RT_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] names(RT_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] RT <- as.matrix(RT_raw) mode(RT) <- 'integer' - + FV_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] names(FV_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] FV <- as.matrix(FV_raw) - - + + pos <- dt[, (ncol(dt) - dim + 1):ncol(dt)] colnames(pos) <- as.character(seq_len(dim)) - + maxRT <- max(RT) finalFV <- min(FV) - + idxs_avail <- dt[['V1']] idxs_replaced <- dt[['V6']] - + idxs_final <- setdiff(idxs_avail, idxs_replaced) - + idx_final_best <- idxs_final[[which.min(FV[idxs_final])]] final_pos <- as.numeric(pos[idx_final_best, ]) # if (sum(FV == finalFV) > 1) { @@ -843,16 +845,16 @@ read_single_file_SOS <- function(file) { # else { # final_pos <- as.numeric(pos[which.min(FV), ]) # } - + PAR <- list( # 'position' = list(pos), 'final_position' = list(final_pos), 'by_FV' = NULL, 'by_RT' = NULL ) - - - + + + object <- list() class(object) <- c('DataSet', class(object)) object$RT <- RT @@ -872,7 +874,7 @@ read_single_file_SOS <- function(file) { #' Read DataSetList of SOS-based data -#' +#' #' Read directory containing .txt files in SOS format and extract information as a DataSetList #' #' @param dir The path to the directory file @@ -887,21 +889,21 @@ read_datasetlist_SOS <- function(dir, corrections_files = NULL) { algIds <- list() suites <- list() maximizations <- list() - + idx <- 1 - + corrs <- as.data.table(rbindlist(lapply(corrections_files, fread))) - + for (f in list.files(dir, recursive = T, pattern = "*.txt", full.names = T)) { if (f %in% corrections_files) next ds <- read_single_file_SOS(f) - + dims[[idx]] <- attr(ds, 'DIM') funcIds[[idx]] <- attr(ds, 'funcId') algIds[[idx]] <- attr(ds, 'algId') suites[[idx]] <- attr(ds, 'suite') maximizations[[idx]] <- attr(ds, 'maximization') - + if (nrow(corrs) > 0) { fn <- substr(basename(f), 1, nchar(basename(f)) - 4) corr_opts <- corrs[V1 == fn, ] @@ -920,7 +922,7 @@ read_datasetlist_SOS <- function(dir, corrections_files = NULL) { else warning(paste0("No boundary corrections ratio found for ", fn)) } - + res[[idx]] <- ds idx <- idx + 1 } @@ -929,13 +931,13 @@ read_datasetlist_SOS <- function(dir, corrections_files = NULL) { attr(res, 'funcId') <- funcIds attr(res, 'algId') <- algIds attr(res, 'ID_attributes') <- c('algId') - + suite <- unique(suites) maximization <- unique(maximizations) if (length(suite) != 1 || length(maximization) != 1) { warning("Multipe different suites detected!") } - + attr(res, 'suite') <- suite attr(res, 'maximization') <- maximization res @@ -943,7 +945,7 @@ read_datasetlist_SOS <- function(dir, corrections_files = NULL) { } #' Find corrections-files in SOS-based folder -#' +#' #' Read directory containing .txt files in SOS format and extract the corrections-files #' #' @param path The path to the directory file