diff --git a/.Rbuildignore b/.Rbuildignore index c3b56406..3313b0a0 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,11 +1,11 @@ -^\.travis\.yml$ -^.*\.Rproj$ -^\.Rproj\.user$ -^cran-comments\.md$ -^NEWS\.md$ -^\.vscode$ -^\.vs*$ -^_config\.yml$ -^LICENSE\.md$ -^misc*$ -^\.github$ +^\.travis\.yml$ +^.*\.Rproj$ +^\.Rproj\.user$ +^cran-comments\.md$ +^NEWS\.md$ +^\.vscode$ +^\.vs*$ +^_config\.yml$ +^LICENSE\.md$ +^misc*$ +^\.github$ diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 92c9158c..c7c52dcf 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,91 +1,91 @@ -# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag. -# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions -on: - push: - branches: - - main - - master - pull_request: - branches: - - main - - master - -name: R-CMD-check - -jobs: - R-CMD-check: - runs-on: ${{ matrix.config.os }} - - name: ${{ matrix.config.os }} (${{ matrix.config.r }}) - - strategy: - fail-fast: false - matrix: - config: - - {os: windows-latest, r: 'release'} - - {os: macOS-latest, r: 'release'} - - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} - - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} - - env: - R_REMOTES_NO_ERRORS_FROM_WARNINGS: true - RSPM: ${{ matrix.config.rspm }} - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/checkout@v2 - - - uses: r-lib/actions/setup-r@v1 - with: - r-version: ${{ matrix.config.r }} - - - uses: r-lib/actions/setup-pandoc@v1 - - - name: Query dependencies - run: | - install.packages('remotes') - saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) - writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") - shell: Rscript {0} - - - name: Restore R package cache - uses: actions/cache@v2 - with: - path: ${{ env.R_LIBS_USER }} - key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} - restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- - - - name: Install system dependencies - if: runner.os == 'Linux' - run: | - while read -r cmd - do - eval sudo $cmd - done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') - - - name: Install X11 dependencies on MacOS - if: runner.os == 'macOS' - run: | - brew install xquartz --cask - - - name: Install dependencies - run: | - remotes::install_deps(dependencies = TRUE) - remotes::install_bioc('ComplexHeatmap') - remotes::install_cran("rcmdcheck") - shell: Rscript {0} - - - name: Check - env: - _R_CHECK_CRAN_INCOMING_REMOTE_: false - run: | - options(crayon.enabled = TRUE) - rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") - shell: Rscript {0} - - - name: Upload check results - if: failure() - uses: actions/upload-artifact@main - with: - name: ${{ runner.os }}-r${{ matrix.config.r }}-results - path: check +# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag. +# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + +name: R-CMD-check + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: windows-latest, r: 'release'} + - {os: macOS-latest, r: 'release'} + - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + RSPM: ${{ matrix.config.rspm }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v1 + with: + r-version: ${{ matrix.config.r }} + + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Query dependencies + run: | + install.packages('remotes') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Restore R package cache + uses: actions/cache@v2 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- + + - name: Install system dependencies + if: runner.os == 'Linux' + run: | + while read -r cmd + do + eval sudo $cmd + done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') + + - name: Install X11 dependencies on MacOS + if: runner.os == 'macOS' + run: | + brew install xquartz --cask + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_bioc('ComplexHeatmap') + remotes::install_cran("rcmdcheck") + shell: Rscript {0} + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_REMOTE_: false + run: | + options(crayon.enabled = TRUE) + rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") + shell: Rscript {0} + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@main + with: + name: ${{ runner.os }}-r${{ matrix.config.r }}-results + path: check diff --git a/.gitignore b/.gitignore index dfa0e501..466bab22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,26 @@ -*.csv -*.o -*.so -*.dll -*.log -*.RData -*.Rhistory -*.Rbuildignore -*.bk -*.pdf -*.html -.* - -!/.Rbuildignore -!/.gitignore -__pycache__ -rawdata/ -test/ -rsconnect/ -tmp/ -tmp2/ -rawdata/ -extract_data.R -.Rproj.user -*.Rproj -.Rhistory +*.csv +*.o +*.so +*.dll +*.log +*.RData +*.Rhistory +*.Rbuildignore +*.bk +*.pdf +*.html +.* + +!/.Rbuildignore +!/.gitignore +__pycache__ +rawdata/ +test/ +rsconnect/ +tmp/ +tmp2/ +rawdata/ +extract_data.R +.Rproj.user +*.Rproj +.Rhistory diff --git a/DESCRIPTION b/DESCRIPTION index 9d169a77..85a4e4ae 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,57 +1,59 @@ -Package: IOHanalyzer -Type: Package -Title: Data Analysis Part of 'IOHprofiler' -Version: 0.1.5.2 -Author: Hao Wang [cre, aut], Diederick Vermetten [aut], Carola Doerr [aut], Thomas Bäck [aut] -Maintainer: Hao Wang -Authors@R: c( - person("Hao", "Wang", email = "h.wang@liacs.leidenuniv.nl", role = c("cre","aut")), - person("Diederick", "Vermetten", email="d.vermetten@gmail.com", role = "aut"), - person("Carola", "Doerr", email = "Carola.Doerr@mpi-inf.mpg.de", role = "aut"), - person("Thomas", "Bäck", email="t.h.w.baeck@liacs.leidenuniv.nl", role = "aut")) -Description: The data analysis module for the Iterative Optimization Heuristics - Profiler ('IOHprofiler'). This module provides statistical analysis methods for the - benchmark data generated by optimization heuristics, which can be visualized through a - web-based interface. The benchmark data is usually generated by the - experimentation module, called 'IOHexperimenter'. 'IOHanalyzer' also supports - the widely used 'COCO' (Comparing Continuous Optimisers) data format for benchmarking. -License: BSD_3_clause + file LICENSE -Encoding: UTF-8 -LazyData: true -URL: http://iohprofiler.liacs.nl, https://github.com/IOHprofiler/IOHAnalyzer -BugReports: https://github.com/IOHprofiler/IOHAnalyzer/issues -Imports: - magrittr, - dplyr, - data.table, - ggplot2, - plotly, - colorspace, - colorRamps, - RColorBrewer, - shiny, - markdown, - reshape2, - shinyjs, - colourpicker, - bsplus, - DT, - kableExtra, - stringi, - httr, - knitr -LinkingTo: Rcpp -SystemRequirements: C++11 -RoxygenNote: 7.0.0 -Suggests: - Rcpp, - testthat, - withr, - shinydashboard, - ComplexHeatmap, - grid, - keyring, - PlayerRatings, - xtable, - igraph -Depends: R (>= 2.10) +Package: IOHanalyzer +Type: Package +Title: Data Analysis Part of 'IOHprofiler' +Version: 0.1.6.0 +Author: Hao Wang [cre, aut], Diederick Vermetten [aut], Carola Doerr [aut], Thomas Bäck [aut] +Maintainer: Hao Wang +Authors@R: c( + person("Hao", "Wang", email = "h.wang@liacs.leidenuniv.nl", role = c("cre","aut")), + person("Diederick", "Vermetten", email="d.vermetten@gmail.com", role = "aut"), + person("Carola", "Doerr", email = "Carola.Doerr@mpi-inf.mpg.de", role = "aut"), + person("Thomas", "Bäck", email="t.h.w.baeck@liacs.leidenuniv.nl", role = "aut")) +Description: The data analysis module for the Iterative Optimization Heuristics + Profiler ('IOHprofiler'). This module provides statistical analysis methods for the + benchmark data generated by optimization heuristics, which can be visualized through a + web-based interface. The benchmark data is usually generated by the + experimentation module, called 'IOHexperimenter'. 'IOHanalyzer' also supports + the widely used 'COCO' (Comparing Continuous Optimisers) data format for benchmarking. +License: BSD_3_clause + file LICENSE +Encoding: UTF-8 +LazyData: true +URL: http://iohprofiler.liacs.nl, https://github.com/IOHprofiler/IOHAnalyzer +BugReports: https://github.com/IOHprofiler/IOHAnalyzer/issues +Imports: + magrittr, + dplyr, + data.table, + ggplot2, + plotly, + colorspace, + colorRamps, + RColorBrewer, + shiny, + markdown, + reshape2, + shinyjs, + colourpicker, + bsplus, + DT, + kableExtra, + stringi, + httr, + knitr, + methods, + rjson +LinkingTo: Rcpp +SystemRequirements: C++11 +RoxygenNote: 7.0.0 +Suggests: + Rcpp, + testthat, + withr, + shinydashboard, + ComplexHeatmap, + grid, + keyring, + PlayerRatings, + xtable, + igraph +Depends: R (>= 2.10) diff --git a/LICENSE b/LICENSE index 772d7517..b94f835f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,3 @@ -YEAR: 2019 -COPYRIGHT HOLDER: Hao Wang +YEAR: 2019 +COPYRIGHT HOLDER: Hao Wang ORGANIZATION: Leiden University \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index d2a3b4d3..c170dbc6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,33 +1,33 @@ -# License - -This application is governed by the __BSD 3-Clause license__. - -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# License + +This application is governed by the __BSD 3-Clause license__. + +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/NAMESPACE b/NAMESPACE index 70b680ae..4cde42f3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,230 +1,241 @@ -# Generated by roxygen2: do not edit by hand - -S3method("==",DataSet) -S3method("[",DataSetList) -S3method(AUC,ECDF) -S3method(ECDF,DataSet) -S3method(ECDF,DataSetList) -S3method(Plot.FV.Aggregated,DataSetList) -S3method(Plot.FV.ECDF_AUC,DataSetList) -S3method(Plot.FV.ECDF_Per_Target,DataSetList) -S3method(Plot.FV.ECDF_Single_Func,DataSetList) -S3method(Plot.FV.Histogram,DataSetList) -S3method(Plot.FV.Multi_Func,DataSetList) -S3method(Plot.FV.PDF,DataSetList) -S3method(Plot.FV.Parameters,DataSetList) -S3method(Plot.FV.Single_Func,DataSetList) -S3method(Plot.RT.Aggregated,DataSetList) -S3method(Plot.RT.ECDF_AUC,DataSetList) -S3method(Plot.RT.ECDF_Multi_Func,DataSetList) -S3method(Plot.RT.ECDF_Per_Target,DataSetList) -S3method(Plot.RT.ECDF_Single_Func,DataSetList) -S3method(Plot.RT.Histogram,DataSetList) -S3method(Plot.RT.Multi_Func,DataSetList) -S3method(Plot.RT.PMF,DataSetList) -S3method(Plot.RT.Parameters,DataSetList) -S3method(Plot.RT.Single_Func,DataSetList) -S3method(Plot.Stats.Glicko2_Candlestick,DataSetList) -S3method(Plot.Stats.Significance_Graph,DataSetList) -S3method(Plot.Stats.Significance_Heatmap,DataSetList) -S3method(arrange,DataSetList) -S3method(as.character,DataSet) -S3method(c,DataSet) -S3method(c,DataSetList) -S3method(get_ERT,DataSet) -S3method(get_ERT,DataSetList) -S3method(get_FV_overview,DataSet) -S3method(get_FV_overview,DataSetList) -S3method(get_FV_sample,DataSet) -S3method(get_FV_sample,DataSetList) -S3method(get_FV_summary,DataSet) -S3method(get_FV_summary,DataSetList) -S3method(get_PAR_name,DataSet) -S3method(get_PAR_sample,DataSet) -S3method(get_PAR_sample,DataSetList) -S3method(get_PAR_summary,DataSet) -S3method(get_PAR_summary,DataSetList) -S3method(get_RT_overview,DataSet) -S3method(get_RT_overview,DataSetList) -S3method(get_RT_sample,DataSet) -S3method(get_RT_sample,DataSetList) -S3method(get_RT_summary,DataSet) -S3method(get_RT_summary,DataSetList) -S3method(get_maxRT,DataSet) -S3method(get_maxRT,DataSetList) -S3method(get_overview,DataSet) -S3method(get_overview,DataSetList) -S3method(max_ERTs,DataSetList) -S3method(mean_FVs,DataSetList) -S3method(pairwise.test,DataSetList) -S3method(pairwise.test,list) -S3method(print,DataSet) -S3method(print,DataSetList) -S3method(subset,DataSetList) -S3method(summary,DataSet) -S3method(summary,DataSetList) -export(AUC) -export(DataSet) -export(DataSetList) -export(ECDF) -export(IOH_plot_ly_default) -export(Plot.FV.Aggregated) -export(Plot.FV.ECDF_AUC) -export(Plot.FV.ECDF_Per_Target) -export(Plot.FV.ECDF_Single_Func) -export(Plot.FV.Histogram) -export(Plot.FV.Multi_Func) -export(Plot.FV.PDF) -export(Plot.FV.Parameters) -export(Plot.FV.Single_Func) -export(Plot.Performviz) -export(Plot.RT.Aggregated) -export(Plot.RT.ECDF_AUC) -export(Plot.RT.ECDF_Multi_Func) -export(Plot.RT.ECDF_Per_Target) -export(Plot.RT.ECDF_Single_Func) -export(Plot.RT.Histogram) -export(Plot.RT.Multi_Func) -export(Plot.RT.PMF) -export(Plot.RT.Parameters) -export(Plot.RT.Single_Func) -export(Plot.Stats.Glicko2_Candlestick) -export(Plot.Stats.Significance_Graph) -export(Plot.Stats.Significance_Heatmap) -export(SP) -export(arrange) -export(bootstrap_RT) -export(cat.DataSet) -export(check_dsc_configured) -export(check_format) -export(clean_DataSetList) -export(fast_RT_samples) -export(generate_data.AUC) -export(generate_data.Aggr) -export(generate_data.ECDF) -export(generate_data.ECDF_raw) -export(generate_data.PMF) -export(generate_data.Parameters) -export(generate_data.Single_Function) -export(generate_data.hist) -export(get_ECDF_targets) -export(get_ERT) -export(get_FV_overview) -export(get_FV_sample) -export(get_FV_summary) -export(get_PAR_name) -export(get_PAR_sample) -export(get_PAR_summary) -export(get_RT_overview) -export(get_RT_sample) -export(get_RT_summary) -export(get_algId) -export(get_color_scheme) -export(get_color_scheme_dt) -export(get_default_ECDF_targets) -export(get_dim) -export(get_dsc_omnibus) -export(get_dsc_posthoc) -export(get_dsc_rank) -export(get_funcId) -export(get_funvals) -export(get_line_style) -export(get_marg_contrib_ecdf) -export(get_maxRT) -export(get_overview) -export(get_parId) -export(get_runtimes) -export(get_shapley_values) -export(get_target_dt) -export(glicko2_ranking) -export(max_ERTs) -export(mean_FVs) -export(pairwise.test) -export(plot_general_data) -export(read_index_file) -export(register_DSC) -export(runServer) -export(save_plotly) -export(save_table) -export(scan_index_file) -export(seq_FV) -export(seq_RT) -export(set_DSC_credentials) -export(set_color_scheme) -importFrom(RColorBrewer,brewer.pal) -importFrom(colorRamps,primary.colors) -importFrom(colorspace,sequential_hcl) -importFrom(data.table,":=") -importFrom(data.table,as.data.table) -importFrom(data.table,copy) -importFrom(data.table,data.table) -importFrom(data.table,frank) -importFrom(data.table,fread) -importFrom(data.table,is.data.table) -importFrom(data.table,melt) -importFrom(data.table,rbindlist) -importFrom(data.table,setnames) -importFrom(data.table,setorderv) -importFrom(dplyr,"%>%") -importFrom(dplyr,mutate) -importFrom(ggplot2,aes) -importFrom(ggplot2,element_text) -importFrom(ggplot2,facet_wrap) -importFrom(ggplot2,geom_jitter) -importFrom(ggplot2,geom_line) -importFrom(ggplot2,geom_ribbon) -importFrom(ggplot2,geom_violin) -importFrom(ggplot2,ggplot) -importFrom(ggplot2,guides) -importFrom(ggplot2,scale_color_manual) -importFrom(ggplot2,scale_colour_manual) -importFrom(ggplot2,scale_fill_manual) -importFrom(ggplot2,scale_x_continuous) -importFrom(ggplot2,scale_x_log10) -importFrom(ggplot2,theme) -importFrom(ggplot2,theme_grey) -importFrom(ggplot2,theme_set) -importFrom(grDevices,col2rgb) -importFrom(grDevices,colors) -importFrom(grDevices,nclass.FD) -importFrom(graphics,hist) -importFrom(httr,POST) -importFrom(httr,add_headers) -importFrom(httr,authenticate) -importFrom(httr,content) -importFrom(knitr,kable) -importFrom(magrittr,"%<>%") -importFrom(magrittr,mod) -importFrom(magrittr,set_colnames) -importFrom(magrittr,set_names) -importFrom(magrittr,set_rownames) -importFrom(plotly,add_annotations) -importFrom(plotly,add_trace) -importFrom(plotly,layout) -importFrom(plotly,orca) -importFrom(plotly,plot_ly) -importFrom(plotly,rename_) -importFrom(plotly,subplot) -importFrom(reshape2,acast) -importFrom(shiny,req) -importFrom(stats,dt) -importFrom(stats,ecdf) -importFrom(stats,integrate) -importFrom(stats,ks.test) -importFrom(stats,median) -importFrom(stats,p.adjust) -importFrom(stats,quantile) -importFrom(stats,rgeom) -importFrom(stats,sd) -importFrom(stringi,stri_detect_fixed) -importFrom(stringi,stri_detect_regex) -importFrom(stringi,stri_locate_all) -importFrom(stringi,stri_rand_strings) -importFrom(stringi,stri_replace) -importFrom(utils,data) -importFrom(utils,head) -importFrom(utils,read.csv) -importFrom(utils,tail) -importFrom(utils,type.convert) -importFrom(utils,write.csv) -useDynLib(IOHanalyzer) +# Generated by roxygen2: do not edit by hand + +S3method("==",DataSet) +S3method("[",DataSetList) +S3method(AUC,ECDF) +S3method(ECDF,DataSet) +S3method(ECDF,DataSetList) +S3method(Plot.FV.Aggregated,DataSetList) +S3method(Plot.FV.ECDF_AUC,DataSetList) +S3method(Plot.FV.ECDF_Per_Target,DataSetList) +S3method(Plot.FV.ECDF_Single_Func,DataSetList) +S3method(Plot.FV.Histogram,DataSetList) +S3method(Plot.FV.Multi_Func,DataSetList) +S3method(Plot.FV.PDF,DataSetList) +S3method(Plot.FV.Parameters,DataSetList) +S3method(Plot.FV.Single_Func,DataSetList) +S3method(Plot.RT.Aggregated,DataSetList) +S3method(Plot.RT.ECDF_AUC,DataSetList) +S3method(Plot.RT.ECDF_Multi_Func,DataSetList) +S3method(Plot.RT.ECDF_Per_Target,DataSetList) +S3method(Plot.RT.ECDF_Single_Func,DataSetList) +S3method(Plot.RT.Histogram,DataSetList) +S3method(Plot.RT.Multi_Func,DataSetList) +S3method(Plot.RT.PMF,DataSetList) +S3method(Plot.RT.Parameters,DataSetList) +S3method(Plot.RT.Single_Func,DataSetList) +S3method(Plot.Stats.Glicko2_Candlestick,DataSetList) +S3method(Plot.Stats.Significance_Graph,DataSetList) +S3method(Plot.Stats.Significance_Heatmap,DataSetList) +S3method(arrange,DataSetList) +S3method(as.character,DataSet) +S3method(c,DataSet) +S3method(c,DataSetList) +S3method(get_ERT,DataSet) +S3method(get_ERT,DataSetList) +S3method(get_FV_overview,DataSet) +S3method(get_FV_overview,DataSetList) +S3method(get_FV_sample,DataSet) +S3method(get_FV_sample,DataSetList) +S3method(get_FV_summary,DataSet) +S3method(get_FV_summary,DataSetList) +S3method(get_PAR_name,DataSet) +S3method(get_PAR_sample,DataSet) +S3method(get_PAR_sample,DataSetList) +S3method(get_PAR_summary,DataSet) +S3method(get_PAR_summary,DataSetList) +S3method(get_RT_overview,DataSet) +S3method(get_RT_overview,DataSetList) +S3method(get_RT_sample,DataSet) +S3method(get_RT_sample,DataSetList) +S3method(get_RT_summary,DataSet) +S3method(get_RT_summary,DataSetList) +S3method(get_id,DataSet) +S3method(get_id,DataSetList) +S3method(get_maxRT,DataSet) +S3method(get_maxRT,DataSetList) +S3method(get_overview,DataSet) +S3method(get_overview,DataSetList) +S3method(max_ERTs,DataSetList) +S3method(mean_FVs,DataSetList) +S3method(pairwise.test,DataSetList) +S3method(pairwise.test,list) +S3method(print,DataSet) +S3method(print,DataSetList) +S3method(subset,DataSet) +S3method(subset,DataSetList) +S3method(summary,DataSet) +S3method(summary,DataSetList) +export(AUC) +export(DataSet) +export(DataSetList) +export(ECDF) +export(IOH_plot_ly_default) +export(Plot.FV.Aggregated) +export(Plot.FV.ECDF_AUC) +export(Plot.FV.ECDF_Per_Target) +export(Plot.FV.ECDF_Single_Func) +export(Plot.FV.Histogram) +export(Plot.FV.Multi_Func) +export(Plot.FV.PDF) +export(Plot.FV.Parameters) +export(Plot.FV.Single_Func) +export(Plot.Performviz) +export(Plot.RT.Aggregated) +export(Plot.RT.ECDF_AUC) +export(Plot.RT.ECDF_Multi_Func) +export(Plot.RT.ECDF_Per_Target) +export(Plot.RT.ECDF_Single_Func) +export(Plot.RT.Histogram) +export(Plot.RT.Multi_Func) +export(Plot.RT.PMF) +export(Plot.RT.Parameters) +export(Plot.RT.Single_Func) +export(Plot.Stats.Glicko2_Candlestick) +export(Plot.Stats.Significance_Graph) +export(Plot.Stats.Significance_Heatmap) +export(SP) +export(arrange) +export(bootstrap_RT) +export(cat.DataSet) +export(change_id) +export(check_dsc_configured) +export(check_format) +export(clean_DataSetList) +export(fast_RT_samples) +export(generate_data.AUC) +export(generate_data.Aggr) +export(generate_data.ECDF) +export(generate_data.ECDF_raw) +export(generate_data.PMF) +export(generate_data.Parameters) +export(generate_data.Single_Function) +export(generate_data.hist) +export(get_ECDF_targets) +export(get_ERT) +export(get_FV_overview) +export(get_FV_sample) +export(get_FV_summary) +export(get_PAR_name) +export(get_PAR_sample) +export(get_PAR_summary) +export(get_RT_overview) +export(get_RT_sample) +export(get_RT_summary) +export(get_algId) +export(get_color_scheme) +export(get_color_scheme_dt) +export(get_default_ECDF_targets) +export(get_dim) +export(get_dsc_omnibus) +export(get_dsc_posthoc) +export(get_dsc_rank) +export(get_funcId) +export(get_funcName) +export(get_funvals) +export(get_id) +export(get_line_style) +export(get_marg_contrib_ecdf) +export(get_maxRT) +export(get_overview) +export(get_parId) +export(get_runtimes) +export(get_shapley_values) +export(get_static_attribute_values) +export(get_static_attributes) +export(get_target_dt) +export(glicko2_ranking) +export(max_ERTs) +export(mean_FVs) +export(pairwise.test) +export(plot_general_data) +export(read_index_file) +export(register_DSC) +export(runServer) +export(save_plotly) +export(save_table) +export(scan_index_file) +export(seq_FV) +export(seq_RT) +export(set_DSC_credentials) +export(set_color_scheme) +importFrom(RColorBrewer,brewer.pal) +importFrom(colorRamps,primary.colors) +importFrom(colorspace,sequential_hcl) +importFrom(data.table,":=") +importFrom(data.table,as.data.table) +importFrom(data.table,copy) +importFrom(data.table,data.table) +importFrom(data.table,frank) +importFrom(data.table,fread) +importFrom(data.table,is.data.table) +importFrom(data.table,melt) +importFrom(data.table,rbindlist) +importFrom(data.table,setnames) +importFrom(data.table,setorderv) +importFrom(data.table,transpose) +importFrom(dplyr,"%>%") +importFrom(dplyr,mutate) +importFrom(ggplot2,aes) +importFrom(ggplot2,element_text) +importFrom(ggplot2,facet_wrap) +importFrom(ggplot2,geom_jitter) +importFrom(ggplot2,geom_line) +importFrom(ggplot2,geom_ribbon) +importFrom(ggplot2,geom_violin) +importFrom(ggplot2,ggplot) +importFrom(ggplot2,guides) +importFrom(ggplot2,scale_color_manual) +importFrom(ggplot2,scale_colour_manual) +importFrom(ggplot2,scale_fill_manual) +importFrom(ggplot2,scale_x_continuous) +importFrom(ggplot2,scale_x_log10) +importFrom(ggplot2,theme) +importFrom(ggplot2,theme_grey) +importFrom(ggplot2,theme_set) +importFrom(grDevices,col2rgb) +importFrom(grDevices,colors) +importFrom(grDevices,nclass.FD) +importFrom(graphics,hist) +importFrom(httr,POST) +importFrom(httr,add_headers) +importFrom(httr,authenticate) +importFrom(httr,content) +importFrom(knitr,kable) +importFrom(magrittr,"%<>%") +importFrom(magrittr,mod) +importFrom(magrittr,set_colnames) +importFrom(magrittr,set_names) +importFrom(magrittr,set_rownames) +importFrom(methods,hasArg) +importFrom(plotly,add_annotations) +importFrom(plotly,add_trace) +importFrom(plotly,layout) +importFrom(plotly,orca) +importFrom(plotly,plot_ly) +importFrom(plotly,rename_) +importFrom(plotly,subplot) +importFrom(reshape2,acast) +importFrom(rjson,fromJSON) +importFrom(shiny,req) +importFrom(stats,dt) +importFrom(stats,ecdf) +importFrom(stats,integrate) +importFrom(stats,ks.test) +importFrom(stats,median) +importFrom(stats,p.adjust) +importFrom(stats,quantile) +importFrom(stats,rgeom) +importFrom(stats,sd) +importFrom(stringi,stri_detect_fixed) +importFrom(stringi,stri_detect_regex) +importFrom(stringi,stri_locate_all) +importFrom(stringi,stri_rand_strings) +importFrom(stringi,stri_replace) +importFrom(utils,data) +importFrom(utils,head) +importFrom(utils,read.csv) +importFrom(utils,tail) +importFrom(utils,type.convert) +importFrom(utils,write.csv) +useDynLib(IOHanalyzer) diff --git a/NEWS.md b/NEWS.md index bb931de7..e6c693da 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,2 +1,2 @@ -# IOHanalyzer 0.1 -* First submission to CRAN +# IOHanalyzer 0.1 +* First submission to CRAN diff --git a/R/DataSet.R b/R/DataSet.R index 59d6f147..0149c098 100644 --- a/R/DataSet.R +++ b/R/DataSet.R @@ -1,940 +1,1054 @@ -#' Constructor of S3 class 'DataSet' -#' -#' DataSet contains the following attributes -#' * funId -#' * DIM -#' * algId -#' * datafile -#' * instance -#' * maxEvals -#' * finalFunEvals -#' * comment -#' * Additional attributes based on the original format -#' -#' @param info A List. Contains a set of in a *.info file. -#' @param verbose Logical. -#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? -#' Set to NULL to determine automatically based on format -#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" -#' @param subsampling Logical. Whether *.cdat files are subsampled? -#' -#' @return A S3 object 'DataSet' -#' @export -#' @examples -#' path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') -#' info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) -#' DataSet(info[[1]]) -DataSet <- function(info, verbose = F, maximization = NULL, format = IOHprofiler, - subsampling = FALSE) { - if (!is.null(info)) { - path <- dirname(info$datafile) - suite <- toupper(info$suite) - - # for an unknown suite, to detect the format - if (is.null(suite) || length(suite) == 0) { - if (verbose) - warning("Suite-name not provided in .info-file, taking best guess based on - the format of data-files.") - suite <- switch(format, - IOHprofiler = "Unknown", - COCO = "BBOB", - BIOBJ_COCO = "biobj-bbob", - TWO_COL = "Unknown") - } - - if (is.null(maximization) || maximization == AUTOMATIC) { - #TODO: Better way to deal with capitalization of attributes - if (!is.null(info$maximization)) maximization <- info$maximization - else if (!is.null(info$Maximization)) maximization <- info$Maximization - else if (!is.null(suite)) { - if (verbose) - warning("maximization or minimization not specified in .info-file, - taking best guess based on the suite-name.") - if (grepl("\\w*bbob\\w*", suite, ignore.case = T) != 0) - maximization <- FALSE - else - maximization <- TRUE - } - else { - warning("Can't detect maximization based on suite-attribute, setting to - minimization by default") - maximization <- FALSE # default to minimization - } - } - - if(!(isTRUE(maximization) || isFALSE(maximization))) - warning("unclear whether we should maximize or minimize.") - - datBaseName <- sub(pattern = '(.*)\\..*$', replacement = '\\1', basename(info$datafile)) - datFile <- file.path(path, paste0(datBaseName, '.dat')) - tdatFile <- file.path(path, paste0(datBaseName, '.tdat')) - cdatFile <- file.path(path, paste0(datBaseName, '.cdat')) - - # NOTE: preference on data file for the alignment by RT: cdat > tdat > dat - if (file.exists(cdatFile)) - fvFile <- cdatFile - else if (file.exists(tdatFile)) - fvFile <- tdatFile - else if (file.exists(datFile)) - fvFile <- datFile - else - stop('No datafiles found, please verify the integrity of the chosen files') - - # NOTE: preference on data file for the alignment by FV: dat > tdat > cdat - if (file.exists(datFile)) - rtFile <- datFile - else if (file.exists(tdatFile)) - rtFile <- tdatFile - else if (file.exists(cdatFile)) - # TODO: perhaps turn on `subsampling` here as this would take quite some time - rtFile <- cdatFile - - read_raw <- switch( - format, - IOHprofiler = read_dat, - COCO = read_dat__COCO, - BIOBJ_COCO = read_dat__BIOBJ_COCO, - TWO_COL = read_dat # TODO: perhaps rename `TWO_COL` or to use a better naming - # scheme for all format names - ) - - RT_raw <- read_raw(rtFile, subsampling) - FV_raw <- read_raw(fvFile, subsampling) - - if (is.null(maximization)) { - if (verbose) - warning("Did not find maximization / minimization, auto-detecting based on - function value progression") - # TODO: idxTarget should be set depending on the data format - idxTarget <- 2 - cond <- unique(lapply(FV_raw, function(FV) FV[1, idxTarget] >= FV[nrow(FV), idxTarget])) - if (length(cond) > 1) - stop('The detected maximization differs in multiple runs') - maximization <- cond - } - - RT <- align_running_time(RT_raw, format = format, maximization = maximization) - FV <- align_function_value(FV_raw, format = format) - - PAR <- list( - 'by_FV' = RT[names(RT) != 'RT'], - 'by_RT' = FV[names(FV) != 'FV'] - ) - - RT <- RT$RT - mode(RT) <- 'integer' - FV <- FV$FV - - if (format %in% c(IOHprofiler)) { - # try to save some memory here... - FV <- tryCatch({ - .FV <- FV - mode(.FV) <- 'integer' - if (all(FV == .FV)) .FV - else FV - }, - warning = function(w) FV) # in case the type coercion gives a warning - } - - # TODO: add more data sanity checks - maxRT <- set_names(sapply(RT_raw, function(d) d[nrow(d), idxEvals]), NULL) - # Fix for old-format files which do not store used runtime in .dat-files - maxRT <- pmax(maxRT, info$maxRT) - if (any(maxRT != info$maxRT) && verbose) - warning('Inconsitent maxRT in *.info file and *.cdat file') - - # TODO: clean up these if-statements: Function to set idxTarget and n_data_column? - # `idxTarget` is a global variable? - if (format == TWO_COL) - finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget - 1]), NULL) - else - finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget]), NULL) - - if (any(finalFV != info$finalFV) && verbose) - warning('Inconsitent finalFvalue in *.info file and *.dat file') - - if (length(info$instance) != length(RT_raw)) { - if (verbose) - warning('The number of instances found in the info is inconsistent with the data!') - info$instance <- seq(length(RT_raw)) - } - - do.call( - function(...) - structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), - c(info, list(maxRT = maxRT, finalFV = finalFV, format = format, - maximization = maximization, suite = suite)) - ) - } - else - structure(list(), class = c('DataSet', 'list')) -} - -#' S3 concatenation function for DataSet -#' -#' @description Concatenation for DataSets. Combines multiple runs from separate DataSets -#' into a single DataSet object if all provided arguments have the same dimension, function ID and -#' algorithm ID, and each contains only a single run. Currently does not support parameter tracking -#' -#' @param ... The DataSets to concatenate -#' @return A new DataSet -#' @export -#' @examples -#' c(dsl[[1]], dsl[[1]]) -c.DataSet <- function(...) { - dsl <- list(...) - - if (length(dsl) == 1) dsl <- dsl[[1]] - dsl <- dsl[sapply(dsl, length) != 0] - - if (length(dsl) == 0) return() - if (length(dsl) == 1) return(dsl[[1]]) - - for (ds in dsl) { - if (!any((class(ds)) == 'DataSet')) - stop("Operation only possible when all arguments are DataSets") - } - - fixed_attrs <- c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') - info <- list() - for (attr_str in fixed_attrs) { - temp <- unique(unlist(lapply(dsl, function(x) attr(x, attr_str)))) - if (length(temp) > 1) { - stop(paste0("Attempted to add datasets with different ", attr_str, - "-attributes! Tis is not allowed, please keep them as separate DataSets!")) - } - info <- c(info,temp) - } - names(info) <- fixed_attrs - - for (attr_str in names(attributes(dsl[[1]]))) { - if (attr_str %in% fixed_attrs || attr_str %in% c("names", "class")) next - temp <- unlist(lapply(dsl, function(x) attr(x, attr_str))) - if (length(unique(temp)) == 1) - temp <- unique(temp) - else - temp <- list(temp_name = temp) - names(temp) <- attr_str - info <- c(info, temp) - } - - format <- info[['format']] #attr(dsl[[1]], "format") - - RT_raw <- unlist(lapply(dsl, function(ds) { - lapply(seq_len(ncol(ds$RT)), function(cnr) { - rt_temp <- as.matrix(ds$RT[, cnr]) - cbind(rt_temp, as.numeric(rownames(ds$RT))) - }) - }), recursive = F) - - RT <- align_running_time(RT_raw, format = "TWO_COL", maximization = info$maximization)$RT - FV <- align_function_value(RT_raw, format = "TWO_COL")$FV - - # TODO: to deal with cases where aligned parameters are present in original DataSets - PAR <- list( - 'by_FV' = RT[names(RT) != 'RT'], - 'by_RT' = FV[names(FV) != 'FV'] - ) - - # Unaligned parameters - for (par_name in names(dsl[[1]]$PAR)) { - if (!par_name %in% c('by_FV', 'by_RT')) - PAR[[par_name]] <- unlist(lapply(dsl, function(x) {x$PAR[[par_name]]}), recursive = F) - } - - do.call( - function(...) - structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), - c(info) - ) -} - -#' S3 generic print operator for DataSet -#' -#' @param x A DataSet object -#' @param ... Arguments passed to other methods -#' -#' @return A short description of the DataSet -#' @examples -#' print(dsl[[1]]) -#' @export -print.DataSet <- function(x, ...) { - cat(as.character.DataSet(x, ...)) -} - -#' S3 generic cat operator for DataSet -#' -#' @param x A DataSet object -#' -#' @return A short description of the DataSet -#' @export -#' @examples -#' cat.DataSet(dsl[[1]]) -cat.DataSet <- function(x) cat(as.character(x)) - -#' S3 generic as.character operator for DataSet -#' -#' @param x A DataSet object -#' @param verbose Verbose mode, currently not implemented -#' @param ... Arguments passed to other methods -#' -#' @return A short description of the DataSet -#' @export -#' @examples -#' as.character(dsl[[1]]) -as.character.DataSet <- function(x, verbose = F, ...) { - # TODO: implement the verbose mode - sprintf('DataSet(%s on f%s %dD)', attr(x, 'algId'), attr(x, 'funcId'), - attr(x, 'DIM')) -} - -#' S3 generic summary operator for DataSet -#' -#' @param object A DataSet object -#' @param ... Arguments passed to other methods -#' -#' @return A summary of the DataSet containing both function-value and runtime based statistics. -#' @examples -#' summary(dsl[[1]]) -#' @export -summary.DataSet <- function(object, ...) { - ds_attr <- attributes(object) - cat('DataSet Object:\n') - cat(sprintf('Source: %s\n', ds_attr$src)) - cat(sprintf('Algorithm: %s\n', ds_attr$algId)) - cat(sprintf('Function ID: %s\n', ds_attr$funcId)) - cat(sprintf('Dimension: %dD\n', ds_attr$DIM)) - - n_instance <- length(ds_attr$instance) - if (n_instance >= 15) { - inst <- paste0(paste(ds_attr$instance[1:7], collapse = ','), - ',...,', - paste(ds_attr$instance[(n_instance - 7):n_instance], collapse = ',')) - cat(sprintf('%d instance found: %s\n\n', n_instance, inst)) - } - else - cat(sprintf('%d instance found: %s\n\n', n_instance, paste(ds_attr$instance, collapse = ','))) - - cat('runtime summary:\n') - function_values <- as.numeric(rownames(object$RT)) - RT.summary <- get_RT_summary(object, function_values) - print(RT.summary) - cat('\n') - - cat('function value summary:\n') - runtimes <- as.numeric(rownames(object$FV)) - if (length(runtimes) > 100) { - runtimes <- runtimes[seq(1, length(runtimes), length.out = 100)] - } - - FV.summary <- get_FV_summary(object, runtimes) - print(FV.summary) - cat('\n') - - cat(paste('Attributes:', paste0(names(ds_attr), collapse = ', '))) -} - -#' S3 generic == operator for DataSets -#' -#' @param dsL A `DataSet` object -#' @param dsR A `DataSet` object -#' -#' -#' @return True if the DataSets contain the same function, dimension and algorithm, -#' and have the exact same attributes -#' @examples -#' dsl[[1]] == dsl[[2]] -#' @export -`==.DataSet` <- function(dsL, dsR) { - if (length(dsL) == 0 || length(dsR) == 0) - return(FALSE) - - for (attr_str in names(attributes(dsL))) { - if (any(attr(dsL, attr_str) != attr(dsR, attr_str))) return(FALSE) - } - return(TRUE) -} - -#' Get Expected RunTime -#' -#' @param ds A DataSet or DataSetList object -#' @param budget Optional; overwrites the budget found in ds for ERT-calculation -#' @param ... Arguments passed to other methods -#' @param ftarget The function target(s) for which to get the ERT -#' -#' @return A data.table containing the runtime samples for each provided target -#' function value -#' @examples -#' get_ERT(dsl, 14) -#' get_ERT(dsl[[1]], 14) -#' @export -#' -get_ERT <- function(ds, ftarget, budget, ...) UseMethod("get_ERT", ds) - -#' Get RunTime Sample -#' -#' @param ds A DataSet or DataSetList object -#' @param ftarget A Numerical vector. Function values at which runtime values are consumed -#' @param ... Arguments passed to other methods -#' -#' -#' @return A data.table containing the runtime samples for each provided target -#' function value -#' @examples -#' get_RT_sample(dsl, 14) -#' get_RT_sample(dsl[[1]], 14) -#' @export -get_RT_sample <- function(ds, ftarget, ...) UseMethod("get_RT_sample", ds) - -#' Get RunTime Summary -#' -#' @param ds A DataSet or DataSetList object -#' @param budget Optional; overwrites the budget found in ds for ERT-calculation -#' @param ... Arguments passed to other methods -#' @param ftarget The function target(s) for which to get the runtime summary -#' -#' @return A data.table containing the runtime statistics for each provided target -#' function value -#' @examples -#' get_RT_summary(dsl, 14) -#' get_RT_summary(dsl[[1]], 14) -#' @export -get_RT_summary <- function(ds, ftarget, budget, ...) UseMethod("get_RT_summary", ds) - -#' Get Funtion Value Samples -#' -#' @param ds A DataSet or DataSetList object -#' @param runtime A Numerical vector. Runtimes at which function values are reached -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the function value samples for each provided -#' target runtime -#' @examples -#' get_FV_sample(dsl, 100) -#' get_FV_sample(dsl[[1]], 100) -#' @export -get_FV_sample <- function(ds, ...) UseMethod("get_FV_sample", ds) - -#' Get Function Value Summary -#' -#' @param ds A DataSet or DataSetList object -#' @param runtime A Numerical vector. Runtimes at which function values are reached -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the function value statistics for each provided -#' target runtime value -#' @examples -#' get_FV_summary(dsl, 100) -#' get_FV_summary(dsl[[1]], 100) -#' @export -get_FV_summary <- function(ds, ...) UseMethod("get_FV_summary", ds) - -#' Get Parameter Value Samples -#' -#' @param ds A DataSet or DataSetList object -#' @param idxValue A Numerical vector. Index values at which parameter values are observed. -#' The index value can either take its value in the range of running times, or function values. -#' Such a value type is signified by `which` parameter. -#' @param ... Arguments passed to other methods -#' -#' @return A data.table object containing parameter values aligned at each given target value -#' @examples -#' get_PAR_sample(dsl, 14) -#' get_PAR_sample(dsl[[1]], 14) -#' @export -get_PAR_sample <- function(ds, idxValue, ...) UseMethod("get_PAR_sample", ds) - -#' Get Parameter Value Summary -#' -#' @param ds A DataSet or DataSetList object -#' @param idxValue A Numerical vector. Index values at which parameter values are observed. -#' The index value can either take its value in the range of running times, or function values. -#' Such a value type is signified by `which` parameter. -#' @param ... Arguments passed to other methods -#' -#' @return A data.table object containing basic statistics of parameter values aligned at each given target value -#' @examples -#' get_PAR_summary(dsl, 14) -#' get_PAR_summary(dsl[[1]], 14) -#' @export -get_PAR_summary <- function(ds, idxValue, ...) UseMethod("get_PAR_summary", ds) - -#' Get the parameter names of the algorithm -#' -#' @param ds A DataSet object -#' @param which a string takes it value in `c('by_FV', 'by_RT')`, indicating the -#' parameters aligned against the running time (RT) or function value (FV). `'by_FV'` -#' is the default value. -#' @return a character list of paramter names, if recorded in the data set -#' @examples -#' get_PAR_name(dsl[[1]]) -#' @export -get_PAR_name <- function(ds, which) UseMethod("get_PAR_name", ds) - -#' Get Function Value condensed overview -#' -#' @param ds A `DataSet` or `DataSetList` object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the algorithm ID, best, worst and mean reached function -#' values, the number of runs and available budget for the DataSet -#' @examples -#' get_FV_overview(dsl) -#' get_FV_overview(dsl[[1]]) -#' get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') -#' @export -get_FV_overview <- function(ds, ...) UseMethod("get_FV_overview", ds) - -#' Get Runtime Value condensed overview -#' -#' @param ds A DataSet or DataSetList object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing the algorithm ID, minimum and maximum used evaluations, -#' number of runs and available budget for the DataSet -#' @examples -#' get_RT_overview(dsl) -#' get_RT_overview(dsl[[1]]) -#' @export -get_RT_overview <- function(ds, ...) UseMethod("get_RT_overview", ds) - -#' Get condensed overview of datasets -#' -#' @param ds A DataSet or DataSetList object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table containing some basic information about the provided DataSet(List) -#' @examples -#' get_overview(dsl) -#' get_overview(dsl[[1]]) -#' @export -get_overview <- function(ds, ...) UseMethod("get_overview", ds) - -#' @rdname get_FV_overview -#' @export -get_FV_overview.DataSet <- function(ds, ...) { - data <- ds$FV - runs <- ncol(data) - last_row <- data[nrow(data), ] - budget <- max(attr(ds, 'maxRT')) - maximization <- attr(ds, 'maximization') - - op <- ifelse(maximization, max, min) - op_inv <- ifelse(maximization, min, max) - - best_fv <- op(last_row, na.rm = T) - worst_recorded_fv <- op_inv(data, na.rm = T) - worst_fv <- op_inv(last_row, na.rm = T) - mean_fv <- mean(last_row, na.rm = T) - median_fv <- median(last_row, na.rm = T) - runs_reached <- sum(last_row == best_fv) - - data.table(algId = attr(ds, 'algId'), - DIM = attr(ds, 'DIM'), - funcId = attr(ds, 'funcId'), - `worst recorded` = worst_recorded_fv, - `worst reached` = worst_fv, - `best reached` = best_fv, - `mean reached` = mean_fv, - `median reached` = median_fv, - runs = runs, - `succ` = runs_reached, - budget = budget) -} - -#' @rdname get_RT_overview -#' @export -#' -get_RT_overview.DataSet <- function(ds, ...) { - - if (!is.null(attr(ds, "format")) && attr(ds, "format") == NEVERGRAD) { - data <- ds$FV - budget <- max(attr(ds, 'maxRT')) - runs <- ncol(data) - min_rt <- rownames(data) %>% as.integer %>% min - max_rt <- budget - } - - else{ - data <- ds$RT - runs <- ncol(data) - budget <- max(attr(ds, 'maxRT')) - min_rt <- min(data, na.rm = T) - max_rt <- max(data, na.rm = T) - } - - data.table(algId = attr(ds, 'algId'), - DIM = attr(ds, 'DIM'), - funcId = attr(ds, 'funcId'), - `miminal runtime` = min_rt, - `maximal runtime` = max_rt, - `runs` = runs, - `Budget` = budget) -} - -#' @rdname get_overview -#' @export -#' -get_overview.DataSet <- function(ds, ...) { - data <- ds$FV - runs <- ncol(data) - - budget <- max(attr(ds, 'maxRT')) - if (!is.null(ds$RT) && length(ds$RT) > 0) { - max_rt <- max(ds$RT, na.rm = T) - budget <- max(budget, max_rt) - } - else max_rt <- budget - - last_row <- data[nrow(data), ] - maximization <- attr(ds, 'maximization') - - op <- ifelse(maximization, max, min) - op_inv <- ifelse(maximization, min, max) - - best_fv <- op(last_row, na.rm = T) - worst_recorded_fv <- op_inv(data, na.rm = T) - worst_fv <- op_inv(last_row, na.rm = T) - mean_fv <- mean(last_row, na.rm = T) - median_fv <- median(last_row, na.rm = T) - runs_reached <- sum(last_row == best_fv) - - data.table(`algId` = attr(ds, 'algId'), - `DIM` = attr(ds, 'DIM'), - `funcId` = attr(ds, 'funcId'), - `runs` = runs, - `best reached` = best_fv, - `succ` = runs_reached, - `budget` = budget, - `max evals used` = max_rt, - `worst recorded` = worst_recorded_fv, - `worst reached` = worst_fv, - `mean reached` = mean_fv, - `median reached` = median_fv - ) - -} - -#' @rdname get_ERT -#' @export -#' -get_ERT.DataSet <- function(ds, ftarget, budget = NULL, ...) { - data <- ds$RT - if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') - else maxRT <- as.numeric(budget) - algId <- attr(ds, 'algId') - maximization <- attr(ds, 'maximization') - - ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) - FValues <- as.numeric(rownames(data)) - idx <- seq_along(FValues) - op <- ifelse(maximization, `>=`, `<=`) - - matched <- sapply(ftarget, function(f) idx[`op`(FValues, f)][1]) - - if (is.list(matched)) - return(data.table()) - - data <- data[matched, , drop = FALSE] - dt <- as.data.table(cbind(algId, ftarget, SP(data, maxRT)$ERT)) - colnames(dt) <- c('algId', 'target', 'ERT') - dt -} - -#' @rdname get_RT_summary -#' @export -#' -get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { - data <- ds$RT - if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') - else maxRT <- as.numeric(budget) - algId <- attr(ds, 'algId') - maximization <- attr(ds, 'maximization') - - ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) - FValues <- as.numeric(rownames(data)) - idx <- seq_along(FValues) - op <- ifelse(maximization, `>=`, `<=`) - - matched <- sapply( - ftarget, - function(f) { - idx[`op`(FValues, f)][1] - } - ) - - if (is.list(matched)) { - return(data.table()) - } - - data <- data[matched, , drop = FALSE] - apply(data, 1, IOHanalyzer_env$D_quantile) %>% - t %>% - as.data.table %>% - cbind(as.data.table(SP(data, maxRT))) %>% - cbind(algId, ftarget, - apply(data, 1, .mean), - apply(data, 1, .median), - apply(data, 1, .sd), .) %>% - set_colnames(c('algId', 'target', 'mean', 'median', - 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), - 'ERT', 'runs', 'ps')) -} - -#' Get the maximal running time -#' -#' @param ds A DataSet or DataSetList object -#' @param ... Arguments passed to other methods -#' -#' @return A data.table object containing the algorithm ID and the running time -#' when the algorithm terminates in each run -#' @examples -#' get_maxRT(dsl) -#' get_maxRT(dsl[[1]]) -#' @export -get_maxRT <- function(ds, ...) UseMethod("get_maxRT", ds) - -#' @rdname get_maxRT -#' @param output The format of the outputted table: 'wide' or 'long' -#' @export -#' -get_maxRT.DataSet <- function(ds, output = 'wide', ...) { - algId <- attr(ds, 'algId') - N <- ncol(ds$RT) - res <- t(c(algId, attr(ds, 'maxRT'))) %>% - as.data.table %>% - set_colnames(c('algId', paste0('run.', seq(N)))) - - if (output == 'long') { - res <- melt(res, id = 'algId', variable.name = 'run', value.name = 'maxRT') - res[, run := as.numeric(gsub('run.', '', run)) %>% as.integer - ][, maxRT := as.integer(maxRT) - ][order(run)] - } - res -} - -#' @rdname get_RT_sample -#' @param output A character determining the format of output data.table: 'wide' or 'long' -#' @export -get_RT_sample.DataSet <- function(ds, ftarget, output = 'wide', ...) { - data <- ds$RT - N <- ncol(data) - algId <- attr(ds, 'algId') - maximization <- attr(ds, 'maximization') - - ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) - FValues <- as.double(rownames(data)) - idx <- seq_along(FValues) - op <- ifelse(maximization, `>=`, `<=`) - - matched <- sapply( - ftarget, - function(f) { - idx[`op`(FValues, f)][1] - } - ) - - res <- cbind(algId, ftarget, as.data.table(data[matched, , drop = FALSE])) %>% - set_colnames(c('algId', 'target', paste0('run.', seq(N)))) - - if (output == 'long') { - # TODO: option to not add run etc to speed up performance of ECDF calculation? - res <- melt(res, id = c('algId', 'target'), variable.name = 'run', value.name = 'RT') - res[, run := as.integer(as.numeric(gsub('run.', '', run))) - ][, RT := as.integer(RT) - ][order(target, run)] - } - res -} - -#' Function to get just the RT samples needed, without any formatting to improve speed -#' @param RT_mat A matrix containing the RT-values of a dataset -#' @param target Which target-value to use -#' @param maximization Whether maximization is needed or not -#' @export -fast_RT_samples <- function(RT_mat, target, maximization = F) { - if (maximization) - idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) >= target] - else - idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) <= target] - if (length(idxs) > 0) { - return(RT_mat[idxs[[1]], ]) - } - return(rep(NA, 15)) -} - -#' @rdname get_FV_summary -#' @export -#' -get_FV_summary.DataSet <- function(ds, runtime, ...) { - data <- ds$FV - NC <- ncol(data) - NR <- nrow(data) - algId <- attr(ds, 'algId') - maximization <- attr(ds, 'maximization') - - runtime <- sort(as.numeric(unique(c(runtime)))) - RT <- as.numeric(rownames(data)) - idx <- seq_along(RT) - - matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) - data <- data[matched, , drop = FALSE] - - cbind(algId, runtime, NC, - apply(data, 1, .mean), - apply(data, 1, .median), - apply(data, 1, .sd), - as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile)))) %>% - set_colnames(c('algId', 'runtime', 'runs', 'mean', 'median', 'sd', - paste0(getOption("IOHanalyzer.quantiles") * 100, '%'))) -} - -#' @rdname get_FV_sample -#' @param output A String. The format of the output data: 'wide' or 'long' -#' -#' @export -#' -get_FV_sample.DataSet <- function(ds, runtime, output = 'wide', ...) { - data <- ds$FV - N <- ncol(data) - n_row <- nrow(data) - algId <- attr(ds, 'algId') - maximization <- attr(ds, 'maximization') - - runtime <- sort(as.numeric(unique(c(runtime)))) - RT <- as.numeric(rownames(data)) - idx <- seq_along(RT) - - matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) - res <- cbind(algId, runtime, as.data.table(data[matched, , drop = FALSE])) %>% - set_colnames(c('algId', 'runtime', paste0('run.', seq(N)))) - - if (output == 'long') { - res <- melt(res, id = c('algId', 'runtime'), variable.name = 'run', value.name = 'f(x)') - res[, run := as.integer(as.numeric(gsub('run.', '', run))) - ][order(runtime, run)] - } - res -} - -#' @rdname get_PAR_name -#' @export -#' -get_PAR_name.DataSet <- function(ds, which = 'by_FV') { - names(ds$PAR[[which]]) -} - -#' @rdname get_PAR_summary -#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved -#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -#' is the default value. -#' @export -get_PAR_summary.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', ...) { - if (which == 'by_FV') { - RefValues <- as.numeric(rownames(ds$RT)) - ds_par <- ds$PAR$by_FV - idx_name <- 'target' - } - else if (which == 'by_RT') { - RefValues <- as.numeric(rownames(ds$FV)) - ds_par <- ds$PAR$by_RT - idx_name <- 'runtime' - } - - idx <- seq_along(RefValues) - algId <- attr(ds, 'algId') - par_name <- get_PAR_name(ds, which = which) - - if (parId != 'all') - par_name <- intersect(par_name, parId) - if (length(par_name) == 0) - return(NULL) - - maximization <- attr(ds, 'maximization') - cond <- maximization || which == 'by_RT' - op <- ifelse(cond, `>=`, `<=`) - idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) - - matched <- sapply( - idxValue, - function(f) { - idx[`op`(RefValues, f)][1] - } - ) - - lapply(par_name, - function(par) { - data <- ds_par[[par]][matched, , drop = FALSE] - df <- cbind( - algId, par, idxValue, - apply(data, 1, function(x) length(x[!is.na(x)])), - apply(data, 1, .mean), - apply(data, 1, .median), - apply(data, 1, .sd), - as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile))) - ) - colnames(df) <- c('algId', 'parId', idx_name, 'runs', 'mean', 'median', 'sd', - paste0(getOption("IOHanalyzer.quantiles") * 100, '%')) - df - }) %>% - rbindlist -} - -#' @rdname get_PAR_sample -#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved -#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -#' is the default value. -#' @param output A character. The format of the output data: 'wide' or 'long' -#' @export -get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', - output = 'wide', ...) { - N <- length(attr(ds, 'instance')) - if (which == 'by_FV') { - RefValues <- as.numeric(rownames(ds$RT)) - ds_par <- ds$PAR$by_FV - idx_name <- 'target' - } - else if (which == 'by_RT') { - RefValues <- as.numeric(rownames(ds$FV)) - ds_par <- ds$PAR$by_RT - idx_name <- 'runtime' - } - - idx <- seq_along(RefValues) - algId <- attr(ds, 'algId') - par_name <- get_PAR_name(ds, which = which) - - if (parId != 'all') - par_name <- intersect(par_name, parId) - if (length(par_name) == 0) - return(NULL) - - maximization <- attr(ds, 'maximization') - cond <- maximization || which == 'by_RT' - op <- ifelse(cond, `>=`, `<=`) - idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) - matched <- sapply(idxValue, function(f) idx[`op`(RefValues, f)][1]) - - res <- lapply(par_name, - function(parId) { - data <- ds_par[[parId]] - data <- as.data.table(data[matched, , drop = FALSE]) - data <- cbind(algId, parId, idxValue, data) - colnames(data) <- c('algId', 'parId', idx_name, paste0('run.', seq(N))) - data - }) - res <- rbindlist(res) - - if (output == 'long') { - res <- melt(res, id = c('algId', 'parId', idx_name), variable.name = 'run', value.name = 'value') - res[, run := as.integer(as.numeric(gsub('run.', '', run)))] - if (which == 'by_FV') - res[order(target, run)] - else if (which == 'by_RT') - res[order(runtime, run)] - } - res -} +#' Constructor of S3 class 'DataSet' +#' +#' DataSet contains the following attributes +#' * funId +#' * DIM +#' * algId +#' * datafile +#' * instance +#' * maxEvals +#' * finalFunEvals +#' * comment +#' * Additional attributes based on the original format +#' +#' @param info A List. Contains a set of in a *.info file. +#' @param verbose Logical. +#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? +#' Set to NULL to determine automatically based on format +#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" +#' @param subsampling Logical. Whether *.cdat files are subsampled? +#' +#' @return A S3 object 'DataSet' +#' @export +#' @examples +#' path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') +#' info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) +#' DataSet(info[[1]]) +DataSet <- function(info, verbose = F, maximization = NULL, format = IOHprofiler, + subsampling = FALSE) { + if (!is.null(info)) { + path <- dirname(info$datafile) + suite <- toupper(info$suite) + + # for an unknown suite, to detect the format + if (is.null(suite) || length(suite) == 0) { + if (verbose) + warning("Suite-name not provided in .info-file, taking best guess based on + the format of data-files.") + suite <- switch(format, + IOHprofiler = "Unknown", + COCO = "BBOB", + BIOBJ_COCO = "biobj-bbob", + TWO_COL = "Unknown") + } + + if (is.null(maximization) || maximization == AUTOMATIC) { + #TODO: Better way to deal with capitalization of attributes + if (!is.null(info$maximization)) maximization <- info$maximization + else if (!is.null(info$Maximization)) maximization <- info$Maximization + else if (!is.null(suite)) { + if (verbose) + warning("maximization or minimization not specified in .info-file, + taking best guess based on the suite-name.") + if (grepl("\\w*bbob\\w*", suite, ignore.case = T) != 0) + maximization <- FALSE + else + maximization <- TRUE + } + else { + warning("Can't detect maximization based on suite-attribute, setting to + minimization by default") + maximization <- FALSE # default to minimization + } + } + + if(!(isTRUE(maximization) || isFALSE(maximization))) + warning("unclear whether we should maximize or minimize.") + + datBaseName <- sub(pattern = '(.*)\\..*$', replacement = '\\1', basename(info$datafile)) + datFile <- file.path(path, paste0(datBaseName, '.dat')) + tdatFile <- file.path(path, paste0(datBaseName, '.tdat')) + cdatFile <- file.path(path, paste0(datBaseName, '.cdat')) + + # NOTE: preference on data file for the alignment by RT: cdat > tdat > dat + if (file.exists(cdatFile)) + fvFile <- cdatFile + else if (file.exists(tdatFile)) + fvFile <- tdatFile + else if (file.exists(datFile)) + fvFile <- datFile + else + stop('No datafiles found, please verify the integrity of the chosen files') + + # NOTE: preference on data file for the alignment by FV: dat > tdat > cdat + if (file.exists(datFile)) + rtFile <- datFile + else if (file.exists(tdatFile)) + rtFile <- tdatFile + else if (file.exists(cdatFile)) + # TODO: perhaps turn on `subsampling` here as this would take quite some time + rtFile <- cdatFile + + read_raw <- switch( + format, + IOHprofiler = read_dat, + COCO = read_dat__COCO, + BIOBJ_COCO = read_dat__BIOBJ_COCO, + TWO_COL = read_dat # TODO: perhaps rename `TWO_COL` or to use a better naming + # scheme for all format names + ) + + RT_raw <- read_raw(rtFile, subsampling) + FV_raw <- read_raw(fvFile, subsampling) + + if (is.null(maximization)) { + if (verbose) + warning("Did not find maximization / minimization, auto-detecting based on + function value progression") + # TODO: idxTarget should be set depending on the data format + idxTarget <- 2 + cond <- unique(lapply(FV_raw, function(FV) FV[1, idxTarget] >= FV[nrow(FV), idxTarget])) + if (length(cond) > 1) + stop('The detected maximization differs in multiple runs') + maximization <- cond + } + + RT <- align_running_time(RT_raw, format = format, maximization = maximization) + FV <- align_function_value(FV_raw, format = format) + + PAR <- list( + 'by_FV' = RT[names(RT) != 'RT'], + 'by_RT' = FV[names(FV) != 'FV'] + ) + + RT <- RT$RT + mode(RT) <- 'integer' + FV <- FV$FV + + if (format %in% c(IOHprofiler)) { + # try to save some memory here... + FV <- tryCatch({ + .FV <- FV + mode(.FV) <- 'integer' + if (all(FV == .FV)) .FV + else FV + }, + warning = function(w) FV) # in case the type coercion gives a warning + } + + # TODO: add more data sanity checks + maxRT <- set_names(sapply(RT_raw, function(d) d[nrow(d), idxEvals]), NULL) + # Fix for old-format files which do not store used runtime in .dat-files + maxRT <- pmax(maxRT, info$maxRT) + if (any(maxRT != info$maxRT) && verbose) + warning('Inconsitent maxRT in *.info file and *.cdat file') + + # TODO: clean up these if-statements: Function to set idxTarget and n_data_column? + # `idxTarget` is a global variable? + if (format == TWO_COL) + finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget - 1]), NULL) + else + finalFV <- set_names(sapply(FV_raw, function(d) d[nrow(d), idxTarget]), NULL) + + if (any(finalFV != info$finalFV) && verbose) + warning('Inconsitent finalFvalue in *.info file and *.dat file') + + if (length(info$instance) != length(RT_raw)) { + if (verbose) + warning('The number of instances found in the info is inconsistent with the data!') + info$instance <- seq(length(RT_raw)) + } + + do.call( + function(...) + structure(list(RT = RT, FV = FV, PAR = PAR), class = c('DataSet', 'list'), ...), + c(info, list(maxRT = maxRT, finalFV = finalFV, format = format, + maximization = maximization, suite = suite, ID = info$algId)) + ) + } + else + structure(list(), class = c('DataSet', 'list')) +} + +#' S3 concatenation function for DataSet +#' +#' @description Concatenation for DataSets. Combines multiple runs from separate DataSets +#' into a single DataSet object if all provided arguments have the same dimension, function ID and +#' algorithm ID, and each contains only a single run. Currently does not support parameter tracking +#' +#' @param ... The DataSets to concatenate +#' @return A new DataSet +#' @export +#' @examples +#' c(dsl[[1]], dsl[[1]]) +c.DataSet <- function(...) { + dsl <- list(...) + + if (length(dsl) == 1) + dsl <- dsl[[1]] + dsl <- dsl[sapply(dsl, length) != 0] + + if (length(dsl) == 0) + return() + if (length(dsl) == 1) + return(dsl[[1]]) + + for (ds in dsl) { + if (!any((class(ds)) == 'DataSet')) + stop("Operation only possible when all arguments are DataSets") + } + + fixed_attrs <- + c('suite', 'maximization', 'DIM', 'funcId', 'algId', 'format') + info <- list() + for (attr_str in fixed_attrs) { + temp <- unique(unlist(lapply(dsl, function(x) + attr(x, attr_str)))) + if (length(temp) > 1) { + stop( + paste0( + "Attempted to add datasets with different ", + attr_str, + "-attributes! This is not supported, please keep them as separate DataSets!" + ) + ) + } + info <- c(info, temp) + } + names(info) <- fixed_attrs + + #Record number of runs to make masks of static attributes + nr_runs <- sapply(dsl, function(x) + ncol(x$FV)) + for (attr_str in names(attributes(dsl[[1]]))) { + if (attr_str %in% fixed_attrs || + attr_str %in% c("names", "class")) + next + temp <- unlist(lapply(dsl, function(x) + attr(x, attr_str))) + if (length(unique(temp)) == 1) + temp <- unique(temp) + else { + if (length(temp) == length(nr_runs)) + temp <- list(temp_name = rep(temp, nr_runs)) + else + temp <- list(temp_name = temp) + } + names(temp) <- attr_str + info <- c(info, temp) + } + + format <- info[['format']] #attr(dsl[[1]], "format") + + RT_raw <- unlist(lapply(dsl, function(ds) { + lapply(seq_len(ncol(ds$RT)), function(cnr) { + rt_temp <- as.matrix(ds$RT[, cnr]) + cbind(rt_temp, as.numeric(rownames(ds$RT))) + }) + }), recursive = F) + + RT <- + align_running_time(RT_raw, + format = "TWO_COL", + maximization = info$maximization)$RT + FV <- align_function_value(RT_raw, format = "TWO_COL")$FV + + # TODO: to deal with cases where aligned parameters are present in original DataSets + PAR <- list('by_FV' = RT[names(RT) != 'RT'], + 'by_RT' = FV[names(FV) != 'FV']) + + # Unaligned parameters + for (par_name in names(dsl[[1]]$PAR)) { + if (!par_name %in% c('by_FV', 'by_RT')) + PAR[[par_name]] <- + unlist(lapply(dsl, function(x) { + x$PAR[[par_name]] + }), recursive = F) + } + + do.call(function(...) + structure(list( + RT = RT, FV = FV, PAR = PAR + ), class = c('DataSet', 'list'), ...), + c(info)) +} + +#' S3 subset function for DataSet +#' +#' @description Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet +#' and turned into a new DataSet object. +#' +#' @param x The DataSet from which to get a subset +#' @param mask The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs +#' present in the provided dataset object x. +#' @param ... Arguments passed to underlying subset method (not yet supported) +#' +#' @return A new DataSet +#' @export +#' @examples +#' subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) +subset.DataSet <- function(x, mask, ...) { + if (length(mask) != ncol(x$FV)) + stop(paste("The input DataSet has", ncol(x$FV), "runs while the input mask array has length", length(mask))) + + info <- list() + for (attr_str in names(attributes(x))) { + if (attr_str %in% c('names', 'class')) + next + temp <- attr(x, attr_str) + if (length(unique(temp)) == 1) + temp <- unique(temp) + else { + if (length(temp) == length(mask)) + temp <- list(temp[mask]) + else{ + warning( + paste0( + "Attribute detected (", + attr_str, + ") with incorrect length for the mask-based subsetting!" + ) + ) + next + } + } + names(temp) <- attr_str + info <- c(info, temp) + } + + format <- info[['format']] + + RT <- as.matrix(x$RT[, mask]) + FV <- as.matrix(x$FV[, mask]) + + PAR <- list( + 'by_FV' = ifelse(ncol(x$PAR$by_FV) == length(mask), x$PAR$by_FV[, mask], NULL), + 'by_RT' = ifelse(ncol(x$PAR$by_RT) == length(mask), x$PAR$by_RT[, mask], NULL) + ) + + do.call(function(...) + structure(list( + RT = RT, FV = FV, PAR = PAR + ), class = c('DataSet', 'list'), ...), + c(info)) +} + +#' S3 generic print operator for DataSet +#' +#' @param x A DataSet object +#' @param ... Arguments passed to other methods +#' +#' @return A short description of the DataSet +#' @examples +#' print(dsl[[1]]) +#' @export +print.DataSet <- function(x, ...) { + cat(as.character.DataSet(x, ...)) +} + +#' S3 generic cat operator for DataSet +#' +#' @param x A DataSet object +#' +#' @return A short description of the DataSet +#' @export +#' @examples +#' cat.DataSet(dsl[[1]]) +cat.DataSet <- function(x) cat(as.character(x)) + +#' S3 generic as.character operator for DataSet +#' +#' @param x A DataSet object +#' @param verbose Verbose mode, currently not implemented +#' @param ... Arguments passed to other methods +#' +#' @return A short description of the DataSet +#' @export +#' @examples +#' as.character(dsl[[1]]) +as.character.DataSet <- function(x, verbose = F, ...) { + # TODO: implement the verbose mode + sprintf('DataSet(%s on f%s %dD)', attr(x, 'algId'), attr(x, 'funcId'), + attr(x, 'DIM')) +} + +#' S3 generic summary operator for DataSet +#' +#' @param object A DataSet object +#' @param ... Arguments passed to other methods +#' +#' @return A summary of the DataSet containing both function-value and runtime based statistics. +#' @examples +#' summary(dsl[[1]]) +#' @export +summary.DataSet <- function(object, ...) { + ds_attr <- attributes(object) + cat('DataSet Object:\n') + cat(sprintf('Source: %s\n', ds_attr$src)) + cat(sprintf('Algorithm: %s\n', ds_attr$algId)) + cat(sprintf('Function ID: %s\n', ds_attr$funcId)) + cat(sprintf('Dimension: %dD\n', ds_attr$DIM)) + + n_instance <- length(ds_attr$instance) + if (n_instance >= 15) { + inst <- paste0(paste(ds_attr$instance[1:7], collapse = ','), + ',...,', + paste(ds_attr$instance[(n_instance - 7):n_instance], collapse = ',')) + cat(sprintf('%d instance found: %s\n\n', n_instance, inst)) + } + else + cat(sprintf('%d instance found: %s\n\n', n_instance, paste(ds_attr$instance, collapse = ','))) + + cat('runtime summary:\n') + function_values <- as.numeric(rownames(object$RT)) + RT.summary <- get_RT_summary(object, function_values) + print(RT.summary) + cat('\n') + + cat('function value summary:\n') + runtimes <- as.numeric(rownames(object$FV)) + if (length(runtimes) > 100) { + runtimes <- runtimes[seq(1, length(runtimes), length.out = 100)] + } + + FV.summary <- get_FV_summary(object, runtimes) + print(FV.summary) + cat('\n') + + cat(paste('Attributes:', paste0(names(ds_attr), collapse = ', '))) +} + +#' S3 generic == operator for DataSets +#' +#' @param dsL A `DataSet` object +#' @param dsR A `DataSet` object +#' +#' +#' @return True if the DataSets contain the same function, dimension and algorithm, +#' and have the exact same attributes +#' @examples +#' dsl[[1]] == dsl[[2]] +#' @export +`==.DataSet` <- function(dsL, dsR) { + if (length(dsL) == 0 || length(dsR) == 0) + return(FALSE) + + for (attr_str in names(attributes(dsL))) { + if (any(attr(dsL, attr_str) != attr(dsR, attr_str))) return(FALSE) + } + return(TRUE) +} + +#' Get Expected RunTime +#' +#' @param ds A DataSet or DataSetList object +#' @param budget Optional; overwrites the budget found in ds for ERT-calculation +#' @param ... Arguments passed to other methods +#' @param ftarget The function target(s) for which to get the ERT +#' +#' @return A data.table containing the runtime samples for each provided target +#' function value +#' @examples +#' get_ERT(dsl, 14) +#' get_ERT(dsl[[1]], 14) +#' @export +#' +get_ERT <- function(ds, ftarget, budget, ...) UseMethod("get_ERT", ds) + +#' Get RunTime Sample +#' +#' @param ds A DataSet or DataSetList object +#' @param ftarget A Numerical vector. Function values at which runtime values are consumed +#' @param ... Arguments passed to other methods +#' +#' +#' @return A data.table containing the runtime samples for each provided target +#' function value +#' @examples +#' get_RT_sample(dsl, 14) +#' get_RT_sample(dsl[[1]], 14) +#' @export +get_RT_sample <- function(ds, ftarget, ...) UseMethod("get_RT_sample", ds) + +#' Get RunTime Summary +#' +#' @param ds A DataSet or DataSetList object +#' @param budget Optional; overwrites the budget found in ds for ERT-calculation +#' @param ... Arguments passed to other methods +#' @param ftarget The function target(s) for which to get the runtime summary +#' +#' @return A data.table containing the runtime statistics for each provided target +#' function value +#' @examples +#' get_RT_summary(dsl, 14) +#' get_RT_summary(dsl[[1]], 14) +#' @export +get_RT_summary <- function(ds, ftarget, budget, ...) UseMethod("get_RT_summary", ds) + +#' Get Funtion Value Samples +#' +#' @param ds A DataSet or DataSetList object +#' @param runtime A Numerical vector. Runtimes at which function values are reached +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the function value samples for each provided +#' target runtime +#' @examples +#' get_FV_sample(dsl, 100) +#' get_FV_sample(dsl[[1]], 100) +#' @export +get_FV_sample <- function(ds, ...) UseMethod("get_FV_sample", ds) + +#' Get Function Value Summary +#' +#' @param ds A DataSet or DataSetList object +#' @param runtime A Numerical vector. Runtimes at which function values are reached +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the function value statistics for each provided +#' target runtime value +#' @examples +#' get_FV_summary(dsl, 100) +#' get_FV_summary(dsl[[1]], 100) +#' @export +get_FV_summary <- function(ds, ...) UseMethod("get_FV_summary", ds) + +#' Get Parameter Value Samples +#' +#' @param ds A DataSet or DataSetList object +#' @param idxValue A Numerical vector. Index values at which parameter values are observed. +#' The index value can either take its value in the range of running times, or function values. +#' Such a value type is signified by `which` parameter. +#' @param ... Arguments passed to other methods +#' +#' @return A data.table object containing parameter values aligned at each given target value +#' @examples +#' get_PAR_sample(dsl, 14) +#' get_PAR_sample(dsl[[1]], 14) +#' @export +get_PAR_sample <- function(ds, idxValue, ...) UseMethod("get_PAR_sample", ds) + +#' Get Parameter Value Summary +#' +#' @param ds A DataSet or DataSetList object +#' @param idxValue A Numerical vector. Index values at which parameter values are observed. +#' The index value can either take its value in the range of running times, or function values. +#' Such a value type is signified by `which` parameter. +#' @param ... Arguments passed to other methods +#' +#' @return A data.table object containing basic statistics of parameter values aligned at each given target value +#' @examples +#' get_PAR_summary(dsl, 14) +#' get_PAR_summary(dsl[[1]], 14) +#' @export +get_PAR_summary <- function(ds, idxValue, ...) UseMethod("get_PAR_summary", ds) + +#' Get the parameter names of the algorithm +#' +#' @param ds A DataSet object +#' @param which a string takes it value in `c('by_FV', 'by_RT')`, indicating the +#' parameters aligned against the running time (RT) or function value (FV). `'by_FV'` +#' is the default value. +#' @return a character list of paramter names, if recorded in the data set +#' @examples +#' get_PAR_name(dsl[[1]]) +#' @export +get_PAR_name <- function(ds, which) UseMethod("get_PAR_name", ds) + +#' Get Function Value condensed overview +#' +#' @param ds A `DataSet` or `DataSetList` object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the algorithm ID, best, worst and mean reached function +#' values, the number of runs and available budget for the DataSet +#' @examples +#' get_FV_overview(dsl) +#' get_FV_overview(dsl[[1]]) +#' get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') +#' @export +get_FV_overview <- function(ds, ...) UseMethod("get_FV_overview", ds) + +#' Get Runtime Value condensed overview +#' +#' @param ds A DataSet or DataSetList object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing the algorithm ID, minimum and maximum used evaluations, +#' number of runs and available budget for the DataSet +#' @examples +#' get_RT_overview(dsl) +#' get_RT_overview(dsl[[1]]) +#' @export +get_RT_overview <- function(ds, ...) UseMethod("get_RT_overview", ds) + +#' Get condensed overview of datasets +#' +#' @param ds A DataSet or DataSetList object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table containing some basic information about the provided DataSet(List) +#' @examples +#' get_overview(dsl) +#' get_overview(dsl[[1]]) +#' @export +get_overview <- function(ds, ...) UseMethod("get_overview", ds) + +#' Get condensed overview of datasets +#' +#' Get the unique identifiers for each DataSet in the provided DataSetList +#' +#' If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +#' this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility +#' +#' @param ds The DataSetList +#' @param ... Arguments passed to other methods +#' +#' @return The list of unique identiefiers present in dsl +#' @examples +#' get_id(dsl) +#' get_id(dsl[[1]]) +#' @export +get_id <- function(ds, ...) UseMethod("get_id", ds) + +#' @rdname get_FV_overview +#' @export +get_FV_overview.DataSet <- function(ds, ...) { + data <- ds$FV + runs <- ncol(data) + last_row <- data[nrow(data), ] + budget <- max(attr(ds, 'maxRT')) + maximization <- attr(ds, 'maximization') + + op <- ifelse(maximization, max, min) + op_inv <- ifelse(maximization, min, max) + + best_fv <- op(last_row, na.rm = T) + worst_recorded_fv <- op_inv(data, na.rm = T) + worst_fv <- op_inv(last_row, na.rm = T) + mean_fv <- mean(last_row, na.rm = T) + median_fv <- median(last_row, na.rm = T) + runs_reached <- sum(last_row == best_fv) + + data.table(ID = get_id(ds), + DIM = attr(ds, 'DIM'), + funcId = attr(ds, 'funcId'), + `worst recorded` = worst_recorded_fv, + `worst reached` = worst_fv, + `best reached` = best_fv, + `mean reached` = mean_fv, + `median reached` = median_fv, + runs = runs, + `succ` = runs_reached, + budget = budget) +} + +#' @rdname get_RT_overview +#' @export +#' +get_RT_overview.DataSet <- function(ds, ...) { + + if (!is.null(attr(ds, "format")) && attr(ds, "format") == NEVERGRAD) { + data <- ds$FV + budget <- max(attr(ds, 'maxRT')) + runs <- ncol(data) + min_rt <- rownames(data) %>% as.integer %>% min + max_rt <- budget + } + + else{ + data <- ds$RT + runs <- ncol(data) + budget <- max(attr(ds, 'maxRT')) + min_rt <- min(data, na.rm = T) + max_rt <- max(data, na.rm = T) + } + + data.table(ID = get_id(ds), + DIM = attr(ds, 'DIM'), + funcId = attr(ds, 'funcId'), + `miminal runtime` = min_rt, + `maximal runtime` = max_rt, + `runs` = runs, + `Budget` = budget) +} + +#' @rdname get_overview +#' @export +#' +get_overview.DataSet <- function(ds, ...) { + data <- ds$FV + runs <- ncol(data) + + budget <- max(attr(ds, 'maxRT')) + if (!is.null(ds$RT) && length(ds$RT) > 0) { + max_rt <- max(ds$RT, na.rm = T) + budget <- max(budget, max_rt) + } + else max_rt <- budget + + last_row <- data[nrow(data), ] + maximization <- attr(ds, 'maximization') + + op <- ifelse(maximization, max, min) + op_inv <- ifelse(maximization, min, max) + + best_fv <- op(last_row, na.rm = T) + worst_recorded_fv <- op_inv(data, na.rm = T) + worst_fv <- op_inv(last_row, na.rm = T) + mean_fv <- mean(last_row, na.rm = T) + median_fv <- median(last_row, na.rm = T) + runs_reached <- sum(last_row == best_fv) + + data.table(ID = get_id(ds), + `DIM` = attr(ds, 'DIM'), + `funcId` = attr(ds, 'funcId'), + `runs` = runs, + `best reached` = best_fv, + `succ` = runs_reached, + `budget` = budget, + `max evals used` = max_rt, + `worst recorded` = worst_recorded_fv, + `worst reached` = worst_fv, + `mean reached` = mean_fv, + `median reached` = median_fv + ) + +} + +#' @rdname get_ERT +#' @export +#' +get_ERT.DataSet <- function(ds, ftarget, budget = NULL, ...) { + data <- ds$RT + if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') + else maxRT <- as.numeric(budget) + algId <- attr(ds, 'algId') + maximization <- attr(ds, 'maximization') + + ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) + FValues <- as.numeric(rownames(data)) + idx <- seq_along(FValues) + op <- ifelse(maximization, `>=`, `<=`) + + matched <- sapply(ftarget, function(f) idx[`op`(FValues, f)][1]) + + if (is.list(matched)) + return(data.table()) + + data <- data[matched, , drop = FALSE] + dt <- as.data.table(cbind(get_id(ds), ftarget, SP(data, maxRT)$ERT)) + colnames(dt) <- c('ID', 'target', 'ERT') + dt +} + +#' @rdname get_RT_summary +#' @export +#' +get_RT_summary.DataSet <- function(ds, ftarget, budget = NULL, ...) { + data <- ds$RT + if (is.null(budget) || is.na(budget)) maxRT <- attr(ds, 'maxRT') + else maxRT <- as.numeric(budget) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) + FValues <- as.numeric(rownames(data)) + idx <- seq_along(FValues) + op <- ifelse(maximization, `>=`, `<=`) + + matched <- sapply( + ftarget, + function(f) { + idx[`op`(FValues, f)][1] + } + ) + + if (is.list(matched)) { + return(data.table()) + } + + data <- data[matched, , drop = FALSE] + dt_temp <- apply(data, 1, IOHanalyzer_env$D_quantile) %>% + t %>% + as.data.table %>% + cbind(as.data.table(SP(data, maxRT))) %>% + cbind(ID, ftarget, + apply(data, 1, .mean), + apply(data, 1, .median), + apply(data, 1, .sd), .) %>% + set_colnames(c('ID', 'target', 'mean', 'median', + 'sd', paste0(getOption("IOHanalyzer.quantiles") * 100, '%'), + 'ERT', 'runs', 'ps')) + dt_temp +} + +#' Get the maximal running time +#' +#' @param ds A DataSet or DataSetList object +#' @param ... Arguments passed to other methods +#' +#' @return A data.table object containing the algorithm ID and the running time +#' when the algorithm terminates in each run +#' @examples +#' get_maxRT(dsl) +#' get_maxRT(dsl[[1]]) +#' @export +get_maxRT <- function(ds, ...) UseMethod("get_maxRT", ds) + +#' @rdname get_maxRT +#' @param output The format of the outputted table: 'wide' or 'long' +#' @export +#' +get_maxRT.DataSet <- function(ds, output = 'wide', ...) { + ID <- get_id(ds) + N <- ncol(ds$RT) + res <- t(c(ID, attr(ds, 'maxRT'))) %>% + as.data.table %>% + set_colnames(c('ID', paste0('run.', seq(N)))) + + if (output == 'long') { + res <- melt(res, id = 'ID', variable.name = 'run', value.name = 'maxRT') + res[, run := as.numeric(gsub('run.', '', run)) %>% as.integer + ][, maxRT := as.integer(maxRT) + ][order(run)] + } + res +} + +#' @rdname get_RT_sample +#' @param output A character determining the format of output data.table: 'wide' or 'long' +#' @export +get_RT_sample.DataSet <- function(ds, ftarget, output = 'wide', ...) { + data <- ds$RT + N <- ncol(data) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + ftarget <- sort(as.double(unique(c(ftarget))), decreasing = !maximization) + FValues <- as.double(rownames(data)) + idx <- seq_along(FValues) + op <- ifelse(maximization, `>=`, `<=`) + + matched <- sapply( + ftarget, + function(f) { + idx[`op`(FValues, f)][1] + } + ) + + res <- cbind(ID, ftarget, as.data.table(data[matched, , drop = FALSE])) %>% + set_colnames(c('ID', 'target', paste0('run.', seq(N)))) + + if (output == 'long') { + # TODO: option to not add run etc to speed up performance of ECDF calculation? + res <- melt(res, id = c('ID', 'target'), variable.name = 'run', value.name = 'RT') + res[, run := as.integer(as.numeric(gsub('run.', '', run))) + ][, RT := as.integer(RT) + ][order(target, run)] + } + res +} + +#' Function to get just the RT samples needed, without any formatting to improve speed +#' @param RT_mat A matrix containing the RT-values of a dataset +#' @param target Which target-value to use +#' @param maximization Whether maximization is needed or not +#' @export +fast_RT_samples <- function(RT_mat, target, maximization = F) { + if (maximization) + idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) >= target] + else + idxs <- seq_along(rownames(RT_mat))[as.double(rownames(RT_mat)) <= target] + if (length(idxs) > 0) { + return(RT_mat[idxs[[1]], ]) + } + return(rep(NA, 15)) +} + +#' @rdname get_FV_summary +#' @export +#' +get_FV_summary.DataSet <- function(ds, runtime, ...) { + data <- ds$FV + NC <- ncol(data) + NR <- nrow(data) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + runtime <- sort(as.numeric(unique(c(runtime)))) + RT <- as.numeric(rownames(data)) + idx <- seq_along(RT) + + matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) + data <- data[matched, , drop = FALSE] + + cbind(ID, runtime, NC, + apply(data, 1, .mean), + apply(data, 1, .median), + apply(data, 1, .sd), + as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile)))) %>% + set_colnames(c('ID', 'runtime', 'runs', 'mean', 'median', 'sd', + paste0(getOption("IOHanalyzer.quantiles") * 100, '%'))) +} + +#' @rdname get_FV_sample +#' @param output A String. The format of the output data: 'wide' or 'long' +#' +#' @export +#' +get_FV_sample.DataSet <- function(ds, runtime, output = 'wide', ...) { + data <- ds$FV + N <- ncol(data) + n_row <- nrow(data) + ID <- get_id(ds) + maximization <- attr(ds, 'maximization') + + runtime <- sort(as.numeric(unique(c(runtime)))) + RT <- as.numeric(rownames(data)) + idx <- seq_along(RT) + + matched <- sapply(runtime, function(r) rev(idx[r >= RT])[1]) + res <- cbind(ID, runtime, as.data.table(data[matched, , drop = FALSE])) %>% + set_colnames(c('ID', 'runtime', paste0('run.', seq(N)))) + + if (output == 'long') { + res <- melt(res, id = c('ID', 'runtime'), variable.name = 'run', value.name = 'f(x)') + res[, run := as.integer(as.numeric(gsub('run.', '', run))) + ][order(runtime, run)] + } + res +} + +#' @rdname get_PAR_name +#' @export +#' +get_PAR_name.DataSet <- function(ds, which = 'by_FV') { + names(ds$PAR[[which]]) +} + +#' @rdname get_PAR_summary +#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved +#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +#' is the default value. +#' @export +get_PAR_summary.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', ...) { + if (which == 'by_FV') { + RefValues <- as.numeric(rownames(ds$RT)) + ds_par <- ds$PAR$by_FV + idx_name <- 'target' + } + else if (which == 'by_RT') { + RefValues <- as.numeric(rownames(ds$FV)) + ds_par <- ds$PAR$by_RT + idx_name <- 'runtime' + } + + idx <- seq_along(RefValues) + ID <- get_id(ds) + par_name <- get_PAR_name(ds, which = which) + + if (parId != 'all') + par_name <- intersect(par_name, parId) + if (length(par_name) == 0) + return(NULL) + + maximization <- attr(ds, 'maximization') + cond <- maximization || which == 'by_RT' + op <- ifelse(cond, `>=`, `<=`) + idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) + + matched <- sapply( + idxValue, + function(f) { + idx[`op`(RefValues, f)][1] + } + ) + + lapply(par_name, + function(par) { + data <- ds_par[[par]][matched, , drop = FALSE] + df <- cbind( + ID, par, idxValue, + apply(data, 1, function(x) length(x[!is.na(x)])), + apply(data, 1, .mean), + apply(data, 1, .median), + apply(data, 1, .sd), + as.data.table(t(apply(data, 1, IOHanalyzer_env$C_quantile))) + ) + colnames(df) <- c('ID', 'parId', idx_name, 'runs', 'mean', 'median', 'sd', + paste0(getOption("IOHanalyzer.quantiles") * 100, '%')) + df + }) %>% + rbindlist +} + +#' @rdname get_PAR_sample +#' @param parId A character vector. Either 'all' or the name of parameters to be retrieved +#' @param which A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +#' retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +#' is the default value. +#' @param output A character. The format of the output data: 'wide' or 'long' +#' @export +get_PAR_sample.DataSet <- function(ds, idxValue, parId = 'all', which = 'by_FV', + output = 'wide', ...) { + N <- length(attr(ds, 'instance')) + if (which == 'by_FV') { + RefValues <- as.numeric(rownames(ds$RT)) + ds_par <- ds$PAR$by_FV + idx_name <- 'target' + } + else if (which == 'by_RT') { + RefValues <- as.numeric(rownames(ds$FV)) + ds_par <- ds$PAR$by_RT + idx_name <- 'runtime' + } + + idx <- seq_along(RefValues) + ID <- get_id(ds) + par_name <- get_PAR_name(ds, which = which) + + if (parId != 'all') + par_name <- intersect(par_name, parId) + if (length(par_name) == 0) + return(NULL) + + maximization <- attr(ds, 'maximization') + cond <- maximization || which == 'by_RT' + op <- ifelse(cond, `>=`, `<=`) + idxValue <- sort(as.numeric(c(idxValue)), decreasing = !cond) + matched <- sapply(idxValue, function(f) idx[`op`(RefValues, f)][1]) + + res <- lapply(par_name, + function(parId) { + data <- ds_par[[parId]] + data <- as.data.table(data[matched, , drop = FALSE]) + data <- cbind(ID, parId, idxValue, data) + colnames(data) <- c('ID', 'parId', idx_name, paste0('run.', seq(N))) + data + }) + res <- rbindlist(res) + + if (output == 'long') { + res <- melt(res, id = c('ID', 'parId', idx_name), variable.name = 'run', value.name = 'value') + res[, run := as.integer(as.numeric(gsub('run.', '', run)))] + if (which == 'by_FV') + res[order(target, run)] + else if (which == 'by_RT') + res[order(runtime, run)] + } + res +} + +#' @rdname get_id +#' @export +get_id.DataSet <- function(ds, ...) { + temp <- attr(ds, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'change_id' function.") + return(attr(ds, 'algId')) + } + return(unique(temp)) +} \ No newline at end of file diff --git a/R/DataSetList.R b/R/DataSetList.R index 3e4818b5..265605bf 100644 --- a/R/DataSetList.R +++ b/R/DataSetList.R @@ -1,1294 +1,1441 @@ -#' Read all the data file in a folfer -# -#' @param path Path to the data files. Will look for all .info-files in this directory and use -#' the corresponding datafiles to create the DataSetList -#' @param verbose Logical. -#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? -#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" -#' @param subsampling Logical. Whether *.cdat files are subsampled? -#' @param print_fun Function used to print output when in verbose mode -#' @noRd -#' @return A DataSetList object -read_dir <- function(path, verbose = T, print_fun = NULL, maximization = TRUE, - format = IOHprofiler, subsampling = FALSE) { - DataSetList( - path, - verbose, - print_fun, - maximization = maximization, - format = format, - subsampling = subsampling - ) - } - -# TODO: improve documentation of this S3 class -#' S3 constructor of the 'DataSetList' -#' -#' Attributes -#' funId -#' DIM -#' algId -# -#' @param path Path to the data files. Will look for all .info-files in this directory and use -#' the corresponding datafiles to create the DataSetList -#' @param verbose Logical. -#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? -#' @param format A character. The format of data source, options are: -#' \itemize{ -#' \item'IOHProfiler' -#' \item'COCO' -#' \item'TWO_COL' -#' \item'COCO_BIOBJ' -#' \item'NEVERGRAD' -#' \item'SOS' -#' } -#' These formats are specified in more detail in our github wiki. -#' @param subsampling Logical. Whether *.cdat files are subsampled? -#' @param print_fun Function used to print output when in verbose mode -#' -#' @return A DataSetList object -#' @export -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -#' DataSetList(path) -DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization = NULL, - format = IOHprofiler, subsampling = FALSE) { - if (is.null(path)) - return(structure(list(), class = c('DataSetList', 'list'))) - - path <- trimws(path) - if (format == NEVERGRAD) { - if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") - return(read_nevergrad(path)) - else - indexFiles <- - file.path(path, list.files(path, pattern = '.csv', recursive = T)) - } - else if (format == SOS) { - return(read_datasetlist_SOS(path, locate_corrections_files(path))) - } - else - indexFiles <- scan_index_file(path) - - if (is.null(print_fun)) - print_fun <- cat - - object <- list() - class(object) <- c('DataSetList', class(object)) - DIM <- c() - algId <- c() - funcId <- c() - suites <- c() - maximizations <- c() - i <- 1 - - for (file in indexFiles) { - if (verbose) { - print_fun(paste('Processing', file, '...\n')) - } - - if (format == NEVERGRAD) { - dsl <- read_nevergrad(file) - object %<>% c(., dsl) - suites[i] <- NEVERGRAD - } - else { - indexInfo <- read_index_file(file) - if (verbose) - print_fun(sprintf(' algorithm %s...\n', indexInfo[[1]]$algId)) - - for (info in indexInfo) { - if (verbose) { - print_fun( - sprintf( - ' %d instances on f%s %dD...\n', - length(info$instance), - info$funcId, - info$DIM - ) - ) - } - - copy_flag <- TRUE - data <- DataSet(info, maximization = maximization, - format = format, subsampling = subsampling) - - DIM[i] <- attr(data, 'DIM') - funcId[i] <- attr(data, 'funcId') - algId[i] <- attr(data, 'algId') - - # TODO: double-check the following treatment on `instance`!!! - instance <- attr(data, 'instance') #Was instance without index? - suites[i] <- attr(data, 'suite') - maximizations[i] <- attr(data, 'maximization') - - if (length(object) != 0) { - idx <- which(sapply(object, function(obj) obj == data)) - for (k in idx) { - instance_ <- attr(object[[k]], 'instance') - if (all(instance == instance_)) { - copy_flag <- FALSE - warning('duplicated instances!') - break - } - - if (length(intersect(instance, instance_)) != 0) { - warning('duplicated instances!') - } - } - } - - if (copy_flag) { - object[[i]] <- data - i <- i + 1 - } - } - } - - if (verbose) - print_fun("\n") - } - - # TODO: sort all DataSet by multiple attributes: algId, funcId and DIM - if (format != NEVERGRAD) { - attr(object, 'DIM') <- DIM - attr(object, 'funcId') <- funcId - attr(object, 'algId') <- algId - } - - suite <- unique(suites) - maximization <- unique(maximizations) - if (length(maximization) != 1) { - warning("Multipe different optimization types detected!") - } - - attr(object, 'suite') <- suite - attr(object, 'maximization') <- maximization - clean_DataSetList(object) -} - - -#' Clean DataSetList object by concatenating DataSets -#' -#' Concatenates all DataSets with the same algorith name, function id and dimension -#' -#' @param dsList The DataSetList object to clean -#' @export -#' @examples -#' clean_DataSetList(dsl) -clean_DataSetList <- function(dsList) { - #To ensure no uninitialized variables are present - .I <- NULL - cases <- mapply( - function(...) paste0(list(...), collapse = ','), - attr(dsList, 'funcId'), - attr(dsList, 'DIM'), - attr(dsList, 'algId'), - SIMPLIFY = T, - USE.NAMES = F - ) - - dt <- as.data.table(cases)[, list(list(.I)), by = cases] - idx_to_del <- c() - - for (idx in dt$V1) { - if (length(idx) > 1) { - dsList[[idx[1]]] <- c.DataSet(dsList[idx]) - idx_to_del <- c(idx_to_del, idx[-1]) - } - } - dsList[idx_to_del] <- NULL - - if (length(idx_to_del) > 0) { - attr(dsList, 'DIM') <- sapply(dsList, function(ds) attr(ds, 'DIM')) - attr(dsList, 'funcId') <- sapply(dsList, function(ds) attr(ds, 'funcId')) - attr(dsList, 'algId') <- sapply(dsList, function(ds) attr(ds, 'algId')) - } - dsList -} - -#' S3 concatenation function for DataSetList -#' -#' @param ... The DataSetLists to concatenate -#' @return A new DataSetList -#' @export -#' @examples -#' c(dsl[1], dsl[3]) -c.DataSetList <- function(...) { - # TODO: maybe remove duplicated dataset in the further - # remove the empty list first - dsl <- list(...) - dsl <- dsl[sapply(dsl, length) != 0] - - if (length(dsl) == 0) - return() - - object <- unlist(dsl, recursive = F) - if (!any((class(object)) == 'DataSetList')) - class(object) <- c('DataSetList', class(object)) - - for (attr_str in c('DIM', 'funcId', 'algId')) { - attr(object, attr_str) <- unlist(lapply(dsl, function(x) attr(x, attr_str))) - } - - # Deal with Suites on the datasetlist level - attr(object, "suite") <- unique( - unlist( - lapply(dsl, function(x) attr(x, "suite")) - ) - ) - - # These attributes NEED to be the same across the datasetlist - for (attr_str in c('maximization')) { - temp <- unique( - unlist(lapply(dsl, function(x) attr(x, attr_str))) - ) - - if (length(temp) > 1) { - stop(paste0("Attempted to add datasetlists with different ", attr_str, - "-attributes! This will lead to errors when processing - this data!")) - } - attr(object, attr_str) <- temp[[1]] - } - object -} - -#' S3 extraction function for DataSetList -#' -#' @param x The DataSetList to use -#' @param i The indices to extract -#' @param drop Currently unused parameter -#' @return The DataSetList of the DataSets at indices i of DataSetList x -#' @export -#' @examples -#' dsl[c(1, 3)] -`[.DataSetList` <- function(x, i, drop = FALSE) { - # remove the attributes firstly - obj <- unclass(x)[i] - class(obj) <- c('DataSetList', class(obj)) - - # also slice the attributes accordingly - attr(obj, 'DIM') <- attr(x, 'DIM')[i] - attr(obj, 'funcId') <- attr(x, 'funcId')[i] - attr(obj, 'algId') <- attr(x, 'algId')[i] - attr(obj, 'suite') <- attr(x, 'suite') - attr(obj, 'maximization') <- attr(x, 'maximization') - - obj -} - -#' S3 print function for DataSetList -#' -#' @param x The DataSetList to print -#' @param ... Arguments for underlying print function? -#' @export -#' @examples -#' print(dsl) -print.DataSetList <- function(x, ...) { - cat('DataSetList:\n') - cat(paste0('Suite: ', attr(x, 'suite'), '\n')) - N <- length(x) - - # TODO: add an option for the following numbers: 15 and 5 - if (N <= 15) - idx <- seq_along(x) - else - idx <- c(1:5, '---', (N-4):N) - idx <- format(idx, justify = 'right') - - for (i in idx) { - if (trimws(i) == '---') - cat(paste0(i), '\n') - else { - .i <- as.integer(trimws(i)) - cat(sprintf('%s: %s\n', i, as.character(x[[.i]]))) - } - } -} - -# TODO: consistent use of ds, data, dsList etc. -# TODO: make decision on `DIM` or `dim`? funcId or fId or fID? algID / algid / algId? - -#' S3 summary function for DataSetList -#' -#' Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every -#' DataSet in the DataSetList -#' @param object The DataSetList to print -#' @param ... Arguments for underlying summary function? -#' @export -#' @examples -#' summary(dsl) -summary.DataSetList <- function(object, ...) { - as.data.frame( - t( - sapply( - object, - function(d) { - list( - suite = attr(d, 'suite'), - funcId = attr(d, 'funcId'), - DIM = attr(d, 'DIM'), - algId = attr(d, 'algId'), - datafile = attr(d, 'datafile'), - comment = attr(d, 'comment') - ) - } - ) - ) - ) -} - -#' S3 sort function for DataSetList -#' -#' Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). -#' Default is as ascending, can be made descending by adding a - in front of the attribute. -#' Sorting accross multiple attributes is supported, in the order they are specified. -#' -#' @param dsl The DataSetList to sort -#' @param ... attribute by which `dsl` is sorted. Multiple attributes can be specified. -#' @export -#' @examples -#' arrange(dsl, DIM, -funcId, algId) -arrange <- function(dsl, ...) UseMethod('arrange', dsl) - -#' @rdname arrange -#' @export -#' -arrange.DataSetList <- function(dsl, ...) { - cols <- substitute(list(...))[-1L] - if (identical(as.character(cols), "NULL")) - return(dsl) - - cols <- as.list(cols) - order <- as.list(rep(1L, length(cols))) - - for (i in seq_along(cols)) { - v <- as.list(cols[[i]]) - if (length(v) > 1L) { - if (v[[1L]] == '-') order[[i]] <- -1L - v <- v[[-1L]] - } - - v <- as.character(v) - if (v %in% c('DIM', 'funcId', 'algId')) - cols[[i]] <- v - else { - cols[[i]] <- NULL - order[[i]] <- NULL - } - } - - cols <- unlist(cols, use.names = F) - order <- unlist(order) - x <- c(list(1:length(dsl)), lapply(cols, function(col) attr(dsl, col))) - names(x) <- c('index', cols) - - DT <- rbindlist(list(x)) - data.table::setorderv(DT, cols, order) - idx <- DT[[1]] - dsl <- dsl[idx] - - # TODO: perhaps we do not need those attributes at all... - for (v in c('DIM', 'funcId', 'algId')) - attr(dsl, v) <- sapply(dsl, function(d) attr(d, v)) - dsl -} - -#' @rdname get_ERT -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' -get_ERT.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), attr(ds, 'funcId'), get_ERT(ds, ftarget, budget)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - })) - } - -#' @rdname get_RT_summary -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_RT_summary.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_RT_summary(ds, ftarget, budget)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - })) - } - -#' @rdname get_RT_sample -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' -#' @export -get_RT_sample.DataSetList <- function(ds, ftarget, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_RT_sample(ds, ftarget, ...)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - }), - fill = T) - } - -#' @rdname get_maxRT -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' -#' @export -get_maxRT.DataSetList <- function(ds, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds_) { - res <- - cbind(attr(ds_, 'DIM'), attr(ds_, 'funcId'), get_maxRT(ds_, ...)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - }), - fill = T) -} - -#' @rdname get_FV_summary -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' -get_FV_summary.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_FV_summary(ds, runtime)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - })) -} - -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' @rdname get_FV_overview -#' -get_FV_overview.DataSetList <- function(ds, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) - get_FV_overview(ds))) - -} - -#' @rdname get_RT_overview -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_RT_overview.DataSetList <- function(ds, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) - get_RT_overview(ds))) -} - -#' @rdname get_overview -#' @export -get_overview.DataSetList <- function(ds, ...) { - df <- rbindlist(lapply(ds, function(ds) - get_overview(ds))) - if (length(get_funcId(ds)) > 1 || length(get_dim(ds)) > 1) { - p2 <- df[,lapply(.SD, mean, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('mean reached')] - if (attr(ds, 'maximization')) { - p1 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'best reached')] - p3 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('worst recorded', 'worst reached')] - } - else { - p1 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('best reached')] - p3 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'worst recorded', 'worst reached')] - } - return(merge(merge(p1,p2),p3)) - } - else { - return(df) - } -} - -#' @rdname get_FV_sample -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -#' -get_FV_sample.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) { - res <- - cbind(attr(ds, 'DIM'), - attr(ds, 'funcId'), - get_FV_sample(ds, runtime, ...)) - colnames(res)[1] <- 'DIM' - colnames(res)[2] <- 'funcId' - res - }), - fill = T) - } - -#' @rdname get_PAR_summary -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_PAR_summary.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) get_PAR_summary(ds, idxValue, ...))) -} - -#' @rdname get_PAR_sample -#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. -#' @export -get_PAR_sample.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { - if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") - if (algorithm != 'all') - ds <- subset(ds, algId == algorithm) - - rbindlist(lapply(ds, function(ds) get_PAR_sample(ds, idxValue, ...)), fill = T) -} - -#' Get all dimensions present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A sorted list of all unique dimensions which occur in the DataSetList -#' @export -#' @examples -#' get_dim(dsl) -get_dim <- function(dsList) { - sort(unique(sapply(dsList, function(d) attr(d, 'DIM')))) -} - -#' Get all function ids present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A sorted list of all unique function ids which occur in the DataSetList -#' @export -#' @examples -#' get_funcId(dsl) -get_funcId <- function(dsList) { - ll <- unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcId'))))) - - # TODO: what if the function ID is a double value? - # those are coerced to integers now - if (is.integer(ll)) return(sort(ll)) - - lli <- suppressWarnings(as.integer(ll)) - if (any(is.na(lli))) return(sort(ll)) - - if (all((lli >= 0L) & (lli <= 1000000000L))) return(ll[order(lli)]) - - # TODO: should this be even allowed? - sort(lli) -} - -#' Get all algorithm ids present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A sorted list of all unique algorithm ids which occur in the DataSetList -#' @export -#' @examples -#' get_algId(dsl) -get_algId <- function(dsList) { - unique(sapply(dsList, function(d) attr(d, 'algId'))) -} - -#' Get all parameter ids present in a DataSetList -#' -#' @param dsList The DataSetList -#' @param which A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned -#' by the running time (RT) or the function value (FV). Note that parameters in each case are -#' not necessary the same. -#' -#' @return A sorted list of all unique parameter ids which occur in the DataSetList -#' @export -#' @examples -#' get_parId(dsl) -get_parId <- function(dsList, which = 'by_FV') { - unique( - unlist( - lapply(dsList, - function(d) { - if (which == 'by_FV') - names(d$PAR$by_FV) - else if (which == 'by_RT') - names(d$PAR$by_RT) - } - ) - ) - ) -} - -# TODO: let the user choose/detect whether the problem is subject to maximization -# and determine whether to sort the function values in ascending/desceding order - -#' Get all function values present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A list matrices of all function values which occur in the DataSetList -#' @export -#' @examples -#' get_funvals(dsl) -get_funvals <- function(dsList) { - if (length(dsList) == 0) - return(NULL) - if (length(dsList[[1]]$RT) == 0) { - x <- sort(unique(as.numeric(unlist( - lapply(dsList, function(x) - as.vector(x$FV)) - )))) - } - else - x <- (sort(unique(as.numeric(unlist( - lapply(dsList, function(x) - rownames(x$RT)) - ))))) - x[!is.na(x) & !is.infinite(x)] -} - -#' Get all runtime values present in a DataSetList -#' -#' @param dsList The DataSetLsit -#' -#' @return A list matrices of all runtime values which occur in the DataSetList -#' @export -#' @examples -#' get_runtimes(dsl) -get_runtimes <- function(dsList) { - sort(unique(as.numeric(unlist( - lapply(dsList, function(x) - rownames(x$FV)) - )))) -} - -# #' Get the best function value reached in a DataSetList -# #' -# #' @param dsList The DataSetLsit -# #' -# #' @return A list matrices of all runtime values which occur in the DataSetList -# #' @export -# get_best_targets <- function(dsList, by = 'funcId', maximize = T) { -# targets <- c() -# funcIds <- get_funcId(dsList) - -# for (i in seq_along(aggr_attr)) { -# data <- subset(dsList, funcId == funcIds[i]) - -# Fall <- get_funvals(data) -# Fval <- ifelse(maximize, max(Fall), min(Fall)) -# targets <- c(targets, Fval) -# } -# targets -# } - -# TODO: the attribute list should als be sliced here... -#' Filter a DataSetList by some criteria -#' -#' @param x The DataSetLsit -#' @param ... The condition to filter on. Can be any expression which assigns True or False -#' to a DataSet object, such as DIM == 625 or funcId == 2 -#' -#' @return The filtered DataSetList -#' @export -#' @examples -#' subset(dsl, funcId == 1) -subset.DataSetList <- function(x, ...) { - condition_call <- substitute(list(...)) - enclos <- parent.frame() - idx <- sapply(x, - function(ds) - all(unlist( - eval(condition_call, attributes(ds), enclos) - ))) - x[idx] -} - -#' Save DataTable in multiple formats -#' -#' @param df The DataTable to store -#' @param file String. The name of the figure file, with the extension of the required file-format -#' @param format Optional, string. Overwrites the extension of the `file` parameter. If not specified while -#' file does not have an extension, it defaults to csv -#' -#' @export -#' @examples -#' df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -#' save_table(df, tempfile(fileext = ".md")) -save_table <- function(df, file, format = NULL) { - if (is.null(format)) { - format <- tools::file_ext(file) - } - if (format == 'TeX' || format == 'tex') { - if (requireNamespace('xtable', quietly = T)) - print(xtable::xtable(df), file = file) - else - write(kable(df, format = 'latex'), file) - } else if (format == 'Markdown' || format == 'md') { - write(kable(df, format = 'markdown'), file) - } else if (format == 'html') { - write(kable(df, format = "html"), file) - } - else { #Default to csv - write.csv(df, file, row.names = F) - } -} - -#' Generation of default ECDF-targets -#' -#' @param dsList The DataSetList object for which to generate the targets -#' @param type The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, -#' equal for all functions / dimensions) -#' @param number_targets The amount of targets to generate -#' -#' @return A data.table with 3 columns: funcId, DIM and target -#' @export -#' @examples -#' get_ECDF_targets(dsl, 'linear', 10) -get_ECDF_targets <- function(dsList, type = "log-linear", number_targets = 10) { - funcIds <- get_funcId(dsList) - dims <- get_dim(dsList) - - dt <- rbindlist(apply(expand.grid(funcIds, dims), 1, function(x) { - if (type == 'bbob') { - fseq <- rev(seq_FV(c(100,1e-8), length.out = 51, scale = 'log')) - } - else { - dsl <- subset(dsList, funcId == x[[1]] && DIM == x[[2]]) - if (length(dsl) == 0) - return(NULL) - fall <- get_funvals(dsl) - if (length(fall) < 2) - return(NULL) - - fseq <- seq_FV(fall, length.out = number_targets, scale = ifelse(type == "log-linear", 'log', 'linear')) - } - data.table(funcId = x[[1]], DIM = x[[2]], target = fseq) - })) - dt -} - -### __________________________ Rewritten data generation functions _______________________ ### -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param start Optional start value (Runtime or target value) -#' @param stop Optional end value (Runtime or target value) -#' @param scale_log Wheterh to use logarithmic scaling or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param include_opts Whether or not to also include the best value hit by each algorithm to -#' the generated datapoints -#' @param budget Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works -#' in fixed_target mode. -#' -#' @export -#' @examples -#' generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, - scale_log = F, which = 'by_RT', include_opts = F, budget = NULL) { - - if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1 ) { - #Required because target generation is included in this function, - #which needs to be done on a per-function basis - stop("Multiple functions / dimensions are present in provided DataSetList. - Please call this function for each individual function/dimension pair instead.") - } - - by_rt <- (which == 'by_RT') - - if (by_rt) - all <- get_funvals(dsList) - else - all <- get_runtimes(dsList) - - maximization <- attr(dsList, 'maximization') - - if (is.null(maximization)) maximization <- T - if (is.null(start)) start <- min(all) - if (is.null(stop)) stop <- max(all) - - if (by_rt) { - Xseq <- seq_FV(all, start, stop, length.out = 60, - scale = ifelse(scale_log, 'log', 'linear')) - if (include_opts) { - for (algid in get_algId(dsList)) { - if (maximization) - Xseq <- c(Xseq, max(get_funvals(subset(dsList, algId == algid)))) - else - Xseq <- c(Xseq, min(get_funvals(subset(dsList, algId == algid)))) - } - Xseq <- unique(sort(Xseq)) - } - dt <- get_RT_summary(dsList, ftarget = Xseq, budget = budget) - } - else { - Xseq <- seq_RT(all, start, stop, length.out = 60, - scale = ifelse(scale_log, 'log', 'linear')) - if (include_opts) { - for (algid in get_algId(dsList)) { - Xseq <- c(Xseq, max(get_funvals(subset(dsList, algId == algid)))) - } - Xseq <- unique(sort(Xseq)) - } - dt <- get_FV_summary(dsList, Xseq) - } - - - dt[, `:=`(upper = mean + sd, lower = mean - sd)] - return(dt) -} - -#' Generate dataframe of a single function/dimension pair for creating PDF or PMF plots -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param target The target value (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -generate_data.PMF <- function(dsList, target, which = 'by_RT') { - if (which == 'by_RT') - return(get_RT_sample(dsList, target, output = 'long')) - return(get_FV_sample(dsList, target, output = 'long')) -} - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param target The target value (Runtime or target value) -#' @param use.equal.bins Whether all bins should be equal size for each algorithm or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -generate_data.hist <- function(dsList, target, use.equal.bins = F, which = 'by_RT') { - width <- NULL # Set local binding to remove warnings - - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) - stop('This function is only available a single function/dimension pair at a time.') - - if (use.equal.bins) { - if (which == 'by_RT') - res1 <- hist( - get_RT_sample(dsList, target, output = 'long')$RT, - breaks = nclass.FD, - plot = F - ) - else - res1 <- hist( - get_FV_sample(dsList, target, output = 'long')$`f(x)`, - breaks = nclass.FD, - plot = F - ) - } - - dt <- as.data.table( - rbindlist( - lapply( - dsList, - function(ds) { - algId <- attr(ds, 'algId') - - if (which == 'by_RT') - data <- get_RT_sample(ds, target, output = 'long')$RT - else if (which == 'by_FV') - data <- get_FV_sample(ds, target, output = 'long')$`f(x)` - else - stop('Invalid argument for parameter `which`.') - - if (sum(!is.na(data)) < 2) - return(NULL) - - if (use.equal.bins) - breaks <- res1$breaks - else - breaks <- nclass.FD - - res <- hist(data, breaks = breaks, plot = F) - breaks <- res$breaks - - plot_text <- paste0( - 'count: ', res$counts, '
breaks: [', - breaks[-length(breaks)], ',', breaks[-1], ']' - ) - - plot_data <- data.frame( - x = res$mids, - y = res$counts, - algId = algId, - width = breaks[2] - breaks[1], - text = plot_text - ) - } - ) %>% { - `[`(., !vapply(., is.null, logical(1))) - } - ) - ) - dt -} - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param scale_log Wheterh to use logarithmic scaling or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param use_full_range Whether or not to use the full range of the x-axis or cut it off as soon as -#' all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension -#' -#' @export -#' @examples -#' generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) -generate_data.ECDF <- function(dsList, targets, scale_log = F, which = 'by_RT', use_full_range = TRUE) { - V1 <- NULL #Set local binding to remove warnings - by_rt <- which == 'by_RT' - if (by_rt) { - RT <- get_runtimes(dsList) - if (!use_full_range) { - if (length(unique(get_funcId(dsList))) > 1 || length(unique(get_dim(dsList))) > 1) { - warning("The limiting of x-values is only supported in the case of 1 function 1 dimension!") - } - else { - maxRT <- as.integer(max(get_RT_summary(dsList, targets)[,'98%']) * 1.1) #Slight buffer for nicer plotting - RT <- c(min(RT), maxRT) - } - } - x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) - - #TODO: Some scaling by dimension? - - if (!is.data.table(targets)) { - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) - stop("Targets provided are not in data.table format, while multiple functions / dimensions - are present in provided DataSetList.") - targets <- data.table( - target = targets, - funcId = get_funcId(dsList), - DIM = get_dim(dsList) - ) - } - } - else { - FV <- get_funvals(dsList) - x <- unique(seq_FV(FV, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) - } - - dt <- as.data.table(rbindlist(lapply(dsList, function(df) { - algId <- attr(df, 'algId') - if (by_rt) { - temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] - targets_ <- temp[funcId == attr(df, 'funcId')][['target']] - } - else - targets_ <- targets - m <- lapply(targets_, function(target) { - if (by_rt) - data <- get_RT_sample(df, target, output = 'long')$RT - else - data <- get_FV_sample(df, target, output = 'long')$`f(x)` - - if (all(is.na(data))) - return(rep(0, length(x))) - fun <- ecdf(data) - if (is.function(fun)) fun(x) else NA - }) %>% - do.call(rbind, .) - - data.frame(x = x, - mean = apply(m, 2, . %>% mean(na.rm = T)), - sd = apply(m, 2, . %>% sd(na.rm = T))) %>% - mutate(upper = mean + sd, lower = mean - sd, algId = algId) - }))) - dt[, mean(mean), by = .(x, algId)][, .(mean = V1, algId = algId, x = x)] -} - - -#' Generate dataframe containing the AUC for any ECDF-curves -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param scale_log Whether to use logarithmic scaling or not -#' @param dt_ecdf A data table of the ECDF to avoid needless recomputations. Will take preference if it -#' is provided together with dsList and targets -#' @param multiple_x Boolean, whether to get only the total AUC or get stepwise AUC values -#' -#' @export -#' @examples -#' generate_data.AUC(dsl, get_ECDF_targets(dsl)) -#' generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) -generate_data.AUC <- function(dsList, targets, scale_log = F, which = 'by_RT', dt_ecdf = NULL, - multiple_x = FALSE) { - idx <- auc_contrib <- mean_pre <- mean_post <- x <- x_pre <- auc <- NULL - if (is.null(dt_ecdf)) { - if (length(dsList) == 0 || is.null(targets)) - return(NULL) - dt_ecdf <- generate_data.ECDF(dsList, targets, scale_log, which) - } - max_idx <- nrow(unique(dt_ecdf[,'x'])) - dt_ecdf[, idx := seq(max_idx), by = 'algId'] - dt3 = copy(dt_ecdf) - dt3[, idx := idx - 1] - dt_merged = merge(dt_ecdf, dt3, by = c('algId', 'idx')) - colnames(dt_merged) <- c("algId", "idx", "mean_pre", "x_pre", "mean_post", "x") - dt_merged[, auc_contrib := ((mean_pre + mean_post)/2)*(x - x_pre)] - dt_merged[, auc := cumsum(auc_contrib)/x, by = 'algId'] - #TODO: just for max x - if (multiple_x) - return(dt_merged[, c('algId','x','auc') ]) - return(dt_merged[idx == (max_idx - 1), c('algId','x','auc') ]) -} - - -# #' Generate dataframe of a single function/dimension pair -# #' -# #' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -# #' -# #' @param dsList The DataSetList object -# #' @param targets A list of the target value for which to calculate the AUC (Runtime or target value) -# #' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -# #' -# #' @export -# #' @examples -# #' generate_data.AUC(subset(dsl, funcId == 1), c(12, 16)) -# generate_data.AUC <- function(dsList, targets, which = 'by_RT') { -# if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { -# stop("This function is only available a single function/dimension pair at a time.") -# } -# by_rt <- which == 'by_RT' -# -# if (by_rt) -# RT.max <- sapply(dsList, function(ds) max(attr(ds, 'maxRT'))) %>% max -# else { -# funevals.max <- sapply(dsList, function(ds) max(attr(ds, 'finalFV'))) %>% max -# funevals.min <- sapply(dsList, function(ds) min(attr(ds, 'finalFV'))) %>% min -# } -# -# as.data.table(rbindlist(lapply(dsList, function(df) { -# algId <- attr(df, 'algId') -# if (by_rt) -# auc <- sapply(targets, function(fv) { -# ECDF(df, fv) %>% AUC(from = 1, to = RT.max) -# }) -# else { -# funs <- lapply(targets, function(r) { -# get_FV_sample(df, r, output = 'long')$'f(x)' %>% { -# if (all(is.na(.))) NULL -# else { -# f <- ecdf(.) -# attr(f, 'min') <- min(.) -# attr(f, 'max') <- max(.) -# f -# } -# } -# }) -# -# auc <- sapply(funs, -# function(fun) { -# if (is.null(fun)) 0 -# else{ -# if (attr(df, 'maximization')) -# integrate(fun, lower = attr(fun, 'min') - 1, upper = funevals.max, -# subdivisions = 1e3) %>% {'$'(., 'value') / funevals.max} -# else -# integrate(fun, lower = funevals.min, upper = attr(fun, 'max') + 1, -# subdivisions = 1e3) %>% {'$'(., 'value') / (attr(fun, 'max') + 1)} -# } -# }) -# } -# data.frame(x = targets, AUC = auc, algId = algId) -# }))) -# } - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param scale_log Wheterh to use logarithmic scaling or not -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.Parameters(subset(dsl, funcId == 1)) -generate_data.Parameters <- function(dsList, which = 'by_RT', scale_log = F) { - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { - stop("This function is only available a single function/dimension pair at a time.") - } - if (which == 'by_RT') { - rtall <- get_runtimes(dsList) - - rtseq <- seq_RT(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) - req(rtseq) - - dt <- get_PAR_summary(dsList, rtseq, which = 'by_RT') - } - else if (which == 'by_FV') { - rtall <- get_funvals(dsList) - - rtseq <- seq_FV(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) - req(rtseq) - - dt <- get_PAR_summary(dsList, rtseq, which = 'by_FV') - } - else stop("Invalid value for parameter `which`") - dt[, `:=`(upper = mean + sd, lower = mean - sd)] -} - -#' Generate dataframe of a single function/dimension pair -#' -#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -#' -#' @param dsList The DataSetList object -#' @param aggr_on Which attribute to use for aggregation. Either 'funcId' or 'DIM' -#' @param targets Optional list of target values (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' -#' @export -#' @examples -#' generate_data.Aggr(dsl) -generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which = 'by_RT') { - maximize <- attr(dsList, 'maximization') - variable <- fid <- value <- NULL #Set local binding to remove warnings - by_rt <- which == 'by_RT' - - if (is.null(targets)) { - targets <- get_target_dt(dsList, which) - } - - aggr_attr <- if (aggr_on == 'funcId') get_funcId(dsList) else get_dim(dsList) - N <- length(get_algId(dsList)) - - dt <- rbindlist(lapply(aggr_attr, function(agg_val) { - if (by_rt) { - if (aggr_on == 'funcId') - dt <- get_RT_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) - else - dt <- get_RT_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) - dt[, c('algId', value = 'ERT', 'funcId', 'DIM')] - setnames(dt, 'ERT', 'value') - } - else{ - if (aggr_on == 'funcId') - dt <- get_FV_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) - else - dt <- get_FV_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) - dt[, c('algId', value = 'mean', 'funcId', 'DIM')] - setnames(dt, 'mean', 'value') - } - })) - - if (by_rt) order_sel <- 1 - else order_sel <- -1*(maximize*2 - 1) - - dt[, rank := frank(order_sel*value, na.last = T), by = .(DIM, funcId)] - return(dt) -} - - - -#' Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of -#' -#' This provides an unaggregated version of the function `generate_data.ECDF`. -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param scale_log Wheterh to use logarithmic scaling or not -#' -#' @export -#' @examples -#' generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) -generate_data.ECDF_raw <- function(dsList, targets, scale_log = F) { - V1 <- NULL #Set local binding to remove warnings - - #Get the x-coordinates at which to calculate the ECDF - RT <- get_runtimes(dsList) - x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) - #TODO: Some scaling by dimension? - - #Get targets to use - if (!is.data.table(targets)) { - if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) - stop("Targets provided are not in data.table format, while multiple functions / dimensions - are present in provided DataSetList.") - targets <- data.table( - target = targets, - funcId = get_funcId(dsList), - DIM = get_dim(dsList) - ) - } - - dt <- as.data.table(rbindlist(lapply(dsList, function(df) { - algId <- attr(df, 'algId') - temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] - targets_ <- temp[funcId == attr(df, 'funcId')][['target']] - - m <- lapply(targets_, function(target) { - data <- fast_RT_samples(df$RT, target, attr(df, 'maximization')) - if (all(is.na(data))) - hit <- (rep(0, length(x))) - else { - fun <- ecdf(data) - hit <- if (is.function(fun)) fun(x) else NA - } - data.table(hit = hit, rt = x, target = target, funcId = attr(df, 'funcId'), DIM = attr(df, 'DIM'), algId = attr(df, 'algId')) - }) %>% - do.call(rbind, .) - m - }))) - dt +#' Read all the data file in a folfer +# +#' @param path Path to the data files. Will look for all .info-files in this directory and use +#' the corresponding datafiles to create the DataSetList +#' @param verbose Logical. +#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? +#' @param format A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL" +#' @param subsampling Logical. Whether *.cdat files are subsampled? +#' @param print_fun Function used to print output when in verbose mode +#' @noRd +#' @return A DataSetList object +read_dir <- function(path, verbose = T, print_fun = NULL, maximization = TRUE, + format = IOHprofiler, subsampling = FALSE) { + DataSetList( + path, + verbose, + print_fun, + maximization = maximization, + format = format, + subsampling = subsampling + ) + } + +# TODO: improve documentation of this S3 class +#' S3 constructor of the 'DataSetList' +#' +#' Attributes +#' funId +#' DIM +#' algId +# +#' @param path Path to the data files. Will look for all .info-files in this directory and use +#' the corresponding datafiles to create the DataSetList +#' @param verbose Logical. +#' @param maximization Logical. Whether the underlying optimization algorithm performs a maximization? +#' @param format A character. The format of data source, options are: +#' \itemize{ +#' \item'IOHProfiler' +#' \item'COCO' +#' \item'TWO_COL' +#' \item'COCO_BIOBJ' +#' \item'NEVERGRAD' +#' \item'SOS' +#' } +#' These formats are specified in more detail in our github wiki. +#' @param subsampling Logical. Whether *.cdat files are subsampled? +#' @param print_fun Function used to print output when in verbose mode +#' @param full_aggregation If True, individual DataSets are aggregated as much as possible: all DataSets +#' with the same algorithmname, function id and dimension are combined together. This leads to information loss +#' related to static variables, so only use if that information is not required. +#' +#' @return A DataSetList object +#' @export +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +#' DataSetList(path) +DataSetList <- function(path = NULL, verbose = T, print_fun = NULL, maximization = NULL, + format = IOHprofiler, subsampling = FALSE, full_aggregation = TRUE) { + if (is.null(path)) + return(structure(list(), class = c('DataSetList', 'list'))) + + path <- trimws(path) + if (format == NEVERGRAD) { + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") + return(read_nevergrad(path)) + else + indexFiles <- + file.path(path, list.files(path, pattern = '.csv', recursive = T)) + } + else if (format == SOS) { + return(read_datasetlist_SOS(path, locate_corrections_files(path))) + } + else + indexFiles <- scan_index_file(path) + + if (is.null(print_fun)) + print_fun <- cat + + object <- list() + class(object) <- c('DataSetList', class(object)) + DIM <- c() + algId <- c() + funcId <- c() + suites <- c() + maximizations <- c() + i <- 1 + + for (file in indexFiles) { + if (verbose) { + print_fun(paste('Processing', file, '...\n')) + } + + if (format == NEVERGRAD) { + dsl <- read_nevergrad(file) + object %<>% c(., dsl) + suites[i] <- NEVERGRAD + } + else { + indexInfo <- read_index_file(file) + if (verbose) + print_fun(sprintf(' algorithm %s...\n', indexInfo[[1]]$algId)) + + for (info in indexInfo) { + if (verbose) { + print_fun( + sprintf( + ' %d instances on f%s %dD...\n', + length(info$instance), + info$funcId, + info$DIM + ) + ) + } + + copy_flag <- TRUE + data <- DataSet(info, maximization = maximization, + format = format, subsampling = subsampling) + + DIM[i] <- attr(data, 'DIM') + funcId[i] <- attr(data, 'funcId') + algId[i] <- attr(data, 'algId') + + # TODO: double-check the following treatment on `instance`!!! + instance <- attr(data, 'instance') #Was instance without index? + suites[i] <- attr(data, 'suite') + maximizations[i] <- attr(data, 'maximization') + + if (length(object) != 0) { + idx <- which(sapply(object, function(obj) obj == data)) + for (k in idx) { + instance_ <- attr(object[[k]], 'instance') + if (all(instance == instance_)) { + copy_flag <- FALSE + warning('duplicated instances!') + break + } + + if (length(intersect(instance, instance_)) != 0) { + warning('duplicated instances!') + } + } + } + + if (copy_flag) { + object[[i]] <- data + i <- i + 1 + } + } + } + + if (verbose) + print_fun("\n") + } + + # TODO: sort all DataSet by multiple attributes: algId, funcId and DIM + if (format != NEVERGRAD) { + attr(object, 'DIM') <- DIM + attr(object, 'funcId') <- funcId + attr(object, 'algId') <- algId + } + + suite <- unique(suites) + maximization <- unique(maximizations) + if (length(maximization) != 1) { + warning("Multipe different optimization types detected!") + } + + attr(object, 'suite') <- suite + attr(object, 'maximization') <- maximization + attr(object, 'ID') <- attr(object, 'algId') + attr(object, 'ID_attributes') <- c('algId') + if (full_aggregation) + clean_DataSetList(object) + else + object +} + + +#' Clean DataSetList object by concatenating DataSets +#' +#' Concatenates all DataSets with the same ID, function id and dimension +#' +#' @param dsList The DataSetList object to clean +#' @export +#' @examples +#' clean_DataSetList(dsl) +clean_DataSetList <- function(dsList) { + #To ensure no uninitialized variables are present + .I <- NULL + cases <- mapply( + function(...) paste0(list(...), collapse = ','), + attr(dsList, 'funcId'), + attr(dsList, 'DIM'), + attr(dsList, 'algId'), + attr(dsList, 'ID'), + SIMPLIFY = T, + USE.NAMES = F + ) + + dt <- as.data.table(cases)[, list(list(.I)), by = cases] + idx_to_del <- c() + + for (idx in dt$V1) { + if (length(idx) > 1) { + dsList[[idx[1]]] <- c.DataSet(dsList[idx]) + idx_to_del <- c(idx_to_del, idx[-1]) + } + } + dsList[idx_to_del] <- NULL + + if (length(idx_to_del) > 0) { + attr(dsList, 'DIM') <- sapply(dsList, function(ds) attr(ds, 'DIM')) + attr(dsList, 'funcId') <- sapply(dsList, function(ds) attr(ds, 'funcId')) + attr(dsList, 'algId') <- sapply(dsList, function(ds) attr(ds, 'algId')) + attr(dsList, 'ID') <- sapply(dsList, function(ds) attr(ds, 'ID')) + } + dsList +} + +#' S3 concatenation function for DataSetList +#' +#' @param ... The DataSetLists to concatenate +#' @return A new DataSetList +#' @export +#' @examples +#' c(dsl[1], dsl[3]) +c.DataSetList <- function(...) { + # TODO: maybe remove duplicated dataset in the further + # remove the empty list first + dsl <- list(...) + dsl <- dsl[sapply(dsl, length) != 0] + + if (length(dsl) == 0) + return() + + object <- unlist(dsl, recursive = F) + if (!any((class(object)) == 'DataSetList')) + class(object) <- c('DataSetList', class(object)) + + for (attr_str in c('DIM', 'funcId', 'algId', 'ID')) { + attr(object, attr_str) <- unlist(lapply(dsl, function(x) attr(x, attr_str))) + } + + # Deal with Suites on the datasetlist level + attr(object, "suite") <- unique( + unlist( + lapply(dsl, function(x) attr(x, "suite")) + ) + ) + + # These attributes NEED to be the same across the datasetlist + for (attr_str in c('maximization', 'ID_attributes')) { + temp <- unique( + unlist(lapply(dsl, function(x) attr(x, attr_str))) + ) + + if (length(temp) > 1) { + stop(paste0("Attempted to add datasetlists with different ", attr_str, + "-attributes! This will lead to errors when processing + this data!")) + } + attr(object, attr_str) <- temp[[1]] + } + object +} + +#' S3 extraction function for DataSetList +#' +#' @param x The DataSetList to use +#' @param i The indices to extract +#' @param drop Currently unused parameter +#' @return The DataSetList of the DataSets at indices i of DataSetList x +#' @export +#' @examples +#' dsl[c(1, 3)] +`[.DataSetList` <- function(x, i, drop = FALSE) { + # remove the attributes firstly + obj <- unclass(x)[i] + class(obj) <- c('DataSetList', class(obj)) + + # also slice the attributes accordingly + attr(obj, 'DIM') <- attr(x, 'DIM')[i] + attr(obj, 'ID') <- attr(x, 'ID')[i] + attr(obj, 'funcId') <- attr(x, 'funcId')[i] + attr(obj, 'algId') <- attr(x, 'algId')[i] + attr(obj, 'suite') <- attr(x, 'suite') + attr(obj, 'maximization') <- attr(x, 'maximization') + + obj +} + +#' S3 print function for DataSetList +#' +#' @param x The DataSetList to print +#' @param ... Arguments for underlying print function? +#' @export +#' @examples +#' print(dsl) +print.DataSetList <- function(x, ...) { + cat('DataSetList:\n') + cat(paste0('Suite: ', attr(x, 'suite'), '\n')) + N <- length(x) + + # TODO: add an option for the following numbers: 15 and 5 + if (N <= 15) + idx <- seq_along(x) + else + idx <- c(1:5, '---', (N-4):N) + idx <- format(idx, justify = 'right') + + for (i in idx) { + if (trimws(i) == '---') + cat(paste0(i), '\n') + else { + .i <- as.integer(trimws(i)) + cat(sprintf('%s: %s\n', i, as.character(x[[.i]]))) + } + } +} + +# TODO: consistent use of ds, data, dsList etc. +# TODO: make decision on `DIM` or `dim`? funcId or fId or fID? algID / algid / algId? + +#' S3 summary function for DataSetList +#' +#' Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every +#' DataSet in the DataSetList +#' @param object The DataSetList to print +#' @param ... Arguments for underlying summary function? +#' @export +#' @examples +#' summary(dsl) +summary.DataSetList <- function(object, ...) { + as.data.frame( + t( + sapply( + object, + function(d) { + list( + suite = attr(d, 'suite'), + funcId = attr(d, 'funcId'), + DIM = attr(d, 'DIM'), + ID = attr(d, 'ID'), + datafile = attr(d, 'datafile'), + comment = attr(d, 'comment') + ) + } + ) + ) + ) +} + +#' S3 sort function for DataSetList +#' +#' Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). +#' Default is as ascending, can be made descending by adding a - in front of the attribute. +#' Sorting accross multiple attributes is supported, in the order they are specified. +#' +#' @param dsl The DataSetList to sort +#' @param ... attribute by which `dsl` is sorted. Multiple attributes can be specified. +#' @export +#' @examples +#' arrange(dsl, DIM, -funcId, algId) +arrange <- function(dsl, ...) UseMethod('arrange', dsl) + +#' @rdname arrange +#' @export +#' +arrange.DataSetList <- function(dsl, ...) { + cols <- substitute(list(...))[-1L] + if (identical(as.character(cols), "NULL")) + return(dsl) + + cols <- as.list(cols) + order <- as.list(rep(1L, length(cols))) + + for (i in seq_along(cols)) { + v <- as.list(cols[[i]]) + if (length(v) > 1L) { + if (v[[1L]] == '-') order[[i]] <- -1L + v <- v[[-1L]] + } + + v <- as.character(v) + if (v %in% c('DIM', 'funcId', 'ID')) + cols[[i]] <- v + else { + cols[[i]] <- NULL + order[[i]] <- NULL + } + } + + cols <- unlist(cols, use.names = F) + order <- unlist(order) + x <- c(list(1:length(dsl)), lapply(cols, function(col) attr(dsl, col))) + names(x) <- c('index', cols) + + DT <- rbindlist(list(x)) + data.table::setorderv(DT, cols, order) + idx <- DT[[1]] + dsl <- dsl[idx] + + # TODO: perhaps we do not need those attributes at all... + for (v in c('DIM', 'funcId', 'ID')) + attr(dsl, v) <- sapply(dsl, function(d) attr(d, v)) + dsl +} + +#' @rdname get_ERT +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' +get_ERT.DataSetList <- function(ds, ftarget, budget = NULL, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), attr(ds, 'funcId'), get_ERT(ds, ftarget, budget)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + })) + } + +#' @rdname get_RT_summary +#' @export +get_RT_summary.DataSetList <- function(ds, ftarget, budget = NULL, ...) { + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_RT_summary(ds, ftarget, budget)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + })) + } + +#' @rdname get_RT_sample +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' +#' @export +get_RT_sample.DataSetList <- function(ds, ftarget, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_RT_sample(ds, ftarget, ...)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + }), + fill = T) + } + +#' @rdname get_maxRT +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' +#' @export +get_maxRT.DataSetList <- function(ds, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds_) { + res <- + cbind(attr(ds_, 'DIM'), attr(ds_, 'funcId'), get_maxRT(ds_, ...)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + }), + fill = T) +} + +#' @rdname get_FV_summary +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' +get_FV_summary.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_FV_summary(ds, runtime)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + })) +} + +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' @rdname get_FV_overview +#' +get_FV_overview.DataSetList <- function(ds, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) + get_FV_overview(ds))) + +} + +#' @rdname get_RT_overview +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +get_RT_overview.DataSetList <- function(ds, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) + get_RT_overview(ds))) +} + +#' @rdname get_overview +#' @export +get_overview.DataSetList <- function(ds, ...) { + df <- rbindlist(lapply(ds, function(ds) + get_overview(ds))) + if (length(get_funcId(ds)) > 1 || length(get_dim(ds)) > 1) { + p2 <- df[,lapply(.SD, mean, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('mean reached')] + if (attr(ds, 'maximization')) { + p1 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'best reached')] + p3 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('worst recorded', 'worst reached')] + } + else { + p1 <- df[,lapply(.SD, min, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('best reached')] + p3 <- df[,lapply(.SD, max, na.rm = TRUE), by = c('DIM', 'funcId'), .SDcols = c('budget', 'worst recorded', 'worst reached')] + } + return(merge(merge(p1,p2),p3)) + } + else { + return(df) + } +} + +#' @rdname get_FV_sample +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +#' +get_FV_sample.DataSetList <- function(ds, runtime, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) { + res <- + cbind(attr(ds, 'DIM'), + attr(ds, 'funcId'), + get_FV_sample(ds, runtime, ...)) + colnames(res)[1] <- 'DIM' + colnames(res)[2] <- 'funcId' + res + }), + fill = T) + } + +#' @rdname get_PAR_summary +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +get_PAR_summary.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) get_PAR_summary(ds, idxValue, ...))) +} + +#' @rdname get_PAR_sample +#' @param algorithm DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider. +#' @export +get_PAR_sample.DataSetList <- function(ds, idxValue, algorithm = 'all', ...) { + if (!missing("algorithm")) warning("Argument 'algorithm' is deprecated and will be removed in the next release of IOHanalyzer.") + if (algorithm != 'all') + ds <- subset(ds, algId == algorithm) + + rbindlist(lapply(ds, function(ds) get_PAR_sample(ds, idxValue, ...)), fill = T) +} + +#' @rdname get_id +#' @export +get_id.DataSetList <- function(ds, ...) { + temp <- attr(ds, 'ID') + if (is.null(temp)) { + warning("No ID attribute set, returning the algId's instead. (from 1.6.0 onwards, ID attributes are always added + to new datasets, see the 'change_id' function.") + return(get_algId(ds)) + } + return(unique(temp)) +} + +#' Get all dimensions present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A sorted list of all unique dimensions which occur in the DataSetList +#' @export +#' @examples +#' get_dim(dsl) +get_dim <- function(dsList) { + sort(unique(sapply(dsList, function(d) attr(d, 'DIM')))) +} + +#' Get all function ids present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A sorted list of all unique function ids which occur in the DataSetList +#' @export +#' @examples +#' get_funcId(dsl) +get_funcId <- function(dsList) { + ll <- unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcId'))))) + + # TODO: what if the function ID is a double value? + # those are coerced to integers now + if (is.integer(ll)) return(sort(ll)) + + lli <- suppressWarnings(as.integer(ll)) + if (any(is.na(lli))) return(sort(ll)) + + if (all((lli >= 0L) & (lli <= 1000000000L))) return(ll[order(lli)]) + + # TODO: should this be even allowed? + sort(lli) +} + +#' Get all function names present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list of all unique function names which occur in the DataSetList +#' @export +#' @examples +#' get_funcName(dsl) +get_funcName <- function(dsList) { + unique(unname(unlist(sapply(dsList, function(d) attr(d, 'funcName'))))) +} + +#' Get all algorithm ids present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A sorted list of all unique algorithm ids which occur in the DataSetList +#' @export +#' @examples +#' get_algId(dsl) +get_algId <- function(dsList) { + unique(sapply(dsList, function(d) attr(d, 'algId'))) +} + +#' Get all parameter ids present in a DataSetList +#' +#' @param dsList The DataSetList +#' @param which A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned +#' by the running time (RT) or the function value (FV). Note that parameters in each case are +#' not necessary the same. +#' +#' @return A sorted list of all unique parameter ids which occur in the DataSetList +#' @export +#' @examples +#' get_parId(dsl) +get_parId <- function(dsList, which = 'by_FV') { + unique( + unlist( + lapply(dsList, + function(d) { + if (which == 'by_FV') + names(d$PAR$by_FV) + else if (which == 'by_RT') + names(d$PAR$by_RT) + } + ) + ) + ) +} + +# TODO: let the user choose/detect whether the problem is subject to maximization +# and determine whether to sort the function values in ascending/desceding order + +#' Get all function values present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list matrices of all function values which occur in the DataSetList +#' @export +#' @examples +#' get_funvals(dsl) +get_funvals <- function(dsList) { + if (length(dsList) == 0) + return(NULL) + if (length(dsList[[1]]$RT) == 0) { + x <- sort(unique(as.numeric(unlist( + lapply(dsList, function(x) + as.vector(x$FV)) + )))) + } + else + x <- (sort(unique(as.numeric(unlist( + lapply(dsList, function(x) + rownames(x$RT)) + ))))) + x[!is.na(x) & !is.infinite(x)] +} + +#' Get all runtime values present in a DataSetList +#' +#' @param dsList The DataSetLsit +#' +#' @return A list matrices of all runtime values which occur in the DataSetList +#' @export +#' @examples +#' get_runtimes(dsl) +get_runtimes <- function(dsList) { + sort(unique(as.numeric(unlist( + lapply(dsList, function(x) + rownames(x$FV)) + )))) +} + +#' Get all attributes which can be used to subset a DataSetList +#' +#' @param dsl The DataSetList +#' +#' @return The list of available attributes +#' @export +#' @examples +#' get_static_attributes(dsl) +get_static_attributes <- function(dsl) { + full_names <- unique(unlist(lapply(dsl, function(ds) {names(attributes(ds))}))) + + reserved_attributes <- c("names", "class", "suite", "maximization", "algInfo", "comment", + "datafile", "maxRT", "finalFV", "format") + setdiff(full_names, reserved_attributes) +} + +#' Get all options for a specific attribute which can be used to subset a DataSetList +#' +#' This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. +#' Note the only attributes returned by `get_static_attributes` are supported in this funcion +#' +#' @param dsl The DataSetList +#' @param attribute the name of the attribute for which to get the available options in dsl +#' @return The list of options for the specified attribute +#' @export +#' @examples +#' get_static_attribute_values(dsl, 'funcId') +get_static_attribute_values <- function(dsl, attribute) { + unique(unlist(lapply(dsl, function(ds) {attr(ds, attribute)}))) +} + +#' Filter a DataSetList by some criteria +#' +#' @param x The DataSetList +#' @param ... The conditions to filter on. Can be any expression which assigns True or False +#' to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes +#' (funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should +#' be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a +#' string to be parsed. This allows exectution of subset commands on arbitrary variables in code. +#' +#' @return The filtered DataSetList +#' @export +#' @examples +#' subset(dsl, funcId == 1) +#' subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes +#' subset(dsl, instance == 1) +#' subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes +#' subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND +subset.DataSetList <- function(x, ...) { + enclos <- parent.frame() + if (hasArg('text')) { + text <- list(...)$text + condition_call <- parse(text = text) + } else { + condition_call <- substitute(list(...)) + condition_call <- condition_call[2:length(condition_call)] + } + + obj <- lapply(x, + function(ds){ + mask <- tryCatch(expr = { + mask <- NULL + for (idx in seq(length(condition_call))) { + mask_temp <- unlist( + eval(condition_call[[idx]], attributes(ds), enclos = enclos) + ) + if (is.null(mask)) mask <- mask_temp + else { + if (length(mask_temp) == 1 && !mask_temp) { + mask <- F + } else if (length(mask_temp) == 1) { + mask <- mask + } else if (length(mask_temp) == length(mask) || length(mask) == 1) { + mask <- mask & mask_temp + } else { + stop("Error creating mask") + } + + } + } + mask + }, error = function(e) {F}) + + if (length(mask) == 1 && mask) return(ds) + else if (length(mask) == 1 || !any(mask)) return(NULL) + return(subset(ds, mask)) + }) + + class(obj) <- c('DataSetList', class(obj)) + obj <- Filter(Negate(is.null), obj) + + # also slice the attributes accordingly + attr(obj, 'suite') <- attr(x, 'suite') + attr(obj, 'maximization') <- attr(x, 'maximization') + attr(obj, 'DIM') <- sapply(obj, function(ds) attr(ds, 'DIM')) + attr(obj, 'funcId') <- sapply(obj, function(ds) attr(ds, 'funcId')) + attr(obj, 'algId') <- sapply(obj, function(ds) attr(ds, 'algId')) + unique_ids <- unlist(sapply(obj, function(ds) attr(ds, 'unique_id'))) + if (!any(is.null(unique_ids))) { + attr(obj, 'unique_ids') <- unique_ids + } + return(obj) +} + +#' Add unique identifiers to each DataSet in the provided DataSetList based on static attributes +#' +#' Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to +#' ensure each dataset has exactly one unique identifier. +#' Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. +#' +#' @param dsl The DataSetList +#' @param attrs The list of attributes to combine into a unique identifier +#' @return A new DataSetList object where the split has been done based on the provided attributes, and the unique +#' identifier has been added. +#' @export +#' @examples +#' change_id(dsl, c('instance')) +change_id <- function(dsl, attrs) { + if (length(dsl) == 0) return(dsl) + if (!all(attrs %in% get_static_attributes(dsl))) stop("Selected attributes are not usable to create unique ids") + grid <- expand.grid(lapply(attrs, function(x){get_static_attribute_values(dsl, x)})) + colnames(grid) <- attrs + + dsl_new <- DataSetList() + attr_vals <- c() + for (x in transpose(grid)) { + #TODO: on Windows, UTF-8 Characters get converted into characters, which then + #don't interact correctly with the parse used in 'subset' function, leading to empty datasetlist objects. + #I'm not aware of any way to fix this, so UFT-8 characters should be avoided in ID for now. + conditions <- paste0(unlist(lapply(seq(length(attrs)), function(idx) { + paste0(attrs[[idx]], ' == "', x[[idx]], '"') + })), collapse = " & ") + dsl_temp <- subset(dsl, text = conditions) + if (length(attrs) == 1) + attr_val <- x + else + attr_val <- paste0(x, collapse = "_") + + attr_vals <- c(attr_vals, rep(attr_val, length(dsl_temp))) + dsl_new <- c(dsl_new, dsl_temp) + } + attr(dsl_new, 'ID_attributes') <- attrs + attr(dsl_new, 'ID') <- attr_vals + for (idx in seq(length(dsl_new))) { + attr(dsl_new[[idx]], 'ID') <- attr_vals[[idx]] + } + class(dsl_new) <- c("DataSetList", "list") + attr(dsl_new, 'suite') <- attr(dsl, 'suite') + attr(dsl_new, 'maximization') <- attr(dsl, 'maximization') + + attr(dsl_new, 'DIM') <- lapply(dsl_new, function(x) attr(x, 'DIM')) + attr(dsl_new, 'funcId') <- lapply(dsl_new, function(x) attr(x, 'funcId')) + attr(dsl_new, 'algId') <- lapply(dsl_new, function(x) attr(x, 'algId')) + dsl_new +} + + + + +#' Save DataTable in multiple formats +#' +#' @param df The DataTable to store +#' @param file String. The name of the figure file, with the extension of the required file-format +#' @param format Optional, string. Overwrites the extension of the `file` parameter. If not specified while +#' file does not have an extension, it defaults to csv +#' +#' @export +#' @examples +#' df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +#' save_table(df, tempfile(fileext = ".md")) +save_table <- function(df, file, format = NULL) { + if (is.null(format)) { + format <- tools::file_ext(file) + } + if (format == 'TeX' || format == 'tex') { + if (requireNamespace('xtable', quietly = T)) + print(xtable::xtable(df), file = file) + else + write(kable(df, format = 'latex'), file) + } else if (format == 'Markdown' || format == 'md') { + write(kable(df, format = 'markdown'), file) + } else if (format == 'html') { + write(kable(df, format = "html"), file) + } + else { #Default to csv + write.csv(df, file, row.names = F) + } +} + +#' Generation of default ECDF-targets +#' +#' @param dsList The DataSetList object for which to generate the targets +#' @param type The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, +#' equal for all functions / dimensions) +#' @param number_targets The amount of targets to generate +#' +#' @return A data.table with 3 columns: funcId, DIM and target +#' @export +#' @examples +#' get_ECDF_targets(dsl, 'linear', 10) +get_ECDF_targets <- function(dsList, type = "log-linear", number_targets = 10) { + funcIds <- get_funcId(dsList) + dims <- get_dim(dsList) + + dt <- rbindlist(apply(expand.grid(funcIds, dims), 1, function(x) { + if (type == 'bbob') { + fseq <- rev(seq_FV(c(100,1e-8), length.out = 51, scale = 'log')) + } + else { + dsl <- subset(dsList, funcId == x[[1]] && DIM == x[[2]]) + if (length(dsl) == 0) + return(NULL) + fall <- get_funvals(dsl) + if (length(fall) < 2) + return(NULL) + + fseq <- seq_FV(fall, length.out = number_targets, scale = ifelse(type == "log-linear", 'log', 'linear')) + } + data.table(funcId = x[[1]], DIM = x[[2]], target = fseq) + })) + dt +} + +### __________________________ Rewritten data generation functions _______________________ ### +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param start Optional start value (Runtime or target value) +#' @param stop Optional end value (Runtime or target value) +#' @param scale_log Wheterh to use logarithmic scaling or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param include_opts Whether or not to also include the best value hit by each algorithm to +#' the generated datapoints +#' @param budget Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works +#' in fixed_target mode. +#' +#' @export +#' @examples +#' generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +generate_data.Single_Function <- function(dsList, start = NULL, stop = NULL, + scale_log = F, which = 'by_RT', include_opts = F, budget = NULL) { + + if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1 ) { + #Required because target generation is included in this function, + #which needs to be done on a per-function basis + stop("Multiple functions / dimensions are present in provided DataSetList. + Please call this function for each individual function/dimension pair instead.") + } + + by_rt <- (which == 'by_RT') + + if (by_rt) + all <- get_funvals(dsList) + else + all <- get_runtimes(dsList) + + maximization <- attr(dsList, 'maximization') + + if (is.null(maximization)) maximization <- T + if (is.null(start)) start <- min(all) + if (is.null(stop)) stop <- max(all) + + if (by_rt) { + Xseq <- seq_FV(all, start, stop, length.out = 60, + scale = ifelse(scale_log, 'log', 'linear')) + if (include_opts) { + for (uid in get_id(dsList)) { + if (maximization) + Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) + else + Xseq <- c(Xseq, min(get_funvals(subset(dsList, ID == uid)))) + } + Xseq <- unique(sort(Xseq)) + } + dt <- get_RT_summary(dsList, ftarget = Xseq, budget = budget) + } + else { + Xseq <- seq_RT(all, start, stop, length.out = 60, + scale = ifelse(scale_log, 'log', 'linear')) + if (include_opts) { + for (uid in get_id(dsList)) { + Xseq <- c(Xseq, max(get_funvals(subset(dsList, ID == uid)))) + } + Xseq <- unique(sort(Xseq)) + } + dt <- get_FV_summary(dsList, Xseq) + } + + + dt[, `:=`(upper = mean + sd, lower = mean - sd)] + return(dt) +} + +#' Generate dataframe of a single function/dimension pair for creating PDF or PMF plots +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param target The target value (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +generate_data.PMF <- function(dsList, target, which = 'by_RT') { + if (which == 'by_RT') + return(get_RT_sample(dsList, target, output = 'long')) + return(get_FV_sample(dsList, target, output = 'long')) +} + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param target The target value (Runtime or target value) +#' @param use.equal.bins Whether all bins should be equal size for each algorithm or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +generate_data.hist <- function(dsList, target, use.equal.bins = F, which = 'by_RT') { + width <- NULL # Set local binding to remove warnings + + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) + stop('This function is only available a single function/dimension pair at a time.') + + if (use.equal.bins) { + if (which == 'by_RT') + res1 <- hist( + get_RT_sample(dsList, target, output = 'long')$RT, + breaks = nclass.FD, + plot = F + ) + else + res1 <- hist( + get_FV_sample(dsList, target, output = 'long')$`f(x)`, + breaks = nclass.FD, + plot = F + ) + } + + dt <- as.data.table( + rbindlist( + lapply( + dsList, + function(ds) { + ID <- get_id(ds) + + if (which == 'by_RT') + data <- get_RT_sample(ds, target, output = 'long')$RT + else if (which == 'by_FV') + data <- get_FV_sample(ds, target, output = 'long')$`f(x)` + else + stop('Invalid argument for parameter `which`.') + + if (sum(!is.na(data)) < 2) + return(NULL) + + if (use.equal.bins) + breaks <- res1$breaks + else + breaks <- nclass.FD + + res <- hist(data, breaks = breaks, plot = F) + breaks <- res$breaks + + plot_text <- paste0( + 'count: ', res$counts, '
breaks: [', + breaks[-length(breaks)], ',', breaks[-1], ']' + ) + + plot_data <- data.frame( + x = res$mids, + y = res$counts, + ID = ID, + width = breaks[2] - breaks[1], + text = plot_text + ) + } + ) %>% { + `[`(., !vapply(., is.null, logical(1))) + } + ) + ) + dt +} + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param scale_log Wheterh to use logarithmic scaling or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param use_full_range Whether or not to use the full range of the x-axis or cut it off as soon as +#' all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension +#' +#' @export +#' @examples +#' generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) +generate_data.ECDF <- function(dsList, targets, scale_log = F, which = 'by_RT', use_full_range = TRUE) { + V1 <- NULL #Set local binding to remove warnings + by_rt <- which == 'by_RT' + if (by_rt) { + RT <- get_runtimes(dsList) + if (!use_full_range) { + if (length(unique(get_funcId(dsList))) > 1 || length(unique(get_dim(dsList))) > 1) { + warning("The limiting of x-values is only supported in the case of 1 function 1 dimension!") + } + else { + maxRT <- as.integer(max(get_RT_summary(dsList, targets)[,'98%']) * 1.1) #Slight buffer for nicer plotting + RT <- c(min(RT), maxRT) + } + } + x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) + + #TODO: Some scaling by dimension? + + if (!is.data.table(targets)) { + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) + stop("Targets provided are not in data.table format, while multiple functions / dimensions + are present in provided DataSetList.") + targets <- data.table( + target = targets, + funcId = get_funcId(dsList), + DIM = get_dim(dsList) + ) + } + } + else { + FV <- get_funvals(dsList) + x <- unique(seq_FV(FV, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) + } + + dt <- as.data.table(rbindlist(lapply(dsList, function(df) { + ID <- get_id(df) + if (by_rt) { + temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] + targets_ <- temp[funcId == attr(df, 'funcId')][['target']] + } + else + targets_ <- targets + m <- lapply(targets_, function(target) { + if (by_rt) + data <- get_RT_sample(df, target, output = 'long')$RT + else + data <- get_FV_sample(df, target, output = 'long')$`f(x)` + + if (all(is.na(data))) + return(rep(0, length(x))) + fun <- ecdf(data) + if (is.function(fun)) fun(x) else NA + }) %>% + do.call(rbind, .) + + data.frame(x = x, + mean = apply(m, 2, . %>% mean(na.rm = T)), + sd = apply(m, 2, . %>% sd(na.rm = T))) %>% + mutate(upper = mean + sd, lower = mean - sd, ID = ID) + }))) + dt[, mean(mean), by = .(x, ID)][, .(mean = V1, ID = ID, x = x)] +} + + +#' Generate dataframe containing the AUC for any ECDF-curves +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param scale_log Whether to use logarithmic scaling or not +#' @param dt_ecdf A data table of the ECDF to avoid needless recomputations. Will take preference if it +#' is provided together with dsList and targets +#' @param multiple_x Boolean, whether to get only the total AUC or get stepwise AUC values +#' +#' @export +#' @examples +#' generate_data.AUC(dsl, get_ECDF_targets(dsl)) +#' generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) +generate_data.AUC <- function(dsList, targets, scale_log = F, which = 'by_RT', dt_ecdf = NULL, + multiple_x = FALSE) { + idx <- auc_contrib <- mean_pre <- mean_post <- x <- x_pre <- auc <- NULL + if (is.null(dt_ecdf)) { + if (length(dsList) == 0 || is.null(targets)) + return(NULL) + dt_ecdf <- generate_data.ECDF(dsList, targets, scale_log, which) + } + max_idx <- nrow(unique(dt_ecdf[,'x'])) + dt_ecdf[, idx := seq(max_idx), by = 'ID'] + dt3 = copy(dt_ecdf) + dt3[, idx := idx - 1] + dt_merged = merge(dt_ecdf, dt3, by = c('ID', 'idx')) + colnames(dt_merged) <- c("ID", "idx", "mean_pre", "x_pre", "mean_post", "x") + dt_merged[, auc_contrib := ((mean_pre + mean_post)/2)*(x - x_pre)] + dt_merged[, auc := cumsum(auc_contrib)/x, by = 'ID'] + #TODO: just for max x + if (multiple_x) + return(dt_merged[, c('ID','x','auc') ]) + return(dt_merged[idx == (max_idx - 1), c('ID','x','auc') ]) +} + + +# #' Generate dataframe of a single function/dimension pair +# #' +# #' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +# #' +# #' @param dsList The DataSetList object +# #' @param targets A list of the target value for which to calculate the AUC (Runtime or target value) +# #' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +# #' +# #' @export +# #' @examples +# #' generate_data.AUC(subset(dsl, funcId == 1), c(12, 16)) +# generate_data.AUC <- function(dsList, targets, which = 'by_RT') { +# if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { +# stop("This function is only available a single function/dimension pair at a time.") +# } +# by_rt <- which == 'by_RT' +# +# if (by_rt) +# RT.max <- sapply(dsList, function(ds) max(attr(ds, 'maxRT'))) %>% max +# else { +# funevals.max <- sapply(dsList, function(ds) max(attr(ds, 'finalFV'))) %>% max +# funevals.min <- sapply(dsList, function(ds) min(attr(ds, 'finalFV'))) %>% min +# } +# +# as.data.table(rbindlist(lapply(dsList, function(df) { +# algId <- attr(df, 'algId') +# if (by_rt) +# auc <- sapply(targets, function(fv) { +# ECDF(df, fv) %>% AUC(from = 1, to = RT.max) +# }) +# else { +# funs <- lapply(targets, function(r) { +# get_FV_sample(df, r, output = 'long')$'f(x)' %>% { +# if (all(is.na(.))) NULL +# else { +# f <- ecdf(.) +# attr(f, 'min') <- min(.) +# attr(f, 'max') <- max(.) +# f +# } +# } +# }) +# +# auc <- sapply(funs, +# function(fun) { +# if (is.null(fun)) 0 +# else{ +# if (attr(df, 'maximization')) +# integrate(fun, lower = attr(fun, 'min') - 1, upper = funevals.max, +# subdivisions = 1e3) %>% {'$'(., 'value') / funevals.max} +# else +# integrate(fun, lower = funevals.min, upper = attr(fun, 'max') + 1, +# subdivisions = 1e3) %>% {'$'(., 'value') / (attr(fun, 'max') + 1)} +# } +# }) +# } +# data.frame(x = targets, AUC = auc, algId = algId) +# }))) +# } + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param scale_log Wheterh to use logarithmic scaling or not +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.Parameters(subset(dsl, funcId == 1)) +generate_data.Parameters <- function(dsList, which = 'by_RT', scale_log = F) { + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1) { + stop("This function is only available a single function/dimension pair at a time.") + } + if (which == 'by_RT') { + rtall <- get_runtimes(dsList) + + rtseq <- seq_RT(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) + req(rtseq) + + dt <- get_PAR_summary(dsList, rtseq, which = 'by_RT') + } + else if (which == 'by_FV') { + rtall <- get_funvals(dsList) + + rtseq <- seq_FV(rtall, length.out = 50, scale = ifelse(scale_log, 'log', 'linear')) + req(rtseq) + + dt <- get_PAR_summary(dsList, rtseq, which = 'by_FV') + } + else stop("Invalid value for parameter `which`") + dt[, `:=`(upper = mean + sd, lower = mean - sd)] +} + +#' Generate dataframe of a single function/dimension pair +#' +#' This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +#' +#' @param dsList The DataSetList object +#' @param aggr_on Which attribute to use for aggregation. Either 'funcId' or 'DIM' +#' @param targets Optional list of target values (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' +#' @export +#' @examples +#' generate_data.Aggr(dsl) +generate_data.Aggr <- function(dsList, aggr_on = 'funcId', targets = NULL, which = 'by_RT') { + maximize <- attr(dsList, 'maximization') + variable <- fid <- value <- NULL #Set local binding to remove warnings + by_rt <- which == 'by_RT' + + if (is.null(targets)) { + targets <- get_target_dt(dsList, which) + } + + aggr_attr <- if (aggr_on == 'funcId') get_funcId(dsList) else get_dim(dsList) + N <- length(get_id(dsList)) + + dt <- rbindlist(lapply(aggr_attr, function(agg_val) { + if (by_rt) { + if (aggr_on == 'funcId') + dt <- get_RT_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) + else + dt <- get_RT_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) + dt[, c('ID', value = 'ERT', 'funcId', 'DIM')] + setnames(dt, 'ERT', 'value') + } + else{ + if (aggr_on == 'funcId') + dt <- get_FV_summary(subset(dsList, funcId == agg_val), targets[funcId == agg_val][['target']]) + else + dt <- get_FV_summary(subset(dsList, DIM == agg_val), targets[DIM == agg_val][['target']]) + dt[, c('ID', value = 'mean', 'funcId', 'DIM')] + setnames(dt, 'mean', 'value') + } + })) + + if (by_rt) order_sel <- 1 + else order_sel <- -1*(maximize*2 - 1) + + dt[, rank := frank(order_sel*value, na.last = T), by = .(DIM, funcId)] + return(dt) +} + + + +#' Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of +#' +#' This provides an unaggregated version of the function `generate_data.ECDF`. +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param scale_log Wheterh to use logarithmic scaling or not +#' +#' @export +#' @examples +#' generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) +generate_data.ECDF_raw <- function(dsList, targets, scale_log = F) { + V1 <- NULL #Set local binding to remove warnings + + #Get the x-coordinates at which to calculate the ECDF + RT <- get_runtimes(dsList) + x <- unique(seq_RT(RT, length.out = 50, scale = ifelse(scale_log, 'log', 'linear'))) + #TODO: Some scaling by dimension? + + #Get targets to use + if (!is.data.table(targets)) { + if (length(get_funcId(dsList)) > 1 || length(get_dim(dsList)) > 1 ) + stop("Targets provided are not in data.table format, while multiple functions / dimensions + are present in provided DataSetList.") + targets <- data.table( + target = targets, + funcId = get_funcId(dsList), + DIM = get_dim(dsList) + ) + } + + dt <- as.data.table(rbindlist(lapply(dsList, function(df) { + ID <- get_id(df) + temp <- targets[DIM == attr(df, 'DIM'), c('target', 'funcId')] + targets_ <- temp[funcId == attr(df, 'funcId')][['target']] + + m <- lapply(targets_, function(target) { + data <- fast_RT_samples(df$RT, target, attr(df, 'maximization')) + if (all(is.na(data))) + hit <- (rep(0, length(x))) + else { + fun <- ecdf(data) + hit <- if (is.function(fun)) fun(x) else NA + } + data.table(hit = hit, rt = x, target = target, funcId = attr(df, 'funcId'), DIM = attr(df, 'DIM'), ID = ID) + }) %>% + do.call(rbind, .) + m + }))) + dt } \ No newline at end of file diff --git a/R/IOHanalyzer-deprecated.R b/R/IOHanalyzer-deprecated.R index 304aaa68..6b5366d9 100644 --- a/R/IOHanalyzer-deprecated.R +++ b/R/IOHanalyzer-deprecated.R @@ -1,196 +1,196 @@ -##IOHanalyzer-deprecated.R -#' @title Deprecated function in package \pkg{IOHanalyzer} -#' @description The functions listed below are deprecated and will be defunct in -#' the near future. When possible, alternative functions with similar -#' functionality are also mentioned. -#' @name IOHanlyzer-deprecated -#' @keywords internal -NULL - -#' Get the ERT-values for all DataSets in a DataSetList at certain targets -#' -#' @param dsList The DataSetLsit -#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. -#' @param targets Predifined target function-values. Should be one for each function/dimension -#' @param maximize Whether the DataSetList is from a maximization or minimization problem -#' -#' @return A data.table containing ERT-values -#' @export -#' @examples -#' max_ERTs(dsl) -max_ERTs <- - function(dsList, - aggr_on = 'funcId', - targets = NULL, - maximize = T) - UseMethod("max_ERTs", dsList) - -#TODO: rename this function! this function needs to be rewritten -#' @rdname max_ERTs -#' @export -max_ERTs.DataSetList <- - function(dsList, - aggr_on = 'funcId', - targets = NULL, - maximize = T) { - .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. - Please use the suggested function instead.") - N <- length(get_algId(dsList)) - - aggr_attr <- - if (aggr_on == 'funcId') - get_funcId(dsList) - else - get_dim(dsList) - if (!is.null(targets) && - length(targets) != length(aggr_attr)) - targets <- NULL - - second_aggr <- - if (aggr_on == 'funcId') - get_dim(dsList) - else - get_funcId(dsList) - if (length(second_aggr) > 1) - return(NULL) - - erts <- seq(0, 0, length.out = length(get_algId(dsList))) - names(erts) <- get_algId(dsList) - - for (j in seq_along(aggr_attr)) { - dsList_filetered <- - if (aggr_on == 'funcId') - subset(dsList, funcId == aggr_attr[[j]]) - else - subset(dsList, DIM == aggr_attr[[j]]) - - if (is.null(targets)) { - Fall <- get_funvals(dsList_filetered) - Fval <- ifelse(maximize, max(Fall), min(Fall)) - } - else - Fval <- targets[[j]] - summary <- get_RT_summary(dsList_filetered, ftarget = Fval) - ert <- summary$ERT - names(ert) <- summary$algId - erts <- rbind(erts, ert[get_algId(dsList)]) - } - return(erts[-1, ]) - } - -#' Get the expected function-values for all DataSets in a DataSetList at certain runtimes -#' -#' @param dsList The DataSetLsit -#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. -#' @param runtimes Predifined target runtimes-values. Should be one for each function/dimension -#' -#' @return A data.table containing expected fucntion-values -#' @export -#' @examples -#' mean_FVs(dsl) -mean_FVs <- - function(dsList, - aggr_on = 'funcId', - runtimes = NULL) - UseMethod("mean_FVs", dsList) - -#' @rdname mean_FVs -#' @export -mean_FVs.DataSetList <- - function(dsList, - aggr_on = 'funcId', - runtimes = NULL) { - N <- length(get_algId(dsList)) - .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. - Please use the suggested function instead.") - - aggr_attr <- - if (aggr_on == 'funcId') - get_funcId(dsList) - else - get_dim(dsList) - if (!is.null(runtimes) && - length(runtimes) != length(aggr_attr)) - targets <- NULL - - second_aggr <- - if (aggr_on == 'funcId') - get_dim(dsList) - else - get_funcId(dsList) - if (length(second_aggr) > 1) - return(NULL) - - erts <- seq(0, 0, length.out = length(get_algId(dsList))) - names(erts) <- get_algId(dsList) - - for (j in seq_along(aggr_attr)) { - dsList_filetered <- - if (aggr_on == 'funcId') - subset(dsList, funcId == aggr_attr[[j]]) - else - subset(dsList, DIM == aggr_attr[[j]]) - - if (is.null(runtimes)) { - RTall <- get_runtimes(dsList_filetered) - RTval <- max(RTall) - } - else - RTval <- runtimes[[j]] - summary <- get_FV_summary(dsList_filetered, runtime = RTval) - ert <- summary$mean - names(ert) <- summary$algId - erts <- rbind(erts, ert[get_algId(dsList)]) - } - return(erts[-1, ]) - } - -# TODO: review / re-write this function -#TODO: inconsistent use of format_func gives slightly different results between -#generated and uploaded targets -#' Generate ECDF targets for a DataSetList -#' -#' @param data A DataSetList -#' @param format_func function to format the targets -#' -#' @return a vector of targets -#' @export -#' @examples -#' get_default_ECDF_targets(dsl) -get_default_ECDF_targets <- function(data, format_func = as.integer) { - .Deprecated("get_ECDF_targets", msg = "This function will be deprecated in the next release of IOHanalyzer. - Please use the suggested function instead.") - - funcIds <- get_funcId(data) - dims <- get_dim(data) - - targets <- list() - names <- list() - for (i in seq_along(funcIds)) { - Id <- funcIds[[i]] - data_sub <- subset(data, funcId == Id) - - for (j in seq_along(dims)) { - dim <- dims[[j]] - data_subsub <- subset(data_sub, DIM == dim) - if (length(data_subsub) == 0) break - fall <- get_funvals(data_subsub) - if (length(fall) < 2) break #TODO: double check why this can happen in nevergrad? - #TODO: Account for minimization / maximization - fmin <- min(fall) - fmax <- max(fall) - - fseq <- seq_FV(fall, fmin, fmax, length.out = 10) %>% - sapply(format_func) - targets <- append(targets, list(fseq)) - - if (length(funcIds) == 1) { - names <- append(names, dim) - } else if (length(dims) == 1) { - names <- append(names, Id) - } else - names <- append(names, paste0(Id, ";", dim)) - } - } - targets %>% set_names(names) +##IOHanalyzer-deprecated.R +#' @title Deprecated function in package \pkg{IOHanalyzer} +#' @description The functions listed below are deprecated and will be defunct in +#' the near future. When possible, alternative functions with similar +#' functionality are also mentioned. +#' @name IOHanlyzer-deprecated +#' @keywords internal +NULL + +#' Get the ERT-values for all DataSets in a DataSetList at certain targets +#' +#' @param dsList The DataSetLsit +#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. +#' @param targets Predifined target function-values. Should be one for each function/dimension +#' @param maximize Whether the DataSetList is from a maximization or minimization problem +#' +#' @return A data.table containing ERT-values +#' @export +#' @examples +#' max_ERTs(dsl) +max_ERTs <- + function(dsList, + aggr_on = 'funcId', + targets = NULL, + maximize = T) + UseMethod("max_ERTs", dsList) + +#TODO: rename this function! this function needs to be rewritten +#' @rdname max_ERTs +#' @export +max_ERTs.DataSetList <- + function(dsList, + aggr_on = 'funcId', + targets = NULL, + maximize = T) { + .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. + Please use the suggested function instead.") + N <- length(get_algId(dsList)) + + aggr_attr <- + if (aggr_on == 'funcId') + get_funcId(dsList) + else + get_dim(dsList) + if (!is.null(targets) && + length(targets) != length(aggr_attr)) + targets <- NULL + + second_aggr <- + if (aggr_on == 'funcId') + get_dim(dsList) + else + get_funcId(dsList) + if (length(second_aggr) > 1) + return(NULL) + + erts <- seq(0, 0, length.out = length(get_algId(dsList))) + names(erts) <- get_algId(dsList) + + for (j in seq_along(aggr_attr)) { + dsList_filetered <- + if (aggr_on == 'funcId') + subset(dsList, funcId == aggr_attr[[j]]) + else + subset(dsList, DIM == aggr_attr[[j]]) + + if (is.null(targets)) { + Fall <- get_funvals(dsList_filetered) + Fval <- ifelse(maximize, max(Fall), min(Fall)) + } + else + Fval <- targets[[j]] + summary <- get_RT_summary(dsList_filetered, ftarget = Fval) + ert <- summary$ERT + names(ert) <- summary$algId + erts <- rbind(erts, ert[get_algId(dsList)]) + } + return(erts[-1, ]) + } + +#' Get the expected function-values for all DataSets in a DataSetList at certain runtimes +#' +#' @param dsList The DataSetLsit +#' @param aggr_on Whether to aggregate on 'funcId' or 'DIM'. +#' @param runtimes Predifined target runtimes-values. Should be one for each function/dimension +#' +#' @return A data.table containing expected fucntion-values +#' @export +#' @examples +#' mean_FVs(dsl) +mean_FVs <- + function(dsList, + aggr_on = 'funcId', + runtimes = NULL) + UseMethod("mean_FVs", dsList) + +#' @rdname mean_FVs +#' @export +mean_FVs.DataSetList <- + function(dsList, + aggr_on = 'funcId', + runtimes = NULL) { + N <- length(get_algId(dsList)) + .Deprecated("generate_data.Aggr", msg = "This function will be deprecated in the next release of IOHanalyzer. + Please use the suggested function instead.") + + aggr_attr <- + if (aggr_on == 'funcId') + get_funcId(dsList) + else + get_dim(dsList) + if (!is.null(runtimes) && + length(runtimes) != length(aggr_attr)) + targets <- NULL + + second_aggr <- + if (aggr_on == 'funcId') + get_dim(dsList) + else + get_funcId(dsList) + if (length(second_aggr) > 1) + return(NULL) + + erts <- seq(0, 0, length.out = length(get_algId(dsList))) + names(erts) <- get_algId(dsList) + + for (j in seq_along(aggr_attr)) { + dsList_filetered <- + if (aggr_on == 'funcId') + subset(dsList, funcId == aggr_attr[[j]]) + else + subset(dsList, DIM == aggr_attr[[j]]) + + if (is.null(runtimes)) { + RTall <- get_runtimes(dsList_filetered) + RTval <- max(RTall) + } + else + RTval <- runtimes[[j]] + summary <- get_FV_summary(dsList_filetered, runtime = RTval) + ert <- summary$mean + names(ert) <- summary$algId + erts <- rbind(erts, ert[get_algId(dsList)]) + } + return(erts[-1, ]) + } + +# TODO: review / re-write this function +#TODO: inconsistent use of format_func gives slightly different results between +#generated and uploaded targets +#' Generate ECDF targets for a DataSetList +#' +#' @param data A DataSetList +#' @param format_func function to format the targets +#' +#' @return a vector of targets +#' @export +#' @examples +#' get_default_ECDF_targets(dsl) +get_default_ECDF_targets <- function(data, format_func = as.integer) { + .Deprecated("get_ECDF_targets", msg = "This function will be deprecated in the next release of IOHanalyzer. + Please use the suggested function instead.") + + funcIds <- get_funcId(data) + dims <- get_dim(data) + + targets <- list() + names <- list() + for (i in seq_along(funcIds)) { + Id <- funcIds[[i]] + data_sub <- subset(data, funcId == Id) + + for (j in seq_along(dims)) { + dim <- dims[[j]] + data_subsub <- subset(data_sub, DIM == dim) + if (length(data_subsub) == 0) break + fall <- get_funvals(data_subsub) + if (length(fall) < 2) break #TODO: double check why this can happen in nevergrad? + #TODO: Account for minimization / maximization + fmin <- min(fall) + fmax <- max(fall) + + fseq <- seq_FV(fall, fmin, fmax, length.out = 10) %>% + sapply(format_func) + targets <- append(targets, list(fseq)) + + if (length(funcIds) == 1) { + names <- append(names, dim) + } else if (length(dims) == 1) { + names <- append(names, Id) + } else + names <- append(names, paste0(Id, ";", dim)) + } + } + targets %>% set_names(names) } \ No newline at end of file diff --git a/R/IOHanalyzer.R b/R/IOHanalyzer.R index 8a5fd72d..4f6bd829 100644 --- a/R/IOHanalyzer.R +++ b/R/IOHanalyzer.R @@ -1,126 +1,129 @@ -#' @importFrom stats dt ecdf integrate median quantile sd rgeom ks.test p.adjust -#' @importFrom grDevices col2rgb colors nclass.FD -#' @importFrom graphics hist -#' @importFrom utils data head read.csv tail type.convert write.csv -#' @importFrom dplyr %>% mutate -#' @importFrom magrittr set_names set_rownames set_colnames %<>% mod -#' @importFrom colorspace sequential_hcl -#' @importFrom RColorBrewer brewer.pal -#' @importFrom colorRamps primary.colors -#' @importFrom data.table as.data.table rbindlist data.table fread := melt is.data.table -#' @importFrom data.table setorderv frank setnames rbindlist copy -#' @importFrom plotly add_annotations add_trace orca plot_ly rename_ subplot layout -#' @importFrom ggplot2 aes geom_jitter geom_line geom_ribbon geom_violin ggplot element_text -#' @importFrom ggplot2 guides scale_color_manual scale_colour_manual scale_fill_manual -#' @importFrom ggplot2 scale_x_continuous scale_x_log10 facet_wrap theme_set theme_grey theme -#' @importFrom shiny req -#' @importFrom stringi stri_detect_regex stri_detect_fixed stri_locate_all stri_replace -#' @importFrom stringi stri_rand_strings -#' @importFrom httr POST add_headers content authenticate -#' @importFrom reshape2 acast -#' @importFrom knitr kable -#' @useDynLib IOHanalyzer -NULL -# Ugly hack, but appears to be required to appease CRAN -utils::globalVariables(c(".", "algId", "run", "ERT", "RT", "group", - "DIM", "Fvalue", "lower", "upper", "target", "format", - "runtime", "parId", "instance", "input", "funcId", - "budget", "dimension", "loss", "name", "optimizer_name", - "rescale", "maxRT", "algnames", ".SD", "function_class")) - -options(shiny.port = 4242) - -.onLoad <- function(libname, pkgname) { - op <- options() - op.IOHanalyzer <- list( - IOHanalyzer.quantiles = c(2, 5, 10, 25, 50, 75, 90, 95, 98) / 100., - IOHanalyzer.max_samples = 100, - IOHanalyzer.backend = 'plotly', - IOHanalyzer.bgcolor = 'rgb(230,230,230)', - IOHanalyzer.gridcolor = 'rgb(255,255,255)', - IOHanalyzer.tickcolor = 'rgb(51,51,51)', - IOHanalyzer.figure_width = 1000, - IOHanalyzer.figure_height = 1000, - IOHanalyzer.legend_location = 'below', - IOHanalyzer.legend_fontsize = 13, - IOHanalyzer.custom_legend_x = 0.5, - IOHanalyzer.custom_legend_y = -0.2, - IOHanalyzer.label_fontsize = 16, - IOHanalyzer.title_fontsize = 16, - IOHanalyzer.tick_fontsize = 12, - IOHanalyzer.linewidth = 2, - IOHanalyzer.markersize = 4, - IOHanalyzer.max_colors = 2 #Set to 2 since colorbrewer only works with >= 3 colors - ) - toset <- !(names(op.IOHanalyzer) %in% names(op)) - if (any(toset)) options(op.IOHanalyzer[toset]) - - invisible() -} - -IOHanalyzer_env <- new.env(parent = emptyenv()) - -.mean <- function(x) mean(x, na.rm = T) -.median <- function(x) median(x, na.rm = T) -.sd <- function(x) sd(x, na.rm = T) -.sum <- function(x) sum(x, na.rm = T) - -# Quantile function for discrete values -IOHanalyzer_env$D_quantile <- function(x, pct = NULL) { - if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") - tryCatch( - quantile(x, pct, names = F, type = 3, na.rm = T), - error = function(e) rep(NA, length(pct)), - warning = function(w) rep(NA, length(pct)) - ) -} - -# Quantile function for real values -IOHanalyzer_env$C_quantile <- function(x, pct = NULL) { - if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") - tryCatch( - quantile(x, pct, names = F, na.rm = T), - error = function(e) rep(NA, length(pct)), - warning = function(w) rep(NA, length(pct)) - ) -} - -IOHprofiler <- 'IOHprofiler' -COCO <- 'COCO' -BIBOJ_COCO <- 'BIBOJ_COCO' -TWO_COL <- 'TWO_COL' -AUTOMATIC <- 'AUTOMATIC' -NEVERGRAD <- 'NEVERGRAD' -SOS <- 'SOS' - -#' IOHanalyzer: Data Analysis Part of IOHprofiler -#' -#' The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). -#' This module provides statistical analysis methods for the benchmark data generated by -#' optimization heuristics, which can be visualized through a -#' web-based interface. The benchmark data is usually generated by the -#' experimentation module, called IOHexperimenter. IOHanalyzer also supports -#' the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. -#' -#' @section Functions: -#' The IOHanalyzer consists of 3 main functionalities: -#' \itemize{ -#' \item Reading and alligning data from different heuristics, such as IOHExperimenter. -#' This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions -#' \item Processing and summarizing this data -#' \item Creating various plots -#' } -#' -#' @docType package -#' @name IOHanalyzer -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -#' dsList <- DataSetList(path) -#' summary(dsList) -#' Plot.RT.Single_Func(dsList[1]) -#' -#' @examples -#' \dontrun{ -#' runServer() -#' } +#' @importFrom stats dt ecdf integrate median quantile sd rgeom ks.test p.adjust +#' @importFrom grDevices col2rgb colors nclass.FD +#' @importFrom graphics hist +#' @importFrom utils data head read.csv tail type.convert write.csv +#' @importFrom dplyr %>% mutate +#' @importFrom magrittr set_names set_rownames set_colnames %<>% mod +#' @importFrom colorspace sequential_hcl +#' @importFrom RColorBrewer brewer.pal +#' @importFrom colorRamps primary.colors +#' @importFrom data.table as.data.table rbindlist data.table fread := melt is.data.table +#' @importFrom data.table setorderv frank setnames rbindlist copy transpose +#' @importFrom plotly add_annotations add_trace orca plot_ly rename_ subplot layout +#' @importFrom ggplot2 aes geom_jitter geom_line geom_ribbon geom_violin ggplot element_text +#' @importFrom ggplot2 guides scale_color_manual scale_colour_manual scale_fill_manual +#' @importFrom ggplot2 scale_x_continuous scale_x_log10 facet_wrap theme_set theme_grey theme +#' @importFrom shiny req +#' @importFrom stringi stri_detect_regex stri_detect_fixed stri_locate_all stri_replace +#' @importFrom stringi stri_rand_strings +#' @importFrom httr POST add_headers content authenticate +#' @importFrom reshape2 acast +#' @importFrom knitr kable +#' @importFrom methods hasArg +#' @importFrom rjson fromJSON +#' @useDynLib IOHanalyzer +NULL +# Ugly hack, but appears to be required to appease CRAN +utils::globalVariables(c(".", "algId", "run", "ERT", "RT", "group", + "DIM", "Fvalue", "lower", "upper", "target", "format", + "runtime", "parId", "instance", "input", "funcId", + "budget", "dimension", "loss", "name", "optimizer_name", + "rescale", "maxRT", "algnames", ".SD", "function_class", "ID", "ids")) + +options(shiny.port = 4242) + +.onLoad <- function(libname, pkgname) { + op <- options() + op.IOHanalyzer <- list( + IOHanalyzer.ID_vars = c("algId"), + IOHanalyzer.quantiles = c(2, 5, 10, 25, 50, 75, 90, 95, 98) / 100., + IOHanalyzer.max_samples = 100, + IOHanalyzer.backend = 'plotly', + IOHanalyzer.bgcolor = 'rgb(230,230,230)', + IOHanalyzer.gridcolor = 'rgb(255,255,255)', + IOHanalyzer.tickcolor = 'rgb(51,51,51)', + IOHanalyzer.figure_width = 1000, + IOHanalyzer.figure_height = 1000, + IOHanalyzer.legend_location = 'below', + IOHanalyzer.legend_fontsize = 13, + IOHanalyzer.custom_legend_x = 0.5, + IOHanalyzer.custom_legend_y = -0.2, + IOHanalyzer.label_fontsize = 16, + IOHanalyzer.title_fontsize = 16, + IOHanalyzer.tick_fontsize = 12, + IOHanalyzer.linewidth = 2, + IOHanalyzer.markersize = 4, + IOHanalyzer.max_colors = 2 #Set to 2 since colorbrewer only works with >= 3 colors + ) + toset <- !(names(op.IOHanalyzer) %in% names(op)) + if (any(toset)) options(op.IOHanalyzer[toset]) + + invisible() +} + +IOHanalyzer_env <- new.env(parent = emptyenv()) + +.mean <- function(x) mean(x, na.rm = T) +.median <- function(x) median(x, na.rm = T) +.sd <- function(x) sd(x, na.rm = T) +.sum <- function(x) sum(x, na.rm = T) + +# Quantile function for discrete values +IOHanalyzer_env$D_quantile <- function(x, pct = NULL) { + if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") + tryCatch( + quantile(x, pct, names = F, type = 3, na.rm = T), + error = function(e) rep(NA, length(pct)), + warning = function(w) rep(NA, length(pct)) + ) +} + +# Quantile function for real values +IOHanalyzer_env$C_quantile <- function(x, pct = NULL) { + if (is.null(pct)) pct <- getOption("IOHanalyzer.quantiles") + tryCatch( + quantile(x, pct, names = F, na.rm = T), + error = function(e) rep(NA, length(pct)), + warning = function(w) rep(NA, length(pct)) + ) +} + +IOHprofiler <- 'IOHprofiler' +COCO <- 'COCO' +BIBOJ_COCO <- 'BIBOJ_COCO' +TWO_COL <- 'TWO_COL' +AUTOMATIC <- 'AUTOMATIC' +NEVERGRAD <- 'NEVERGRAD' +SOS <- 'SOS' + +#' IOHanalyzer: Data Analysis Part of IOHprofiler +#' +#' The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). +#' This module provides statistical analysis methods for the benchmark data generated by +#' optimization heuristics, which can be visualized through a +#' web-based interface. The benchmark data is usually generated by the +#' experimentation module, called IOHexperimenter. IOHanalyzer also supports +#' the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. +#' +#' @section Functions: +#' The IOHanalyzer consists of 3 main functionalities: +#' \itemize{ +#' \item Reading and alligning data from different heuristics, such as IOHExperimenter. +#' This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions +#' \item Processing and summarizing this data +#' \item Creating various plots +#' } +#' +#' @docType package +#' @name IOHanalyzer +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +#' dsList <- DataSetList(path) +#' summary(dsList) +#' Plot.RT.Single_Func(dsList[1]) +#' +#' @examples +#' \dontrun{ +#' runServer() +#' } NULL \ No newline at end of file diff --git a/R/RcppExports.R b/R/RcppExports.R index 81825ee7..0b8ddd7f 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -1,31 +1,31 @@ -# Generated by using Rcpp::compileAttributes() -> do not edit by hand -# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 - -align_by_target_inner_loop <- function(t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) { - .Call('_IOHanalyzer_align_by_target_inner_loop', PACKAGE = 'IOHanalyzer', t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) -} - -c_impute <- function(x, y, rowname) { - .Call('_IOHanalyzer_c_impute', PACKAGE = 'IOHanalyzer', x, y, rowname) -} - -c_impute_running_time <- function(index, value, FV, maximization) { - .Call('_IOHanalyzer_c_impute_running_time', PACKAGE = 'IOHanalyzer', index, value, FV, maximization) -} - -#' Align a list of data set by function values -#' -#' @param data the data -#' @param FV Function values -#' @param idxValue index of the function values -#' @param maximization Boolean -#' @param idxTarget index of the target -#' @noRd -c_align_running_time <- function(data, FV, idxValue, maximization, idxTarget) { - .Call('_IOHanalyzer_c_align_running_time', PACKAGE = 'IOHanalyzer', data, FV, idxValue, maximization, idxTarget) -} - -c_read_dat <- function(dat, NC, leading) { - .Call('_IOHanalyzer_c_read_dat', PACKAGE = 'IOHanalyzer', dat, NC, leading) -} - +# Generated by using Rcpp::compileAttributes() -> do not edit by hand +# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 + +align_by_target_inner_loop <- function(t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) { + .Call('_IOHanalyzer_align_by_target_inner_loop', PACKAGE = 'IOHanalyzer', t, idxEvals, idxTarget, data, index, next_lines, curr_eval, maximization) +} + +c_impute <- function(x, y, rowname) { + .Call('_IOHanalyzer_c_impute', PACKAGE = 'IOHanalyzer', x, y, rowname) +} + +c_impute_running_time <- function(index, value, FV, maximization) { + .Call('_IOHanalyzer_c_impute_running_time', PACKAGE = 'IOHanalyzer', index, value, FV, maximization) +} + +#' Align a list of data set by function values +#' +#' @param data the data +#' @param FV Function values +#' @param idxValue index of the function values +#' @param maximization Boolean +#' @param idxTarget index of the target +#' @noRd +c_align_running_time <- function(data, FV, idxValue, maximization, idxTarget) { + .Call('_IOHanalyzer_c_align_running_time', PACKAGE = 'IOHanalyzer', data, FV, idxValue, maximization, idxTarget) +} + +c_read_dat <- function(dat, NC, leading) { + .Call('_IOHanalyzer_c_read_dat', PACKAGE = 'IOHanalyzer', dat, NC, leading) +} + diff --git a/R/data.R b/R/data.R index da9552b8..aa917beb 100644 --- a/R/data.R +++ b/R/data.R @@ -1,17 +1,17 @@ -#' Example DataSetList used in tests / examples -#' -#' A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D -#' -#' @format DataSetList -#' @examples -#' summary(dsl) -"dsl" - -#' Larger example DataSetList used in tests / examples -#' -#' A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D -#' -#' @format DataSetList -#' @examples -#' summary(dsl_large) +#' Example DataSetList used in tests / examples +#' +#' A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D +#' +#' @format DataSetList +#' @examples +#' summary(dsl) +"dsl" + +#' Larger example DataSetList used in tests / examples +#' +#' A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D +#' +#' @format DataSetList +#' @examples +#' summary(dsl_large) "dsl_large" \ No newline at end of file diff --git a/R/plot.R b/R/plot.R index 90fb1018..6ac88d88 100644 --- a/R/plot.R +++ b/R/plot.R @@ -1,369 +1,369 @@ -# font No. 1... -f1 <- list( - family = 'Old Standard TT, serif', - size = 11, - color = 'black' -) - -# font No. 2... -f2 <- list( - family = 'Old Standard TT, serif', - size = 13, - color = 'black' -) - -# font No. 3... -f3 <- function() { - list( - family = 'Old Standard TT, serif', - size = getOption("IOHanalyzer.tick_fontsize", default = 12), - color = 'black' - ) -} - -legend_right <- function() { - list(x = 1.01, y = 1, orientation = 'v', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -legend_inside <- function() { - list(x = .01, y = 1, orientation = 'v', - bgcolor = 'rgba(255, 255, 255, 0)', - bordercolor = 'rgba(255, 255, 255, 0)', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -legend_inside2 <- function() { - list(x = 0.7, y = 0.1, orientation = 'v', - bgcolor = 'rgba(255, 255, 255, 0.5)', - bordercolor = 'rgba(255, 255, 255, 0.8)', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -#TODO: Make the y-value configurable -legend_below <- function() { - list(y = -0.2, orientation = 'h', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - -legend_custom <- function() { - list(x = getOption("IOHanalyzer.custom_legend_x", default = 0.5), - y = getOption("IOHanalyzer.custom_legend_y", default = -0.2), orientation = 'h', - font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), - family = 'Old Standard TT, serif')) -} - - -legend_location <- function(){ - opt <- getOption('IOHanalyzer.legend_location', default = 'below') - if (opt == 'outside_right') return(legend_right()) - else if (opt == 'inside_left') return(legend_inside()) - else if (opt == 'inside_right') return(legend_inside2()) - else if (opt == 'below') return(legend_below()) - else if (opt == 'custom') return(legend_custom()) - # else if (opt == 'below2') return(legend_below2()) - else warning(paste0("The selected legend option (", opt, ") is not implemented")) -} - -# TODO: create font object as above for title, axis... - -#' Template for creating plots in the IOHanalyzer-style -#' -#' @param title Title for the plot -#' @param x.title X-axis label -#' @param y.title Y-axis label -#' -#' @export -#' @examples -#' IOH_plot_ly_default("Example plot","x-axis","y-axis") -IOH_plot_ly_default <- function(title = NULL, x.title = NULL, y.title = NULL) { - plot_ly() %>% - layout(title = list(text = title, - font = list(size = getOption("IOHanalyzer.title_fontsize", default = 16), - family = 'Old Standard TT, serif')), - autosize = T, hovermode = 'compare', - legend = legend_location(), - paper_bgcolor = 'rgb(255,255,255)', - plot_bgcolor = getOption('IOHanalyzer.bgcolor'), - font = list(size = getOption("IOHanalyzer.label_fontsize", default = 16), - family = 'Old Standard TT, serif'), - autosize = T, - showlegend = T, - xaxis = list( - # title = list(text = x.title, font = f3), - title = x.title, - gridcolor = getOption('IOHanalyzer.gridcolor'), - showgrid = TRUE, - showline = FALSE, - showticklabels = TRUE, - tickcolor = getOption('IOHanalyzer.tickcolor'), - ticks = 'outside', - ticklen = 9, - tickfont = f3(), - exponentformat = 'e', - zeroline = F), - yaxis = list( - # title = list(text = y.title, font = f3), - title = y.title, - gridcolor = getOption('IOHanalyzer.gridcolor'), - showgrid = TRUE, - showline = FALSE, - showticklabels = TRUE, - tickcolor = getOption('IOHanalyzer.tickcolor'), - ticks = 'outside', - ticklen = 9, - tickfont = f3(), - exponentformat = 'e', - zeroline = F)) -} - -t <- theme_grey() + - theme(text = element_text(size = 15), - plot.title = element_text(hjust = 0.5) - # legend.position = c(0.15, 0.7), # legend position - # legend.key = element_blank(), # no small box around symbol - # legend.key.size = unit(1.3, "line"), # bigger symbols - # legend.background = element_rect(color = alpha("black", 0.5), - # fill = alpha('blue', 0.0), - # size = 1, - # linetype = "solid") - ) -theme_set(t) - -gg_beanplot <- function(mapping, data, p = NULL, width = 3, fill = 'grey', - colour = 'grey', alpha = 1, kernel = 'gaussian', bw = 'SJ', - draw_quantiles = NULL, trim = TRUE, na.rm = FALSE, - show.legend = NA, point.shape = 20, show.sample = T, - show.violin = T, linetype = 'solid') { - - set.seed(42) - x <- as.character(mapping$x) - y <- as.character(mapping$y) - df <- data[, c(x, y)] %>% rename_(.dots = c('x' = x, 'y' = y)) - - if (!is.numeric(df$x)) - df$x <- tryCatch(as.numeric(df$x), # in case x is a factor... - warning = function(w) return(match(x, as.factor(x)))) - - if (is.null(p)) - p <- ggplot() - - if (show.violin) - p <- p + geom_violin(data = data, mapping = mapping, trim = trim, - draw_quantiles = draw_quantiles, bw = bw, - kernel = kernel, scale = 'width', - width = width, alpha = alpha) - if (show.sample) - p <- p + geom_jitter(data = df, aes(x, y), height = 0, width = width / 2, - alpha = 0.45, shape = point.shape, size = 3.5) - # geom_segment(aes(x = x - width / 2.2, xend = x + width / 2.2, y = y, yend = y), - # df, col = 'black', size = 0.2, alpha = 0.3, linetype = linetype) - p -} - - -Set1 <- function(n) colorspace::sequential_hcl(n, h = c(360, 40), c. = c(100, NA, 90), l = c(28, 90), - power = c(1, 1.1), gamma = NULL, fixup = TRUE, - alpha = 1)#, palette = NULL, rev = FALSE) - -Set2 <- function(n) colorspace::sequential_hcl(n, c(261, 26), c. = c(50, NA, 70), l = c(54, 77), - power = c(0.5, NA), gamma = NULL, - fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) - -Set3 <- function(n) colorspace::sequential_hcl(n, c(-88, 59), c. = c(60, 75, 55), l = c(40, 90), - power = c(0.1, 1.2), gamma = NULL, - fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) - -IOHanalyzer_env$used_colorscheme <- Set2 -IOHanalyzer_env$alg_colors <- NULL - -#' Set the colorScheme of the IOHanalyzer plots -#' -#' @param schemename Three default colorschemes are implemented: -#' \itemize{ -#' \item Default -#' \item Variant 1 -#' \item Variant 2 -#' \item Variant 3 -#' } -#' And it is also possible to select "Custom", which allows uploading of a custom set of colors -#' @param algnames The names of the algorithms for which to set the colors -#' @param path The path to the file containing the colors to use. Only used if -#' schemename is "Custom" -#' -#' @export -#' -#' @examples -#' set_color_scheme("Default", get_algId(dsl)) -set_color_scheme <- function(schemename, algnames, path = NULL){ - if (schemename == "Custom" && !is.null(path)) { - dt <- fread(path, header = T) - if (any(colnames(dt) != c("algnames", "colors", "linestyles"))) { - warning("Incorrect file-format has been uploaded.") - } - else - IOHanalyzer_env$alg_colors <- dt - return() - } - else { - if (schemename == "Default") { - options(IOHanalyzer.max_colors = 2) - } - else { - if (schemename == "Variant 1") IOHanalyzer_env$used_colorscheme <- Set1 - else if (schemename == "Variant 2") IOHanalyzer_env$used_colorscheme <- Set2 - else if (schemename == "Variant 3") IOHanalyzer_env$used_colorscheme <- Set3 - options(IOHanalyzer.max_colors = length(algnames)) - } - create_color_scheme(algnames) - } -} - -#' Get datatable of current color (and linestyle) scheme to file -#' -#' @return data.table object with 3 columns: algnames, colors, linestyles -#' @export -#' @examples -#' get_color_scheme_dt() -get_color_scheme_dt <- function(){ - return(IOHanalyzer_env$alg_colors) -} - -#' Helper function to create default color scheme -#' -#' @noRd -create_color_scheme <- function(algnames) { - if (length(algnames) == 0) { - return(NULL) - } - colors <- color_palettes(length(algnames)) - linestyles <- rep(c("solid", "dash", "dot"), ceiling(length(colors)/3))[1:length(colors)] - IOHanalyzer_env$alg_colors <- data.table(algnames, colors, linestyles) -} - -#' Get colors according to the current colorScheme of the IOHanalyzer -#' -#' @param algnames_in List of algorithms for which to get colors -#' -#' @export -#' -#' @examples -#' get_color_scheme(get_algId(dsl)) -get_color_scheme <- function(algnames_in){ - if (is.null(IOHanalyzer_env$alg_colors)) - create_color_scheme(algnames_in) - cdt <- IOHanalyzer_env$alg_colors - colors <- subset(cdt, algnames %in% algnames_in)[['colors']] - if (is.null(colors) || length(colors) != length(algnames_in)) { - return(color_palettes(length(algnames_in))) - } - names(colors) <- algnames_in - return(colors) -} - - -#' Get line styles according to the current styleScheme of the IOHanalyzer -#' -#' @param algnames_in List of algorithms for which to get linestyles -#' -#' @export -#' -#' @examples -#' get_line_style(get_algId(dsl)) -get_line_style <- function(algnames_in){ - if (is.null(IOHanalyzer_env$alg_colors)) - create_color_scheme(algnames_in) - cdt <- IOHanalyzer_env$alg_colors - linestyles <- subset(cdt, algnames %in% algnames_in)[['linestyles']] - if (is.null(linestyles) || length(linestyles) != length(algnames_in)) { - return(rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), - ceiling(length(algnames_in)/3))[1:length(algnames_in)]) - } - return(linestyles) -} - -# TODO: incoporate more colors -color_palettes <- function(ncolor) { - # TODO: FIX IT! - max_colors <- getOption("IOHanalyzer.max_colors", 2) - if (ncolor <= max_colors) return(IOHanalyzer_env$used_colorscheme(ncolor)) - - brewer <- function(n) { - colors <- RColorBrewer::brewer.pal(n, 'Spectral') - colors[colors == "#FFFFBF"] <- "#B2B285" - colors[colors == "#E6F598"] <- "#86FF33" - colors[colors == '#FEE08B'] <- "#FFFF33" - colors - } - - color_fcts <- c(colorRamps::primary.colors, IOHanalyzer_env$used_colorscheme) - - n <- min(11, ncolor) - colors <- brewer(n) - ncolor <- ncolor - n - - i <- 1 - while (ncolor > 0) { - n <- min(8, ncolor) - if (i > length(color_fcts)) { - colors <- c(colors, colorRamps::primary.colors(ncolor)) - break - } else { - colors <- c(colors, color_fcts[[i]](n)) - ncolor <- ncolor - n - } - i <- i + 1 - } - colors -} - -#' Save plotly figure in multiple format -#' -#' NOTE: This function requires orca to be installed -#' -#' @param p plotly object. The plot to be saved -#' @param file String. The name of the figure file, with the extension of the required file-format -#' @param width Optional. Width of the figure -#' @param height Optional. Height of the figure -#' @param ... Additional arguments for orca -#' @export -#' @examples -#' \dontrun{ -#' p <- Plot.RT.Single_Func(dsl[1]) -#' save_plotly(p, 'example_file.png') -#' } -save_plotly <- function(p, file, width = NULL, height = NULL, ...) { - - if (!requireNamespace("withr", quietly = TRUE)) { - stop("Package \"withr\" needed for this function to work. Please install it.", - call. = FALSE) - } - des <- dirname(file) - file <- basename(file) - format <- tools::file_ext(file) - - pwd <- tempdir() - if (is.null(width)) width <- getOption("IOHanalyzer.figure_width", default = NULL) - if (is.null(height)) height <- getOption("IOHanalyzer.figure_height", default = NULL) - - if (format %in% c('svg', 'png', 'jpeg', 'webp', 'pdf', 'eps')) - withr::with_dir(pwd, orca(p, file, format = format, width = width, height = height, ...)) - else { - file_svg <- paste0(file, '.svg') - withr::with_dir(pwd, orca(p, file_svg, format = 'svg', width = width, height = height, ...)) - invisible( - system( - paste( - 'inkscape', file.path(pwd,file_svg), - paste0('--export-', format, ' ', file.path(pwd, file)) - ), - intern = T - ) - ) - } - file.rename(file.path(pwd, file), file.path(des, file)) -} +# font No. 1... +f1 <- list( + family = 'Old Standard TT, serif', + size = 11, + color = 'black' +) + +# font No. 2... +f2 <- list( + family = 'Old Standard TT, serif', + size = 13, + color = 'black' +) + +# font No. 3... +f3 <- function() { + list( + family = 'Old Standard TT, serif', + size = getOption("IOHanalyzer.tick_fontsize", default = 12), + color = 'black' + ) +} + +legend_right <- function() { + list(x = 1.01, y = 1, orientation = 'v', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +legend_inside <- function() { + list(x = .01, y = 1, orientation = 'v', + bgcolor = 'rgba(255, 255, 255, 0)', + bordercolor = 'rgba(255, 255, 255, 0)', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +legend_inside2 <- function() { + list(x = 0.7, y = 0.1, orientation = 'v', + bgcolor = 'rgba(255, 255, 255, 0.5)', + bordercolor = 'rgba(255, 255, 255, 0.8)', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +#TODO: Make the y-value configurable +legend_below <- function() { + list(y = -0.2, orientation = 'h', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + +legend_custom <- function() { + list(x = getOption("IOHanalyzer.custom_legend_x", default = 0.5), + y = getOption("IOHanalyzer.custom_legend_y", default = -0.2), orientation = 'h', + font = list(size = getOption("IOHanalyzer.legend_fontsize", default = 18), + family = 'Old Standard TT, serif')) +} + + +legend_location <- function(){ + opt <- getOption('IOHanalyzer.legend_location', default = 'below') + if (opt == 'outside_right') return(legend_right()) + else if (opt == 'inside_left') return(legend_inside()) + else if (opt == 'inside_right') return(legend_inside2()) + else if (opt == 'below') return(legend_below()) + else if (opt == 'custom') return(legend_custom()) + # else if (opt == 'below2') return(legend_below2()) + else warning(paste0("The selected legend option (", opt, ") is not implemented")) +} + +# TODO: create font object as above for title, axis... + +#' Template for creating plots in the IOHanalyzer-style +#' +#' @param title Title for the plot +#' @param x.title X-axis label +#' @param y.title Y-axis label +#' +#' @export +#' @examples +#' IOH_plot_ly_default("Example plot","x-axis","y-axis") +IOH_plot_ly_default <- function(title = NULL, x.title = NULL, y.title = NULL) { + plot_ly() %>% + layout(title = list(text = title, + font = list(size = getOption("IOHanalyzer.title_fontsize", default = 16), + family = 'Old Standard TT, serif')), + autosize = T, hovermode = 'compare', + legend = legend_location(), + paper_bgcolor = 'rgb(255,255,255)', + plot_bgcolor = getOption('IOHanalyzer.bgcolor'), + font = list(size = getOption("IOHanalyzer.label_fontsize", default = 16), + family = 'Old Standard TT, serif'), + autosize = T, + showlegend = T, + xaxis = list( + # title = list(text = x.title, font = f3), + title = x.title, + gridcolor = getOption('IOHanalyzer.gridcolor'), + showgrid = TRUE, + showline = FALSE, + showticklabels = TRUE, + tickcolor = getOption('IOHanalyzer.tickcolor'), + ticks = 'outside', + ticklen = 9, + tickfont = f3(), + exponentformat = 'e', + zeroline = F), + yaxis = list( + # title = list(text = y.title, font = f3), + title = y.title, + gridcolor = getOption('IOHanalyzer.gridcolor'), + showgrid = TRUE, + showline = FALSE, + showticklabels = TRUE, + tickcolor = getOption('IOHanalyzer.tickcolor'), + ticks = 'outside', + ticklen = 9, + tickfont = f3(), + exponentformat = 'e', + zeroline = F)) +} + +t <- theme_grey() + + theme(text = element_text(size = 15), + plot.title = element_text(hjust = 0.5) + # legend.position = c(0.15, 0.7), # legend position + # legend.key = element_blank(), # no small box around symbol + # legend.key.size = unit(1.3, "line"), # bigger symbols + # legend.background = element_rect(color = alpha("black", 0.5), + # fill = alpha('blue', 0.0), + # size = 1, + # linetype = "solid") + ) +theme_set(t) + +gg_beanplot <- function(mapping, data, p = NULL, width = 3, fill = 'grey', + colour = 'grey', alpha = 1, kernel = 'gaussian', bw = 'SJ', + draw_quantiles = NULL, trim = TRUE, na.rm = FALSE, + show.legend = NA, point.shape = 20, show.sample = T, + show.violin = T, linetype = 'solid') { + + set.seed(42) + x <- as.character(mapping$x) + y <- as.character(mapping$y) + df <- data[, c(x, y)] %>% rename_(.dots = c('x' = x, 'y' = y)) + + if (!is.numeric(df$x)) + df$x <- tryCatch(as.numeric(df$x), # in case x is a factor... + warning = function(w) return(match(x, as.factor(x)))) + + if (is.null(p)) + p <- ggplot() + + if (show.violin) + p <- p + geom_violin(data = data, mapping = mapping, trim = trim, + draw_quantiles = draw_quantiles, bw = bw, + kernel = kernel, scale = 'width', + width = width, alpha = alpha) + if (show.sample) + p <- p + geom_jitter(data = df, aes(x, y), height = 0, width = width / 2, + alpha = 0.45, shape = point.shape, size = 3.5) + # geom_segment(aes(x = x - width / 2.2, xend = x + width / 2.2, y = y, yend = y), + # df, col = 'black', size = 0.2, alpha = 0.3, linetype = linetype) + p +} + + +Set1 <- function(n) colorspace::sequential_hcl(n, h = c(360, 40), c. = c(100, NA, 90), l = c(28, 90), + power = c(1, 1.1), gamma = NULL, fixup = TRUE, + alpha = 1)#, palette = NULL, rev = FALSE) + +Set2 <- function(n) colorspace::sequential_hcl(n, c(261, 26), c. = c(50, NA, 70), l = c(54, 77), + power = c(0.5, NA), gamma = NULL, + fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) + +Set3 <- function(n) colorspace::sequential_hcl(n, c(-88, 59), c. = c(60, 75, 55), l = c(40, 90), + power = c(0.1, 1.2), gamma = NULL, + fixup = TRUE, alpha = 1)#, palette = NULL, rev = FALSE) + +IOHanalyzer_env$used_colorscheme <- Set2 +IOHanalyzer_env$id_colors <- NULL + +#' Set the colorScheme of the IOHanalyzer plots +#' +#' @param schemename Three default colorschemes are implemented: +#' \itemize{ +#' \item Default +#' \item Variant 1 +#' \item Variant 2 +#' \item Variant 3 +#' } +#' And it is also possible to select "Custom", which allows uploading of a custom set of colors +#' @param ids The names of the algorithms (or custom ids, see `change_id`) for which to set the colors +#' @param path The path to the file containing the colors to use. Only used if +#' schemename is "Custom" +#' +#' @export +#' +#' @examples +#' set_color_scheme("Default", get_algId(dsl)) +set_color_scheme <- function(schemename, ids, path = NULL){ + if (schemename == "Custom" && !is.null(path)) { + dt <- fread(path, header = T) + if (any(colnames(dt) != c("ids", "colors", "linestyles"))) { + warning("Incorrect file-format has been uploaded.") + } + else + IOHanalyzer_env$id_colors <- dt + return() + } + else { + if (schemename == "Default") { + options(IOHanalyzer.max_colors = 2) + } + else { + if (schemename == "Variant 1") IOHanalyzer_env$used_colorscheme <- Set1 + else if (schemename == "Variant 2") IOHanalyzer_env$used_colorscheme <- Set2 + else if (schemename == "Variant 3") IOHanalyzer_env$used_colorscheme <- Set3 + options(IOHanalyzer.max_colors = length(ids)) + } + create_color_scheme(ids) + } +} + +#' Get datatable of current color (and linestyle) scheme to file +#' +#' @return data.table object with 3 columns: ids, colors, linestyles +#' @export +#' @examples +#' get_color_scheme_dt() +get_color_scheme_dt <- function(){ + return(IOHanalyzer_env$id_colors) +} + +#' Helper function to create default color scheme +#' +#' @noRd +create_color_scheme <- function(ids) { + if (length(ids) == 0) { + return(NULL) + } + colors <- color_palettes(length(ids)) + linestyles <- rep(c("solid", "dash", "dot"), ceiling(length(colors)/3))[1:length(colors)] + IOHanalyzer_env$id_colors <- data.table(ids, colors, linestyles) +} + +#' Get colors according to the current colorScheme of the IOHanalyzer +#' +#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get colors +#' +#' @export +#' +#' @examples +#' get_color_scheme(get_algId(dsl)) +get_color_scheme <- function(ids_in){ + if (is.null(IOHanalyzer_env$id_colors)) + create_color_scheme(ids_in) + cdt <- IOHanalyzer_env$id_colors + colors <- subset(cdt, ids %in% ids_in)[['colors']] + if (is.null(colors) || length(colors) != length(ids_in)) { + return(color_palettes(length(ids_in))) + } + names(colors) <- ids_in + return(colors) +} + + +#' Get line styles according to the current styleScheme of the IOHanalyzer +#' +#' @param ids_in List of algorithms (or custom ids, see `change_id`) for which to get linestyles +#' +#' @export +#' +#' @examples +#' get_line_style(get_algId(dsl)) +get_line_style <- function(ids_in){ + if (is.null(IOHanalyzer_env$id_colors)) + create_color_scheme(ids_in) + cdt <- IOHanalyzer_env$id_colors + linestyles <- subset(cdt, ids %in% ids_in)[['linestyles']] + if (is.null(linestyles) || length(linestyles) != length(ids_in)) { + return(rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), + ceiling(length(ids_in)/3))[1:length(ids_in)]) + } + return(linestyles) +} + +# TODO: incoporate more colors +color_palettes <- function(ncolor) { + # TODO: FIX IT! + max_colors <- getOption("IOHanalyzer.max_colors", 2) + if (ncolor <= max_colors) return(IOHanalyzer_env$used_colorscheme(ncolor)) + + brewer <- function(n) { + colors <- RColorBrewer::brewer.pal(n, 'Spectral') + colors[colors == "#FFFFBF"] <- "#B2B285" + colors[colors == "#E6F598"] <- "#86FF33" + colors[colors == '#FEE08B'] <- "#FFFF33" + colors + } + + color_fcts <- c(colorRamps::primary.colors, IOHanalyzer_env$used_colorscheme) + + n <- min(11, ncolor) + colors <- brewer(n) + ncolor <- ncolor - n + + i <- 1 + while (ncolor > 0) { + n <- min(8, ncolor) + if (i > length(color_fcts)) { + colors <- c(colors, colorRamps::primary.colors(ncolor)) + break + } else { + colors <- c(colors, color_fcts[[i]](n)) + ncolor <- ncolor - n + } + i <- i + 1 + } + colors +} + +#' Save plotly figure in multiple format +#' +#' NOTE: This function requires orca to be installed +#' +#' @param p plotly object. The plot to be saved +#' @param file String. The name of the figure file, with the extension of the required file-format +#' @param width Optional. Width of the figure +#' @param height Optional. Height of the figure +#' @param ... Additional arguments for orca +#' @export +#' @examples +#' \dontrun{ +#' p <- Plot.RT.Single_Func(dsl[1]) +#' save_plotly(p, 'example_file.png') +#' } +save_plotly <- function(p, file, width = NULL, height = NULL, ...) { + + if (!requireNamespace("withr", quietly = TRUE)) { + stop("Package \"withr\" needed for this function to work. Please install it.", + call. = FALSE) + } + des <- dirname(file) + file <- basename(file) + format <- tools::file_ext(file) + + pwd <- tempdir() + if (is.null(width)) width <- getOption("IOHanalyzer.figure_width", default = NULL) + if (is.null(height)) height <- getOption("IOHanalyzer.figure_height", default = NULL) + + if (format %in% c('svg', 'png', 'jpeg', 'webp', 'pdf', 'eps')) + withr::with_dir(pwd, orca(p, file, format = format, width = width, height = height, ...)) + else { + file_svg <- paste0(file, '.svg') + withr::with_dir(pwd, orca(p, file_svg, format = 'svg', width = width, height = height, ...)) + invisible( + system( + paste( + 'inkscape', file.path(pwd,file_svg), + paste0('--export-', format, ' ', file.path(pwd, file)) + ), + intern = T + ) + ) + } + file.rename(file.path(pwd, file), file.path(des, file)) +} diff --git a/R/plotDataSetList.R b/R/plotDataSetList.R index 76f9ba6c..0360fa6e 100644 --- a/R/plotDataSetList.R +++ b/R/plotDataSetList.R @@ -1,1417 +1,1417 @@ -symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", - "triangle-up-open", "triangle-down-open") - -get_legends <- function(dsList) { - N <- length(dsList) - legends <- sapply(dsList, function(d) attr(d, 'algId')) - - if (length(unique(legends)) < N) { - funcId <- sapply(dsList, function(d) attr(d, 'funcId')) - if (length(unique(funcId)) > 1) - legends <- paste0(legends, '-F', funcId) - } - - if (length(unique(legends)) < N) { - DIM <- sapply(dsList, function(d) attr(d, 'DIM')) - if (length(unique(DIM)) > 1) - legends <- paste0(legends, '-', DIM, 'D') - } - legends -} - -insert_best_parts <- function(from_data, to_data, best_is_min) { - if (all(is.na(from_data))) - to_data - else - if (best_is_min) - pmin(from_data, to_data, na.rm = T) - else - pmax(from_data, to_data, na.rm = T) -} - -generate_rbga <- function(color,a){ - paste0('rgba(', paste0(color, collapse = ','), ',', a,')') -} - -grad_functions <- c( - scaled_edges = function(count, amount, intensity){ - scale <- (intensity + 1)/2 - color_end <- floor(scale*amount*2) - if (count < color_end) - 1/color_end - else - 0 - } - , - fixed_edges = function(count, amount, intensity) { - scale <- (intensity + 1) / 2 - color_center <- floor(scale * amount) + 1 - if (count <= color_center) - 1 / (2 * color_center) - else - 1 / (2 * (amount - color_center)) - } -) - -#S3 generics -# TODO: decide which parameters need to be in the generics - -#' Plot lineplot of the ERTs of a DataSetList -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param Fstart The starting function value. -#' @param Fstop The final function value. -#' @param show.ERT Whether or not to show the ERT-values -#' @param show.CI Whether or not to show the standard deviations -#' @param show.mean Whether or not to show the mean hitting times -#' @param show.median Whether or not to show the median hitting times -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) -#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' -#' @param includeOpts Whether or not to include all best points reached by each algorithm -#' @param p Existing plot to which to add the current data -#' @return A plot of ERT-values of the DataSetList -#' @export -#' @examples -#' Plot.RT.Single_Func(subset(dsl, funcId == 1)) -Plot.RT.Single_Func <- function(dsList, Fstart = NULL, Fstop = NULL, - show.ERT = T, show.CI = F, show.mean = F, - show.median = F, backend = NULL, - scale.xlog = F, scale.ylog = F, scale.reverse = F, - includeOpts = F, p = NULL) - UseMethod("Plot.RT.Single_Func", dsList) -#' Plot lineplot of the expected function values of a DataSetList -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param RTstart The starting runtime value. -#' @param RTstop The final runtime value. -#' @param show.CI Whether or not to show the standard deviations -#' @param show.mean Whether or not to show the mean runtimes -#' @param show.median Whether or not to show the median runtimes -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) -#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' -#' -#' @return A plot of ERT-values of the DataSetList -#' @export -#' @examples -#' Plot.FV.Single_Func(subset(dsl, funcId == 1)) -Plot.FV.Single_Func <- function(dsList, RTstart = NULL, RTstop = NULL, show.CI = F, show.mean = T, - show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F) UseMethod("Plot.FV.Single_Func", dsList) -#' Plot probablity mass function of the runtimes of a DataSetList at a certain target function value -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value. -#' @param show.sample Whether or not to show the individual runtime samples -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' -#' -#' @return A plot of the probablity mass function of the runtimes at a the -#' target function value of the DataSetList -#' @export -#' @examples -#' Plot.RT.PMF(subset(dsl, funcId == 1), 14) -Plot.RT.PMF <- function(dsList, ftarget, show.sample = F, scale.ylog = F, backend = NULL) - UseMethod("Plot.RT.PMF", dsList) -#' Plot histograms of the runtimes of a DataSetList at a certain target function value -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value. -#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either -#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. -#' @param use.equal.bins Whether to determine one bin size for all plots or have individual -#' bin sizes for each algorithm -#' -#' @return A plot of the histograms of the runtimes at a the -#' target function value of the DataSetList -#' @export -#' @examples -#' Plot.RT.Histogram(subset(dsl, funcId == 1), 14) -Plot.RT.Histogram <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F) - UseMethod("Plot.RT.Histogram", dsList) -#' Plot the empirical cumulative distriburtion as a function of the running times of -#' a DataSetList at certain target function values -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftargets The target function values -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the running times of the DataSetList at the target function values -#' @export -#' @examples -#' Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) -Plot.RT.ECDF_Per_Target <- function(dsList, ftargets, scale.xlog = F) - UseMethod("Plot.RT.ECDF_Per_Target", dsList) -#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of -#' a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param fstart The starting function value -#' @param fstop The final function value -#' @param fstep The spacing between starting and final function values -#' @param show.per_target Whether or not to show the individual ECDF-curves for each target -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the running times of the DataSetList -#' @export -#' @examples -#' Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) -Plot.RT.ECDF_Single_Func <- function(dsList, fstart = NULL, fstop = NULL, - fstep = NULL, show.per_target = F, - scale.xlog = F) UseMethod("Plot.RT.ECDF_Single_Func", dsList) -#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param fstart The starting function value -#' @param fstop The final function value -#' @param fstep The spacing between starting and final function values -#' @param fval_formatter Function to format the function-value labels -#' -#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList -#' @export -#' @examples -#' Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) -Plot.RT.ECDF_AUC <- function(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, - fval_formatter = as.integer) UseMethod("Plot.RT.ECDF_AUC", dsList) -#' Plot probablity density function of the function values of a DataSetList at -#' a certain target runtime -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param runtime The target runtime -#' @param show.sample Whether or not to show the individual function value samples -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' -#' @return A plot of the probablity density function of the runtimes at a the -#' target function value of the DataSetList -#' @export -#' @examples -#' Plot.FV.PDF(subset(dsl, funcId == 1), 100) -Plot.FV.PDF <- function(dsList, runtime, show.sample = F, scale.ylog = F) - UseMethod("Plot.FV.PDF", dsList) -#' Plot histograms of the function values of a DataSetList at a certain target runtime -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param runtime The target runtime -#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either -#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. -#' @param use.equal.bins Whether to determine one bin size for all plots or have individual -#' bin sizes for each algorithm -#' -#' @return A plot of the histograms of the function values at a the -#' target runtime of the DataSetList -#' @export -#' @examples -#' Plot.FV.Histogram(subset(dsl, funcId == 1), 100) -Plot.FV.Histogram <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F) - UseMethod("Plot.FV.Histogram", dsList) -#' Plot the empirical cumulative distriburtion as a function of the target values of -#' a DataSetList at certain target runtimes -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param runtimes The target runtimes -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the fucntion values of the DataSetList at the target runtimes -#' @export -#' @examples -#' Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) -Plot.FV.ECDF_Per_Target <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F) - UseMethod("Plot.FV.ECDF_Per_Target", dsList) -#' Plot the aggregated empirical cumulative distriburtion as a function of the function values of -#' a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param rt_min The starting runtime -#' @param rt_max The final runtime -#' @param rt_step The spacing between starting and final runtimes -#' @param show.per_target Whether or not to show the individual ECDF-curves for each runtime -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the function values of the DataSetList -#' @export -#' @examples -#' Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) -Plot.FV.ECDF_Single_Func <- function(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL, scale.xlog = F, - show.per_target = F, scale.reverse = F) - UseMethod("Plot.FV.ECDF_Single_Func", dsList) -#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param rt_min The starting runtime -#' @param rt_max The final runtime -#' @param rt_step The spacing between starting and final runtimes -#' -#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList -#' @export -#' @examples -#' Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) -Plot.FV.ECDF_AUC <- function(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL) UseMethod("Plot.FV.ECDF_AUC", dsList) -#' Plot the parameter values recorded in a DataSetList (aligned by funcion value) -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param f_min The starting function value. -#' @param f_max The final function value. -#' @param show.mean Whether or not to show the mean parameter values -#' @param show.median Whether or not to show the median parameter values -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param algids Which algorithms from dsList to use -#' @param par_name Which parameters to create plots for; set to NULL to use all -#' parameters found in dsList. -#' @param show.CI Whether or not to show the standard deviation -#' -#' @return A plot of for every recorded parameter in the DataSetList -#' @export -#' @examples -#' Plot.RT.Parameters(subset(dsl, funcId == 1)) -Plot.RT.Parameters <- function(dsList, f_min = NULL, f_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) UseMethod("Plot.RT.Parameters", dsList) -#' Plot the parameter values recorded in a DataSetList (aligned by budget) -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param rt_min The starting budget value. -#' @param rt_max The final budget value. -#' @param show.mean Whether or not to show the mean parameter values -#' @param show.median Whether or not to show the median parameter values -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param algids Which algorithms from dsList to use -#' @param par_name Which parameters to create plots for; set to NULL to use all -#' parameters found in dsList. -#' @param show.CI Whether or not to show the standard deviation -#' -#' @return A plot of for every recorded parameter in the DataSetList -#' @export -#' @examples -#' Plot.FV.Parameters(subset(dsl, funcId == 1)) -Plot.FV.Parameters <- function(dsList, rt_min = NULL, rt_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) UseMethod("Plot.FV.Parameters", dsList) -#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of -#' a DataSetList. Aggregated over multiple functions or dimensions. -#' -#' @param dsList A DataSetList. -#' @param targets The target function values. Specified in a data.frame, as can be generated -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' by the function 'get_ECDF_targets' -#' -#' @return A plot of the empirical cumulative distriburtion as a function of -#' the running times of the DataSetList -#' @export -#' @examples -#' Plot.RT.ECDF_Multi_Func(dsl) -Plot.RT.ECDF_Multi_Func <- function(dsList, targets = NULL, scale.xlog = F) - UseMethod("Plot.RT.ECDF_Multi_Func", dsList) -#' Plot ERT-plots for multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) -#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. -#' -#' @return A plot of ERT-values of the DataSetList -#' @export -#' @examples -#' Plot.RT.Multi_Func(dsl) -Plot.RT.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, scale.reverse = F, - backend = NULL) - UseMethod("Plot.RT.Multi_Func", dsList) -#' Plot ERT-based comparison over multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' -#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param maximize Wheter or not to the data is of a maximization problem -#' @param targets Custom list of function-value targets, one for each function or dimension. -#' @param use_rank Wheter to use a ranking system. If False, the actual ERT-values will be used. -#' @param erts Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function -#' of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating -#' this plot. -#' @param inf.action How to handle infinite ERTs ('overlap' or 'jitter') -#' @return A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList -#' @export -#' @examples -#' Plot.RT.Aggregated(dsl) -Plot.RT.Aggregated <- function(dsList, aggr_on = 'funcId', targets = NULL, - plot_mode = 'radar', use_rank = F, scale.ylog = T, maximize = T, - erts = NULL, inf.action = 'overlap') - UseMethod("Plot.RT.Aggregated", dsList) -#' Plot expected function value-based comparison over multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' -#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param runtimes Custom list of function-value targets, one for each function or dimension. -#' @param use_rank Wheter to use a ranking system. If False, the actual expected function- -#' values will be used. -#' @param fvs Pre-calculated expected function-values for the provided runtimes Created by the -#' max_ERTs function of DataSetList. Can be provided to prevent needless computation -#' in recalculating ERTs when recreating this plot. -#' -#' @return A plot of expected function value-based comparison on the provided functions -#' or dimensions of the DataSetList -#' @export -#' @examples -#' Plot.FV.Aggregated(dsl) -Plot.FV.Aggregated <- function(dsList, aggr_on = 'funcId', runtimes = NULL, plot_mode = 'radar', - use_rank = F, scale.ylog = T, fvs = NULL) - UseMethod("Plot.FV.Aggregated", dsList) - -#' Plot FV-plots for multiple functions or dimensions -#' -#' @param dsList A DataSetList (should consist of only one function OR dimension). -#' @param scale.xlog Whether or not to scale the x-axis logaritmically -#' @param scale.ylog Whether or not to scale the y-axis logaritmically -#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. -#' -#' @return A plot of Function-values of the DataSetList -#' @export -#' @examples -#' Plot.FV.Multi_Func(dsl) -Plot.FV.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, backend = NULL) - UseMethod("Plot.FV.Multi_Func", dsList) - -#' Plot a heatmap showing the statistically different algorithms -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value to use -#' @param alpha The cutoff for statistical significance -#' @param bootstrap.size The amound of bootstrapped samples used -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' -#' @return A heatmap showing the statistical significance between algorithms -#' @export -#' @examples -#' Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) -Plot.Stats.Significance_Heatmap <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, - which = 'by_FV') - UseMethod("Plot.Stats.Significance_Heatmap", dsList) - -#' Plot a network graph showing the statistically different algorithms -#' -#' @param dsList A DataSetList (should consist of only one function and dimension). -#' @param ftarget The target function value to use -#' @param alpha The cutoff for statistical significance -#' @param bootstrap.size The amound of bootstrapped samples used -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' -#' @return A graph showing the statistical significance between algorithms -#' @export -#' @examples -#' Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) -Plot.Stats.Significance_Graph <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, - which = 'by_FV') - UseMethod("Plot.Stats.Significance_Graph", dsList) - -#' Create a candlestick plot of Glicko2-rankings -#' -#' @param dsList A DataSetList -#' @param nr_rounds The number of rounds in the tournament -#' @param glicko2_rank_df Optional. Dataframe containing the glicko2 rating to avoid needless recalculation. -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' @param target_dt Optional: data table containing the targets for each function and dimension -#' -#' @export -#' @examples -#' Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) -Plot.Stats.Glicko2_Candlestick <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, - which = 'by_FV', target_dt = NULL) - UseMethod("Plot.Stats.Glicko2_Candlestick", dsList) - - -##Implementations - -#' @rdname Plot.RT.Single_Func -#' @export -Plot.RT.Single_Func.DataSetList <- function(dsList, Fstart = NULL, Fstop = NULL, - show.ERT = T, show.CI = T, show.mean = F, - show.median = F, backend = NULL, - scale.xlog = F, scale.ylog = F, - scale.reverse = F, includeOpts = F, p = NULL) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - if (backend == 'plotly') { - data <- generate_data.Single_Function(dsList, Fstart, Fstop, scale.xlog, 'by_RT', includeOpts) - - y_attrs <- c() - if (show.ERT) y_attrs <- c(y_attrs, 'ERT') - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, - scale.ylog = scale.ylog, p = p, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - show_legend <- F - } - if (show.CI) { - p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - } - } - # } else if (backend == 'ggplot2') { - # dt[, 'group' := paste(algId, funcId, DIM, sep = '-')] - # p <- ggplot(data = dt, aes(group = `group`, colour = `group`)) - # - # if (show.CI) p <- p + geom_ribbon(aes(target, ymin = lower, ymax = upper, fill = `group`), - # alpha = 0.2, colour = NA) - # if (show.ERT) p <- p + geom_line(aes(target, ERT), size = 1.2) - # if (show.mean) p <- p + geom_line(aes(target, mean), linetype = 'dashed') - # if (show.median) p <- p + geom_line(aes(target, median), linetype = 'dotted') - # - # p <- p + - # scale_color_manual(values = colors) + - # scale_fill_manual(values = colors) - # } - return(p) -} - -#' @rdname Plot.FV.Single_Func -#' @export -Plot.FV.Single_Func.DataSetList <- function(dsList, RTstart = NULL, RTstop = NULL, - show.CI = F, - show.mean = T, show.median = F, - backend = NULL, - scale.xlog = F, scale.ylog = F, - scale.reverse = F) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - if (backend == 'plotly') { - data <- generate_data.Single_Function(dsList, RTstart, RTstop, scale.xlog, 'by_FV') - - y_attrs <- c() - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, - scale.ylog = scale.ylog, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - show_legend <- F - } - else - p <- NULL - if (show.CI) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, - scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = scale.reverse) - } - - } - # } else if (backend == 'ggplot2') { - # fce[, 'group' := paste(algId, funcId, DIM, sep = '-')] - # p <- ggplot(data = fce, aes(group = `group`, colour = `group`)) - # - # if (show.mean) p <- p + geom_line(aes(runtime, mean), linetype = 'dashed') - # if (show.median) p <- p + geom_line(aes(runtime, median), linetype = 'dotted') - # - # p <- p + - # scale_color_manual(values = colors) + - # scale_fill_manual(values = colors) - # - # #TODO: add individual run etc - # } - return(p) -} - -#' @rdname Plot.RT.PMF -#' @export -Plot.RT.PMF.DataSetList <- function(dsList, ftarget, show.sample = F, - scale.ylog = F, backend = NULL){ - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - data <- generate_data.PMF(dsList, ftarget, 'by_RT') - - plot_general_data(data, 'algId', 'RT', scale.ylog = scale.ylog, - x_title = "Algorithm", y_title = "Function Evaluations") -} - -#' @rdname Plot.RT.Histogram -#' @export -Plot.RT.Histogram.DataSetList <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F){ - if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { - warning("Invalid dataset uploaded. Please ensure the datasetlist contains data - from only one function and only one dimension.") - return(NULL) - } - data <- generate_data.hist(dsList, ftarget, use.equal.bins, 'by_RT') - - subplot_attr <- if (plot_mode == 'subplot') 'algId' else NULL - plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Function Evaluations", - y_title = "Runs") -} - -#' @rdname Plot.RT.ECDF_Per_Target -#' @export -Plot.RT.ECDF_Per_Target.DataSetList <- function(dsList, ftargets, scale.xlog = F){ - req(length(ftargets) != 0) - data <- generate_data.ECDF(dsList, ftargets, scale.xlog) - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of runs", scale.xlog = scale.xlog, show.legend = T) -} - -#' @rdname Plot.RT.ECDF_Single_Func -#' @export -Plot.RT.ECDF_Single_Func.DataSetList <- function(dsList, fstart = NULL, fstop = NULL, - fstep = NULL, show.per_target = F, - scale.xlog = F) { - - targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep) - req(targets) - - data <- generate_data.ECDF(dsList, targets, scale.xlog) - - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of (run, target) pairs", - scale.xlog = scale.xlog, show.legend = T) -} - -#' @rdname Plot.RT.ECDF_AUC -#' @export -Plot.RT.ECDF_AUC.DataSetList <- function(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, - fval_formatter = as.integer) { - - targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep, length.out = 10) - req(targets) - - data <- generate_data.AUC(dsList, targets, multiple_x = TRUE) - - plot_general_data(data, 'x', 'auc', 'radar') -} - -#' @rdname Plot.FV.PDF -#' @export -Plot.FV.PDF.DataSetList <- function(dsList, runtime, show.sample = F, scale.ylog = F){ - - data <- generate_data.PMF(dsList, runtime, 'by_FV') - - plot_general_data(data, 'algId', 'f(x)', scale.ylog = scale.ylog, - x_title = "Algorithm", y_title = "Target Value") -} - -#' @rdname Plot.FV.Histogram -#' @export -Plot.FV.Histogram.DataSetList <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F){ - if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { - warning("Invalid dataset uploaded. Please ensure the datasetlist contains data - from only one function and only one dimension.") - return(NULL) - } - data <- generate_data.hist(dsList, runtime, use.equal.bins, 'by_FV') - - subplot_attr <- if (plot_mode == 'subplot') 'algId' else NULL - plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Target Values", - y_title = "Runs") -} - -#' @rdname Plot.FV.ECDF_Per_Target -#' @export -Plot.FV.ECDF_Per_Target.DataSetList <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F){ - #TODO: Fvals in legend need to be formatted properly - runtimes <- runtimes[!is.na(runtimes)] - req(length(runtimes) != 0) - - data <- generate_data.ECDF(dsList, runtimes, scale.xlog, which = 'by_FV') - - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of runs", scale.xlog = scale.xlog, - show.legend = T, - scale.reverse = scale.reverse) -} - -#' @rdname Plot.FV.ECDF_Single_Func -#' @export -Plot.FV.ECDF_Single_Func.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL, scale.xlog = F, - show.per_target = F, scale.reverse = F){ - - targets <- seq_RT(get_funvals(dsList), rt_min, rt_max, rt_step) - req(targets) - data <- generate_data.ECDF(dsList, targets, scale.xlog, which = 'by_FV') - - plot_general_data(data, 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of (run, target) pairs", - scale.xlog = scale.xlog, - scale.reverse = scale.reverse, show.legend = T) -} - -#' @rdname Plot.FV.ECDF_AUC -#' @export -Plot.FV.ECDF_AUC.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, rt_step = NULL) { - targets <- seq_RT(get_runtimes(dsList), rt_min, rt_max, rt_step, length.out = 10) - req(targets) - data <- generate_data.AUC(dsList, targets, which = 'by_FV', multiple_x = TRUE) - - plot_general_data(data, 'x', 'auc', 'radar') -} - -#' @rdname Plot.RT.Parameters -#' @export -Plot.RT.Parameters.DataSetList <- function(dsList, f_min = NULL, f_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) { - data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_FV') - - y_attrs <- c() - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - show_legend <- F - } - else - p <- NULL - if (show.CI) { - p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - } - p -} - - -#' @rdname Plot.FV.Parameters -#' @export -Plot.FV.Parameters.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, - algids = 'all', par_name = NULL, - scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, - show.CI = F) { - data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_RT') - - y_attrs <- c() - if (show.mean) y_attrs <- c(y_attrs, 'mean') - if (show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - show_legend <- F - } - else - p <- NULL - if (show.CI) { - p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = scale.ylog, subplot_attr = 'parId', - scale.xlog = scale.xlog) - } - p -} - -#' @rdname Plot.RT.ECDF_Multi_Func -#' @export -Plot.RT.ECDF_Multi_Func.DataSetList <- function(dsList, targets = NULL, - scale.xlog = F) { - if (is.null(targets) || !is.data.table(targets)) { - targets <- get_ECDF_targets(dsList) - } - - data <- generate_data.ECDF(dsList, targets, scale.xlog) - - plot_general_data(data, 'x', 'mean', 'line', - scale.xlog = scale.xlog, - x_title = "Function Evaluations", - y_title = "Proportion of (run, target, ...) pairs", - show.legend = T) -} - -#' @rdname Plot.RT.Multi_Func -#' @export -Plot.RT.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, - scale.ylog = F, - scale.reverse = F, - backend = NULL) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - data <- rbindlist(lapply(get_funcId(dsList), function(fid) { - generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, - which = 'by_RT') - })) - - plot_general_data(data, x_attr = 'target', y_attr = 'ERT', - subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, - scale.ylog = scale.ylog, x_title = 'Best-so-far f(x)', - y_title = 'ERT', show.legend = T, - scale.reverse = scale.reverse) -} - -#' @rdname Plot.FV.Multi_Func -#' @export -Plot.FV.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, - scale.ylog = F, - backend = NULL) { - if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') - - data <- rbindlist(lapply(get_funcId(dsList), function(fid) { - generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, - which = 'by_FV') - })) - - plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', - subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, - scale.ylog = scale.ylog, x_title = 'Runtime', - y_title = 'Best-so-far f(x)', show.legend = T) -} - -#' @rdname Plot.RT.Aggregated -#' @export -Plot.RT.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', targets = NULL, - plot_mode = 'radar', use_rank = F, - scale.ylog = T, maximize = T, erts = NULL, - inf.action = 'overlap') { - targets <- get_target_dt(dsList) - data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) - y_attr <- if (use_rank) 'rank' else 'value' - y_title <- if (use_rank) 'Rank' else 'ERT' - plot_general_data(data, type = plot_mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = scale.ylog, - inf.action = inf.action) -} - -#' @rdname Plot.FV.Aggregated -#' @export -Plot.FV.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', runtimes = NULL, - plot_mode = 'radar', use_rank = F, - scale.ylog = T, fvs = NULL){ - targets <- get_target_dt(dsList, which = 'by_FV') - data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, which = 'by_FV') - y_attr <- if (use_rank) 'rank' else 'value' - y_title <- if (use_rank) 'Rank' else 'Best-so-far f(x)' - plot_general_data(data, type = plot_mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = scale.ylog) -} - -#' @rdname Plot.Stats.Significance_Heatmap -#' @export -Plot.Stats.Significance_Heatmap.DataSetList <- function(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = 'by_FV'){ - if (length(get_dim(dsList)) != 1 || - length(get_funcId(dsList)) != 1 || - length(get_algId(dsList)) < 2) - return(NULL) - - p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) - y <- p_matrix <= alpha - colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), - col = c('blue', 'blue', 'white', 'white', 'red', 'red') - ) - heatmap <- y - t(y) - - p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', - xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) - p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), - xaxis = list(tickangle = 45)) - p -} - -#' Helper function for Plot.Stats.Significance_Graph -#' -#' @param x x -#' @param start default is 0 -#' @param direction default is 1 -#' -#' @noRd -radian.rescale <- function(x, start=0, direction=1) { - c.rotate <- function(x) (x + start) %% (2 * pi) * direction - c.rotate((2 * pi * (x - min(x)) / (max(x) - min(x)))) -} - -#' @rdname Plot.Stats.Significance_Graph -#' @export -Plot.Stats.Significance_Graph.DataSetList <- function(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = 'by_FV'){ - if (!requireNamespace("igraph", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || length(get_algId(dsList)) < 2) { - return(NULL) - } - p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) - g <- igraph::graph_from_adjacency_matrix(p_matrix <= alpha, mode = 'directed', diag = F) - lab.locs <- radian.rescale(x = 1:nrow(p_matrix), direction = -1, start = 0) - - igraph::plot.igraph(g, layout = igraph::layout.circle(g), vertex.size = 10, edge.arrow.size = 1, - vertex.label.color = 'black', - vertex.label.dist = 2, - vertex.label.cex = 1, - vertex.label.degree = lab.locs) -} - -#' @rdname Plot.Stats.Glicko2_Candlestick -#' @export -Plot.Stats.Glicko2_Candlestick.DataSetList <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, - which = 'by_FV', target_dt = NULL) { - df <- glicko2_rank_df - - if (is.null(df)) { - df <- glicko2_ranking(dsList, nr_rounds, which, target_dt = target_dt)$ratings - algIds <- df$Player$algId - } - else{ - algIds <- df$algId - } - p <- IOH_plot_ly_default(title = "Glicko2-rating", - x.title = "Algorithm", - y.title = "Rating") - df$Rating %<>% as.numeric - df$Deviation %<>% as.numeric - high <- df$Rating + 3*df$Deviation - low <- df$Rating - 3*df$Deviation - open <- df$Rating + df$Deviation - close <- df$Rating - df$Deviation - - N <- length(df$Rating) - colors <- get_color_scheme(algIds) - if (length(colors != N)) { - colors <- get_color_scheme(get_algId(dsList)) - } - - for (i in seq(N)) { - # rgba_str <- paste0('rgba(', paste0(col2rgb(colors[i]), collapse = ','), ',0.52)') - color <- list(line = list(color = colors[[i]])) - p %<>% add_trace(type = "candlestick", x = algIds[[i]], open = open[[i]], close = close[[i]], - high = high[[i]], low = low[[i]], legendgroup = algIds[[i]], - name = algIds[[i]], increasing = color, decreasing = color, - hovertext = paste0(format(df$Rating[[i]], digits = 3), '+-', - format(df$Deviation[[i]], digits = 3)), - hoverinfo = "text") - } - p %<>% layout(xaxis = list(rangeslider = list(visible = F))) - p -} - -### _______________________ Rewritten plotting function ____________________ ### - -#' Add transparancy to named list of colors -#' -#' @param colors Named list of colors (in hex-notation) -#' @param percentage The percentage of opacity. 0 is fully transparant, 1 is fully opaque -#' -#' @noRd -add_transparancy <- function(colors, percentage){ - hex_val <- format(as.hexmode(as.integer(255 * percentage)), upper.case = T, width = 2) - sapply(colors, function(col) { col <- paste0('#', substr(col, 2, 7), hex_val) }) -} - -#' General function for plotting within IOHanalyzer -#' -#' @param df The dataframe containing the data to plot. It should contain at least two columns: -#' 'x_attr' and 'y_attr' -#' @param x_attr The column to specify the x_axis. Default is 'algId' -#' @param legend_attr Default is 'algId' This is also used for the selection of colorschemes -#' @param y_attr The column to specify the y_axis -#' @param type The type of plot to use. Currently available: 'violin', 'line', 'radar', -#' 'bar', hist' and 'ribbon' -#' @param upper_attr When using ribbon-plot, this can be used to create a shaded area. -#' Only works in combination with`lower_attr` and `type` == 'ribbon' -#' @param lower_attr When using ribbon-plot, this can be used to create a shaded area. -#' Only works in combination with`upper_attr` and `type` == 'ribbon' -#' @param subplot_attr Which attribute of the dataframe to use for creating subplots -#' @param scale.xlog Logarithmic scaling of x-axis -#' @param scale.ylog Logarithmic scaling of y-axis -#' @param scale.reverse Decreasing or increasing x-axis -#' @param x_title Title of x-axis. Defaults to x_attr -#' @param y_title Title of x-axis. Defaults to x_attr -#' @param plot_title Title of x-axis. Defaults to no title -#' @param p A previously existing plot on which to add traces. If NULL, a new canvas is created -#' @param show.legend Whether or not to include a legend -#' @param inf.action How to deal with infinite values. Can be 'none', 'overlap' or 'jitter' -#' @param ... Additional parameters for the add_trace function -#' -#' @export -plot_general_data <- function(df, x_attr = 'algId', y_attr = 'vals', type = 'violin', - legend_attr = 'algId', scale.xlog = F, scale.ylog = F, - scale.reverse = F, p = NULL, x_title = NULL, - y_title = NULL, plot_title = NULL, upper_attr = NULL, - lower_attr = NULL, subplot_attr = NULL, show.legend = F, - inf.action = 'none', ...) { - - l <- x <- isinf <- y <- text <- l_orig <- NULL #Set local binding to remove warnings - - #Only allow valid plot types - if (!(type %in% c('violin', 'line', 'radar', 'hist', 'ribbon', 'line+ribbon', 'bar'))) { - stop(paste0("Provided plot type ('", type, "') is not supported")) - } - - #And valid number of y-attributes - if (length(y_attr) == 0) { - stop("At least one y-attribute is needed to plot") - } - - #Deal with subplots - if (!is.null(subplot_attr)) { - if (!subplot_attr %in% colnames(df)) { - stop("Provided subplot-attribut is not a colname of the selected data.table.") - } - colnames(df)[colnames(df) == subplot_attr] <- "subplot_attr" - attrs <- unique(df[, subplot_attr]) - if (length(attrs) == 0) stop("Attempting to create subplots with fewer than 2 unique values of - `subplot_attrs`-column") - if (length(attrs) == 1) return(plot_general_data(df, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, - scale.reverse, p, x_title, y_title, attrs, upper_attr, lower_attr, - show.legend = show.legend, subplot_attr = NULL, ...)) - if (subplot_attr == legend_attr) { - df[, l := subplot_attr] - } - - #Only need one legend for the whole plot - legends_show <- rep(F, length(attrs)) - legends_show[[1]] <- show.legend - names(legends_show) <- attrs - - #Get some number of rows and columns - n_cols <- 1 + ceiling(length(attrs)/10) - n_rows <- ceiling(length(attrs) / n_cols) - - p <- lapply(seq(length(attrs)), function(idx) { - attr_val <- attrs[[idx]] - df_sub <- df[subplot_attr == attr_val] - disp_y <- idx %% n_cols == 1 - disp_x <- idx > (length(attrs) - n_cols) - x.title = if (disp_x) x_title else "" - y.title = if (disp_y) y_title else "" - - #Generate title for the subplots - if (stri_detect_regex(subplot_attr, "(?i)fun")) - sub_title <- paste0('F', attr_val) - else if (stri_detect_regex(subplot_attr, "(?i)dim")) - sub_title <- paste0('D', attr_val) - else - sub_title <- paste0(attr_val) - p <- NULL - if (stri_detect_fixed(type, '+')) { - type1 <- substr(type, 0, stri_locate_all(type, fixed = '+')[[1]][[1]] - 1) - p <- plot_general_data(df_sub, x_attr, y_attr, type1, legend_attr, scale.xlog, scale.ylog, - scale.reverse, NULL, x.title, y.title, plot_title, upper_attr, lower_attr, - show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) - type <- substr(type, stri_locate_all(type, fixed = '+')[[1]][[1]] + 1, nchar(type)) - } - plot_general_data(df_sub, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, - scale.reverse, p, x.title, y.title, plot_title, upper_attr, lower_attr, - show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) %>% - layout( - annotations = list( - text = sub_title, - font = f2, - xref = "paper", yref = "paper", align = "center", - yanchor = "bottom", - xanchor = "center", textangle = 0, - x = 0.5, y = 1, showarrow = FALSE - ) - ) - }) - - p <- subplot( - p, nrows = n_rows, titleX = T, titleY = T, - margin = c(0.02, 0.02, 0.06, 0.06) - ) %>% - layout(title = plot_title) - return(p) - } - - # Replace colnames to have easier matching - if (!x_attr %in% colnames(df) || !all(y_attr %in% colnames(df))) { - stop("Not all provided attributes are colnames of the selected data.table.") - } - colnames(df)[colnames(df) == x_attr] <- "x" - - - if (length(y_attr) == 1 && type != 'line') - colnames(df)[colnames(df) == y_attr] <- "y" - else if (type != 'line') stop("Multiple y-attrs is currently only supported for line-plots") - - if ( !is.null(upper_attr) && !is.null(lower_attr)) { - if (!upper_attr %in% colnames(df) || !lower_attr %in% colnames(df)) { - stop("Provided upper and lower attributes are not colnames of the selected data.table.") - } - colnames(df)[colnames(df) == upper_attr] <- "upper" - colnames(df)[colnames(df) == lower_attr] <- "lower" - } - - if ( x_attr != legend_attr) { - colnames(df)[colnames(df) == legend_attr] <- "l" - xs <- unique(df[['l']]) - } - else{ - xs <- unique(df[['x']]) - } - - #Get color and based on legend-attribute - colors <- get_color_scheme(xs) - if (is.null(names(colors)) || !all(names(colors) %in% xs) ) names(colors) <- xs - - xscale <- if (scale.xlog) 'log' else 'linear' - yscale <- if (scale.ylog) 'log' else 'linear' - - #If new plot is needed, create one. Store in bool to decide if axis scaling is needed. - is_new_plot <- F - if (is.null(p)) { - p <- IOH_plot_ly_default(x.title = ifelse(is.null(x_title), x_attr, x_title), - y.title = ifelse(is.null(y_title), y_attr, y_title), - title = plot_title) - is_new_plot <- T - } - - switch(type, - 'violin' = { - if (legend_attr != x_attr) { - warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") - } - #Update names to aviod numerical legend - if (is.numeric(df[['x']])) { - if (stri_detect_regex(x_attr, "(?i)fun")) - df <- df[, x := paste0('F', sprintf("%02d", x))] - else if (stri_detect_regex(x_attr, "(?i)dim")) - df <- df[, x := paste0('D', as.character(x))] - else - df <- df[, x := as.character(x)] - } - #Update color names as well, since the value changed - names(colors) <- unique(df[['x']]) - - p %<>% - add_trace(data = df, - x = ~x, y = ~y, type = 'violin', - hoveron = "points+kde", - points = F, - pointpos = 1.5, - jitter = 0, - scalemode = 'count', - meanline = list(visible = F), - name = ~x, - colors = colors, - color = ~x, - split = ~x, - line = list(color = 'black', width = 1.1), - box = list(visible = T), - spanmode = 'hard', - showlegend = show.legend, - ... - ) - if (is_new_plot) { - p %<>% layout(yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - }, - 'line' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - - # Force legend to be categorical - df[, l_orig := l] - if (is.numeric(df[['l']])) { - df[, l := paste0('A', l)] - names(colors) <- paste0('A', names(colors)) - } - - #Use linestyles to differentiate traces if only one attribute is selected to be plotted - #TODO: Combine these two options more elegantly - if (length(y_attr) == 1) { - dashes <- get_line_style(xs) - names(dashes) <- xs - colnames(df)[colnames(df) == y_attr] <- "y" - - df[, isinf := is.infinite(y)] - df[, text := as.character(round(y, getOption("IOHanalyzer.precision", 2)))] - - if (inf.action == 'overlap') { - maxval <- max(df[isinf == F, 'y']) - df[['y']][df[['isinf']]] <- 10**(ceiling(log10(maxval)) + 1) - } - else if (inf.action == 'jitter') { - #TODO: Faster way to compute this - maxval <- max(df[isinf == F, 'y']) - for (xval in unique(df[['x']])) { - tempval <- 10**(ceiling(log10(maxval)) + 1) - for (lval in unique(df[['l']])) { - temp <- df[l == lval][x == xval] - if (nrow(temp) > 0 && df[l == lval][x == xval][['isinf']]) { - df[l == lval][x == xval][['y']] <- tempval - tempval <- 1.2 * tempval - } - } - } - } - - suppressWarnings( - p %<>% - add_trace( - data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, - type = 'scatter', mode = 'lines+markers', - linetype = ~l_orig, marker = list(size = getOption('IOHanalyzer.markersize', 4)), - linetypes = dashes, - colors = colors, showlegend = show.legend, - text = ~text, line = list(width = getOption('IOHanalyzer.linewidth', 2)), - hovertemplate = '%{text}', - ... - ) - ) - if (inf.action != 'none') { - p %<>% add_trace(data = df[isinf == T], x = ~x, y = ~y, legendgroup = ~l_orig, name = ~l_orig, - type = 'scatter', mode = 'markers', color = ~l, - marker = list(symbol = 'circle-open', size = 8 + getOption('IOHanalyzer.markersize', 4)), - colors = colors, showlegend = F, text = 'Inf', hoverinfo = 'none', - ... - ) - } - - } - else { - if (inf.action != 'none') { - warning("inf.action is not yet supported for multiple y-attributes") - } - - dashes_full <- rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), - ceiling(length(y_attr)/3))[1:length(y_attr)] - names(dashes_full) <- y_attr - - for (y_atr in y_attr) { - colnames(df)[colnames(df) == y_atr] <- "y" - - #TODO: Figure out how to supress warning about 6 linetypes - dashstyle <- dashes_full[[y_atr]] - suppressWarnings( - p %<>% - add_trace( - data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, - type = 'scatter', mode = 'lines+markers', - marker = list(size = getOption('IOHanalyzer.markersize', 4)), linetype = dashstyle, - colors = colors, showlegend = show.legend, name = ~l, - text = y_atr, line = list(width = getOption('IOHanalyzer.linewidth', 2)), - ... - ) - ) - colnames(df)[colnames(df) == "y"] <- y_atr - show.legend <- F - } - } - if (is_new_plot) { - if (is.numeric(df[['x']])) - p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - else - p %<>% layout(xaxis = list(type = 'category', tickfont = f3(), ticklen = 3), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - - } - }, - 'ribbon' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - if (is.null(upper_attr) || is.null(lower_attr)) { - stop("No upper or lower attribute provided for ribbon-plot") - } - - for (name in xs) { - df_small <- df[l == name] - legend_name <- as.character(name) - rgba_str <- paste0('rgba(', paste0(col2rgb(colors[[name]]), collapse = ','), ',0.2)') - p %<>% - add_trace(data = df_small, x = ~x, y = ~upper, type = 'scatter', mode = 'lines', - line = list(color = rgba_str, width = 0), legendgroup = legend_name, - showlegend = F, name = 'upper', ...) %>% - add_trace(x = ~x, y = ~lower, type = 'scatter', mode = 'lines', - fill = 'tonexty', line = list(color = 'transparent'), legendgroup = legend_name, - fillcolor = rgba_str, showlegend = F, name = 'lower', ...) - } - - - - if (is_new_plot) { - p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - }, - 'radar' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - if (is.numeric(df[['x']])) { - if (stri_detect_regex(x_attr, "(?i)fun")) - df <- df[, x := paste0('F', sprintf("%02d", x))] - else if (stri_detect_regex(x_attr, "(?i)dim")) - df <- df[, x := paste0('D', as.character(x))] - else - df <- df[, x := as.character(x)] - } - df <- df[, col := add_transparancy(colors, 0.4)[l]] - p %<>% - add_trace(data = df, type = 'scatterpolar', r = ~y, - theta = ~x, mode = 'markers', #marker = list(color = 'lightgrey', size=0), - fill = 'toself', connectgaps = T, fillcolor = ~col, color = ~l, colors = colors, - name = ~l, legendgroup = ~l, ...) - if (is_new_plot) { - p %<>% layout(polar = list(radialaxis = list(type = yscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)))) - } - }, - 'hist' = { - if (legend_attr == x_attr) { - stop("Duplicated attribute selected for x-axis and legend.") - } - if (!'width' %in% colnames(df)) { - stop("No 'width'-column included in the provided dataframe. This is required for a histogram-plot") - } - p %<>% - add_trace(data = df, x = ~x, y = ~y, width = ~width, type = 'bar', - name = ~l, text = ~text, hoverinfo = 'text', - colors = add_transparancy(colors, 0.6), color = ~l, - marker = list(line = list(color = 'rgb(8,48,107)')), - ...) - - if (is_new_plot) { - p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, - autorange = ifelse(scale.reverse, "reversed", T)), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - }, - 'bar' = { - if (legend_attr != x_attr) { - warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") - } - colors = add_transparancy(colors, 0.6) - for (xv in xs) { - p %<>% - add_trace(x = xv, y = df[x == xv, y], type = 'bar', - name = xv, - color = colors[xv], - marker = list(line = list(color = 'rgb(8,48,107)')), - ...) - } - - if (is_new_plot) { - p %<>% layout(xaxis = list(tickfont = f3(), ticklen = 3), - yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) - } - } - ) - return(p) -} - - -#' Create the PerformViz plot -#' -#' From the paper: -#' -#' @param DSC_rank_result The result from a call to DSCtool rank service (`get_dsc_rank`) -#' -#' @return A performviz plot -#' @export -#' @examples -#' \dontrun{ -#' Plot.Performviz(get_dsc_rank(dsl)) -#' } -Plot.Performviz <- function(DSC_rank_result) { - if (!requireNamespace("ComplexHeatmap", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - if (!requireNamespace("reshape2", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - if (!requireNamespace("grid", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - mlist <- DSC_rank_result$ranked_matrix - - problem <- NULL #Assign variable to remove warnings - # df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - # function(x) { - # list(algorithm = x$algorithm, rank = x$rank) - # })) - # df_temp[, problem := mlist[[problem_idx]]$problem] - - df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { - df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - function(x) { - list(algorithm = x$algorithm, rank = x$rank) - })) - df_temp[, problem := mlist[[problem_idx]]$problem] - })) - - rank_matrix <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') - df <- rank_matrix - # colnames(df)<-index - # rownames(df)<-vector - # Define some graphics to display the distribution of columns - # library(ComplexHeatmap) - .hist = ComplexHeatmap::anno_histogram(df, gp = grid::gpar(fill = "lightblue")) - .density = ComplexHeatmap::anno_density(df, type = "line", gp = grid::gpar(col = "blue")) - ha_mix_top = ComplexHeatmap::HeatmapAnnotation(hist = .hist, density = .density) - # Define some graphics to display the distribution of rows - .violin = ComplexHeatmap::anno_density(df, type = "violin", - gp = grid::gpar(fill = "lightblue"), which = "row") - .boxplot = ComplexHeatmap::anno_boxplot(df, which = "row") - ha_mix_right = ComplexHeatmap::HeatmapAnnotation(violin = .violin, bxplt = .boxplot, - which = "row", width = grid::unit(4, "cm")) - # Combine annotation with heatmap - heatmap_main <- ComplexHeatmap::Heatmap(df, name = "Ranking", - column_names_gp = grid::gpar(fontsize = 8), - top_annotation = ha_mix_top, - top_annotation_height = grid::unit(3.8, "cm")) - return(ComplexHeatmap::draw( - ComplexHeatmap::`+.AdditiveUnit`(heatmap_main, ha_mix_right)) - ) -} +symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", + "triangle-up-open", "triangle-down-open") + +get_legends <- function(dsList) { + N <- length(dsList) + legends <- sapply(dsList, function(d) get_id(d)) + + if (length(unique(legends)) < N) { + funcId <- sapply(dsList, function(d) attr(d, 'funcId')) + if (length(unique(funcId)) > 1) + legends <- paste0(legends, '-F', funcId) + } + + if (length(unique(legends)) < N) { + DIM <- sapply(dsList, function(d) attr(d, 'DIM')) + if (length(unique(DIM)) > 1) + legends <- paste0(legends, '-', DIM, 'D') + } + legends +} + +insert_best_parts <- function(from_data, to_data, best_is_min) { + if (all(is.na(from_data))) + to_data + else + if (best_is_min) + pmin(from_data, to_data, na.rm = T) + else + pmax(from_data, to_data, na.rm = T) +} + +generate_rbga <- function(color,a){ + paste0('rgba(', paste0(color, collapse = ','), ',', a,')') +} + +grad_functions <- c( + scaled_edges = function(count, amount, intensity){ + scale <- (intensity + 1)/2 + color_end <- floor(scale*amount*2) + if (count < color_end) + 1/color_end + else + 0 + } + , + fixed_edges = function(count, amount, intensity) { + scale <- (intensity + 1) / 2 + color_center <- floor(scale * amount) + 1 + if (count <= color_center) + 1 / (2 * color_center) + else + 1 / (2 * (amount - color_center)) + } +) + +#S3 generics +# TODO: decide which parameters need to be in the generics + +#' Plot lineplot of the ERTs of a DataSetList +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param Fstart The starting function value. +#' @param Fstop The final function value. +#' @param show.ERT Whether or not to show the ERT-values +#' @param show.CI Whether or not to show the standard deviations +#' @param show.mean Whether or not to show the mean hitting times +#' @param show.median Whether or not to show the median hitting times +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) +#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' +#' @param includeOpts Whether or not to include all best points reached by each algorithm +#' @param p Existing plot to which to add the current data +#' @return A plot of ERT-values of the DataSetList +#' @export +#' @examples +#' Plot.RT.Single_Func(subset(dsl, funcId == 1)) +Plot.RT.Single_Func <- function(dsList, Fstart = NULL, Fstop = NULL, + show.ERT = T, show.CI = F, show.mean = F, + show.median = F, backend = NULL, + scale.xlog = F, scale.ylog = F, scale.reverse = F, + includeOpts = F, p = NULL) + UseMethod("Plot.RT.Single_Func", dsList) +#' Plot lineplot of the expected function values of a DataSetList +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param RTstart The starting runtime value. +#' @param RTstop The final runtime value. +#' @param show.CI Whether or not to show the standard deviations +#' @param show.mean Whether or not to show the mean runtimes +#' @param show.median Whether or not to show the median runtimes +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) +#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' +#' +#' @return A plot of ERT-values of the DataSetList +#' @export +#' @examples +#' Plot.FV.Single_Func(subset(dsl, funcId == 1)) +Plot.FV.Single_Func <- function(dsList, RTstart = NULL, RTstop = NULL, show.CI = F, show.mean = T, + show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F) UseMethod("Plot.FV.Single_Func", dsList) +#' Plot probablity mass function of the runtimes of a DataSetList at a certain target function value +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value. +#' @param show.sample Whether or not to show the individual runtime samples +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2' +#' +#' @return A plot of the probablity mass function of the runtimes at a the +#' target function value of the DataSetList +#' @export +#' @examples +#' Plot.RT.PMF(subset(dsl, funcId == 1), 14) +Plot.RT.PMF <- function(dsList, ftarget, show.sample = F, scale.ylog = F, backend = NULL) + UseMethod("Plot.RT.PMF", dsList) +#' Plot histograms of the runtimes of a DataSetList at a certain target function value +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value. +#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either +#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. +#' @param use.equal.bins Whether to determine one bin size for all plots or have individual +#' bin sizes for each algorithm +#' +#' @return A plot of the histograms of the runtimes at a the +#' target function value of the DataSetList +#' @export +#' @examples +#' Plot.RT.Histogram(subset(dsl, funcId == 1), 14) +Plot.RT.Histogram <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F) + UseMethod("Plot.RT.Histogram", dsList) +#' Plot the empirical cumulative distriburtion as a function of the running times of +#' a DataSetList at certain target function values +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftargets The target function values +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the running times of the DataSetList at the target function values +#' @export +#' @examples +#' Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) +Plot.RT.ECDF_Per_Target <- function(dsList, ftargets, scale.xlog = F) + UseMethod("Plot.RT.ECDF_Per_Target", dsList) +#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of +#' a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param fstart The starting function value +#' @param fstop The final function value +#' @param fstep The spacing between starting and final function values +#' @param show.per_target Whether or not to show the individual ECDF-curves for each target +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the running times of the DataSetList +#' @export +#' @examples +#' Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) +Plot.RT.ECDF_Single_Func <- function(dsList, fstart = NULL, fstop = NULL, + fstep = NULL, show.per_target = F, + scale.xlog = F) UseMethod("Plot.RT.ECDF_Single_Func", dsList) +#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param fstart The starting function value +#' @param fstop The final function value +#' @param fstep The spacing between starting and final function values +#' @param fval_formatter Function to format the function-value labels +#' +#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList +#' @export +#' @examples +#' Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) +Plot.RT.ECDF_AUC <- function(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, + fval_formatter = as.integer) UseMethod("Plot.RT.ECDF_AUC", dsList) +#' Plot probablity density function of the function values of a DataSetList at +#' a certain target runtime +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param runtime The target runtime +#' @param show.sample Whether or not to show the individual function value samples +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' +#' @return A plot of the probablity density function of the runtimes at a the +#' target function value of the DataSetList +#' @export +#' @examples +#' Plot.FV.PDF(subset(dsl, funcId == 1), 100) +Plot.FV.PDF <- function(dsList, runtime, show.sample = F, scale.ylog = F) + UseMethod("Plot.FV.PDF", dsList) +#' Plot histograms of the function values of a DataSetList at a certain target runtime +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param runtime The target runtime +#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either +#' 'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm. +#' @param use.equal.bins Whether to determine one bin size for all plots or have individual +#' bin sizes for each algorithm +#' +#' @return A plot of the histograms of the function values at a the +#' target runtime of the DataSetList +#' @export +#' @examples +#' Plot.FV.Histogram(subset(dsl, funcId == 1), 100) +Plot.FV.Histogram <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F) + UseMethod("Plot.FV.Histogram", dsList) +#' Plot the empirical cumulative distriburtion as a function of the target values of +#' a DataSetList at certain target runtimes +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param runtimes The target runtimes +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the fucntion values of the DataSetList at the target runtimes +#' @export +#' @examples +#' Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) +Plot.FV.ECDF_Per_Target <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F) + UseMethod("Plot.FV.ECDF_Per_Target", dsList) +#' Plot the aggregated empirical cumulative distriburtion as a function of the function values of +#' a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param rt_min The starting runtime +#' @param rt_max The final runtime +#' @param rt_step The spacing between starting and final runtimes +#' @param show.per_target Whether or not to show the individual ECDF-curves for each runtime +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization) +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the function values of the DataSetList +#' @export +#' @examples +#' Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) +Plot.FV.ECDF_Single_Func <- function(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL, scale.xlog = F, + show.per_target = F, scale.reverse = F) + UseMethod("Plot.FV.ECDF_Single_Func", dsList) +#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param rt_min The starting runtime +#' @param rt_max The final runtime +#' @param rt_step The spacing between starting and final runtimes +#' +#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList +#' @export +#' @examples +#' Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) +Plot.FV.ECDF_AUC <- function(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL) UseMethod("Plot.FV.ECDF_AUC", dsList) +#' Plot the parameter values recorded in a DataSetList (aligned by funcion value) +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param f_min The starting function value. +#' @param f_max The final function value. +#' @param show.mean Whether or not to show the mean parameter values +#' @param show.median Whether or not to show the median parameter values +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param algids Which algorithms from dsList to use +#' @param par_name Which parameters to create plots for; set to NULL to use all +#' parameters found in dsList. +#' @param show.CI Whether or not to show the standard deviation +#' +#' @return A plot of for every recorded parameter in the DataSetList +#' @export +#' @examples +#' Plot.RT.Parameters(subset(dsl, funcId == 1)) +Plot.RT.Parameters <- function(dsList, f_min = NULL, f_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) UseMethod("Plot.RT.Parameters", dsList) +#' Plot the parameter values recorded in a DataSetList (aligned by budget) +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param rt_min The starting budget value. +#' @param rt_max The final budget value. +#' @param show.mean Whether or not to show the mean parameter values +#' @param show.median Whether or not to show the median parameter values +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param algids Which algorithms from dsList to use +#' @param par_name Which parameters to create plots for; set to NULL to use all +#' parameters found in dsList. +#' @param show.CI Whether or not to show the standard deviation +#' +#' @return A plot of for every recorded parameter in the DataSetList +#' @export +#' @examples +#' Plot.FV.Parameters(subset(dsl, funcId == 1)) +Plot.FV.Parameters <- function(dsList, rt_min = NULL, rt_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) UseMethod("Plot.FV.Parameters", dsList) +#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of +#' a DataSetList. Aggregated over multiple functions or dimensions. +#' +#' @param dsList A DataSetList. +#' @param targets The target function values. Specified in a data.frame, as can be generated +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' by the function 'get_ECDF_targets' +#' +#' @return A plot of the empirical cumulative distriburtion as a function of +#' the running times of the DataSetList +#' @export +#' @examples +#' Plot.RT.ECDF_Multi_Func(dsl) +Plot.RT.ECDF_Multi_Func <- function(dsList, targets = NULL, scale.xlog = F) + UseMethod("Plot.RT.ECDF_Multi_Func", dsList) +#' Plot ERT-plots for multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization) +#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. +#' +#' @return A plot of ERT-values of the DataSetList +#' @export +#' @examples +#' Plot.RT.Multi_Func(dsl) +Plot.RT.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, scale.reverse = F, + backend = NULL) + UseMethod("Plot.RT.Multi_Func", dsList) +#' Plot ERT-based comparison over multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' +#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param maximize Wheter or not to the data is of a maximization problem +#' @param targets Custom list of function-value targets, one for each function or dimension. +#' @param use_rank Wheter to use a ranking system. If False, the actual ERT-values will be used. +#' @param erts Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function +#' of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating +#' this plot. +#' @param inf.action How to handle infinite ERTs ('overlap' or 'jitter') +#' @return A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList +#' @export +#' @examples +#' Plot.RT.Aggregated(dsl) +Plot.RT.Aggregated <- function(dsList, aggr_on = 'funcId', targets = NULL, + plot_mode = 'radar', use_rank = F, scale.ylog = T, maximize = T, + erts = NULL, inf.action = 'overlap') + UseMethod("Plot.RT.Aggregated", dsList) +#' Plot expected function value-based comparison over multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param plot_mode How the plots should be created. Can be 'line' or 'radar' +#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM') +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param runtimes Custom list of function-value targets, one for each function or dimension. +#' @param use_rank Wheter to use a ranking system. If False, the actual expected function- +#' values will be used. +#' @param fvs Pre-calculated expected function-values for the provided runtimes Created by the +#' max_ERTs function of DataSetList. Can be provided to prevent needless computation +#' in recalculating ERTs when recreating this plot. +#' +#' @return A plot of expected function value-based comparison on the provided functions +#' or dimensions of the DataSetList +#' @export +#' @examples +#' Plot.FV.Aggregated(dsl) +Plot.FV.Aggregated <- function(dsList, aggr_on = 'funcId', runtimes = NULL, plot_mode = 'radar', + use_rank = F, scale.ylog = T, fvs = NULL) + UseMethod("Plot.FV.Aggregated", dsList) + +#' Plot FV-plots for multiple functions or dimensions +#' +#' @param dsList A DataSetList (should consist of only one function OR dimension). +#' @param scale.xlog Whether or not to scale the x-axis logaritmically +#' @param scale.ylog Whether or not to scale the y-axis logaritmically +#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'. +#' +#' @return A plot of Function-values of the DataSetList +#' @export +#' @examples +#' Plot.FV.Multi_Func(dsl) +Plot.FV.Multi_Func <- function(dsList, scale.xlog = F, scale.ylog = F, backend = NULL) + UseMethod("Plot.FV.Multi_Func", dsList) + +#' Plot a heatmap showing the statistically different algorithms +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value to use +#' @param alpha The cutoff for statistical significance +#' @param bootstrap.size The amound of bootstrapped samples used +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' +#' @return A heatmap showing the statistical significance between algorithms +#' @export +#' @examples +#' Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) +Plot.Stats.Significance_Heatmap <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, + which = 'by_FV') + UseMethod("Plot.Stats.Significance_Heatmap", dsList) + +#' Plot a network graph showing the statistically different algorithms +#' +#' @param dsList A DataSetList (should consist of only one function and dimension). +#' @param ftarget The target function value to use +#' @param alpha The cutoff for statistical significance +#' @param bootstrap.size The amound of bootstrapped samples used +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' +#' @return A graph showing the statistical significance between algorithms +#' @export +#' @examples +#' Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) +Plot.Stats.Significance_Graph <- function(dsList, ftarget, alpha = 0.01, bootstrap.size = 30, + which = 'by_FV') + UseMethod("Plot.Stats.Significance_Graph", dsList) + +#' Create a candlestick plot of Glicko2-rankings +#' +#' @param dsList A DataSetList +#' @param nr_rounds The number of rounds in the tournament +#' @param glicko2_rank_df Optional. Dataframe containing the glicko2 rating to avoid needless recalculation. +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' @param target_dt Optional: data table containing the targets for each function and dimension +#' +#' @export +#' @examples +#' Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) +Plot.Stats.Glicko2_Candlestick <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, + which = 'by_FV', target_dt = NULL) + UseMethod("Plot.Stats.Glicko2_Candlestick", dsList) + + +##Implementations + +#' @rdname Plot.RT.Single_Func +#' @export +Plot.RT.Single_Func.DataSetList <- function(dsList, Fstart = NULL, Fstop = NULL, + show.ERT = T, show.CI = T, show.mean = F, + show.median = F, backend = NULL, + scale.xlog = F, scale.ylog = F, + scale.reverse = F, includeOpts = F, p = NULL) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + if (backend == 'plotly') { + data <- generate_data.Single_Function(dsList, Fstart, Fstop, scale.xlog, 'by_RT', includeOpts) + + y_attrs <- c() + if (show.ERT) y_attrs <- c(y_attrs, 'ERT') + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, p = p, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + show_legend <- F + } + if (show.CI) { + p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + } + } + # } else if (backend == 'ggplot2') { + # dt[, 'group' := paste(algId, funcId, DIM, sep = '-')] + # p <- ggplot(data = dt, aes(group = `group`, colour = `group`)) + # + # if (show.CI) p <- p + geom_ribbon(aes(target, ymin = lower, ymax = upper, fill = `group`), + # alpha = 0.2, colour = NA) + # if (show.ERT) p <- p + geom_line(aes(target, ERT), size = 1.2) + # if (show.mean) p <- p + geom_line(aes(target, mean), linetype = 'dashed') + # if (show.median) p <- p + geom_line(aes(target, median), linetype = 'dotted') + # + # p <- p + + # scale_color_manual(values = colors) + + # scale_fill_manual(values = colors) + # } + return(p) +} + +#' @rdname Plot.FV.Single_Func +#' @export +Plot.FV.Single_Func.DataSetList <- function(dsList, RTstart = NULL, RTstop = NULL, + show.CI = F, + show.mean = T, show.median = F, + backend = NULL, + scale.xlog = F, scale.ylog = F, + scale.reverse = F) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + if (backend == 'plotly') { + data <- generate_data.Single_Function(dsList, RTstart, RTstop, scale.xlog, 'by_FV') + + y_attrs <- c() + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + show_legend <- F + } + else + p <- NULL + if (show.CI) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, + scale.xlog = scale.xlog, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = scale.reverse) + } + + } + # } else if (backend == 'ggplot2') { + # fce[, 'group' := paste(algId, funcId, DIM, sep = '-')] + # p <- ggplot(data = fce, aes(group = `group`, colour = `group`)) + # + # if (show.mean) p <- p + geom_line(aes(runtime, mean), linetype = 'dashed') + # if (show.median) p <- p + geom_line(aes(runtime, median), linetype = 'dotted') + # + # p <- p + + # scale_color_manual(values = colors) + + # scale_fill_manual(values = colors) + # + # #TODO: add individual run etc + # } + return(p) +} + +#' @rdname Plot.RT.PMF +#' @export +Plot.RT.PMF.DataSetList <- function(dsList, ftarget, show.sample = F, + scale.ylog = F, backend = NULL){ + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + data <- generate_data.PMF(dsList, ftarget, 'by_RT') + + plot_general_data(data, 'ID', 'RT', scale.ylog = scale.ylog, + x_title = "Algorithm", y_title = "Function Evaluations") +} + +#' @rdname Plot.RT.Histogram +#' @export +Plot.RT.Histogram.DataSetList <- function(dsList, ftarget, plot_mode = 'overlay', use.equal.bins = F){ + if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { + warning("Invalid dataset uploaded. Please ensure the datasetlist contains data + from only one function and only one dimension.") + return(NULL) + } + data <- generate_data.hist(dsList, ftarget, use.equal.bins, 'by_RT') + + subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL + plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Function Evaluations", + y_title = "Runs") +} + +#' @rdname Plot.RT.ECDF_Per_Target +#' @export +Plot.RT.ECDF_Per_Target.DataSetList <- function(dsList, ftargets, scale.xlog = F){ + req(length(ftargets) != 0) + data <- generate_data.ECDF(dsList, ftargets, scale.xlog) + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of runs", scale.xlog = scale.xlog, show.legend = T) +} + +#' @rdname Plot.RT.ECDF_Single_Func +#' @export +Plot.RT.ECDF_Single_Func.DataSetList <- function(dsList, fstart = NULL, fstop = NULL, + fstep = NULL, show.per_target = F, + scale.xlog = F) { + + targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep) + req(targets) + + data <- generate_data.ECDF(dsList, targets, scale.xlog) + + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of (run, target) pairs", + scale.xlog = scale.xlog, show.legend = T) +} + +#' @rdname Plot.RT.ECDF_AUC +#' @export +Plot.RT.ECDF_AUC.DataSetList <- function(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, + fval_formatter = as.integer) { + + targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep, length.out = 10) + req(targets) + + data <- generate_data.AUC(dsList, targets, multiple_x = TRUE) + + plot_general_data(data, 'x', 'auc', 'radar') +} + +#' @rdname Plot.FV.PDF +#' @export +Plot.FV.PDF.DataSetList <- function(dsList, runtime, show.sample = F, scale.ylog = F){ + + data <- generate_data.PMF(dsList, runtime, 'by_FV') + + plot_general_data(data, 'ID', 'f(x)', scale.ylog = scale.ylog, + x_title = "Algorithm", y_title = "Target Value") +} + +#' @rdname Plot.FV.Histogram +#' @export +Plot.FV.Histogram.DataSetList <- function(dsList, runtime, plot_mode='overlay', use.equal.bins = F){ + if (length(get_funcId(dsList)) != 1 || length(get_dim(dsList)) != 1) { + warning("Invalid dataset uploaded. Please ensure the datasetlist contains data + from only one function and only one dimension.") + return(NULL) + } + data <- generate_data.hist(dsList, runtime, use.equal.bins, 'by_FV') + + subplot_attr <- if (plot_mode == 'subplot') 'ID' else NULL + plot_general_data(data, 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Target Values", + y_title = "Runs") +} + +#' @rdname Plot.FV.ECDF_Per_Target +#' @export +Plot.FV.ECDF_Per_Target.DataSetList <- function(dsList, runtimes, scale.xlog = F, scale.reverse = F){ + #TODO: Fvals in legend need to be formatted properly + runtimes <- runtimes[!is.na(runtimes)] + req(length(runtimes) != 0) + + data <- generate_data.ECDF(dsList, runtimes, scale.xlog, which = 'by_FV') + + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of runs", scale.xlog = scale.xlog, + show.legend = T, + scale.reverse = scale.reverse) +} + +#' @rdname Plot.FV.ECDF_Single_Func +#' @export +Plot.FV.ECDF_Single_Func.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL, scale.xlog = F, + show.per_target = F, scale.reverse = F){ + + targets <- seq_RT(get_funvals(dsList), rt_min, rt_max, rt_step) + req(targets) + data <- generate_data.ECDF(dsList, targets, scale.xlog, which = 'by_FV') + + plot_general_data(data, 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of (run, target) pairs", + scale.xlog = scale.xlog, + scale.reverse = scale.reverse, show.legend = T) +} + +#' @rdname Plot.FV.ECDF_AUC +#' @export +Plot.FV.ECDF_AUC.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, rt_step = NULL) { + targets <- seq_RT(get_runtimes(dsList), rt_min, rt_max, rt_step, length.out = 10) + req(targets) + data <- generate_data.AUC(dsList, targets, which = 'by_FV', multiple_x = TRUE) + + plot_general_data(data, 'x', 'auc', 'radar') +} + +#' @rdname Plot.RT.Parameters +#' @export +Plot.RT.Parameters.DataSetList <- function(dsList, f_min = NULL, f_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) { + data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_FV') + + y_attrs <- c() + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'target', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + show_legend <- F + } + else + p <- NULL + if (show.CI) { + p <- plot_general_data(data, x_attr = 'target', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + } + p +} + + +#' @rdname Plot.FV.Parameters +#' @export +Plot.FV.Parameters.DataSetList <- function(dsList, rt_min = NULL, rt_max = NULL, + algids = 'all', par_name = NULL, + scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, + show.CI = F) { + data <- generate_data.Parameters(dsList, scale.xlog, which = 'by_RT') + + y_attrs <- c() + if (show.mean) y_attrs <- c(y_attrs, 'mean') + if (show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + show_legend <- F + } + else + p <- NULL + if (show.CI) { + p <- plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = scale.ylog, subplot_attr = 'parId', + scale.xlog = scale.xlog) + } + p +} + +#' @rdname Plot.RT.ECDF_Multi_Func +#' @export +Plot.RT.ECDF_Multi_Func.DataSetList <- function(dsList, targets = NULL, + scale.xlog = F) { + if (is.null(targets) || !is.data.table(targets)) { + targets <- get_ECDF_targets(dsList) + } + + data <- generate_data.ECDF(dsList, targets, scale.xlog) + + plot_general_data(data, 'x', 'mean', 'line', + scale.xlog = scale.xlog, + x_title = "Function Evaluations", + y_title = "Proportion of (run, target, ...) pairs", + show.legend = T) +} + +#' @rdname Plot.RT.Multi_Func +#' @export +Plot.RT.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, + scale.ylog = F, + scale.reverse = F, + backend = NULL) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + data <- rbindlist(lapply(get_funcId(dsList), function(fid) { + generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, + which = 'by_RT') + })) + + plot_general_data(data, x_attr = 'target', y_attr = 'ERT', + subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, + scale.ylog = scale.ylog, x_title = 'Best-so-far f(x)', + y_title = 'ERT', show.legend = T, + scale.reverse = scale.reverse) +} + +#' @rdname Plot.FV.Multi_Func +#' @export +Plot.FV.Multi_Func.DataSetList <- function(dsList, scale.xlog = F, + scale.ylog = F, + backend = NULL) { + if (is.null(backend)) backend <- getOption("IOHanalyzer.backend", default = 'plotly') + + data <- rbindlist(lapply(get_funcId(dsList), function(fid) { + generate_data.Single_Function(subset(dsList, funcId == fid), scale_log = scale.xlog, + which = 'by_FV') + })) + + plot_general_data(data, x_attr = 'runtime', y_attr = 'mean', + subplot_attr = 'funcId', type = 'line', scale.xlog = scale.xlog, + scale.ylog = scale.ylog, x_title = 'Runtime', + y_title = 'Best-so-far f(x)', show.legend = T) +} + +#' @rdname Plot.RT.Aggregated +#' @export +Plot.RT.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', targets = NULL, + plot_mode = 'radar', use_rank = F, + scale.ylog = T, maximize = T, erts = NULL, + inf.action = 'overlap') { + targets <- get_target_dt(dsList) + data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) + y_attr <- if (use_rank) 'rank' else 'value' + y_title <- if (use_rank) 'Rank' else 'ERT' + plot_general_data(data, type = plot_mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = scale.ylog, + inf.action = inf.action) +} + +#' @rdname Plot.FV.Aggregated +#' @export +Plot.FV.Aggregated.DataSetList <- function(dsList, aggr_on = 'funcId', runtimes = NULL, + plot_mode = 'radar', use_rank = F, + scale.ylog = T, fvs = NULL){ + targets <- get_target_dt(dsList, which = 'by_FV') + data <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, which = 'by_FV') + y_attr <- if (use_rank) 'rank' else 'value' + y_title <- if (use_rank) 'Rank' else 'Best-so-far f(x)' + plot_general_data(data, type = plot_mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = scale.ylog) +} + +#' @rdname Plot.Stats.Significance_Heatmap +#' @export +Plot.Stats.Significance_Heatmap.DataSetList <- function(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = 'by_FV'){ + if (length(get_dim(dsList)) != 1 || + length(get_funcId(dsList)) != 1 || + length(get_id(dsList)) < 2) + return(NULL) + + p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) + y <- p_matrix <= alpha + colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), + col = c('blue', 'blue', 'white', 'white', 'red', 'red') + ) + heatmap <- y - t(y) + + p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', + xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) + p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), + xaxis = list(tickangle = 45)) + p +} + +#' Helper function for Plot.Stats.Significance_Graph +#' +#' @param x x +#' @param start default is 0 +#' @param direction default is 1 +#' +#' @noRd +radian.rescale <- function(x, start=0, direction=1) { + c.rotate <- function(x) (x + start) %% (2 * pi) * direction + c.rotate((2 * pi * (x - min(x)) / (max(x) - min(x)))) +} + +#' @rdname Plot.Stats.Significance_Graph +#' @export +Plot.Stats.Significance_Graph.DataSetList <- function(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = 'by_FV'){ + if (!requireNamespace("igraph", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + if (length(get_dim(dsList)) != 1 || length(get_funcId(dsList)) != 1 || length(get_id(dsList)) < 2) { + return(NULL) + } + p_matrix <- pairwise.test(dsList, ftarget, bootstrap.size, which) + g <- igraph::graph_from_adjacency_matrix(p_matrix <= alpha, mode = 'directed', diag = F) + lab.locs <- radian.rescale(x = 1:nrow(p_matrix), direction = -1, start = 0) + + igraph::plot.igraph(g, layout = igraph::layout.circle(g), vertex.size = 10, edge.arrow.size = 1, + vertex.label.color = 'black', + vertex.label.dist = 2, + vertex.label.cex = 1, + vertex.label.degree = lab.locs) +} + +#' @rdname Plot.Stats.Glicko2_Candlestick +#' @export +Plot.Stats.Glicko2_Candlestick.DataSetList <- function(dsList, nr_rounds = 100, glicko2_rank_df = NULL, + which = 'by_FV', target_dt = NULL) { + df <- glicko2_rank_df + + if (is.null(df)) { + df <- glicko2_ranking(dsList, nr_rounds, which, target_dt = target_dt)$ratings + Ids <- df$Player$ID + } + else{ + Ids <- df$ID + } + p <- IOH_plot_ly_default(title = "Glicko2-rating", + x.title = "ID", + y.title = "Rating") + df$Rating %<>% as.numeric + df$Deviation %<>% as.numeric + high <- df$Rating + 3*df$Deviation + low <- df$Rating - 3*df$Deviation + open <- df$Rating + df$Deviation + close <- df$Rating - df$Deviation + + N <- length(df$Rating) + colors <- get_color_scheme(Ids) + if (length(colors != N)) { + colors <- get_color_scheme(get_id(dsList)) + } + + for (i in seq(N)) { + # rgba_str <- paste0('rgba(', paste0(col2rgb(colors[i]), collapse = ','), ',0.52)') + color <- list(line = list(color = colors[[i]])) + p %<>% add_trace(type = "candlestick", x = Ids[[i]], open = open[[i]], close = close[[i]], + high = high[[i]], low = low[[i]], legendgroup = Ids[[i]], + name = Ids[[i]], increasing = color, decreasing = color, + hovertext = paste0(format(df$Rating[[i]], digits = 3), '+-', + format(df$Deviation[[i]], digits = 3)), + hoverinfo = "text") + } + p %<>% layout(xaxis = list(rangeslider = list(visible = F))) + p +} + +### _______________________ Rewritten plotting function ____________________ ### + +#' Add transparancy to named list of colors +#' +#' @param colors Named list of colors (in hex-notation) +#' @param percentage The percentage of opacity. 0 is fully transparant, 1 is fully opaque +#' +#' @noRd +add_transparancy <- function(colors, percentage){ + hex_val <- format(as.hexmode(as.integer(255 * percentage)), upper.case = T, width = 2) + sapply(colors, function(col) { col <- paste0('#', substr(col, 2, 7), hex_val) }) +} + +#' General function for plotting within IOHanalyzer +#' +#' @param df The dataframe containing the data to plot. It should contain at least two columns: +#' 'x_attr' and 'y_attr' +#' @param x_attr The column to specify the x_axis. Default is 'algId' +#' @param legend_attr Default is 'algId' This is also used for the selection of colorschemes +#' @param y_attr The column to specify the y_axis +#' @param type The type of plot to use. Currently available: 'violin', 'line', 'radar', +#' 'bar', hist' and 'ribbon' +#' @param upper_attr When using ribbon-plot, this can be used to create a shaded area. +#' Only works in combination with`lower_attr` and `type` == 'ribbon' +#' @param lower_attr When using ribbon-plot, this can be used to create a shaded area. +#' Only works in combination with`upper_attr` and `type` == 'ribbon' +#' @param subplot_attr Which attribute of the dataframe to use for creating subplots +#' @param scale.xlog Logarithmic scaling of x-axis +#' @param scale.ylog Logarithmic scaling of y-axis +#' @param scale.reverse Decreasing or increasing x-axis +#' @param x_title Title of x-axis. Defaults to x_attr +#' @param y_title Title of x-axis. Defaults to x_attr +#' @param plot_title Title of x-axis. Defaults to no title +#' @param p A previously existing plot on which to add traces. If NULL, a new canvas is created +#' @param show.legend Whether or not to include a legend +#' @param inf.action How to deal with infinite values. Can be 'none', 'overlap' or 'jitter' +#' @param ... Additional parameters for the add_trace function +#' +#' @export +plot_general_data <- function(df, x_attr = 'ID', y_attr = 'vals', type = 'violin', + legend_attr = 'ID', scale.xlog = F, scale.ylog = F, + scale.reverse = F, p = NULL, x_title = NULL, + y_title = NULL, plot_title = NULL, upper_attr = NULL, + lower_attr = NULL, subplot_attr = NULL, show.legend = F, + inf.action = 'none', ...) { + + l <- x <- isinf <- y <- text <- l_orig <- NULL #Set local binding to remove warnings + + #Only allow valid plot types + if (!(type %in% c('violin', 'line', 'radar', 'hist', 'ribbon', 'line+ribbon', 'bar'))) { + stop(paste0("Provided plot type ('", type, "') is not supported")) + } + + #And valid number of y-attributes + if (length(y_attr) == 0) { + stop("At least one y-attribute is needed to plot") + } + + #Deal with subplots + if (!is.null(subplot_attr)) { + if (!subplot_attr %in% colnames(df)) { + stop("Provided subplot-attribut is not a colname of the selected data.table.") + } + colnames(df)[colnames(df) == subplot_attr] <- "subplot_attr" + attrs <- unique(df[, subplot_attr]) + if (length(attrs) == 0) stop("Attempting to create subplots with fewer than 2 unique values of + `subplot_attrs`-column") + if (length(attrs) == 1) return(plot_general_data(df, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, + scale.reverse, p, x_title, y_title, attrs, upper_attr, lower_attr, + show.legend = show.legend, subplot_attr = NULL, ...)) + if (subplot_attr == legend_attr) { + df[, l := subplot_attr] + } + + #Only need one legend for the whole plot + legends_show <- rep(F, length(attrs)) + legends_show[[1]] <- show.legend + names(legends_show) <- attrs + + #Get some number of rows and columns + n_cols <- 1 + ceiling(length(attrs)/10) + n_rows <- ceiling(length(attrs) / n_cols) + + p <- lapply(seq(length(attrs)), function(idx) { + attr_val <- attrs[[idx]] + df_sub <- df[subplot_attr == attr_val] + disp_y <- idx %% n_cols == 1 + disp_x <- idx > (length(attrs) - n_cols) + x.title = if (disp_x) x_title else "" + y.title = if (disp_y) y_title else "" + + #Generate title for the subplots + if (stri_detect_regex(subplot_attr, "(?i)fun")) + sub_title <- paste0('F', attr_val) + else if (stri_detect_regex(subplot_attr, "(?i)dim")) + sub_title <- paste0('D', attr_val) + else + sub_title <- paste0(attr_val) + p <- NULL + if (stri_detect_fixed(type, '+')) { + type1 <- substr(type, 0, stri_locate_all(type, fixed = '+')[[1]][[1]] - 1) + p <- plot_general_data(df_sub, x_attr, y_attr, type1, legend_attr, scale.xlog, scale.ylog, + scale.reverse, NULL, x.title, y.title, plot_title, upper_attr, lower_attr, + show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) + type <- substr(type, stri_locate_all(type, fixed = '+')[[1]][[1]] + 1, nchar(type)) + } + plot_general_data(df_sub, x_attr, y_attr, type, legend_attr, scale.xlog, scale.ylog, + scale.reverse, p, x.title, y.title, plot_title, upper_attr, lower_attr, + show.legend = legends_show[[attr_val]], subplot_attr = NULL, ...) %>% + layout( + annotations = list( + text = sub_title, + font = f2, + xref = "paper", yref = "paper", align = "center", + yanchor = "bottom", + xanchor = "center", textangle = 0, + x = 0.5, y = 1, showarrow = FALSE + ) + ) + }) + + p <- subplot( + p, nrows = n_rows, titleX = T, titleY = T, + margin = c(0.02, 0.02, 0.06, 0.06) + ) %>% + layout(title = plot_title) + return(p) + } + + # Replace colnames to have easier matching + if (!x_attr %in% colnames(df) || !all(y_attr %in% colnames(df))) { + stop("Not all provided attributes are colnames of the selected data.table.") + } + colnames(df)[colnames(df) == x_attr] <- "x" + + + if (length(y_attr) == 1 && type != 'line') + colnames(df)[colnames(df) == y_attr] <- "y" + else if (type != 'line') stop("Multiple y-attrs is currently only supported for line-plots") + + if ( !is.null(upper_attr) && !is.null(lower_attr)) { + if (!upper_attr %in% colnames(df) || !lower_attr %in% colnames(df)) { + stop("Provided upper and lower attributes are not colnames of the selected data.table.") + } + colnames(df)[colnames(df) == upper_attr] <- "upper" + colnames(df)[colnames(df) == lower_attr] <- "lower" + } + + if ( x_attr != legend_attr) { + colnames(df)[colnames(df) == legend_attr] <- "l" + xs <- unique(df[['l']]) + } + else{ + xs <- unique(df[['x']]) + } + + #Get color and based on legend-attribute + colors <- get_color_scheme(xs) + if (is.null(names(colors)) || !all(names(colors) %in% xs) ) names(colors) <- xs + + xscale <- if (scale.xlog) 'log' else 'linear' + yscale <- if (scale.ylog) 'log' else 'linear' + + #If new plot is needed, create one. Store in bool to decide if axis scaling is needed. + is_new_plot <- F + if (is.null(p)) { + p <- IOH_plot_ly_default(x.title = ifelse(is.null(x_title), x_attr, x_title), + y.title = ifelse(is.null(y_title), y_attr, y_title), + title = plot_title) + is_new_plot <- T + } + + switch(type, + 'violin' = { + if (legend_attr != x_attr) { + warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") + } + #Update names to aviod numerical legend + if (is.numeric(df[['x']])) { + if (stri_detect_regex(x_attr, "(?i)fun")) + df <- df[, x := paste0('F', sprintf("%02d", x))] + else if (stri_detect_regex(x_attr, "(?i)dim")) + df <- df[, x := paste0('D', as.character(x))] + else + df <- df[, x := as.character(x)] + } + #Update color names as well, since the value changed + names(colors) <- unique(df[['x']]) + + p %<>% + add_trace(data = df, + x = ~x, y = ~y, type = 'violin', + hoveron = "points+kde", + points = F, + pointpos = 1.5, + jitter = 0, + scalemode = 'count', + meanline = list(visible = F), + name = ~x, + colors = colors, + color = ~x, + split = ~x, + line = list(color = 'black', width = 1.1), + box = list(visible = T), + spanmode = 'hard', + showlegend = show.legend, + ... + ) + if (is_new_plot) { + p %<>% layout(yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + }, + 'line' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + + # Force legend to be categorical + df[, l_orig := l] + if (is.numeric(df[['l']])) { + df[, l := paste0('A', l)] + names(colors) <- paste0('A', names(colors)) + } + + #Use linestyles to differentiate traces if only one attribute is selected to be plotted + #TODO: Combine these two options more elegantly + if (length(y_attr) == 1) { + dashes <- get_line_style(xs) + names(dashes) <- xs + colnames(df)[colnames(df) == y_attr] <- "y" + + df[, isinf := is.infinite(y)] + df[, text := as.character(round(y, getOption("IOHanalyzer.precision", 2)))] + + if (inf.action == 'overlap') { + maxval <- max(df[isinf == F, 'y']) + df[['y']][df[['isinf']]] <- 10**(ceiling(log10(maxval)) + 1) + } + else if (inf.action == 'jitter') { + #TODO: Faster way to compute this + maxval <- max(df[isinf == F, 'y']) + for (xval in unique(df[['x']])) { + tempval <- 10**(ceiling(log10(maxval)) + 1) + for (lval in unique(df[['l']])) { + temp <- df[l == lval][x == xval] + if (nrow(temp) > 0 && df[l == lval][x == xval][['isinf']]) { + df[l == lval][x == xval][['y']] <- tempval + tempval <- 1.2 * tempval + } + } + } + } + + suppressWarnings( + p %<>% + add_trace( + data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, + type = 'scatter', mode = 'lines+markers', + linetype = ~l_orig, marker = list(size = getOption('IOHanalyzer.markersize', 4)), + linetypes = dashes, + colors = colors, showlegend = show.legend, + text = ~text, line = list(width = getOption('IOHanalyzer.linewidth', 2)), + hovertemplate = '%{text}', + ... + ) + ) + if (inf.action != 'none') { + p %<>% add_trace(data = df[isinf == T], x = ~x, y = ~y, legendgroup = ~l_orig, name = ~l_orig, + type = 'scatter', mode = 'markers', color = ~l, + marker = list(symbol = 'circle-open', size = 8 + getOption('IOHanalyzer.markersize', 4)), + colors = colors, showlegend = F, text = 'Inf', hoverinfo = 'none', + ... + ) + } + + } + else { + if (inf.action != 'none') { + warning("inf.action is not yet supported for multiple y-attributes") + } + + dashes_full <- rep(c("solid", "dot", "dash", "longdash", "dashdot", "longdashdot"), + ceiling(length(y_attr)/3))[1:length(y_attr)] + names(dashes_full) <- y_attr + + for (y_atr in y_attr) { + colnames(df)[colnames(df) == y_atr] <- "y" + + #TODO: Figure out how to supress warning about 6 linetypes + dashstyle <- dashes_full[[y_atr]] + suppressWarnings( + p %<>% + add_trace( + data = df, x = ~x, y = ~y, color = ~l, legendgroup = ~l_orig, name = ~l_orig, + type = 'scatter', mode = 'lines+markers', + marker = list(size = getOption('IOHanalyzer.markersize', 4)), linetype = dashstyle, + colors = colors, showlegend = show.legend, name = ~l, + text = y_atr, line = list(width = getOption('IOHanalyzer.linewidth', 2)), + ... + ) + ) + colnames(df)[colnames(df) == "y"] <- y_atr + show.legend <- F + } + } + if (is_new_plot) { + if (is.numeric(df[['x']])) + p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + else + p %<>% layout(xaxis = list(type = 'category', tickfont = f3(), ticklen = 3), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + + } + }, + 'ribbon' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + if (is.null(upper_attr) || is.null(lower_attr)) { + stop("No upper or lower attribute provided for ribbon-plot") + } + + for (name in xs) { + df_small <- df[l == name] + legend_name <- as.character(name) + rgba_str <- paste0('rgba(', paste0(col2rgb(colors[[name]]), collapse = ','), ',0.2)') + p %<>% + add_trace(data = df_small, x = ~x, y = ~upper, type = 'scatter', mode = 'lines', + line = list(color = rgba_str, width = 0), legendgroup = legend_name, + showlegend = F, name = 'upper', ...) %>% + add_trace(x = ~x, y = ~lower, type = 'scatter', mode = 'lines', + fill = 'tonexty', line = list(color = 'transparent'), legendgroup = legend_name, + fillcolor = rgba_str, showlegend = F, name = 'lower', ...) + } + + + + if (is_new_plot) { + p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + }, + 'radar' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + if (is.numeric(df[['x']])) { + if (stri_detect_regex(x_attr, "(?i)fun")) + df <- df[, x := paste0('F', sprintf("%02d", x))] + else if (stri_detect_regex(x_attr, "(?i)dim")) + df <- df[, x := paste0('D', as.character(x))] + else + df <- df[, x := as.character(x)] + } + df <- df[, col := add_transparancy(colors, 0.4)[l]] + p %<>% + add_trace(data = df, type = 'scatterpolar', r = ~y, + theta = ~x, mode = 'markers', #marker = list(color = 'lightgrey', size=0), + fill = 'toself', connectgaps = T, fillcolor = ~col, color = ~l, colors = colors, + name = ~l, legendgroup = ~l, ...) + if (is_new_plot) { + p %<>% layout(polar = list(radialaxis = list(type = yscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)))) + } + }, + 'hist' = { + if (legend_attr == x_attr) { + stop("Duplicated attribute selected for x-axis and legend.") + } + if (!'width' %in% colnames(df)) { + stop("No 'width'-column included in the provided dataframe. This is required for a histogram-plot") + } + p %<>% + add_trace(data = df, x = ~x, y = ~y, width = ~width, type = 'bar', + name = ~l, text = ~text, hoverinfo = 'text', + colors = add_transparancy(colors, 0.6), color = ~l, + marker = list(line = list(color = 'rgb(8,48,107)')), + ...) + + if (is_new_plot) { + p %<>% layout(xaxis = list(type = xscale, tickfont = f3(), ticklen = 3, + autorange = ifelse(scale.reverse, "reversed", T)), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + }, + 'bar' = { + if (legend_attr != x_attr) { + warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name") + } + colors = add_transparancy(colors, 0.6) + for (xv in xs) { + p %<>% + add_trace(x = xv, y = df[x == xv, y], type = 'bar', + name = xv, + color = colors[xv], + marker = list(line = list(color = 'rgb(8,48,107)')), + ...) + } + + if (is_new_plot) { + p %<>% layout(xaxis = list(tickfont = f3(), ticklen = 3), + yaxis = list(type = yscale, tickfont = f3(), ticklen = 3)) + } + } + ) + return(p) +} + + +#' Create the PerformViz plot +#' +#' From the paper: +#' +#' @param DSC_rank_result The result from a call to DSCtool rank service (`get_dsc_rank`) +#' +#' @return A performviz plot +#' @export +#' @examples +#' \dontrun{ +#' Plot.Performviz(get_dsc_rank(dsl)) +#' } +Plot.Performviz <- function(DSC_rank_result) { + if (!requireNamespace("ComplexHeatmap", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + if (!requireNamespace("reshape2", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + if (!requireNamespace("grid", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + mlist <- DSC_rank_result$ranked_matrix + + problem <- NULL #Assign variable to remove warnings + # df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + # function(x) { + # list(algorithm = x$algorithm, rank = x$rank) + # })) + # df_temp[, problem := mlist[[problem_idx]]$problem] + + df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { + df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + function(x) { + list(algorithm = x$algorithm, rank = x$rank) + })) + df_temp[, problem := mlist[[problem_idx]]$problem] + })) + + rank_matrix <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') + df <- rank_matrix + # colnames(df)<-index + # rownames(df)<-vector + # Define some graphics to display the distribution of columns + # library(ComplexHeatmap) + .hist = ComplexHeatmap::anno_histogram(df, gp = grid::gpar(fill = "lightblue")) + .density = ComplexHeatmap::anno_density(df, type = "line", gp = grid::gpar(col = "blue")) + ha_mix_top = ComplexHeatmap::HeatmapAnnotation(hist = .hist, density = .density) + # Define some graphics to display the distribution of rows + .violin = ComplexHeatmap::anno_density(df, type = "violin", + gp = grid::gpar(fill = "lightblue"), which = "row") + .boxplot = ComplexHeatmap::anno_boxplot(df, which = "row") + ha_mix_right = ComplexHeatmap::HeatmapAnnotation(violin = .violin, bxplt = .boxplot, + which = "row", width = grid::unit(4, "cm")) + # Combine annotation with heatmap + heatmap_main <- ComplexHeatmap::Heatmap(df, name = "Ranking", + column_names_gp = grid::gpar(fontsize = 8), + top_annotation = ha_mix_top, + top_annotation_height = grid::unit(3.8, "cm")) + return(ComplexHeatmap::draw( + ComplexHeatmap::`+.AdditiveUnit`(heatmap_main, ha_mix_right)) + ) +} diff --git a/R/readFiles.R b/R/readFiles.R index 9b91b35a..78348df6 100644 --- a/R/readFiles.R +++ b/R/readFiles.R @@ -1,889 +1,957 @@ -# sourceCpp('src/align.cc') -# sourceCpp('src/read.cc') - -#' Reduce the size of the data set by evenly subsampling the records -#' -#' @param df The data to subsample -#' @param n The amount of samples -#' @return A smaller data.frame -limit.data <- function(df, n) { - N <- nrow(df) - if (N > n) { - idx <- unique(c(1, seq(1, N, length.out = n), N)) - df[idx, ] - } else - df -} - -#' Scan *.info files for IOHProfiler or COCO -#' -#' @param folder The folder containing the .info files -#' @return The paths to all found .info-files -#' @export -#' @note This automatically filetrs our files of size 0 -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -#' scan_index_file(path) -scan_index_file <- function(folder) { - folder <- trimws(folder) - files <- list.files(folder, pattern = '.info$', recursive = T, full.names = T) - files[file.size(files) > 0] -} - -#' Read .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @export -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -#' info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) -read_index_file <- function(fname) { - tryCatch( - read_index_file__IOH(fname), - warning = function(e) read_index_file__COCO(fname), - error = function(e) read_index_file__COCO(fname), - finally = function(e) stop(paste0('Error in reading .info files ', e)) - ) -} - -#' Read IOHprofiler-based .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @noRd -read_index_file__IOH <- function(fname) { - f <- file(fname, 'r') - path <- dirname(fname) - data <- list() - i <- 1 - - while (TRUE) { - # TODO: remove suppressWarnings later - lines <- suppressWarnings(readLines(f, n = 3)) - if (length(lines) == 0) - break - - # TODO: make this quote symbol ' or " as the configurable parameter - # TODO: Fix this - name_value <- read.csv(text = lines[1], header = F, quote = c("\"","'")) %>% - as.list %>% - unlist %>% - as.vector - - header <- name_value %>% - trimws %>% { - regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' - } %>% - unlist %>% - trimws %>% - matrix(nrow = 2) %>% { - ans <- as.list(.[2, ]) - names(ans) <- .[1, ] - for (name in .[1, ]) { - value <- ans[[name]] - ans[[name]] <- gsub("'", '', value) - - if (name == 'maximization') - value <- as.logical(value) - else - value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - - if (!is.na(value)) ans[[name]] <- value - } - ans - } - - record <- trimws(strsplit(lines[3], ',')[[1]]) - - # TODO: this must also be removed... - if (record[2] == "") { - warning(sprintf('File %s is incomplete!', fname)) - finalFVs <- NULL - instances <- NULL - maxRTs <- NULL - } else { - res <- matrix(unlist(strsplit(record[-1], ':')), nrow = 2) - info <- matrix(unlist(strsplit(res[2, ], '\\|')), nrow = 2) - #Check for incorrect usages of reset_problem and remove them - maxRTs <- as.numeric(info[1,]) - idx_correct <- which(maxRTs > 0) - finalFVs <- as.numeric(info[2,])[idx_correct] - instances <- as.numeric(res[1,])[idx_correct] - maxRTs <- maxRTs[idx_correct] - } - - record[1] <- gsub("\\\\", "/", record[1]) - datafile <- file.path(path, record[1]) - - # TODO: check the name of the attributes and fix them! - data[[i]] <- c( - header, - list( - comment = lines[2], - datafile = datafile, - instance = instances, - maxRT = maxRTs, - finalFV = finalFVs - ) - ) - i <- i + 1 - } - close(f) - data -} - -#' Read single-objective COCO-based .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @noRd -read_index_file__COCO <- function(fname) { - f <- file(fname, 'r') - path <- dirname(fname) - data <- list() - i <- 1 - while (TRUE) { - - lines <- suppressWarnings(readLines(f, n = 3)) # read header and comments - if (length(lines) < 3) { - break - } - comment <- lines[2] - name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) - - header <- trimws(name_value) %>% { - regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' - } %>% - unlist %>% - trimws %>% - matrix(nrow = 2) %>% { - ans <- as.list(.[2, ]) - names(ans) <- .[1, ] - for (name in .[1, ]) { - value <- ans[[name]] - ans[[name]] <- gsub("'", '', value) - value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - if (!is.na(value)) - ans[[name]] <- value - } - ans - } - - names(header) <- gsub('algorithm', 'algId', names(header)) - - record <- strsplit(lines[3], ',')[[1]] %>% trimws - - if (length(record) < 2) { - warning(sprintf('File %s is incomplete!', fname)) - res <- NULL - info <- NULL - } else { - res <- matrix(unlist(strsplit(record[-c(1)], ':')), nrow = 2) - info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) - } - - record[1] <- gsub("\\\\", "/", record[1]) - if ('folder' %in% names(header)) - datafile <- file.path(path, header$folder, record[1]) - else - datafile <- file.path(path, record[1]) - - - # TODO: check the name of the attributes and fix them! - data[[i]] <- c( - header, - list( - comment = comment, - datafile = datafile, - instance = as.numeric(res[1, ]), - maxRT = info[1, ], - finalFV = info[2, ] - ) - ) - i <- i + 1 - } - close(f) - data -} - -#' Read bi-objective COCO-based .info files and extract information -#' -#' @param fname The path to the .info file -#' @return The data contained in the .info file -#' @noRd -read_index_file__BIOBJ_COCO <- function(fname) { - f <- file(fname, 'r') - path <- dirname(fname) - data <- list() - i <- 1 - - lines <- suppressWarnings(readLines(f, n = 2)) # read header and comments - comment <- lines[2] - name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) - - header <- trimws(name_value) %>% { - regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' - } %>% - unlist %>% - trimws %>% - matrix(nrow = 2) %>% { - ans <- as.list(.[2, ]) - names(ans) <- .[1, ] - for (name in .[1, ]) { - value <- ans[[name]] - ans[[name]] <- gsub("'", '', value) - value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric - if (!is.na(value)) - ans[[name]] <- value - } - ans - } - - names(header) <- gsub('algorithm', 'algId', names(header)) - while (TRUE) { - # TODO: remove suppressWarnings later - lines <- suppressWarnings(readLines(f, n = 1)) - if (length(lines) == 0) - break - - record <- strsplit(lines[1], ',')[[1]] %>% trimws - - # TODO: this must also be removed... - if (record[4] == "") { - warning(sprintf('File %s is incomplete!', fname)) - res <- NULL - info <- NULL - } else { - res <- matrix(unlist(strsplit(record[-c(1, 2, 3)], ':')), nrow = 2) - info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) - } - - record[3] <- gsub("\\\\", "/", record[3]) - if ('folder' %in% names(header)) - datafile <- file.path(path, header$folder, record[3]) - else - datafile <- file.path(path, record[3]) - - funcId <- trimws(strsplit(record[1], '=')[[1]][2]) - funcId.int <- suppressWarnings(as.integer(funcId.int)) - if(!any(is.na(funcId.int))) { - if(all((funcId.int >= 0L) & (funcId.int <= 1000000000L))) { - funcId <- funcId.int - } - } - - DIM <- as.numeric(trimws(strsplit(record[2], '=')[[1]][2])) - - # TODO: check the name of the attributes and fix them! - data[[i]] <- c( - header, - list( - comment = comment, - funcId = funcId, - DIM = DIM, - datafile = datafile, - instance = as.numeric(res[1, ]), - maxRT = info[1, ], - finalFV = info[2, ] - ) - ) - i <- i + 1 - } - close(f) - data -} - -#' Check the format of data -#' -#' Throws a warning when multiple formats are found in the same folder. -#' -#' @param path The path to the folder to check -#' @return The format of the data in the given folder. Either 'COCO', 'IOHprofiler', -#' 'NEVERGRAD' or 'SOS'. -#' @export -#' @examples -#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -#' check_format(path) -check_format <- function(path) { - if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") - return(NEVERGRAD) - - if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "rds") - return("RDS") - - index_files <- scan_index_file(path) - if (length(index_files) == 0) - return(SOS) - - info <- unlist(lapply(index_files, read_index_file), recursive = F) - datafile <- sapply(info, function(item) item$datafile) - - format <- lapply(datafile, function(file) { - tryCatch({ - if (!file.exists(file)) { - cdatfile <- stri_replace(file, ".cdat", fixed = ".dat") - tdatfile <- stri_replace(file, ".tdat", fixed = ".dat") - if (file.exists(cdatfile)) file <- cdatfile - else file <- tdatfile - } - first_line <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) - }, error = function(e) { - stop("Error detecting data files specified in .info, please verify the - integrity of the provided files.") - }) - - if (startsWith(first_line, '% function') || startsWith(first_line, '% f evaluations')) - COCO - else if (startsWith(first_line, '\"function')) { - n_col <- ncol(fread(file, header = FALSE, sep = ' ', - colClasses = 'character', fill = T, nrows = 1)) - if (n_col == 2) - TWO_COL - else - IOHprofiler - } - else if (first_line == '%') # Bi-objective COCO format... - BIBOJ_COCO - else { - stop("Error detecting file format of file ", file, "; Please verify - the integrity of this file.") - } - }) %>% - unlist %>% - unique - - csv_files <- file.path(path, list.files(path, pattern = '.csv', recursive = T)) - if (length(csv_files) > 0) - format <- c(format, NEVERGRAD) - - txt_files <- file.path(path, list.files(path, pattern = '.txt', recursive = T)) - if (length(txt_files) > 0) - format <- c(format, SOS) - - if (length(format) > 1) { - stop( - paste( - path, - 'contains multiple data formats. This is not allowed for data processing. - Please check the returned dataframe for more information.' - ) - ) - } else - format -} - -#' Read IOHProfiler *.dat files -#' -#' @param fname The path to the .dat file -#' @param subsampling Whether to subsample the data or not -#' @noRd -#' @return A list of data.frames -read_dat <- function(fname, subsampling = FALSE) { - # TODO: use the same data loading method as in read_dat__COCO - df <- fread(fname, header = FALSE, sep = ' ', colClasses = 'character', fill = T) - colnames(df) <- as.character(df[1, ]) - idx <- which(!grepl('\\d+', df[[1]], perl = T)) - - # check for data consistence - header_len <- min(apply(df[idx, ] != "", 1, sum)) - idx <- c(idx, nrow(df) + 1) - df <- df[, 1:header_len] - - # turn off the warnings of the data coersion below - options(warn = -1) - # TOOD: this opeartor is the bottelneck - df <- sapply(df, function(c) {class(c) <- 'numeric'; c}) - options(warn = 0) - - res <- lapply(seq(length(idx) - 1), function(i) { - i1 <- idx[i] + 1 - i2 <- idx[i + 1] - 1 - ans <- df[i1:i2, ] - if (i1 == i2) - ans <- as.matrix(t(ans)) - - # TODO: determine the number of record in the 'efficient mode' - if (subsampling) - ans <- limit.data(ans, n = 500) - else - ans - }) - res -} - -# TODO: this method is deprecated. Remove it later -# TODO: maybe not subsampling for COCO data -#' read COCO '.dat'-like file -#' -#' @param fname The path to the .dat file -#' @param subsampling Whether to subsample the data or not -#' @noRd -#' @return A list of data.frames -read_dat__COCO_ <- function(fname, subsampling = FALSE) { - c_read_dat(path.expand(fname), 7, '%') -} - -#' read COCO '.dat'-like file directly in R -#' -#' @param fname The path to the .dat file -#' @param subsampling Whether to subsample the data or not -#' @noRd -#' @return A list of data.frames -read_dat__COCO <- function(fname, subsampling = FALSE) { - select <- seq(5) - # read the file as a character vector (one string per row) - X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] - idx <- which(startsWith(X, '%')) - X <- gsub('\\s+|\\t', ' ', X, perl = T) - - df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) - idx <- c((idx + 1) - seq_along(idx), nrow(df)) - - lapply(seq(length(idx) - 1), - function(i) { - i1 <- idx[i] - i2 <- idx[i + 1] - 1 - as.matrix(df[i1:i2, ]) - }) -} - -read_dat__BIOBJ_COCO <- function(fname, subsampling = FALSE) { - if (endsWith(fname, '.dat')) - select <- seq(3) - else if (endsWith(fname, '.tdat')) - select <- seq(2) - - # read the file as a character vector (one string per row) - X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] - idx <- which(startsWith(X, '%')) - X <- gsub('\\s+|\\t', ' ', X, perl = T) - - df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) - idx <- which(startsWith(X, '% function')) - idx <- c((idx + 1) - seq_along(idx) * 4, nrow(df)) - - lapply(seq(length(idx) - 1), function(i) { - i1 <- idx[i] - i2 <- idx[i + 1] - 1 - as.matrix(df[i1:i2, ]) - }) -} - -# global variables for the alignment functions -idxEvals <- 1 -idxTarget <- 3 -n_data_column <- 5 - -# TODO: add docs to the following three functions -check_contiguous <- function(data) { - sapply(data, - function(d) { - v <- d[, idxEvals] - N <- length(v) - v[1] == 1 && v[N] == N - }) %>% - all -} - -align_contiguous <- function(data, idx, rownames) { - N <- length(data) - nrow <- length(rownames) - lapply(data, - function(d) { - v <- d[, idx] - r <- nrow - length(v) - if (r > 0) { - v <- c(v, rep(v[length(v)], r)) - } - v - }) %>% - unlist %>% - matrix(nrow = nrow, ncol = N) %>% - set_rownames(rownames) -} - -align_non_contiguous <- function(data, idx, rownames) { - N <- length(data) - nrow <- length(rownames) - lapply(data, - function(d) { - c_impute(d[, idx], d[, idxEvals], rownames) - }) %>% - unlist %>% - matrix(nrow = nrow, ncol = N) %>% - set_rownames(rownames) -} - -#' Align data by runtimes -#' @param data The data to align -#' @param format Whether the data is form IOHprofiler or COCO -#' @param include_param Whether to include the recorded parameters in the alignment -#' @noRd -#' @return Data aligned by the running time -align_running_time <- function(data, format = IOHprofiler, include_param = TRUE, - maximization = TRUE) { - if (format == IOHprofiler) - idxTarget <- 3 - else if (format == COCO) - idxTarget <- 3 - else if (format == BIBOJ_COCO) { - n_data_column <- 3 - idxTarget <- 2 - } - else if (format == TWO_COL) { - n_data_column <- 2 - idxTarget <- 2 - } - - FV <- sort(unique(unlist(lapply(data, function(x) x[, idxTarget]))), - decreasing = !maximization) - n_column <- unique(sapply(data, ncol)) - - if (format == COCO) { - n_param <- 0 - idxValue <- idxEvals - param_names <- NULL - } - else if (format == IOHprofiler) { - n_param <- n_column - n_data_column - if (include_param && n_param > 0) { - param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] - idxValue <- c(idxEvals, (n_data_column + 1):n_column) - } - else { - param_names <- NULL - idxValue <- idxEvals - } - } - else { - param_names <- NULL - idxValue <- idxEvals - } - - res <- c_align_running_time(data, FV, idxValue - 1, maximization, idxTarget - 1) - names(res) <- c('RT', param_names) - res -} - -#' Align data by function values -#' @param data The data to align -#' @param format Whether the data is form IOHprofiler or COCO. -#' @param include_param Whether to include the recorded parameters in the alignment -#' @noRd -#' @return Data aligned by the function value -align_function_value <- function(data, include_param = TRUE, format = IOHprofiler) { - n_column <- unique(sapply(data, ncol)) - stopifnot(length(n_column) == 1) - - if (format == COCO) { - maximization <- FALSE - idxTarget <- 3 - n_param <- 0 - } - else if (format == IOHprofiler) { - maximization <- TRUE - idxTarget <- 3 - n_param <- n_column - n_data_column - } - else if (format == BIBOJ_COCO) { # bi-objective COCO format - maximization <- FALSE - idxTarget <- 2 - n_data_column <- 2 - n_param <- 0 # no parameter is allowed in this case - } - else if (format == TWO_COL) { - maximization <- TRUE - idxTarget <- 2 - n_param <- 0 - } - - if (check_contiguous(data)) { - nrow <- sapply(data, nrow) %>% max - runtime <- seq(nrow) - align_func <- align_contiguous - } else { - runtime <- sort(unique(unlist(lapply(data, function(x) x[, idxEvals])))) - nrow <- length(runtime) - align_func <- align_non_contiguous - } - - FV <- align_func(data, idxTarget, runtime) - include_param <- include_param && (n_param > 0) - - if (include_param) { - param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] - param <- list() - for (i in seq(n_param)) { - name <- param_names[i] - param[[name]] <- align_func(data, i + n_data_column, runtime) - } - } - - if (include_param) { - c(list(FV = FV), param) - } else { - list(FV = FV) - } -} - - -#' Read Nevergrad data -#' -#' Read .csv files in nevergrad format and extract information as a DataSetList -#' -#' @param fname The path to the .csv file -#' @return The DataSetList extracted from the .csv file provided -#' @noRd -read_nevergrad <- function(path){ - dt <- fread(path) - - if (!'name' %in% colnames(dt)) { - dt[, name := function_class] - } - - triplets <- unique(dt[, .(optimizer_name, dimension, name)]) - algIds <- unique(triplets$optimizer_name) - DIMs <- unique(triplets$dimension) - funcIds <- unique(triplets$name) - - res <- list() - - idx <- 1 - - for (i in seq(nrow(triplets))) { - algId <- triplets$optimizer_name[i] - DIM <- triplets$dimension[i] - funcId <- triplets$name[i] - - rescale_name <- 'rescale' - if ( !('rescale' %in% colnames(dt))) { - if ( 'transform' %in% colnames(dt)) - colnames(dt)[colnames(dt) == "transform"] <- "rescale" - else{ - dt$rescale <- NA - } - } - - data <- dt[optimizer_name == algId & dimension == DIM & name == funcId, - .(budget, loss, rescale)] - - for (scaled in unique(data$rescale)) { - if (!is.na(scaled)) { - data_reduced <- data[rescale == scaled, .(budget, loss)] - } - else { - data_reduced <- data[is.na(rescale), .(budget, loss)] - } - - if (!is.na(scaled) && scaled) { - funcId_name <- paste0(funcId, '_rescaled') - } - else { - funcId_name <- funcId - } - - rows <- unique(data_reduced$budget) %>% sort - FV <- lapply(rows, - function(b) { - data_reduced[budget == b, loss] - } - ) %>% - do.call(rbind, .) %>% - set_rownames(rows) - - RT <- list() - - ds <- structure(list(RT = RT, FV = FV), - class = c('DataSet', 'list'), - maxRT = max(rows), - finalFV = min(FV), - format = 'NEVERGRAD', - maximization = FALSE, - algId = algId, - funcId = funcId_name, - DIM = DIM) - res[[idx]] <- ds - idx <- idx + 1 - } - } - class(res) %<>% c('DataSetList') - attr(res, 'DIM') <- DIMs - attr(res, 'funcId') <- funcIds - attr(res, 'algId') <- algIds - attr(res, 'suite') <- 'NEVERGRAD' - attr(res, 'maximization') <- F - res - -} - -#' Read single DataSet of SOS-based data -#' -#' Read single .txt files in SOS format and extract information as a DataSet -#' -#' @param file The path to the .txt file -#' @return The DataSet extracted from the .txt file provided -#' @noRd -read_single_file_SOS <- function(file) { - V1 <- NULL #Local binding to remove CRAN warnings - - algId <- substr(basename(file), 1, stringi::stri_locate_last(basename(file), fixed = 'D')[[1]] - 1) - - dt <- fread(file, header = F) - header <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) - splitted <- header %>% trimws %>% strsplit("\\s+") %>% .[[1]] %>% .[2:length(.)] - info <- list(algId = algId) - for (i in seq_len(length(splitted) / 2)) { - temp <- splitted[[2*i]] - name <- splitted[[2*i - 1]] - if (name == 'function') name <- 'funcId' - if (name == 'dim') name <- 'DIM' - names(temp) <- name - info <- c(info, temp) - } - - dim <- as.numeric(info$DIM) - - - RT_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] - names(RT_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] - RT <- as.matrix(RT_raw) - mode(RT) <- 'integer' - - FV_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] - names(FV_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] - FV <- as.matrix(FV_raw) - - - pos <- dt[, (ncol(dt) - dim + 1):ncol(dt)] - colnames(pos) <- as.character(seq_len(dim)) - - maxRT <- max(RT) - finalFV <- min(FV) - - if (sum(FV == finalFV) > 1) { - #Reconstruct population to determine which best solution is final position - ids_min <- dt[FV_raw == finalFV, V1] - replaced_idxs <- dt[[colnames(dt)[[ncol(dt) - dim]]]] - #If none, take the last one added - pos_idx <- max(ids_min) - for (i in ids_min) { - if (all(replaced_idxs != i)) { - #If multiple, take the first one added - pos_idx <- i - break - } - } - final_pos <- as.numeric(pos[pos_idx, ]) - } - else { - final_pos <- as.numeric(pos[which.min(FV), ]) - } - - PAR <- list( - # 'position' = list(pos), - 'final_position' = list(final_pos), - 'by_FV' = NULL, - 'by_RT' = NULL - ) - - - - object <- list() - class(object) <- c('DataSet', class(object)) - object$RT <- RT - object$FV <- FV - object$PAR <- PAR - attr(object, 'maxRT') <- maxRT - attr(object, 'finalFV') <- finalFV - attr(object, 'format') <- "SOS" - attr(object, 'maximization') <- F - attr(object, 'suite') <- "SOS" - for (i in seq_along(info)) { - attr(object, names(info)[[i]]) <- type.convert(info[[i]], as.is = T) - } - object -} - - -#' Read DataSetList of SOS-based data -#' -#' Read directory containing .txt files in SOS format and extract information as a DataSetList -#' -#' @param dir The path to the directory file -#' @param corrections_file A file containing boundary-correction ratios for the files in `dir` -#' @return The DataSetList extracted from the directory provided -#' @noRd -read_datasetlist_SOS <- function(dir, corrections_files = NULL) { - V1 <- V3 <- V4 <- NULL #Local binding to remove CRAN warnings - res <- list() - dims <- list() - funcIds <- list() - algIds <- list() - suites <- list() - maximizations <- list() - - idx <- 1 - - corrs <- as.data.table(rbindlist(lapply(corrections_files, fread))) - - for (f in list.files(dir, recursive = T, pattern = "*.txt", full.names = T)) { - if (f %in% corrections_files) next - ds <- read_single_file_SOS(f) - - dims[[idx]] <- attr(ds, 'DIM') - funcIds[[idx]] <- attr(ds, 'funcId') - algIds[[idx]] <- attr(ds, 'algId') - suites[[idx]] <- attr(ds, 'suite') - maximizations[[idx]] <- attr(ds, 'maximization') - - if (nrow(corrs) > 0) { - fn <- substr(basename(f), 1, nchar(basename(f)) - 4) - corr_opts <- corrs[V1 == fn, ] - if (stri_detect_fixed(fn, "DE")) { - corr <- corr_opts[V3 == attr(ds, 'F'), ][V4 == attr(ds, 'CR'), 'V2'][['V2']] - } - else if (stri_detect_fixed(fn, "RIS")) { - corr <- corr_opts[['V2']] - } - else { - warning("Unknown algorithm, so skipping lookup of boundary corrections ratio") - corr <- NULL - } - if (length(corr) == 1) - ds$PAR$'corrections' <- corr[[1]] - else - warning(paste0("No boundary corrections ratio found for ", fn)) - } - - res[[idx]] <- ds - idx <- idx + 1 - } - class(res) %<>% c('DataSetList') - attr(res, 'DIM') <- dims - attr(res, 'funcId') <- funcIds - attr(res, 'algId') <- algIds - - suite <- unique(suites) - maximization <- unique(maximizations) - if (length(suite) != 1 || length(maximization) != 1) { - warning("Multipe different suites detected!") - } - - attr(res, 'suite') <- suite - attr(res, 'maximization') <- maximization - res - clean_DataSetList(res) -} - -#' Find corrections-files in SOS-based folder -#' -#' Read directory containing .txt files in SOS format and extract the corrections-files -#' -#' @param path The path to the directory file -#' @return The relative paths to the corection files -#' @noRd -locate_corrections_files <- function(path) { - files <- list.files(path, recursive = T, pattern = "*.txt", full.names = T) - files[stri_detect_fixed(files, 'corrections')] -} +# sourceCpp('src/align.cc') +# sourceCpp('src/read.cc') + +#' Reduce the size of the data set by evenly subsampling the records +#' +#' @param df The data to subsample +#' @param n The amount of samples +#' @return A smaller data.frame +limit.data <- function(df, n) { + N <- nrow(df) + if (N > n) { + idx <- unique(c(1, seq(1, N, length.out = n), N)) + df[idx, ] + } else + df +} + +#' Scan *.info files for IOHProfiler or COCO +#' +#' @param folder The folder containing the .info and .json files +#' @return The paths to all found .info and .json-files +#' @export +#' @note This automatically filetrs our files of size 0 +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +#' scan_index_file(path) +scan_index_file <- function(folder) { + folder <- trimws(folder) + files <- list.files(folder, pattern = '.(info|json)$', recursive = T, full.names = T) + files[file.size(files) > 0] +} + +#' Read .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @export +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +#' info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) +read_index_file <- function(fname) { + format <- tools::file_ext(fname) + if (format == 'json') + read_index_file__json(fname) + else { + tryCatch( + read_index_file__IOH(fname), + warning = function(e) read_index_file__COCO(fname), + error = function(e) read_index_file__COCO(fname), + finally = function(e) stop(paste0('Error in reading .info files ', e)) + ) + } +} + +#' Read IOHprofiler-based .json files and extract information +#' +#' @param fname The path to the json info-file +#' @return The data contained in the json info-file +#' @noRd +read_index_file__json <- function(fname) { + + json_data <- fromJSON(file = fname) + base_dir <- dirname(fname) + exp_attrs <- sapply(json_data$experiment_attributes, function(x) {x}) + + data <- list() + tryCatch({ + fid <- json_data$function_id + fname <- json_data$function_name + suite <- json_data$suite + maximization <- json_data$maximization + algid <- json_data$algorithm$name + }, error = function(e) {return(NULL)}) + + data <- lapply(json_data$scenarios, function(scenario) { + + run_attrs <- list() + + for (run_attr in json_data$run_attributes) { + attr(run_attrs, run_attr) <- sapply(scenario$runs, function(x) x$run_attr) + } + + datafile <- file.path(base_dir, scenario$path) + + temp <- c(list( + funcId = fid, + funcName = fname, + suite = suite, + maximization = maximization, + algId = algid, + DIM = scenario$dimension, + datafile = datafile, + instance = sapply(scenario$runs, function(x) x$instance), + maxRT = sapply(scenario$runs, function(x) x$evals), + finalFV = sapply(scenario$runs, function(x) x$best$y), + final_pos = sapply(scenario$runs, function(x) x$best$x) + ), run_attrs, + exp_attrs) + + }) + data +} + +#' Read IOHprofiler-based .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @noRd +read_index_file__IOH <- function(fname) { + f <- file(fname, 'r') + path <- dirname(fname) + data <- list() + i <- 1 + + while (TRUE) { + # TODO: remove suppressWarnings later + lines <- suppressWarnings(readLines(f, n = 3)) + if (length(lines) == 0) + break + + # TODO: make this quote symbol ' or " as the configurable parameter + # TODO: Fix this + name_value <- read.csv(text = lines[1], header = F, quote = c("\"","'")) %>% + as.list %>% + unlist %>% + as.vector + + header <- name_value %>% + trimws %>% { + regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' + } %>% + unlist %>% + trimws %>% + matrix(nrow = 2) %>% { + ans <- as.list(.[2, ]) + names(ans) <- .[1, ] + for (name in .[1, ]) { + value <- ans[[name]] + ans[[name]] <- gsub("'", '', value) + + if (name == 'maximization') + value <- as.logical(value) + else + value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric + + if (!is.na(value)) ans[[name]] <- value + } + ans + } + + record <- trimws(strsplit(lines[3], ',')[[1]]) + + # TODO: this must also be removed... + if (record[2] == "") { + warning(sprintf('File %s is incomplete!', fname)) + finalFVs <- NULL + instances <- NULL + maxRTs <- NULL + } else { + res <- matrix(unlist(strsplit(record[-1], ':')), nrow = 2) + info <- matrix(unlist(strsplit(res[2, ], '\\|')), nrow = 2) + #Check for incorrect usages of reset_problem and remove them + maxRTs <- as.numeric(info[1,]) + idx_correct <- which(maxRTs > 0) + finalFVs <- as.numeric(info[2,])[idx_correct] + instances <- as.numeric(res[1,])[idx_correct] + maxRTs <- maxRTs[idx_correct] + } + + record[1] <- gsub("\\\\", "/", record[1]) + datafile <- file.path(path, record[1]) + + # TODO: check the name of the attributes and fix them! + data[[i]] <- c( + header, + list( + comment = lines[2], + datafile = datafile, + instance = instances, + maxRT = maxRTs, + finalFV = finalFVs + ) + ) + i <- i + 1 + } + close(f) + data +} + +#' Read single-objective COCO-based .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @noRd +read_index_file__COCO <- function(fname) { + f <- file(fname, 'r') + path <- dirname(fname) + data <- list() + i <- 1 + while (TRUE) { + + lines <- suppressWarnings(readLines(f, n = 3)) # read header and comments + if (length(lines) < 3) { + break + } + comment <- lines[2] + name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) + + header <- trimws(name_value) %>% { + regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' + } %>% + unlist %>% + trimws %>% + matrix(nrow = 2) %>% { + ans <- as.list(.[2, ]) + names(ans) <- .[1, ] + for (name in .[1, ]) { + value <- ans[[name]] + ans[[name]] <- gsub("'", '', value) + value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric + if (!is.na(value)) + ans[[name]] <- value + } + ans + } + + names(header) <- gsub('algorithm', 'algId', names(header)) + + record <- strsplit(lines[3], ',')[[1]] %>% trimws + + if (length(record) < 2) { + warning(sprintf('File %s is incomplete!', fname)) + res <- NULL + info <- NULL + } else { + res <- matrix(unlist(strsplit(record[-c(1)], ':')), nrow = 2) + info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) + } + + record[1] <- gsub("\\\\", "/", record[1]) + if ('folder' %in% names(header)) + datafile <- file.path(path, header$folder, record[1]) + else + datafile <- file.path(path, record[1]) + + + # TODO: check the name of the attributes and fix them! + data[[i]] <- c( + header, + list( + comment = comment, + datafile = datafile, + instance = as.numeric(res[1, ]), + maxRT = info[1, ], + finalFV = info[2, ] + ) + ) + i <- i + 1 + } + close(f) + data +} + +#' Read bi-objective COCO-based .info files and extract information +#' +#' @param fname The path to the .info file +#' @return The data contained in the .info file +#' @noRd +read_index_file__BIOBJ_COCO <- function(fname) { + f <- file(fname, 'r') + path <- dirname(fname) + data <- list() + i <- 1 + + lines <- suppressWarnings(readLines(f, n = 2)) # read header and comments + comment <- lines[2] + name_value <- as.vector(unlist(as.list(read.csv(text = lines[1], header = F, quote = "'")))) + + header <- trimws(name_value) %>% { + regmatches(., regexpr("=", .), invert = T) # match the first appearance of '=' + } %>% + unlist %>% + trimws %>% + matrix(nrow = 2) %>% { + ans <- as.list(.[2, ]) + names(ans) <- .[1, ] + for (name in .[1, ]) { + value <- ans[[name]] + ans[[name]] <- gsub("'", '', value) + value <- suppressWarnings(as.numeric(value)) # convert quoted numeric values to numeric + if (!is.na(value)) + ans[[name]] <- value + } + ans + } + + names(header) <- gsub('algorithm', 'algId', names(header)) + while (TRUE) { + # TODO: remove suppressWarnings later + lines <- suppressWarnings(readLines(f, n = 1)) + if (length(lines) == 0) + break + + record <- strsplit(lines[1], ',')[[1]] %>% trimws + + # TODO: this must also be removed... + if (record[4] == "") { + warning(sprintf('File %s is incomplete!', fname)) + res <- NULL + info <- NULL + } else { + res <- matrix(unlist(strsplit(record[-c(1, 2, 3)], ':')), nrow = 2) + info <- matrix(as.numeric(unlist(strsplit(res[2, ], '\\|'))), nrow = 2) + } + + record[3] <- gsub("\\\\", "/", record[3]) + if ('folder' %in% names(header)) + datafile <- file.path(path, header$folder, record[3]) + else + datafile <- file.path(path, record[3]) + + funcId <- trimws(strsplit(record[1], '=')[[1]][2]) + funcId.int <- suppressWarnings(as.integer(funcId.int)) + if(!any(is.na(funcId.int))) { + if(all((funcId.int >= 0L) & (funcId.int <= 1000000000L))) { + funcId <- funcId.int + } + } + + DIM <- as.numeric(trimws(strsplit(record[2], '=')[[1]][2])) + + # TODO: check the name of the attributes and fix them! + data[[i]] <- c( + header, + list( + comment = comment, + funcId = funcId, + DIM = DIM, + datafile = datafile, + instance = as.numeric(res[1, ]), + maxRT = info[1, ], + finalFV = info[2, ] + ) + ) + i <- i + 1 + } + close(f) + data +} + +#' Check the format of data +#' +#' Throws a warning when multiple formats are found in the same folder. +#' +#' @param path The path to the folder to check +#' @return The format of the data in the given folder. Either 'COCO', 'IOHprofiler', +#' 'NEVERGRAD' or 'SOS'. +#' @export +#' @examples +#' path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +#' check_format(path) +check_format <- function(path) { + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "csv") + return(NEVERGRAD) + + if (sub('[^\\.]*\\.', '', basename(path), perl = T) == "rds") + return("RDS") + + index_files <- scan_index_file(path) + if (length(index_files) == 0) + return(SOS) + + info <- unlist(lapply(index_files, read_index_file), recursive = F) + datafile <- sapply(info, function(item) item$datafile) + + format <- lapply(datafile, function(file) { + tryCatch({ + if (!file.exists(file)) { + cdatfile <- stri_replace(file, ".cdat", fixed = ".dat") + tdatfile <- stri_replace(file, ".tdat", fixed = ".dat") + if (file.exists(cdatfile)) file <- cdatfile + else file <- tdatfile + } + first_line <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) + }, error = function(e) { + stop("Error detecting data files specified in .info, please verify the + integrity of the provided files.") + }) + + if (startsWith(first_line, '% function') || startsWith(first_line, '% f evaluations')) + COCO + else if (startsWith(first_line, '\"function')) { + n_col <- ncol(fread(file, header = FALSE, sep = ' ', + colClasses = 'character', fill = T, nrows = 1)) + if (n_col == 2) + TWO_COL + else + IOHprofiler + } + else if (first_line == '%') # Bi-objective COCO format... + BIBOJ_COCO + else { + stop("Error detecting file format of file ", file, "; Please verify + the integrity of this file.") + } + }) %>% + unlist %>% + unique + + csv_files <- file.path(path, list.files(path, pattern = '.csv', recursive = T)) + if (length(csv_files) > 0) + format <- c(format, NEVERGRAD) + + txt_files <- file.path(path, list.files(path, pattern = '.txt', recursive = T)) + if (length(txt_files) > 0) + format <- c(format, SOS) + + if (length(format) > 1) { + stop( + paste( + path, + 'contains multiple data formats. This is not allowed for data processing. + Please check the returned dataframe for more information.' + ) + ) + } else + format +} + +#' Read IOHProfiler *.dat files +#' +#' @param fname The path to the .dat file +#' @param subsampling Whether to subsample the data or not +#' @noRd +#' @return A list of data.frames +read_dat <- function(fname, subsampling = FALSE) { + # TODO: use the same data loading method as in read_dat__COCO + df <- fread(fname, header = FALSE, sep = ' ', colClasses = 'character', fill = T) + colnames(df) <- as.character(df[1, ]) + idx <- which(!grepl('\\d+', df[[1]], perl = T)) + + # check for data consistence + header_len <- min(apply(df[idx, ] != "", 1, sum)) + idx <- c(idx, nrow(df) + 1) + df <- df[, 1:header_len] + + # turn off the warnings of the data coersion below + options(warn = -1) + # TOOD: this opeartor is the bottelneck + df <- sapply(df, function(c) {class(c) <- 'numeric'; c}) + options(warn = 0) + + res <- lapply(seq(length(idx) - 1), function(i) { + i1 <- idx[i] + 1 + i2 <- idx[i + 1] - 1 + ans <- df[i1:i2, ] + if (i1 == i2) + ans <- as.matrix(t(ans)) + + # TODO: determine the number of record in the 'efficient mode' + if (subsampling) + ans <- limit.data(ans, n = 500) + else + ans + }) + res +} + +# TODO: this method is deprecated. Remove it later +# TODO: maybe not subsampling for COCO data +#' read COCO '.dat'-like file +#' +#' @param fname The path to the .dat file +#' @param subsampling Whether to subsample the data or not +#' @noRd +#' @return A list of data.frames +read_dat__COCO_ <- function(fname, subsampling = FALSE) { + c_read_dat(path.expand(fname), 7, '%') +} + +#' read COCO '.dat'-like file directly in R +#' +#' @param fname The path to the .dat file +#' @param subsampling Whether to subsample the data or not +#' @noRd +#' @return A list of data.frames +read_dat__COCO <- function(fname, subsampling = FALSE) { + select <- seq(5) + # read the file as a character vector (one string per row) + X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] + idx <- which(startsWith(X, '%')) + X <- gsub('\\s+|\\t', ' ', X, perl = T) + + df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) + idx <- c((idx + 1) - seq_along(idx), nrow(df)) + + lapply(seq(length(idx) - 1), + function(i) { + i1 <- idx[i] + i2 <- idx[i + 1] - 1 + as.matrix(df[i1:i2, ]) + }) +} + +read_dat__BIOBJ_COCO <- function(fname, subsampling = FALSE) { + if (endsWith(fname, '.dat')) + select <- seq(3) + else if (endsWith(fname, '.tdat')) + select <- seq(2) + + # read the file as a character vector (one string per row) + X <- fread(fname, header = FALSE, sep = '\n', colClasses = 'character')[[1]] + idx <- which(startsWith(X, '%')) + X <- gsub('\\s+|\\t', ' ', X, perl = T) + + df <- fread(text = X[-idx], header = F, sep = ' ', select = select, fill = T) + idx <- which(startsWith(X, '% function')) + idx <- c((idx + 1) - seq_along(idx) * 4, nrow(df)) + + lapply(seq(length(idx) - 1), function(i) { + i1 <- idx[i] + i2 <- idx[i + 1] - 1 + as.matrix(df[i1:i2, ]) + }) +} + +# global variables for the alignment functions +idxEvals <- 1 +idxTarget <- 3 +n_data_column <- 5 + +# TODO: add docs to the following three functions +check_contiguous <- function(data) { + sapply(data, + function(d) { + v <- d[, idxEvals] + N <- length(v) + v[1] == 1 && v[N] == N + }) %>% + all +} + +align_contiguous <- function(data, idx, rownames) { + N <- length(data) + nrow <- length(rownames) + lapply(data, + function(d) { + v <- d[, idx] + r <- nrow - length(v) + if (r > 0) { + v <- c(v, rep(v[length(v)], r)) + } + v + }) %>% + unlist %>% + matrix(nrow = nrow, ncol = N) %>% + set_rownames(rownames) +} + +align_non_contiguous <- function(data, idx, rownames) { + N <- length(data) + nrow <- length(rownames) + lapply(data, + function(d) { + c_impute(d[, idx], d[, idxEvals], rownames) + }) %>% + unlist %>% + matrix(nrow = nrow, ncol = N) %>% + set_rownames(rownames) +} + +#' Align data by runtimes +#' @param data The data to align +#' @param format Whether the data is form IOHprofiler or COCO +#' @param include_param Whether to include the recorded parameters in the alignment +#' @noRd +#' @return Data aligned by the running time +align_running_time <- function(data, format = IOHprofiler, include_param = TRUE, + maximization = TRUE) { + if (format == IOHprofiler) + idxTarget <- 3 + else if (format == COCO) + idxTarget <- 3 + else if (format == BIBOJ_COCO) { + n_data_column <- 3 + idxTarget <- 2 + } + else if (format == TWO_COL) { + n_data_column <- 2 + idxTarget <- 2 + } + + FV <- sort(unique(unlist(lapply(data, function(x) x[, idxTarget]))), + decreasing = !maximization) + n_column <- unique(sapply(data, ncol)) + + if (format == COCO) { + n_param <- 0 + idxValue <- idxEvals + param_names <- NULL + } + else if (format == IOHprofiler) { + n_param <- n_column - n_data_column + if (include_param && n_param > 0) { + param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] + idxValue <- c(idxEvals, (n_data_column + 1):n_column) + } + else { + param_names <- NULL + idxValue <- idxEvals + } + } + else { + param_names <- NULL + idxValue <- idxEvals + } + + res <- c_align_running_time(data, FV, idxValue - 1, maximization, idxTarget - 1) + names(res) <- c('RT', param_names) + res +} + +#' Align data by function values +#' @param data The data to align +#' @param format Whether the data is form IOHprofiler or COCO. +#' @param include_param Whether to include the recorded parameters in the alignment +#' @noRd +#' @return Data aligned by the function value +align_function_value <- function(data, include_param = TRUE, format = IOHprofiler) { + n_column <- unique(sapply(data, ncol)) + stopifnot(length(n_column) == 1) + + if (format == COCO) { + maximization <- FALSE + idxTarget <- 3 + n_param <- 0 + } + else if (format == IOHprofiler) { + maximization <- TRUE + idxTarget <- 3 + n_param <- n_column - n_data_column + } + else if (format == BIBOJ_COCO) { # bi-objective COCO format + maximization <- FALSE + idxTarget <- 2 + n_data_column <- 2 + n_param <- 0 # no parameter is allowed in this case + } + else if (format == TWO_COL) { + maximization <- TRUE + idxTarget <- 2 + n_param <- 0 + } + + if (check_contiguous(data)) { + nrow <- sapply(data, nrow) %>% max + runtime <- seq(nrow) + align_func <- align_contiguous + } else { + runtime <- sort(unique(unlist(lapply(data, function(x) x[, idxEvals])))) + nrow <- length(runtime) + align_func <- align_non_contiguous + } + + FV <- align_func(data, idxTarget, runtime) + include_param <- include_param && (n_param > 0) + + if (include_param) { + param_names <- colnames(data[[1]])[(n_data_column + 1):n_column] + param <- list() + for (i in seq(n_param)) { + name <- param_names[i] + param[[name]] <- align_func(data, i + n_data_column, runtime) + } + } + + if (include_param) { + c(list(FV = FV), param) + } else { + list(FV = FV) + } +} + + +#' Read Nevergrad data +#' +#' Read .csv files in nevergrad format and extract information as a DataSetList +#' +#' @param fname The path to the .csv file +#' @return The DataSetList extracted from the .csv file provided +#' @noRd +read_nevergrad <- function(path){ + dt <- fread(path) + + if (!'name' %in% colnames(dt)) { + dt[, name := function_class] + } + + triplets <- unique(dt[, .(optimizer_name, dimension, name)]) + algIds <- unique(triplets$optimizer_name) + DIMs <- unique(triplets$dimension) + funcIds <- unique(triplets$name) + + res <- list() + + idx <- 1 + + for (i in seq(nrow(triplets))) { + algId <- triplets$optimizer_name[i] + DIM <- triplets$dimension[i] + funcId <- triplets$name[i] + + rescale_name <- 'rescale' + if ( !('rescale' %in% colnames(dt))) { + if ( 'transform' %in% colnames(dt)) + colnames(dt)[colnames(dt) == "transform"] <- "rescale" + else{ + dt$rescale <- NA + } + } + + data <- dt[optimizer_name == algId & dimension == DIM & name == funcId, + .(budget, loss, rescale)] + + for (scaled in unique(data$rescale)) { + if (!is.na(scaled)) { + data_reduced <- data[rescale == scaled, .(budget, loss)] + } + else { + data_reduced <- data[is.na(rescale), .(budget, loss)] + } + + if (!is.na(scaled) && scaled) { + funcId_name <- paste0(funcId, '_rescaled') + } + else { + funcId_name <- funcId + } + + rows <- unique(data_reduced$budget) %>% sort + FV <- lapply(rows, + function(b) { + data_reduced[budget == b, loss] + } + ) %>% + do.call(rbind, .) %>% + set_rownames(rows) + + RT <- list() + + ds <- structure(list(RT = RT, FV = FV), + class = c('DataSet', 'list'), + maxRT = max(rows), + finalFV = min(FV), + format = 'NEVERGRAD', + maximization = FALSE, + algId = algId, + funcId = funcId_name, + DIM = DIM) + res[[idx]] <- ds + idx <- idx + 1 + } + } + class(res) %<>% c('DataSetList') + attr(res, 'DIM') <- DIMs + attr(res, 'funcId') <- funcIds + attr(res, 'algId') <- algIds + attr(res, 'suite') <- 'NEVERGRAD' + attr(res, 'maximization') <- F + res + +} + +#' Read single DataSet of SOS-based data +#' +#' Read single .txt files in SOS format and extract information as a DataSet +#' +#' @param file The path to the .txt file +#' @return The DataSet extracted from the .txt file provided +#' @noRd +read_single_file_SOS <- function(file) { + V1 <- NULL #Local binding to remove CRAN warnings + + algId <- substr(basename(file), 1, stringi::stri_locate_last(basename(file), fixed = 'D')[[1]] - 1) + + dt <- fread(file, header = F) + header <- scan(file, what = 'character', sep = '\n', n = 1, quiet = T) + splitted <- header %>% trimws %>% strsplit("\\s+") %>% .[[1]] %>% .[2:length(.)] + info <- list(algId = algId) + for (i in seq_len(length(splitted) / 2)) { + temp <- splitted[[2*i]] + name <- splitted[[2*i - 1]] + if (name == 'function') name <- 'funcId' + if (name == 'dim') name <- 'DIM' + names(temp) <- name + info <- c(info, temp) + } + + dim <- as.numeric(info$DIM) + #Hardcoded fix for SB-related data + if (is.null(dim) || length(dim) == 0) { + warning("Dimension not explicitly defined, setting as 30 by default") + dim <- 30 + info$DIM <- dim + } + + RT_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] + names(RT_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] + RT <- as.matrix(RT_raw) + mode(RT) <- 'integer' + + FV_raw <- dt[[colnames(dt)[[ncol(dt) - dim - 2]]]] + names(FV_raw) <- dt[[colnames(dt)[[ncol(dt) - dim - 1]]]] + FV <- as.matrix(FV_raw) + + + pos <- dt[, (ncol(dt) - dim + 1):ncol(dt)] + colnames(pos) <- as.character(seq_len(dim)) + + maxRT <- max(RT) + finalFV <- min(FV) + + idxs_avail <- dt[['V1']] + idxs_replaced <- dt[['V6']] + + idxs_final <- setdiff(idxs_avail, idxs_replaced) + + idx_final_best <- idxs_final[[which.min(FV[idxs_final])]] + final_pos <- as.numeric(pos[idx_final_best, ]) + # if (sum(FV == finalFV) > 1) { + # #Reconstruct population to determine which best solution is final position + # ids_min <- dt[FV_raw == finalFV, V1] + # replaced_idxs <- dt[[colnames(dt)[[ncol(dt) - dim]]]] + # #If none, take the last one added + # pos_idx <- max(ids_min) + # for (i in ids_min) { + # if (all(replaced_idxs != i)) { + # #If multiple, take the first one added + # pos_idx <- i + # break + # } + # } + # final_pos <- as.numeric(pos[pos_idx, ]) + # } + # else { + # final_pos <- as.numeric(pos[which.min(FV), ]) + # } + + PAR <- list( + # 'position' = list(pos), + 'final_position' = list(final_pos), + 'by_FV' = NULL, + 'by_RT' = NULL + ) + + + + object <- list() + class(object) <- c('DataSet', class(object)) + object$RT <- RT + object$FV <- FV + object$PAR <- PAR + attr(object, 'maxRT') <- maxRT + attr(object, 'finalFV') <- finalFV + attr(object, 'format') <- "SOS" + attr(object, 'maximization') <- F + attr(object, 'suite') <- "SOS" + for (i in seq_along(info)) { + attr(object, names(info)[[i]]) <- type.convert(info[[i]], as.is = T) + } + attr(object, 'ID') <- attr(object, 'algId') + object +} + + +#' Read DataSetList of SOS-based data +#' +#' Read directory containing .txt files in SOS format and extract information as a DataSetList +#' +#' @param dir The path to the directory file +#' @param corrections_file A file containing boundary-correction ratios for the files in `dir` +#' @return The DataSetList extracted from the directory provided +#' @noRd +read_datasetlist_SOS <- function(dir, corrections_files = NULL) { + V1 <- V3 <- V4 <- NULL #Local binding to remove CRAN warnings + res <- list() + dims <- list() + funcIds <- list() + algIds <- list() + suites <- list() + maximizations <- list() + + idx <- 1 + + corrs <- as.data.table(rbindlist(lapply(corrections_files, fread))) + + for (f in list.files(dir, recursive = T, pattern = "*.txt", full.names = T)) { + if (f %in% corrections_files) next + ds <- read_single_file_SOS(f) + + dims[[idx]] <- attr(ds, 'DIM') + funcIds[[idx]] <- attr(ds, 'funcId') + algIds[[idx]] <- attr(ds, 'algId') + suites[[idx]] <- attr(ds, 'suite') + maximizations[[idx]] <- attr(ds, 'maximization') + + if (nrow(corrs) > 0) { + fn <- substr(basename(f), 1, nchar(basename(f)) - 4) + corr_opts <- corrs[V1 == fn, ] + if (stri_detect_fixed(fn, "DE")) { + corr <- corr_opts[V3 == attr(ds, 'F'), ][V4 == attr(ds, 'CR'), 'V2'][['V2']] + } + else if (stri_detect_fixed(fn, "RIS")) { + corr <- corr_opts[['V2']] + } + else { + warning("Unknown algorithm, so skipping lookup of boundary corrections ratio") + corr <- NULL + } + if (length(corr) == 1) + ds$PAR$'corrections' <- corr[[1]] + else + warning(paste0("No boundary corrections ratio found for ", fn)) + } + + res[[idx]] <- ds + idx <- idx + 1 + } + class(res) %<>% c('DataSetList') + attr(res, 'DIM') <- dims + attr(res, 'funcId') <- funcIds + attr(res, 'algId') <- algIds + attr(res, 'ID_attributes') <- c('algId') + + suite <- unique(suites) + maximization <- unique(maximizations) + if (length(suite) != 1 || length(maximization) != 1) { + warning("Multipe different suites detected!") + } + + attr(res, 'suite') <- suite + attr(res, 'maximization') <- maximization + res + clean_DataSetList(res) +} + +#' Find corrections-files in SOS-based folder +#' +#' Read directory containing .txt files in SOS format and extract the corrections-files +#' +#' @param path The path to the directory file +#' @return The relative paths to the corection files +#' @noRd +locate_corrections_files <- function(path) { + files <- list.files(path, recursive = T, pattern = "*.txt", full.names = T) + files[stri_detect_fixed(files, 'corrections')] +} diff --git a/R/runServer.R b/R/runServer.R index dd7741e7..00a51868 100644 --- a/R/runServer.R +++ b/R/runServer.R @@ -1,18 +1,18 @@ -#' Create a shiny-server GUI to interactively use the IOHanalyzer -#' @param port Optional; which port the server should be opened at. Defaults -#' to the option set for 'shiny.port' -#' @param open_browser Whether or not to open a browser tab with the -#' IOHanalyzer GUI. Defaults to TRUE. -#' @export -#' @examples -#' \dontrun{ -#' runServer(6563, TRUE) -#' } -runServer <- function(port = getOption('shiny.port'), open_browser = TRUE) { - appDir <- system.file("shiny-server", package = "IOHanalyzer") - if (appDir == "") { - stop("Could not find example directory. Try re-installing `IOHanalyzer`.", call. = FALSE) - } - - shiny::runApp(appDir, port = port, launch.browser = open_browser, display.mode = "normal") -} +#' Create a shiny-server GUI to interactively use the IOHanalyzer +#' @param port Optional; which port the server should be opened at. Defaults +#' to the option set for 'shiny.port' +#' @param open_browser Whether or not to open a browser tab with the +#' IOHanalyzer GUI. Defaults to TRUE. +#' @export +#' @examples +#' \dontrun{ +#' runServer(6563, TRUE) +#' } +runServer <- function(port = getOption('shiny.port'), open_browser = TRUE) { + appDir <- system.file("shiny-server", package = "IOHanalyzer") + if (appDir == "") { + stop("Could not find example directory. Try re-installing `IOHanalyzer`.", call. = FALSE) + } + + shiny::runApp(appDir, port = port, launch.browser = open_browser, display.mode = "normal") +} diff --git a/R/stats.R b/R/stats.R index d27a8ce8..3d57c137 100644 --- a/R/stats.R +++ b/R/stats.R @@ -1,1025 +1,1024 @@ -#' Estimator 'SP' for the Expected Running Time (ERT) -#' -#' @param data A dataframe or matrix. Each row stores the runtime sample points from -#' several runs -#' @param max_runtime The budget to use for calculating ERT. If this is a vector, the largest value is taken. -#' Using this as a vector is being deprecated, and will be removed in a future update -#' -#' @return A list containing ERTs, number of succesfull runs and the succes rate -#' @export -#' @examples -#' SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) -SP <- function(data, max_runtime) { - N <- ncol(data) - if (length(max_runtime) > 1) { - warning("Argument 'max_runtime' will only be available to use as a single numeric value in the next release of IOHanalyzer.") - max_runtime <- max(max_runtime) - } - idx <- is.na(data) | data > max_runtime - succ <- N - rowSums(idx) - succ_rate <- succ / N - data[idx] <- max_runtime - - list(ERT = rowSums(data) / succ, runs = succ, succ_rate = succ_rate) -} - -#' Bootstrapping for running time samples -#' -#' @param x A numeric vector. A sample of the running time. -#' @param max_eval A numeric vector, containing the maximal running time in -#' each run. It should have the same size as x -#' @param bootstrap.size integer, the size of the bootstrapped sample -#' -#' @return A numeric vector of the bootstrapped running time sample -#' @export -#' @examples -#' ds <- dsl[[1]] -#' x <- get_RT_sample(ds, ftarget = 16, output = 'long') -#' max_eval <- get_maxRT(dsl, output = 'long') -#' bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) -bootstrap_RT <- function(x, max_eval, bootstrap.size) { - x_succ <- x[!is.na(x)] - x_unsucc <- max_eval[is.na(x)] - n_succ <- length(x_succ) - n_unsucc <- length(x_unsucc) - - p <- n_succ / length(x) - N <- rgeom(bootstrap.size, p) - - if (n_succ == 0){ - return (rep(Inf, bootstrap.size)) - } - - sapply(N, - function(size) { - if (size > 0) - x <- sum(sample(x_unsucc, size, replace = T)) - else - x <- 0 - x <- x + sample(x_succ, 1, replace = T) - }) -} - -# TODO: remove the bootstrapping part as it does not make much sense here... -#' Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times -#' among a data set -#' -#' @description This function performs a Kolmogorov-Smirnov test on each pair of -#' algorithms in the input x to determine which algorithm gives a significantly -#' smaller running time. The resulting p-values are arranged in a matrix, where -#' each cell (i, j) contains a p-value from the test with alternative hypothesis: -#' the running time of algorithm i is smaller (thus better) than that of j. -#' -#' @param x either a list that contains running time sample for each algorithm as -#' sub-lists, or a DataSetList object -#' @param bootstrap.size integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping -#' @param ... all other options -#' @return A matrix containing p-values of the test -#' @export -#' @examples -#' pairwise.test(subset(dsl, funcId == 1), 16) -pairwise.test <- function(x, ...) UseMethod('pairwise.test', x) - -#' @param max_eval list that contains the maximal running time for each algorithm -#' as sub-lists -#' @export -#' @rdname pairwise.test -pairwise.test.list <- function(x, max_eval, bootstrap.size = 30, ...) { - if ("DataSetList" %in% class(x)) { - class(x) <- rev(class(x)) - pairwise.test.DataSetList(x) - } - - N <- length(x) - p.value <- matrix(NA, N, N) - - for (i in seq(1, N - 1)) { - for (j in seq(i + 1, N)) { - if (bootstrap.size == 0) { - x1 <- x[[i]] - x2 <- x[[j]] - } else { - x1 <- bootstrap_RT(x[[i]], max_eval[[i]], bootstrap.size) - x2 <- bootstrap_RT(x[[j]], max_eval[[j]], bootstrap.size) - } - if (all(is.na(x1))) { - if (all(is.na(x2))) { - next - } - else { - p.value[i, j] <- 1 - p.value[j, i] <- 0 - } - } - else if (all(is.na(x2))) { - p.value[i, j] <- 0 - p.value[j, i] <- 1 - } - else { - options(warn = -1) - p.value[i, j] <- ks.test(x1, x2, alternative = 'greater', exact = F)$p.value - p.value[j, i] <- ks.test(x1, x2, alternative = 'less', exact = F)$p.value - options(warn = 0) - } - } - } - - p.value.adjust <- p.adjust(as.vector(p.value), method = 'holm') - p.value <- matrix(p.value.adjust, N, N) - colnames(p.value) <- rownames(p.value) <- names(x) - p.value -} - -#' @param ftarget float, the target value used to determine the running / hitting -#' @param which wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison -#' time -#' @export -#' @rdname pairwise.test -pairwise.test.DataSetList <- function(x, ftarget, bootstrap.size = 0, which = 'by_FV', ...) { - if (which == 'by_FV') { - dt <- get_RT_sample(x, ftarget, output = 'long') - maxRT <- get_maxRT(x, output = 'long') - maxRT <- split(maxRT$maxRT, maxRT$algId) - s <- split(dt$RT, dt$algId) - } - else if (which == 'by_RT') { - dt <- get_FV_sample(x, ftarget, output = 'long') - maxRT <- NULL - if (bootstrap.size > 0) warning("Bootstrapping is currently not supported for - fixed-budget statistics.") - bootstrap.size = 0 - s <- split(dt$`f(x)`, dt$algId) - } - else stop("Unsupported argument 'which'. Available options are 'by_FV' and 'by_RT'") - - return(pairwise.test.list(s, maxRT, bootstrap.size)) -} - -# TODO: move those two functions to a separate file -# TODO: review / re-write this function -#' Function for generating sequences of function values -#' -#' @param FV A list of function values -#' @param from Starting function value. Will be replaced by min(FV) if it is NULL or too small -#' @param to Stopping function value. Will be replaced by max(FV) if it is NULL or too large -#' @param by Stepsize of the sequence. Will be replaced if it is too small -#' @param length.out Number of values in the sequence. -#' 'by' takes preference if both it and length.out are provided. -#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a -#' linear or log-linear spacing respectively. If NULL, the scale will be predicted -#' based on FV -#' -#' @return A sequence of function values -#' @export -#' @examples -#' FVall <- get_runtimes(dsl) -#' seq_FV(FVall, 10, 16, 1, scale='linear') -seq_FV <- function(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, scale = NULL) { - from <- max(from, min(FV)) - to <- min(to, max(FV)) - - rev_trans <- function(x) x - - # Auto detect scaling - # TODO: Improve this detection (based on FV?). Currently very arbitrary - if (is.null(scale)) { - if (to < 0 || from < 0) - scale <- 'linear' - else if (abs(log10(mean(FV)) - log10(median(FV))) > 1) - scale <- 'log' - else - scale <- 'linear' - } - - if (scale == 'log') { - trans <- log10 - rev_trans <- function(x) 10 ^ x - # TODO: Better way to deal with negative values - # set lowest possible target globally instead of arbitrary 1e-12 - from <- max(1e-12, from) - to <- max(1e-12 ,to) - from <- trans(from) - to <- trans(to) - } - - #Avoid generating too many samples - if (!is.null(by)) { - nr_samples_generated <- (to - from) / by - if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { - by <- NULL - if (is.null(length.out)) - length.out <- getOption("IOHanalyzer.max_samples", default = 100) - } - } - - if (is.null(by) || by > to - from) { - if (is.null(length.out)) { - length.out <- 10 - args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) - } else - args <- list(from = from, to = to, length.out = length.out) - } else - args <- list(from = from, to = to, by = by) - - # tryCatch({ - do.call(seq, args) %>% - c(from, ., to) %>% # always include the starting / ending value - unique %>% - rev_trans - # }, error = function(e) { - # c() - # }) -} - -# TODO: review / re-write this function -#' Function for generating sequences of runtime values -#' -#' @param RT A list of runtime values -#' @param from Starting runtime value. Will be replaced by min(RT) if it is NULL or too small -#' @param to Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large -#' @param by Stepsize of the sequence. Will be replaced if it is too small -#' @param length.out Number of values in the sequence. -#' 'by' takes preference if both it and length.out are provided. -#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a -#' linear or log-linear spacing respectively. -#' -#' @return A sequence of runtime values -#' @export -#' @examples -#' RTall <- get_runtimes(dsl) -#' seq_RT(RTall, 0, 500, length.out=10, scale='log') -seq_RT <- function(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, - scale = 'linear') { - rev_trans <- function(x) x - - # Do this first to avoid the log-values being overwritten. - from <- max(from, min(RT)) - to <- min(to, max(RT)) - - if (scale == 'log') { - RT <- log10(RT) - rev_trans <- function(x) 10 ^ x - if (!is.null(from)) - from <- log10(from) - if (!is.null(to)) - to <- log10(to) - if (!is.null(by)) - by <- log10(by) - } - - #Avoid generating too many samples - if (!is.null(by)) { - nr_samples_generated <- (to - from) / by - if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { - by <- NULL - if (is.null(length.out)) - length.out <- getOption("IOHanalyzer.max_samples", default = 100) - } - } - - # Also reset by if it is too large - if (is.null(by) || by > to - from) { - if (is.null(length.out)) { - length.out <- 10 - args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) - } else - args <- list(from = from, to = to, length.out = length.out) - } else - args <- list(from = from, to = to, by = by) - - do.call(seq, args) %>% - c(from, ., to) %>% # always include the starting / ending value - unique %>% - rev_trans %>% - round -} - -# TODO: implement the empirical p.m.f. for runtime -EPMF <- function() { - -} - -#' Empirical Cumulative Dsitribution Function of Runtime of a single data set -#' -#' @param ds A DataSet or DataSetList object. -#' @param ftarget A Numerical vector. Function values at which runtime values are consumed -#' @param ... Arguments passed to other methods -#' -#' @return a object of type 'ECDF' -#' @export -#' @examples -#' ECDF(dsl,c(12,14)) -#' ECDF(dsl[[1]],c(12,14)) -ECDF <- function(ds, ftarget, ...) UseMethod("ECDF", ds) - -# TODO: also implement the ecdf functions for function values and parameters -#' @rdname ECDF -#' @export -ECDF.DataSet <- function(ds, ftarget, ...) { - runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT - runtime <- runtime[!is.na(runtime)] - if (length(runtime) == 0) - fun <- ecdf(Inf) - else - fun <- ecdf(runtime) - - class(fun)[1] <- 'ECDF' - attr(fun, 'min') <- min(runtime) - attr(fun, 'samples') <- length(runtime) - attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) - fun -} - -# TODO: review / re-write this function -#' @rdname ECDF -#' @export -ECDF.DataSetList <- function(ds, ftarget, ...) { - if (length(ds) == 0) return(NULL) - - dims <- unique(get_dim(ds)) - funcs <- unique(get_funcId(ds)) - - if (is.data.table(ftarget)) { - runtime <- sapply(seq(nrow(ftarget)), function(i) { - if (length(dims) > 1 && length(funcs) > 1) { - names_temp <- ftarget[i][[1]] %>% - strsplit(., ';') - FuncId <- names_temp[[1]][[1]] - Dim <- names_temp[[1]][[2]] - } - else if (length(dims) > 1) { - FuncId <- funcs[[1]] - Dim <- ftarget[i][[1]] - } - else if (length(funcs) > 1) { - FuncId <- ftarget[i][[1]] - Dim <- dims[[1]] - } - data <- subset(ds, funcId == FuncId, DIM == Dim) - if (length(data) == 0) return(NA) - temp_targets <- ftarget[i] %>% - unlist %>% - as.numeric - names(temp_targets) <- NULL - res <- get_RT_sample(data, temp_targets[2:11], output = 'long')$RT - res[is.na(res)] <- Inf - res - }) %>% - unlist - } else if (is.list(ftarget)) { - runtime <- sapply(seq_along(ftarget), function(i) { - if(length(dims) > 1 && length(funcs) >1){ - names_temp <- names(ftarget[i])[[1]] %>% - strsplit(., ';') - FuncId <- names_temp[[1]][[1]] - Dim <- names_temp[[1]][[2]] - } - else if(length(dims) > 1){ - FuncId <- funcs[[1]] - Dim <- names(ftarget[i])[[1]] - } - else if(length(funcs) > 1){ - FuncId <- names(ftarget[i])[[1]] - Dim <- dims[[1]] - } else { - FuncId <- funcs[[1]] - Dim <- dims[[1]] - } - data <- subset(ds, funcId == FuncId, DIM == Dim) - if (length(data) == 0) return(NA) - res <- get_RT_sample(data, ftarget[[i]], output = 'long')$RT - res[is.na(res)] <- Inf - res - }) %>% - unlist - } else { - runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT - runtime[is.na(runtime)] <- Inf - } - - - if (length(runtime) == 0) return(NULL) - - fun <- ecdf(runtime) - class(fun)[1] <- 'ECDF' - attr(fun, 'min') <- min(runtime) - attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) - fun -} - -#' Area Under Curve (Empirical Cumulative Dsitribution Function) -#' -#' @param fun A ECDF object. -#' @param from double. Starting point of the area on x-axis -#' @param to double. Ending point of the area on x-axis -#' -#' @return a object of type 'ECDF' -#' @export -#' @examples -#' ecdf <- ECDF(dsl,c(12,14)) -#' AUC(ecdf, 0, 100) -AUC <- function(fun, from = NULL, to = NULL) UseMethod('AUC', fun) - -#' @rdname AUC -#' @export -AUC.ECDF <- function(fun, from = NULL, to = NULL) { - if (is.null(from)) - from <- attr(fun, 'min') - if (is.null(to)) - to <- attr(fun, 'max') - - if (is.null(fun)) - 0 - else - integrate(fun, lower = from, upper = to, subdivisions = 1e3L)$value / (to - from) -} - -#' Generate datatables of runtime or function value targets for a DataSetList -#' -#' Only one target is generated per (function, dimension)-pair, as opposed to the -#' function `get_default_ECDF_targets`, which generates multiple targets. -#' -#' @param dsList A DataSetList -#' @param which Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets -#' -#' @return a data.table of targets -#' @export -#' @examples -#' get_target_dt(dsl) -get_target_dt <- function(dsList, which = 'by_RT') { - vals <- c() - funcs <- get_funcId(dsList) - dims <- get_dim(dsList) - dt <- as.data.table(expand.grid(funcs, dims)) - colnames(dt) <- c("funcId", "DIM") - if (which == 'by_FV') { - target_func <- get_target_FV - } - else if (which == 'by_RT') { - target_func <- get_target_RT - } - else stop("Invalid argument for `which`; can only be `by_FV` or `by_RT`.") - targets <- apply(dt, 1, - function(x) { - dsl_sub <- subset(dsList, funcId == x[[1]], DIM == x[[2]]) - if (length(dsl_sub) > 0) - target_func(dsl_sub) - else - NULL - }) - dt[, target := targets] - - return(dt[!sapply(dt[['target']], is.null)]) -} - -#' Helper function for `get_target_dt` -#' @noRd -get_target_FV <- function(dsList){ - vals <- get_FV_summary(dsList, Inf)[[paste0(100*getOption("IOHanalyzer.quantiles", 0.02)[[1]], '%')]] - if (is.null(attr(dsList, 'maximization')) || attr(dsList, 'maximization')) { - return(max(vals)) - } - else { - return(min(vals)) - } -} - -#' Helper function for `get_target_dt` -#' @noRd -get_target_RT <- function(dsList){ - return(max(get_runtimes(dsList))) -} - -#' Glicko2 raning of algorithms -#' -#' This procedure ranks algorithms based on a glicko2-procedure. -#' Every round (total nr_rounds), for every function and dimension of the datasetlist, -#' each pair of algorithms competes. This competition samples a random runtime for the -#' provided target (defaults to best achieved target). Whichever algorithm has the lower -#' runtime wins the game. Then, from these games, the glicko2-rating is determined. -#' -#' @param dsl The DataSetList, can contain multiple functions and dimensions, but should have the -#' same algorithms for all of them -#' @param nr_rounds The number of rounds to run. More rounds leads to a more accurate ranking. -#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective -#' @param target_dt Custom data.table target value to use. When NULL, this is selected automatically. -#' @return A dataframe containing the glicko2-ratings and some additional info -#' -#' @export -#' @examples -#' glicko2_ranking(dsl, nr_round = 25) -#' glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') -glicko2_ranking <- function(dsl, nr_rounds = 100, which = 'by_FV', target_dt = NULL){ - if (!requireNamespace("PlayerRatings", quietly = TRUE)) { - stop("Package \"pkg\" needed for this function to work. Please install it.", - call. = FALSE) - } - req(length(get_algId(dsl)) > 1) - - if (!is.null(target_dt) && !('data.table' %in% class(target_dt))) { - warning("Provided `target_dt` argument is not a data.table") - target_dt <- NULL - } - - if (is.null(target_dt)) - target_dt <- get_target_dt(dsl, which) - - if (!(which %in% c('by_FV', 'by_RT'))) - stop("Invalid argument: 'which' can only be 'by_FV' or 'by_RT'") - p1 <- NULL - p2 <- NULL - scores <- NULL - weeks <- NULL - get_dims <- function(){ - dims <- get_dim(dsl) - if (length(dims) > 1) { - dims <- sample(dims) - } - dims - } - get_funcs <- function(){ - funcs <- get_funcId(dsl) - if (length(funcs) > 1) { - funcs <- sample(funcs) - } - funcs - } - n_algs = length(get_algId(dsl)) - alg_names <- NULL - for (k in seq(1,nr_rounds)) { - for (dim in get_dims()) { - targets_temp <- target_dt[target_dt$DIM == dim] - for (fid in get_funcs()) { - dsl_s <- subset(dsl, funcId == fid && DIM == dim) - if (length(dsl_s) == 0) - next - if (which == 'by_FV') { - target <- targets_temp[targets_temp$funcId == fid]$target - x_arr <- get_RT_sample(dsl_s, target) - win_operator <- `<` - } - else { - target <- targets_temp[targets_temp$funcId == fid]$target - x_arr <- get_FV_sample(dsl_s, target) - win_operator <- ifelse(attr(dsl, 'maximization'), `>`, `<`) - } - vals = array(dim = c(n_algs,ncol(x_arr) - 4)) - for (i in seq(1,n_algs)) { - z <- x_arr[i] - y <- as.numeric(z[,5:ncol(x_arr)]) - vals[i,] = y - } - if (is.null(alg_names)) alg_names <- x_arr[,3] - - for (i in seq(1,n_algs)) { - for (j in seq(i,n_algs)) { - if (i == j) next - weeks <- c(weeks, k) - s1 <- sample(vals[i,], 1) - s2 <- sample(vals[j,], 1) - if (is.na(s1)) { - if (is.na(s2)) { - won <- 0.5 - } - else{ - won <- 0 - } - } - else{ - if (is.na(s2)) { - won <- 1 - } - else if (s1 == s2) { - won <- 0.5 #Tie - } - else { - won <- win_operator(s1, s2) - } - } - p1 <- c(p1, i) - p2 <- c(p2, j) - scores <- c(scores, won) - - } - } - } - } - } - # weeks <- seq(1,1,length.out = length(p1)) - games <- data.frame(Weeks = weeks, Player1 = p1, Player2 = p2, Scores = as.numeric(scores)) - lout <- PlayerRatings::glicko2(games, init = c(1500,350,0.06)) - lout$ratings$Player <- alg_names[lout$ratings$Player] - lout -} - - -#' Verify that the credentials for DSCtool have been set -#' -#' This uses the keyring package to store and load credentials. -#' If the keyring package does not exists, it will default to look for -#' a config-file in the 'repository'-folder, under your home directory. -#' This can be changed by setting the option IOHprofiler.config_dir -#' If you already have an account, please call `set_DSC_credentials` -#' with the corresponding username and password. -#' If you don't have an account, you can register for one using `register_DSC` -#' -#' -#' -#' @export -#' @examples -#' check_dsc_configured() -check_dsc_configured <- function() { - if (!requireNamespace("keyring", quietly = TRUE)) { - warning("It is recommended to have the 'keyring' package installed to store - DSCtool settings. Since this package is not found, we default - to a local settings-file instead.") - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - if (!file.exists(paste0(repo_dir, "/config.rds"))) { - return(FALSE) - } - data <- readRDS(paste0(repo_dir, "/config.rds")) - if (is.null(data$DSCusername) || is.null(data$DSCpassword)) { - return(FALSE) - } - return(TRUE) - } - tryCatch({ - username <- keyring::key_get("DSCtool_name") - pwd <- keyring::key_get("DSCtool") - return(TRUE) - }, - error = function(e) { - print("DSCtool connection not yet set. Please use the function `register_DSC` to - create an account to use the DSC tool locally (or use 'add_DSC_credentials' - to use an existing account)") - return(FALSE) - } - ) -} - -#' Register an account to the DSCtool API -#' -#' This uses the keyring package to store and load credentials. -#' If you already have an account, please call `set_DSC_credentials` instead -#' -#' @param name Your name -#' @param username A usename to be identified with. Will be stored on keyring under 'DSCtool_name' -#' @param affiliation Your affiliation (university / company) -#' @param email Your email adress -#' @param password The password to use. If NULL, this will be generated at random. -#' Will be stored on keyring under 'DSCtool' -#' -#' @export -#' @examples -#' \dontrun{ -#' register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") -#' } -register_DSC <- function(name, username, affiliation, email, password = NULL) { - if (is.null(password)) password <- stri_rand_strings(1, 10) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/manage/user" - result_json <- POST(url, add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = list(name = name, affiliation = affiliation, email = email, - username = username, password = password), - encode = "json") - if (result_json$status_code == 200) { - if (!requireNamespace("keyring", quietly = TRUE)) { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - saveRDS(list(DSCusername = username, DSCpassword = password), - paste0(repo_dir, "/config.rds")) - return(TRUE) - } - else { - keyring::key_set_with_value("DSCtool", password = password) - keyring::key_set_with_value("DSCtool_name", password = username) - return(TRUE) - } - } - else { - stop("Something went wrong in registering for DSCtool") - } -} - -#' Register an account to the DSCtool API -#' -#' This uses the keyring package to store and load credentials. -#' If you already have an account, please call `add_DSC_credentials` instead -#' -#' @param username The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name' -#' @param password The password you use on DSCtool. Will be stored on keyring under 'DSCtool' -#' -#' @export -#' @examples -#' \dontrun{set_DSC_credentials('jdoe', 'monkey123')} -set_DSC_credentials <- function(username, password) { - if (!requireNamespace("keyring", quietly = TRUE)) { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - saveRDS(list(DSCusername = username, DSCpassword = password), - paste0(repo_dir, "/config.rds")) - } - else { - keyring::key_set_with_value("DSCtool", password = password) - keyring::key_set_with_value("DSCtool_name", password = username) - } -} - -#' Load stored credentials for DSCtool -#' -#' -#' @noRd -#' @examples -#' \dontrun{ -#' get_DSC_credentials() -#' } -get_DSC_credentials <- function() { - if (!requireNamespace("keyring", quietly = TRUE)) { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - data <- readRDS(paste0(repo_dir, "/config.rds")) - return(list(name = data$DSCusername, pwd = data$DSCpassword)) - } - else { - return(list(name = keyring::key_get("DSCtool_name"), - pwd = keyring::key_get("DSCtool"))) - } -} - -#' Convert a DataSetList to the format needed for the DSCtool -#' -#' -#' @param dsList The DataSetList object -#' @param targets Optional list of target values (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. -#' -#' @noRd -#' @examples -#' convert_to_dsc_compliant(dsl) -convert_to_dsc_compliant <- function(dsList, targets = NULL, which = 'by_RT', - na.correction = NULL) { - maximize <- attr(dsList, 'maximization') - variable <- fid <- value <- NULL #Set local binding to remove warnings - by_rt <- which == 'by_RT' - - if (is.null(targets)) { - targets <- get_target_dt(dsList, which) - } - - final_list <- lapply(get_algId(dsList), function(algname) { - dsl_sub <- subset(dsList, algId == algname) - problems <- lapply(dsl_sub, function(ds) { - target <- targets[funcId == attr(ds, 'funcId') & DIM == attr(ds, 'DIM'), 'target'] - if (!is.numeric(target)) { #TODO: more robust solution - target <- target[[1]][[1]] - } - if (by_rt) { - data <- get_RT_sample(ds, target, output = 'long')$RT - if (any(is.na(data)) & !is.null(na.correction)) { - if (na.correction == 'PAR-1') { - budgets <- attr(ds, 'maxRT') - data[is.na(data)] <- budgets[is.na(data)] - } - else if (na.correction == 'PAR-10') { - budgets <- attr(ds, 'maxRT') - data[is.na(data)] <- 10 * budgets[is.na(data)] - } - else if (na.correction == 'ERT') { - ert <- as.numeric(get_ERT(ds, target)$ERT) - if (is.infinite(ert)) { - ert <- sum(attr(ds, 'maxRT')) - } - data[is.na(data)] <- ert - } - else if (na.correction == 'Remove-na') { - data <- data[!is.na(data)] - } - else { - stop('This value for `na.correction` is not supported') - } - } - } - else { - data <- get_FV_sample(ds, target, output = 'long')$`f(x)` - if (attr(ds, 'maximization')) data <- -1 * data - } - if (any(is.na(data))) { - warning("NA-values detected in data preparation for DSCtool. This will likely result in an - error. Please verify the provided DataSetList.") - } - return(list(name = paste0("F", attr(ds, 'funcId'), "_", attr(ds, 'DIM'),"D"), - data = data)) - }) - return(list(algorithm = algname, problems = problems)) - }) - - - return(final_list) -} - -#' Get the matrix of rankings using the DSCtool api for a DataSetList -#' -#' -#' @param dsList The DataSetList object -#' @param targets Optional list of target values (Runtime or target value) -#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' -#' @param test_type Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests -#' @param alpha Threshold value for statistical significance -#' @param epsilon Minimum threshold to have practical difference between algorithms (eDSC) -#' @param monte_carlo_iterations How many monte-carlo-simulations to perform -#' (set to 0 to use regular DSC) -#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. -#' Options are: -#' - 'NULL': No correction is done. This will likely result in an error, as the DSCtool -#' does not allow for na values -#' - 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) -#' - 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) -#' - 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this -#' reverts to nr_runs * budget -#' - 'Remove-na' Removes all NA values -#' -#' @return A named list containing a ranked-matrix which has the rankin of each algorithm -#' on each problem, as well as a list of which omnibus tests can be used to further process -#' this data. This can be further analyzed using `get_dsc_omnibus` -#' -#' @export -#' @examples -#' get_dsc_rank(dsl) -get_dsc_rank <- function(dsList, targets = NULL, which = 'by_RT', test_type = "AD", alpha = 0.05, - epsilon = 0, monte_carlo_iterations = 0, na.correction = NULL) { - if (!check_dsc_configured()) return(NULL) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/rank" - dsc_list <- convert_to_dsc_compliant(dsList, targets, which, na.correction = na.correction) - json_list <- list(method = list(name = test_type, alpha = alpha), epsilon = epsilon, - data = dsc_list, monte_carlo_iterations = monte_carlo_iterations) - - result_json <- POST(url, - authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), - add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = json_list, encode = "json") - - return(content(result_json)$result) -} - - -#' Perform omnibus statistical tests on the matrix of rankings from the DSCtool api -#' -#' -#' @param res The result of a call to the `get_dsc_rank` -#' @param method Which method to use to do the tests. -#' Has be be one of the allowed ones in `res$valid_methods`. -#' When NULL, the first valid option is chosen by default -#' @param alpha Threshold value for statistical significance -#' -#' @return A named list containing the algorithm means -#' -#' @export -#' @examples -#' get_dsc_omnibus(get_dsc_rank(dsl)) -get_dsc_omnibus <- function(res, method = NULL, alpha = 0.05) { - if (!check_dsc_configured()) return(NULL) - if (is.null(method)) method <- res$valid_methods[[1]] - else req(method %in% res$valid_methods) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/omnibus" - new_json <- list(method = list(name = method, alpha = alpha), - ranked_matrix = res$ranked_matrix, - number_algorithms = res$number_algorithms, - parametric_tests = res$parametric_tests) - result_json <- POST(url, - authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), - add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = new_json, encode = "json") - - return(content(result_json)$result) -} - -#' Perform post-hoc processing on data from DSCtool -#' -#' -#' @param omni_res The result from a call to `get_dsc_omnibus` -#' @param nr_algs The number of algorithms present in `omni_res` -#' @param nr_problems The number of problems present in `omni_res` -#' @param base_algorithm The base algorithm to which the other are compared. -#' This has to be present in `omni_res$algorithm_means` as an `algorithm` property -#' @param method Either 'friedman' or 'friedman-aligned-rank' -#' @param alpha Threshold value for statistical significance -#' -#' @return A named list containing 4 types of analyses: -#' * Zvalue -#' * UnadjustedPValue -#' * Holm -#' * Hochberg -#' -#' @export -#' @examples -#' get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) -get_dsc_posthoc <- function(omni_res, nr_algs, nr_problems, base_algorithm = NULL, - method = "friedman", alpha = 0.05) { - if (!check_dsc_configured()) return(NULL) - if (is.null(base_algorithm)) base_algorithm <- omni_res$algorithm_means[[1]]$algorithm - else req(base_algorithm %in% unlist(omni_res$algorithm_means)) - url <- "https://ws.ijs.si:8443/dsc-1.5/service/posthoc" - new_json <- list(method = list(name = method, alpha = alpha), - algorithm_means = omni_res$algorithm_means, - n = nr_problems, - k = nr_algs, - base_algorithm = base_algorithm) - result_json <- POST(url, - authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), - add_headers(.headers = c('Content-Type' = "application/json", - 'Accept' = "application/json")), - body = new_json, encode = "json") - - return(content(result_json)$result) -} - - -#' Get the marginal contribution of an algorithm to a portfolio -#' -#' Based on the contribution to the ECDF-curve of the VBS of the portfolio -#' -#' @param alg The result from a call to `get_dsc_omnibus` -#' @param perm The permutation of algorithms to which is being contributed -#' @param j At which point in the permutation the contribution should be measured -#' @param dt The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`) -#' -#' @export -#' @examples -#' dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) -#' get_marg_contrib_ecdf(get_algId(dsl)[[1]], get_algId(dsl), 1, dt) -get_marg_contrib_ecdf <- function(alg, perm, j, dt) { - hit <- NULL #Bind to avoid notes - algs <- perm[0:j] - v_pre <- sum(dt[algId %in% algs, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - algs <- perm[0:(j - 1)] - if (j == 1) { - v_post <- 0 - } - else - v_post <- sum(dt[algId %in% algs, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - v_pre - v_post -} - - -#' Get the shapley-values of a portfolio of algorithms -#' -#' Based on the contribution to the ECDF-curve of the VBS of the portfolio -#' -#' @param dsList The DataSetList object -#' @param targets A list or data.table containing the targets per function / dimension. If this is -#' a data.table, it needs columns 'target', 'DIM' and 'funcId' -#' @param scale.log Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not -#' @param group_size How many permutation groups will be considered -#' @param max_perm_size The maximum limit for permutations to be considered -#' @param normalize Whether or not to ensure the resulting values will be in [0,1] -#' -#' @export -#' @examples -#' get_shapley_values(dsl, get_ECDF_targets(dsl)) -get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, max_perm_size = 10, normalize = T){ - hit <- NULL #Bind to avoid notes - algs_full <- get_algId(dsList) - - dt <- generate_data.ECDF_raw(dsList, targets, scale_log = scale.log) - - nr_players <- length(algs_full) - - perms <- lapply(seq_len(nr_players * group_size), function(i) {sample(algs_full)}) - - max_val <- sum(dt[, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) - ### For each algorithm, calculate shapley value - shapleys <- lapply(algs_full, function(alg) { - ### For each group of permutations - temp <- mean(unlist( - lapply(seq_len(max_perm_size), function(j) { - perm_sub <- perms[((group_size * (j - 1)) + 1):(group_size * (j))] - ### - mean(unlist( - lapply(perm_sub, function(perm) { - perm <- replace(perm, c(j, which(perm == alg)), c(alg,perm[[j]])) - get_marg_contrib_ecdf(alg, perm, j, dt) - }) - )) - - }) - )) - if (normalize) - temp <- temp / max_val #Scale so Shapleys always fall between 0 and 1 - temp - }) - - data.table(algId = algs_full, shapley = shapleys) -} - +#' Estimator 'SP' for the Expected Running Time (ERT) +#' +#' @param data A dataframe or matrix. Each row stores the runtime sample points from +#' several runs +#' @param max_runtime The budget to use for calculating ERT. If this is a vector, the largest value is taken. +#' Using this as a vector is being deprecated, and will be removed in a future update +#' +#' @return A list containing ERTs, number of succesfull runs and the succes rate +#' @export +#' @examples +#' SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) +SP <- function(data, max_runtime) { + N <- ncol(data) + if (length(max_runtime) > 1) { + max_runtime <- max(max_runtime) + } + idx <- is.na(data) | data > max_runtime + succ <- N - rowSums(idx) + succ_rate <- succ / N + data[idx] <- max_runtime + + list(ERT = rowSums(data) / succ, runs = succ, succ_rate = succ_rate) +} + +#' Bootstrapping for running time samples +#' +#' @param x A numeric vector. A sample of the running time. +#' @param max_eval A numeric vector, containing the maximal running time in +#' each run. It should have the same size as x +#' @param bootstrap.size integer, the size of the bootstrapped sample +#' +#' @return A numeric vector of the bootstrapped running time sample +#' @export +#' @examples +#' ds <- dsl[[1]] +#' x <- get_RT_sample(ds, ftarget = 16, output = 'long') +#' max_eval <- get_maxRT(dsl, output = 'long') +#' bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) +bootstrap_RT <- function(x, max_eval, bootstrap.size) { + x_succ <- x[!is.na(x)] + x_unsucc <- max_eval[is.na(x)] + n_succ <- length(x_succ) + n_unsucc <- length(x_unsucc) + + p <- n_succ / length(x) + N <- rgeom(bootstrap.size, p) + + if (n_succ == 0){ + return (rep(Inf, bootstrap.size)) + } + + sapply(N, + function(size) { + if (size > 0) + x <- sum(sample(x_unsucc, size, replace = T)) + else + x <- 0 + x <- x + sample(x_succ, 1, replace = T) + }) +} + +# TODO: remove the bootstrapping part as it does not make much sense here... +#' Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times +#' among a data set +#' +#' @description This function performs a Kolmogorov-Smirnov test on each pair of +#' algorithms in the input x to determine which algorithm gives a significantly +#' smaller running time. The resulting p-values are arranged in a matrix, where +#' each cell (i, j) contains a p-value from the test with alternative hypothesis: +#' the running time of algorithm i is smaller (thus better) than that of j. +#' +#' @param x either a list that contains running time sample for each algorithm as +#' sub-lists, or a DataSetList object +#' @param bootstrap.size integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping +#' @param ... all other options +#' @return A matrix containing p-values of the test +#' @export +#' @examples +#' pairwise.test(subset(dsl, funcId == 1), 16) +pairwise.test <- function(x, ...) UseMethod('pairwise.test', x) + +#' @param max_eval list that contains the maximal running time for each algorithm +#' as sub-lists +#' @export +#' @rdname pairwise.test +pairwise.test.list <- function(x, max_eval, bootstrap.size = 30, ...) { + if ("DataSetList" %in% class(x)) { + class(x) <- rev(class(x)) + pairwise.test.DataSetList(x) + } + + N <- length(x) + p.value <- matrix(NA, N, N) + + for (i in seq(1, N - 1)) { + for (j in seq(i + 1, N)) { + if (bootstrap.size == 0) { + x1 <- x[[i]] + x2 <- x[[j]] + } else { + x1 <- bootstrap_RT(x[[i]], max_eval[[i]], bootstrap.size) + x2 <- bootstrap_RT(x[[j]], max_eval[[j]], bootstrap.size) + } + if (all(is.na(x1))) { + if (all(is.na(x2))) { + next + } + else { + p.value[i, j] <- 1 + p.value[j, i] <- 0 + } + } + else if (all(is.na(x2))) { + p.value[i, j] <- 0 + p.value[j, i] <- 1 + } + else { + options(warn = -1) + p.value[i, j] <- ks.test(x1, x2, alternative = 'greater', exact = F)$p.value + p.value[j, i] <- ks.test(x1, x2, alternative = 'less', exact = F)$p.value + options(warn = 0) + } + } + } + + p.value.adjust <- p.adjust(as.vector(p.value), method = 'holm') + p.value <- matrix(p.value.adjust, N, N) + colnames(p.value) <- rownames(p.value) <- names(x) + p.value +} + +#' @param ftarget float, the target value used to determine the running / hitting +#' @param which wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison +#' time +#' @export +#' @rdname pairwise.test +pairwise.test.DataSetList <- function(x, ftarget, bootstrap.size = 0, which = 'by_FV', ...) { + if (which == 'by_FV') { + dt <- get_RT_sample(x, ftarget, output = 'long') + maxRT <- get_maxRT(x, output = 'long') + maxRT <- split(maxRT$maxRT, maxRT$ID) + s <- split(dt$RT, dt$ID) + } + else if (which == 'by_RT') { + dt <- get_FV_sample(x, ftarget, output = 'long') + maxRT <- NULL + if (bootstrap.size > 0) warning("Bootstrapping is currently not supported for + fixed-budget statistics.") + bootstrap.size = 0 + s <- split(dt$`f(x)`, dt$ID) + } + else stop("Unsupported argument 'which'. Available options are 'by_FV' and 'by_RT'") + + return(pairwise.test.list(s, maxRT, bootstrap.size)) +} + +# TODO: move those two functions to a separate file +# TODO: review / re-write this function +#' Function for generating sequences of function values +#' +#' @param FV A list of function values +#' @param from Starting function value. Will be replaced by min(FV) if it is NULL or too small +#' @param to Stopping function value. Will be replaced by max(FV) if it is NULL or too large +#' @param by Stepsize of the sequence. Will be replaced if it is too small +#' @param length.out Number of values in the sequence. +#' 'by' takes preference if both it and length.out are provided. +#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a +#' linear or log-linear spacing respectively. If NULL, the scale will be predicted +#' based on FV +#' +#' @return A sequence of function values +#' @export +#' @examples +#' FVall <- get_runtimes(dsl) +#' seq_FV(FVall, 10, 16, 1, scale='linear') +seq_FV <- function(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, scale = NULL) { + from <- max(from, min(FV)) + to <- min(to, max(FV)) + + rev_trans <- function(x) x + + # Auto detect scaling + # TODO: Improve this detection (based on FV?). Currently very arbitrary + if (is.null(scale)) { + if (to < 0 || from < 0) + scale <- 'linear' + else if (abs(log10(mean(FV)) - log10(median(FV))) > 1) + scale <- 'log' + else + scale <- 'linear' + } + + if (scale == 'log') { + trans <- log10 + rev_trans <- function(x) 10 ^ x + # TODO: Better way to deal with negative values + # set lowest possible target globally instead of arbitrary 1e-12 + from <- max(1e-12, from) + to <- max(1e-12 ,to) + from <- trans(from) + to <- trans(to) + } + + #Avoid generating too many samples + if (!is.null(by)) { + nr_samples_generated <- (to - from) / by + if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { + by <- NULL + if (is.null(length.out)) + length.out <- getOption("IOHanalyzer.max_samples", default = 100) + } + } + + if (is.null(by) || by > to - from) { + if (is.null(length.out)) { + length.out <- 10 + args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) + } else + args <- list(from = from, to = to, length.out = length.out) + } else + args <- list(from = from, to = to, by = by) + + # tryCatch({ + do.call(seq, args) %>% + c(from, ., to) %>% # always include the starting / ending value + unique %>% + rev_trans + # }, error = function(e) { + # c() + # }) +} + +# TODO: review / re-write this function +#' Function for generating sequences of runtime values +#' +#' @param RT A list of runtime values +#' @param from Starting runtime value. Will be replaced by min(RT) if it is NULL or too small +#' @param to Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large +#' @param by Stepsize of the sequence. Will be replaced if it is too small +#' @param length.out Number of values in the sequence. +#' 'by' takes preference if both it and length.out are provided. +#' @param scale Scaling of the sequence. Can be either 'linear' or 'log', indicating a +#' linear or log-linear spacing respectively. +#' +#' @return A sequence of runtime values +#' @export +#' @examples +#' RTall <- get_runtimes(dsl) +#' seq_RT(RTall, 0, 500, length.out=10, scale='log') +seq_RT <- function(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, + scale = 'linear') { + rev_trans <- function(x) x + + # Do this first to avoid the log-values being overwritten. + from <- max(from, min(RT)) + to <- min(to, max(RT)) + + if (scale == 'log') { + RT <- log10(RT) + rev_trans <- function(x) 10 ^ x + if (!is.null(from)) + from <- log10(from) + if (!is.null(to)) + to <- log10(to) + if (!is.null(by)) + by <- log10(by) + } + + #Avoid generating too many samples + if (!is.null(by)) { + nr_samples_generated <- (to - from) / by + if (nr_samples_generated > getOption("IOHanalyzer.max_samples", default = 100)) { + by <- NULL + if (is.null(length.out)) + length.out <- getOption("IOHanalyzer.max_samples", default = 100) + } + } + + # Also reset by if it is too large + if (is.null(by) || by > to - from) { + if (is.null(length.out)) { + length.out <- 10 + args <- list(from = from, to = to, by = (to - from) / (length.out - 1)) + } else + args <- list(from = from, to = to, length.out = length.out) + } else + args <- list(from = from, to = to, by = by) + + do.call(seq, args) %>% + c(from, ., to) %>% # always include the starting / ending value + unique %>% + rev_trans %>% + round +} + +# TODO: implement the empirical p.m.f. for runtime +EPMF <- function() { + +} + +#' Empirical Cumulative Dsitribution Function of Runtime of a single data set +#' +#' @param ds A DataSet or DataSetList object. +#' @param ftarget A Numerical vector. Function values at which runtime values are consumed +#' @param ... Arguments passed to other methods +#' +#' @return a object of type 'ECDF' +#' @export +#' @examples +#' ECDF(dsl,c(12,14)) +#' ECDF(dsl[[1]],c(12,14)) +ECDF <- function(ds, ftarget, ...) UseMethod("ECDF", ds) + +# TODO: also implement the ecdf functions for function values and parameters +#' @rdname ECDF +#' @export +ECDF.DataSet <- function(ds, ftarget, ...) { + runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT + runtime <- runtime[!is.na(runtime)] + if (length(runtime) == 0) + fun <- ecdf(Inf) + else + fun <- ecdf(runtime) + + class(fun)[1] <- 'ECDF' + attr(fun, 'min') <- min(runtime) + attr(fun, 'samples') <- length(runtime) + attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) + fun +} + +# TODO: review / re-write this function +#' @rdname ECDF +#' @export +ECDF.DataSetList <- function(ds, ftarget, ...) { + if (length(ds) == 0) return(NULL) + + dims <- unique(get_dim(ds)) + funcs <- unique(get_funcId(ds)) + + if (is.data.table(ftarget)) { + runtime <- sapply(seq(nrow(ftarget)), function(i) { + if (length(dims) > 1 && length(funcs) > 1) { + names_temp <- ftarget[i][[1]] %>% + strsplit(., ';') + FuncId <- names_temp[[1]][[1]] + Dim <- names_temp[[1]][[2]] + } + else if (length(dims) > 1) { + FuncId <- funcs[[1]] + Dim <- ftarget[i][[1]] + } + else if (length(funcs) > 1) { + FuncId <- ftarget[i][[1]] + Dim <- dims[[1]] + } + data <- subset(ds, funcId == FuncId, DIM == Dim) + if (length(data) == 0) return(NA) + temp_targets <- ftarget[i] %>% + unlist %>% + as.numeric + names(temp_targets) <- NULL + res <- get_RT_sample(data, temp_targets[2:11], output = 'long')$RT + res[is.na(res)] <- Inf + res + }) %>% + unlist + } else if (is.list(ftarget)) { + runtime <- sapply(seq_along(ftarget), function(i) { + if(length(dims) > 1 && length(funcs) >1){ + names_temp <- names(ftarget[i])[[1]] %>% + strsplit(., ';') + FuncId <- names_temp[[1]][[1]] + Dim <- names_temp[[1]][[2]] + } + else if(length(dims) > 1){ + FuncId <- funcs[[1]] + Dim <- names(ftarget[i])[[1]] + } + else if(length(funcs) > 1){ + FuncId <- names(ftarget[i])[[1]] + Dim <- dims[[1]] + } else { + FuncId <- funcs[[1]] + Dim <- dims[[1]] + } + data <- subset(ds, funcId == FuncId, DIM == Dim) + if (length(data) == 0) return(NA) + res <- get_RT_sample(data, ftarget[[i]], output = 'long')$RT + res[is.na(res)] <- Inf + res + }) %>% + unlist + } else { + runtime <- get_RT_sample(ds, ftarget, output = 'long')$RT + runtime[is.na(runtime)] <- Inf + } + + + if (length(runtime) == 0) return(NULL) + + fun <- ecdf(runtime) + class(fun)[1] <- 'ECDF' + attr(fun, 'min') <- min(runtime) + attr(fun, 'max') <- max(runtime) # the sample can be retrieved by knots(fun) + fun +} + +#' Area Under Curve (Empirical Cumulative Dsitribution Function) +#' +#' @param fun A ECDF object. +#' @param from double. Starting point of the area on x-axis +#' @param to double. Ending point of the area on x-axis +#' +#' @return a object of type 'ECDF' +#' @export +#' @examples +#' ecdf <- ECDF(dsl,c(12,14)) +#' AUC(ecdf, 0, 100) +AUC <- function(fun, from = NULL, to = NULL) UseMethod('AUC', fun) + +#' @rdname AUC +#' @export +AUC.ECDF <- function(fun, from = NULL, to = NULL) { + if (is.null(from)) + from <- attr(fun, 'min') + if (is.null(to)) + to <- attr(fun, 'max') + + if (is.null(fun)) + 0 + else + integrate(fun, lower = from, upper = to, subdivisions = 1e3L)$value / (to - from) +} + +#' Generate datatables of runtime or function value targets for a DataSetList +#' +#' Only one target is generated per (function, dimension)-pair, as opposed to the +#' function `get_default_ECDF_targets`, which generates multiple targets. +#' +#' @param dsList A DataSetList +#' @param which Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets +#' +#' @return a data.table of targets +#' @export +#' @examples +#' get_target_dt(dsl) +get_target_dt <- function(dsList, which = 'by_RT') { + vals <- c() + funcs <- get_funcId(dsList) + dims <- get_dim(dsList) + dt <- as.data.table(expand.grid(funcs, dims)) + colnames(dt) <- c("funcId", "DIM") + if (which == 'by_FV') { + target_func <- get_target_FV + } + else if (which == 'by_RT') { + target_func <- get_target_RT + } + else stop("Invalid argument for `which`; can only be `by_FV` or `by_RT`.") + targets <- apply(dt, 1, + function(x) { + dsl_sub <- subset(dsList, funcId == x[[1]], DIM == x[[2]]) + if (length(dsl_sub) > 0) + target_func(dsl_sub) + else + NULL + }) + dt[, target := targets] + + return(dt[!sapply(dt[['target']], is.null)]) +} + +#' Helper function for `get_target_dt` +#' @noRd +get_target_FV <- function(dsList){ + vals <- get_FV_summary(dsList, Inf)[[paste0(100*getOption("IOHanalyzer.quantiles", 0.02)[[1]], '%')]] + if (is.null(attr(dsList, 'maximization')) || attr(dsList, 'maximization')) { + return(max(vals)) + } + else { + return(min(vals)) + } +} + +#' Helper function for `get_target_dt` +#' @noRd +get_target_RT <- function(dsList){ + return(max(get_runtimes(dsList))) +} + +#' Glicko2 raning of algorithms +#' +#' This procedure ranks algorithms based on a glicko2-procedure. +#' Every round (total nr_rounds), for every function and dimension of the datasetlist, +#' each pair of algorithms competes. This competition samples a random runtime for the +#' provided target (defaults to best achieved target). Whichever algorithm has the lower +#' runtime wins the game. Then, from these games, the glicko2-rating is determined. +#' +#' @param dsl The DataSetList, can contain multiple functions and dimensions, but should have the +#' same algorithms for all of them +#' @param nr_rounds The number of rounds to run. More rounds leads to a more accurate ranking. +#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective +#' @param target_dt Custom data.table target value to use. When NULL, this is selected automatically. +#' @return A dataframe containing the glicko2-ratings and some additional info +#' +#' @export +#' @examples +#' glicko2_ranking(dsl, nr_round = 25) +#' glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') +glicko2_ranking <- function(dsl, nr_rounds = 100, which = 'by_FV', target_dt = NULL){ + if (!requireNamespace("PlayerRatings", quietly = TRUE)) { + stop("Package \"pkg\" needed for this function to work. Please install it.", + call. = FALSE) + } + req(length(get_id(dsl)) > 1) + + if (!is.null(target_dt) && !('data.table' %in% class(target_dt))) { + warning("Provided `target_dt` argument is not a data.table") + target_dt <- NULL + } + + if (is.null(target_dt)) + target_dt <- get_target_dt(dsl, which) + + if (!(which %in% c('by_FV', 'by_RT'))) + stop("Invalid argument: 'which' can only be 'by_FV' or 'by_RT'") + p1 <- NULL + p2 <- NULL + scores <- NULL + weeks <- NULL + get_dims <- function(){ + dims <- get_dim(dsl) + if (length(dims) > 1) { + dims <- sample(dims) + } + dims + } + get_funcs <- function(){ + funcs <- get_funcId(dsl) + if (length(funcs) > 1) { + funcs <- sample(funcs) + } + funcs + } + n_algs = length(get_id(dsl)) + alg_names <- NULL + for (k in seq(1,nr_rounds)) { + for (dim in get_dims()) { + targets_temp <- target_dt[target_dt$DIM == dim] + for (fid in get_funcs()) { + dsl_s <- subset(dsl, funcId == fid && DIM == dim) + if (length(dsl_s) == 0) + next + if (which == 'by_FV') { + target <- targets_temp[targets_temp$funcId == fid]$target + x_arr <- get_RT_sample(dsl_s, target) + win_operator <- `<` + } + else { + target <- targets_temp[targets_temp$funcId == fid]$target + x_arr <- get_FV_sample(dsl_s, target) + win_operator <- ifelse(attr(dsl, 'maximization'), `>`, `<`) + } + vals = array(dim = c(n_algs,ncol(x_arr) - 4)) + for (i in seq(1,n_algs)) { + z <- x_arr[i] + y <- as.numeric(z[,5:ncol(x_arr)]) + vals[i,] = y + } + if (is.null(alg_names)) alg_names <- x_arr[,3] + + for (i in seq(1,n_algs)) { + for (j in seq(i,n_algs)) { + if (i == j) next + weeks <- c(weeks, k) + s1 <- sample(vals[i,], 1) + s2 <- sample(vals[j,], 1) + if (is.na(s1)) { + if (is.na(s2)) { + won <- 0.5 + } + else{ + won <- 0 + } + } + else{ + if (is.na(s2)) { + won <- 1 + } + else if (s1 == s2) { + won <- 0.5 #Tie + } + else { + won <- win_operator(s1, s2) + } + } + p1 <- c(p1, i) + p2 <- c(p2, j) + scores <- c(scores, won) + + } + } + } + } + } + # weeks <- seq(1,1,length.out = length(p1)) + games <- data.frame(Weeks = weeks, Player1 = p1, Player2 = p2, Scores = as.numeric(scores)) + lout <- PlayerRatings::glicko2(games, init = c(1500,350,0.06)) + lout$ratings$Player <- alg_names[lout$ratings$Player] + lout +} + + +#' Verify that the credentials for DSCtool have been set +#' +#' This uses the keyring package to store and load credentials. +#' If the keyring package does not exists, it will default to look for +#' a config-file in the 'repository'-folder, under your home directory. +#' This can be changed by setting the option IOHprofiler.config_dir +#' If you already have an account, please call `set_DSC_credentials` +#' with the corresponding username and password. +#' If you don't have an account, you can register for one using `register_DSC` +#' +#' +#' +#' @export +#' @examples +#' check_dsc_configured() +check_dsc_configured <- function() { + if (!requireNamespace("keyring", quietly = TRUE)) { + warning("It is recommended to have the 'keyring' package installed to store + DSCtool settings. Since this package is not found, we default + to a local settings-file instead.") + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + if (!file.exists(paste0(repo_dir, "/config.rds"))) { + return(FALSE) + } + data <- readRDS(paste0(repo_dir, "/config.rds")) + if (is.null(data$DSCusername) || is.null(data$DSCpassword)) { + return(FALSE) + } + return(TRUE) + } + tryCatch({ + username <- keyring::key_get("DSCtool_name") + pwd <- keyring::key_get("DSCtool") + return(TRUE) + }, + error = function(e) { + print("DSCtool connection not yet set. Please use the function `register_DSC` to + create an account to use the DSC tool locally (or use 'add_DSC_credentials' + to use an existing account)") + return(FALSE) + } + ) +} + +#' Register an account to the DSCtool API +#' +#' This uses the keyring package to store and load credentials. +#' If you already have an account, please call `set_DSC_credentials` instead +#' +#' @param name Your name +#' @param username A usename to be identified with. Will be stored on keyring under 'DSCtool_name' +#' @param affiliation Your affiliation (university / company) +#' @param email Your email adress +#' @param password The password to use. If NULL, this will be generated at random. +#' Will be stored on keyring under 'DSCtool' +#' +#' @export +#' @examples +#' \dontrun{ +#' register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") +#' } +register_DSC <- function(name, username, affiliation, email, password = NULL) { + if (is.null(password)) password <- stri_rand_strings(1, 10) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/manage/user" + result_json <- POST(url, add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = list(name = name, affiliation = affiliation, email = email, + username = username, password = password), + encode = "json") + if (result_json$status_code == 200) { + if (!requireNamespace("keyring", quietly = TRUE)) { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + saveRDS(list(DSCusername = username, DSCpassword = password), + paste0(repo_dir, "/config.rds")) + return(TRUE) + } + else { + keyring::key_set_with_value("DSCtool", password = password) + keyring::key_set_with_value("DSCtool_name", password = username) + return(TRUE) + } + } + else { + stop("Something went wrong in registering for DSCtool") + } +} + +#' Register an account to the DSCtool API +#' +#' This uses the keyring package to store and load credentials. +#' If you already have an account, please call `add_DSC_credentials` instead +#' +#' @param username The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name' +#' @param password The password you use on DSCtool. Will be stored on keyring under 'DSCtool' +#' +#' @export +#' @examples +#' \dontrun{set_DSC_credentials('jdoe', 'monkey123')} +set_DSC_credentials <- function(username, password) { + if (!requireNamespace("keyring", quietly = TRUE)) { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + saveRDS(list(DSCusername = username, DSCpassword = password), + paste0(repo_dir, "/config.rds")) + } + else { + keyring::key_set_with_value("DSCtool", password = password) + keyring::key_set_with_value("DSCtool_name", password = username) + } +} + +#' Load stored credentials for DSCtool +#' +#' +#' @noRd +#' @examples +#' \dontrun{ +#' get_DSC_credentials() +#' } +get_DSC_credentials <- function() { + if (!requireNamespace("keyring", quietly = TRUE)) { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + data <- readRDS(paste0(repo_dir, "/config.rds")) + return(list(name = data$DSCusername, pwd = data$DSCpassword)) + } + else { + return(list(name = keyring::key_get("DSCtool_name"), + pwd = keyring::key_get("DSCtool"))) + } +} + +#' Convert a DataSetList to the format needed for the DSCtool +#' +#' +#' @param dsList The DataSetList object +#' @param targets Optional list of target values (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. +#' +#' @noRd +#' @examples +#' convert_to_dsc_compliant(dsl) +convert_to_dsc_compliant <- function(dsList, targets = NULL, which = 'by_RT', + na.correction = NULL) { + maximize <- attr(dsList, 'maximization') + variable <- fid <- value <- NULL #Set local binding to remove warnings + by_rt <- which == 'by_RT' + + if (is.null(targets)) { + targets <- get_target_dt(dsList, which) + } + + final_list <- lapply(get_id(dsList), function(id) { + dsl_sub <- subset(dsList, ID == id) + problems <- lapply(dsl_sub, function(ds) { + target <- targets[funcId == attr(ds, 'funcId') & DIM == attr(ds, 'DIM'), 'target'] + if (!is.numeric(target)) { #TODO: more robust solution + target <- target[[1]][[1]] + } + if (by_rt) { + data <- get_RT_sample(ds, target, output = 'long')$RT + if (any(is.na(data)) & !is.null(na.correction)) { + if (na.correction == 'PAR-1') { + budgets <- attr(ds, 'maxRT') + data[is.na(data)] <- budgets[is.na(data)] + } + else if (na.correction == 'PAR-10') { + budgets <- attr(ds, 'maxRT') + data[is.na(data)] <- 10 * budgets[is.na(data)] + } + else if (na.correction == 'ERT') { + ert <- as.numeric(get_ERT(ds, target)$ERT) + if (is.infinite(ert)) { + ert <- sum(attr(ds, 'maxRT')) + } + data[is.na(data)] <- ert + } + else if (na.correction == 'Remove-na') { + data <- data[!is.na(data)] + } + else { + stop('This value for `na.correction` is not supported') + } + } + } + else { + data <- get_FV_sample(ds, target, output = 'long')$`f(x)` + if (attr(ds, 'maximization')) data <- -1 * data + } + if (any(is.na(data))) { + warning("NA-values detected in data preparation for DSCtool. This will likely result in an + error. Please verify the provided DataSetList.") + } + return(list(name = paste0("F", attr(ds, 'funcId'), "_", attr(ds, 'DIM'),"D"), + data = data)) + }) + return(list(ID = id, problems = problems)) + }) + + + return(final_list) +} + +#' Get the matrix of rankings using the DSCtool api for a DataSetList +#' +#' +#' @param dsList The DataSetList object +#' @param targets Optional list of target values (Runtime or target value) +#' @param which Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV' +#' @param test_type Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests +#' @param alpha Threshold value for statistical significance +#' @param epsilon Minimum threshold to have practical difference between algorithms (eDSC) +#' @param monte_carlo_iterations How many monte-carlo-simulations to perform +#' (set to 0 to use regular DSC) +#' @param na.correction How to deal with missing values. Only used in fixed-target perspective. +#' Options are: +#' - 'NULL': No correction is done. This will likely result in an error, as the DSCtool +#' does not allow for na values +#' - 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) +#' - 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) +#' - 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this +#' reverts to nr_runs * budget +#' - 'Remove-na' Removes all NA values +#' +#' @return A named list containing a ranked-matrix which has the rankin of each algorithm +#' on each problem, as well as a list of which omnibus tests can be used to further process +#' this data. This can be further analyzed using `get_dsc_omnibus` +#' +#' @export +#' @examples +#' get_dsc_rank(dsl) +get_dsc_rank <- function(dsList, targets = NULL, which = 'by_RT', test_type = "AD", alpha = 0.05, + epsilon = 0, monte_carlo_iterations = 0, na.correction = NULL) { + if (!check_dsc_configured()) return(NULL) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/rank" + dsc_list <- convert_to_dsc_compliant(dsList, targets, which, na.correction = na.correction) + json_list <- list(method = list(name = test_type, alpha = alpha), epsilon = epsilon, + data = dsc_list, monte_carlo_iterations = monte_carlo_iterations) + + result_json <- POST(url, + authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), + add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = json_list, encode = "json") + + return(content(result_json)$result) +} + + +#' Perform omnibus statistical tests on the matrix of rankings from the DSCtool api +#' +#' +#' @param res The result of a call to the `get_dsc_rank` +#' @param method Which method to use to do the tests. +#' Has be be one of the allowed ones in `res$valid_methods`. +#' When NULL, the first valid option is chosen by default +#' @param alpha Threshold value for statistical significance +#' +#' @return A named list containing the algorithm means +#' +#' @export +#' @examples +#' get_dsc_omnibus(get_dsc_rank(dsl)) +get_dsc_omnibus <- function(res, method = NULL, alpha = 0.05) { + if (!check_dsc_configured()) return(NULL) + if (is.null(method)) method <- res$valid_methods[[1]] + else req(method %in% res$valid_methods) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/omnibus" + new_json <- list(method = list(name = method, alpha = alpha), + ranked_matrix = res$ranked_matrix, + number_algorithms = res$number_algorithms, + parametric_tests = res$parametric_tests) + result_json <- POST(url, + authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), + add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = new_json, encode = "json") + + return(content(result_json)$result) +} + +#' Perform post-hoc processing on data from DSCtool +#' +#' +#' @param omni_res The result from a call to `get_dsc_omnibus` +#' @param nr_algs The number of algorithms present in `omni_res` +#' @param nr_problems The number of problems present in `omni_res` +#' @param base_algorithm The base algorithm to which the other are compared. +#' This has to be present in `omni_res$algorithm_means` as an `algorithm` property +#' @param method Either 'friedman' or 'friedman-aligned-rank' +#' @param alpha Threshold value for statistical significance +#' +#' @return A named list containing 4 types of analyses: +#' * Zvalue +#' * UnadjustedPValue +#' * Holm +#' * Hochberg +#' +#' @export +#' @examples +#' get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) +get_dsc_posthoc <- function(omni_res, nr_algs, nr_problems, base_algorithm = NULL, + method = "friedman", alpha = 0.05) { + if (!check_dsc_configured()) return(NULL) + if (is.null(base_algorithm)) base_algorithm <- omni_res$algorithm_means[[1]]$algorithm + else req(base_algorithm %in% unlist(omni_res$algorithm_means)) + url <- "https://ws.ijs.si:8443/dsc-1.5/service/posthoc" + new_json <- list(method = list(name = method, alpha = alpha), + algorithm_means = omni_res$algorithm_means, + n = nr_problems, + k = nr_algs, + base_algorithm = base_algorithm) + result_json <- POST(url, + authenticate(get_DSC_credentials()$name, get_DSC_credentials()$pwd), + add_headers(.headers = c('Content-Type' = "application/json", + 'Accept' = "application/json")), + body = new_json, encode = "json") + + return(content(result_json)$result) +} + + +#' Get the marginal contribution of an algorithm to a portfolio +#' +#' Based on the contribution to the ECDF-curve of the VBS of the portfolio +#' +#' @param id The id for which to get the contribution +#' @param perm The permutation of algorithms to which is being contributed +#' @param j At which point in the permutation the contribution should be measured +#' @param dt The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`) +#' +#' @export +#' @examples +#' dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) +#' get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) +get_marg_contrib_ecdf <- function(id, perm, j, dt) { + hit <- NULL #Bind to avoid notes + ids <- perm[0:j] + v_pre <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + ids <- perm[0:(j - 1)] + if (j == 1) { + v_post <- 0 + } + else + v_post <- sum(dt[ID %in% ids, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + v_pre - v_post +} + + +#' Get the shapley-values of a portfolio of algorithms +#' +#' Based on the contribution to the ECDF-curve of the VBS of the portfolio +#' +#' @param dsList The DataSetList object +#' @param targets A list or data.table containing the targets per function / dimension. If this is +#' a data.table, it needs columns 'target', 'DIM' and 'funcId' +#' @param scale.log Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not +#' @param group_size How many permutation groups will be considered +#' @param max_perm_size The maximum limit for permutations to be considered +#' @param normalize Whether or not to ensure the resulting values will be in [0,1] +#' +#' @export +#' @examples +#' get_shapley_values(dsl, get_ECDF_targets(dsl)) +get_shapley_values <- function(dsList, targets, scale.log = T, group_size = 5, max_perm_size = 10, normalize = T){ + hit <- NULL #Bind to avoid notes + ids_full <- get_id(dsList) + + dt <- generate_data.ECDF_raw(dsList, targets, scale_log = scale.log) + + nr_players <- length(ids_full) + + perms <- lapply(seq_len(nr_players * group_size), function(i) {sample(ids_full)}) + + max_val <- sum(dt[, list(hit = max(hit)), by = c('funcId', 'DIM', 'target', 'rt')][['hit']]) + ### For each algorithm, calculate shapley value + shapleys <- lapply(ids_full, function(id) { + ### For each group of permutations + temp <- mean(unlist( + lapply(seq_len(max_perm_size), function(j) { + perm_sub <- perms[((group_size * (j - 1)) + 1):(group_size * (j))] + ### + mean(unlist( + lapply(perm_sub, function(perm) { + perm <- replace(perm, c(j, which(perm == id)), c(id,perm[[j]])) + get_marg_contrib_ecdf(id, perm, j, dt) + }) + )) + + }) + )) + if (normalize) + temp <- temp / max_val #Scale so Shapleys always fall between 0 and 1 + temp + }) + + data.table(ID = ids_full, shapley = shapleys) +} + diff --git a/README.md b/README.md index 2772079d..461ea083 100644 --- a/README.md +++ b/README.md @@ -1,168 +1,168 @@ -# IOHanalyzer - - -[![metacran downloads](https://cranlogs.r-pkg.org/badges/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) -[![CRAN_Status_Badge_version_last_release](https://www.r-pkg.org/badges/version-last-release/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) -[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) -[![R-CMD-check](https://github.com/IOHprofiler/IOHanalyzer/workflows/R-CMD-check/badge.svg)](https://github.com/IOHprofiler/IOHanalyzer/actions) - - -The __performance analyzer__ for **I**terative **O**ptimization **H**euristics (IOHs). - -* __Documentation__: [https://arxiv.org/abs/2007.03953](https://arxiv.org/abs/2007.03953) -* __Wiki page__: [https://iohprofiler.github.io/IOHanalyzer/](https://iohprofiler.github.io/IOHanalyzer/) -* __Bug reports__: [https://github.com/IOHprofiler/IOHAnalyzer/issues](https://github.com/IOHprofiler/IOHAnalyzer/issues) -* __Online service__: [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) -* __General Contact__: [mailto:iohprofiler@liacs.leidenuniv.nl](mailto:iohprofiler@liacs.leidenuniv.nl) -* __Mailing List__: [https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler](https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler) - -![](./misc/demo.gif) - -It _provides_: - -* a web-based graphical user interface (GUI) to analyze and visualize the empirical performance of IOHs, -* interactive plotting, -* statistical evaluation, -* report generation, and -* a command-line interface (CLI) for the `R` console allowing for fine-grained controls. - -It is _built mainly on_: - -* `R` packages [Shiny](https://shiny.rstudio.com/), [Plotly](https://plotly.com/) and [Rcpp](http://www.rcpp.org/). - - -It is _available through_: - -* a free [online service](#server) that you can use right away, or -* the local [installation](#install) of the package. - -## Online Service - -A free server [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) running the stable version of __IOHanalyzer__ is hosted in [Leiden Institute of Advanced Computer Science](https://liacs.leidenuniv.nl/). You're welcome to check it out! - -## Installation - -### Software dependency - -* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). -* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. - -### Stable version - -Please start up an `R` console and install the stable version as: - -```r -R> install.packages('IOHanalyzer') -``` - -which is maintained on [CRAN](https://CRAN.R-project.org/package=IOHanalyzer) (Comprehensive R Archive Network). - -### Lastest version - -The lastest development is always hosted on Github. In case you'd like to try out this version, the `R` package **devtool** is needed: - -```r -R> install.packages('devtools') -R> devtools::install_github('IOHprofiler/IOHanalyzer') -``` - -### Development version - -If you want to run the version on which you develop: - -```r -R> devtools::install_git("/path/to/your/IOHanalyzer/git/repo") -``` - -## Runinng the Web Interface locally - -The IOHanalyzer package can be loaded using the following commands: - -```r -R> library('IOHanalyzer') -R> runServer() -``` - -It should open a browser on the `localhost` server, using a random port number. Of course, you -could also specify the port number directly: - -```r -R> runServer(port = 1234) -``` - -Have fun! For the complete reference on usage, please check out our [Wiki page](https://iohprofiler.github.io/). - -## Host it online - -We provide a docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. - -## Supported Data Format - -Specific formats are required to load your benchmark data to **IOHanalyzer**. If your data sets are generated in the format of - -* **COCO/BBOB** data format as regulated in [https://hal.inria.fr/inria-00362649](https://hal.inria.fr/inria-00362649), -* **Nevergrad** data format (explained in [https://github.com/facebookresearch/nevergrad](https://github.com/facebookresearch/nevergrad)), or -* **IOHprofiler** data format, which is motivated and modified from **COCO** data format, - -then you just need to compress the data folder obtained from the experiment into a __zip__ file and uploaded it. However, **you are encouraged to convert your own benchmark data to the format regulated here!**. The supported data format is specified in [this page](https://iohprofiler.github.io/IOHanalyzer/data/). Please follow the instruction there to convert your data sets. - -## Our Team - -* [Hao Wang](https://www.lip6.fr/actualite/personnes-fiche.php?ident=D2381), Sorbonne Université, LIP6, France. -* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science, The Netherlands. -* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science, The Netherlands. -* [Ofer M. Shir](https://ofersh.github.io/telhai/), Tel-Hai College, Israel. -* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), Sorbonne Université, CNRS, LIP6, France. -* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science, The Netherlands. - -When using IOHprofiler and parts thereof, please kindly cite this work as - -Hao Wang, Diederick Vermettern, Furong Ye, Carola Doerr and Thomas Bäck: _IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic_, arXiv e-prints:2007.03953, 2020. - -```bibtex -@ARTICLE{IOHprofiler, - author = {Hao Wang and Diederick Vermettern and Furong Ye and Carola Doerr and Thomas B{\"a}ck}, - title = {{IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic}}, - journal = {arXiv e-prints:2007.03953}, - archivePrefix = "arXiv", - eprint = {2007.03953}, - year = 2020, - month = July, - keywords = {Computer Science - Neural and Evolutionary Computing}, - url = {https://arxiv.org/abs/2007.03953} -} -``` - -## License - -This application is governed by the __BSD 3-Clause license__. - -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# IOHanalyzer + + +[![metacran downloads](https://cranlogs.r-pkg.org/badges/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) +[![CRAN_Status_Badge_version_last_release](https://www.r-pkg.org/badges/version-last-release/IOHanalyzer)](https://cran.r-project.org/package=IOHanalyzer) +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) +[![R-CMD-check](https://github.com/IOHprofiler/IOHanalyzer/workflows/R-CMD-check/badge.svg)](https://github.com/IOHprofiler/IOHanalyzer/actions) + + +The __performance analyzer__ for **I**terative **O**ptimization **H**euristics (IOHs). + +* __Documentation__: [https://arxiv.org/abs/2007.03953](https://arxiv.org/abs/2007.03953) +* __Wiki page__: [https://iohprofiler.github.io/IOHanalyzer/](https://iohprofiler.github.io/IOHanalyzer/) +* __Bug reports__: [https://github.com/IOHprofiler/IOHAnalyzer/issues](https://github.com/IOHprofiler/IOHAnalyzer/issues) +* __Online service__: [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) +* __General Contact__: [mailto:iohprofiler@liacs.leidenuniv.nl](mailto:iohprofiler@liacs.leidenuniv.nl) +* __Mailing List__: [https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler](https://lists.leidenuniv.nl/mailman/listinfo/iohprofiler) + +![](./misc/demo.gif) + +It _provides_: + +* a web-based graphical user interface (GUI) to analyze and visualize the empirical performance of IOHs, +* interactive plotting, +* statistical evaluation, +* report generation, and +* a command-line interface (CLI) for the `R` console allowing for fine-grained controls. + +It is _built mainly on_: + +* `R` packages [Shiny](https://shiny.rstudio.com/), [Plotly](https://plotly.com/) and [Rcpp](http://www.rcpp.org/). + + +It is _available through_: + +* a free [online service](#server) that you can use right away, or +* the local [installation](#install) of the package. + +## Online Service + +A free server [http://iohprofiler.liacs.nl](http://iohprofiler.liacs.nl) running the stable version of __IOHanalyzer__ is hosted in [Leiden Institute of Advanced Computer Science](https://liacs.leidenuniv.nl/). You're welcome to check it out! + +## Installation + +### Software dependency + +* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). +* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. + +### Stable version + +Please start up an `R` console and install the stable version as: + +```r +R> install.packages('IOHanalyzer') +``` + +which is maintained on [CRAN](https://CRAN.R-project.org/package=IOHanalyzer) (Comprehensive R Archive Network). + +### Lastest version + +The lastest development is always hosted on Github. In case you'd like to try out this version, the `R` package **devtool** is needed: + +```r +R> install.packages('devtools') +R> devtools::install_github('IOHprofiler/IOHanalyzer') +``` + +### Development version + +If you want to run the version on which you develop: + +```r +R> devtools::install_git("/path/to/your/IOHanalyzer/git/repo") +``` + +## Runinng the Web Interface locally + +The IOHanalyzer package can be loaded using the following commands: + +```r +R> library('IOHanalyzer') +R> runServer() +``` + +It should open a browser on the `localhost` server, using a random port number. Of course, you +could also specify the port number directly: + +```r +R> runServer(port = 1234) +``` + +Have fun! For the complete reference on usage, please check out our [Wiki page](https://iohprofiler.github.io/). + +## Host it online + +We provide a docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. + +## Supported Data Format + +Specific formats are required to load your benchmark data to **IOHanalyzer**. If your data sets are generated in the format of + +* **COCO/BBOB** data format as regulated in [https://hal.inria.fr/inria-00362649](https://hal.inria.fr/inria-00362649), +* **Nevergrad** data format (explained in [https://github.com/facebookresearch/nevergrad](https://github.com/facebookresearch/nevergrad)), or +* **IOHprofiler** data format, which is motivated and modified from **COCO** data format, + +then you just need to compress the data folder obtained from the experiment into a __zip__ file and uploaded it. However, **you are encouraged to convert your own benchmark data to the format regulated here!**. The supported data format is specified in [this page](https://iohprofiler.github.io/IOHanalyzer/data/). Please follow the instruction there to convert your data sets. + +## Our Team + +* [Hao Wang](https://www.lip6.fr/actualite/personnes-fiche.php?ident=D2381), Sorbonne Université, LIP6, France. +* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science, The Netherlands. +* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science, The Netherlands. +* [Ofer M. Shir](https://ofersh.github.io/telhai/), Tel-Hai College, Israel. +* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), Sorbonne Université, CNRS, LIP6, France. +* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science, The Netherlands. + +When using IOHprofiler and parts thereof, please kindly cite this work as + +Hao Wang, Diederick Vermettern, Furong Ye, Carola Doerr and Thomas Bäck: _IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic_, arXiv e-prints:2007.03953, 2020. + +```bibtex +@ARTICLE{IOHprofiler, + author = {Hao Wang and Diederick Vermettern and Furong Ye and Carola Doerr and Thomas B{\"a}ck}, + title = {{IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic}}, + journal = {arXiv e-prints:2007.03953}, + archivePrefix = "arXiv", + eprint = {2007.03953}, + year = 2020, + month = July, + keywords = {Computer Science - Neural and Evolutionary Computing}, + url = {https://arxiv.org/abs/2007.03953} +} +``` + +## License + +This application is governed by the __BSD 3-Clause license__. + +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cran-comments.md b/cran-comments.md index e3f92550..ff5d7d15 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,16 +1,16 @@ -## This is a resubmission after a previous rejection -All exported functions needed to have examples documented. This has now been implemented. - -## Test environments -* local windows install, R 3.5.1 and 3.5.3 -* local ubuntu install, R 3.4.4 -* local macOS install, R 3.5.2 - -## R CMD check results -There were no ERRORs or WARNINGs. - -There was 1 NOTE: -* New submission, since this is the first submission of the IOHanalyzer package to CRAN - -## Downstream dependencies -Since this is the first submission of the IOHanalyzer package, there are no downstream dependencies +## This is a resubmission after a previous rejection +All exported functions needed to have examples documented. This has now been implemented. + +## Test environments +* local windows install, R 3.5.1 and 3.5.3 +* local ubuntu install, R 3.4.4 +* local macOS install, R 3.5.2 + +## R CMD check results +There were no ERRORs or WARNINGs. + +There was 1 NOTE: +* New submission, since this is the first submission of the IOHanalyzer package to CRAN + +## Downstream dependencies +Since this is the first submission of the IOHanalyzer package, there are no downstream dependencies diff --git a/data/dsl.rda b/data/dsl.rda index 70d34732..dcfd8838 100644 Binary files a/data/dsl.rda and b/data/dsl.rda differ diff --git a/data/dsl_large.rda b/data/dsl_large.rda index 4266add5..3d3339ab 100644 Binary files a/data/dsl_large.rda and b/data/dsl_large.rda differ diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info b/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info index 15c8c102..aabfcb20 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info +++ b/inst/extdata/ONE_PLUS_LAMDA_EA/IOHprofiler_f1_i1.info @@ -1,3 +1,3 @@ -suite = 'PBO', funcId = 1, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' -% +suite = 'PBO', funcId = 1, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' +% data_f1/IOHprofiler_f1_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 \ No newline at end of file diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat b/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat index 26105167..de8cbf7b 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat +++ b/inst/extdata/ONE_PLUS_LAMDA_EA/data_f1/IOHprofiler_f1_DIM100_i1.dat @@ -1,472 +1,472 @@ -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 -2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 -3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 -4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 -5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 -6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 -7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 -9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 -13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 -23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 -94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 -111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 -2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 -19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 -57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 -72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 -75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 -80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 -3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 -6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 -7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 -20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 -40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 -134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 -7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 -28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 -33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 -113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 -190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 -458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 -47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 -3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 -50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 -4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 -19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 -60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 -184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 -186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 -4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 -115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 -118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 -129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 +2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 +3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 +4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 +5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 +6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 +7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 +9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 +13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 +23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 +94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 +111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 +2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 +19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 +57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 +72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 +75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 +80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 +3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 +6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 +7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 +20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 +40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 +134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 +7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 +28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 +33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 +113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 +190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 +458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 +47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 +3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 +50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 +4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 +19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 +60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 +184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 +186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 +4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 +115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 +118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 +129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info index 129b40c6..f021512e 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fbla_i1.info @@ -1,3 +1,3 @@ -funcId = bla, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' -% -data_fbla/IOHprofiler_fbla_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 +funcId = bla, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' +% +data_fbla/IOHprofiler_fbla_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info index 213c2096..021ca405 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/IOHprofiler_fblubb_i1.info @@ -1,3 +1,3 @@ -funcId = blubb, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' -% -data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 +funcId = blubb, DIM = 100, algId = 'ONE_PLUS_LAMDA_EA', algInfo = 'ONE_PLUS_LAMDA_EA' +% +data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02, 1:5000|1.00000e+02 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat index 26105167..de8cbf7b 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fbla/IOHprofiler_fbla_DIM100_i1.dat @@ -1,472 +1,472 @@ -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 -2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 -3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 -4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 -5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 -6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 -7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 -9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 -13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 -23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 -94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 -111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 -2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 -19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 -57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 -72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 -75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 -80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 -3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 -6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 -7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 -20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 -40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 -134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 -7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 -28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 -33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 -113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 -190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 -458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 -47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 -3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 -50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 -4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 -19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 -60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 -184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 -186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 -4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 -115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 -118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 -129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 +2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 +3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 +4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 +5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 +6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 +7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 +9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 +13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 +23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 +94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 +111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 +2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 +19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 +57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 +72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 +75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 +80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 +3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 +6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 +7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 +20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 +40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 +134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 +7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 +28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 +33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 +113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 +190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 +458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 +47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 +3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 +50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 +4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 +19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 +60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 +184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 +186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 +4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 +115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 +118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 +129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 diff --git a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat index 26105167..de8cbf7b 100644 --- a/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat +++ b/inst/extdata/ONE_PLUS_LAMDA_EA_ws/data_fblubb/IOHprofiler_fblubb_DIM100_i1.dat @@ -1,472 +1,472 @@ -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 -2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 -3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 -4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 -5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 -6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 -7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 -9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 -13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 -23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 -94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 -111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 -2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 -8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 -14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 -19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 -21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 -57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 -72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 -75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 -80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 -3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 -6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 -7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 -18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 -20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 -40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 -134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 -4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 -6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 -7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 -10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 -11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 -18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 -28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 -33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 -113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 -190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 -458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 -2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 -14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 -24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 -47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 -3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 -12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 -17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 -32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 -36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 -46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 -50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 -4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 -8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 -19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 -56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 -60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 -68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 -81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 -184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 -186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 -4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 -7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 -47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 -60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 -76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 -80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 -117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 -122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 -124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 -138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 -155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 -158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 -159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 -190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" -1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 -9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 -18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 -20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 -22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 -23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 -29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 -30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 -35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 -37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 -40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 -43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 -46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 -51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 -64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 -68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 -71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 -78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 -82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 -84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 -91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 -97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 -99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 -101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 -102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 -109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 -115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 -118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 -121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 -124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 -129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 -159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 -169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 -173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 -176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 -192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 -202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 -288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 -302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 -306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 -307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 -329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 -353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 -391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 -686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +3.80000e+01 +3.80000e+01 +3.80000e+01 +3.80000e+01 0.010 1.000 0.000 +2 +3.90000e+01 +3.90000e+01 +3.90000e+01 +3.90000e+01 0.010 1.000 1.000 +3 +4.10000e+01 +4.10000e+01 +4.10000e+01 +4.10000e+01 0.010 1.000 4.000 +4 +4.20000e+01 +4.20000e+01 +4.20000e+01 +4.20000e+01 0.010 1.000 3.000 +5 +4.30000e+01 +4.30000e+01 +4.30000e+01 +4.30000e+01 0.010 1.000 1.000 +6 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 1.000 +7 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 2.000 +9 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +10 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 1.000 +13 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +14 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +17 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +19 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +20 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +21 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 3.000 +23 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +29 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +34 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +39 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +49 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +54 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +55 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +56 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +57 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +60 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +65 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +76 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +82 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 4.000 +94 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +96 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +109 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +110 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 2.000 +111 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +115 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +122 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +130 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +135 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +140 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +151 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +174 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +182 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +212 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +229 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +236 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +245 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +251 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +284 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +328 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +332 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +465 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +573 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +664 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.40000e+01 +4.40000e+01 +4.40000e+01 +4.40000e+01 0.010 1.000 0.000 +2 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 1.000 +8 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +9 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +10 +4.80000e+01 +4.80000e+01 +4.80000e+01 +4.80000e+01 0.010 1.000 1.000 +14 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 2.000 +19 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +20 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 1.000 +21 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +35 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +38 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +41 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +42 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +46 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +52 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 3.000 +57 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +65 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +66 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +69 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +71 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 2.000 +72 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 2.000 +75 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +79 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 2.000 +80 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +90 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +109 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +135 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +150 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +161 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +164 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +165 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +198 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +207 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +212 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +218 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +222 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +233 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +244 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +250 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +289 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +301 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +338 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +342 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +364 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +384 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +396 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +438 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +451 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.50000e+01 +4.50000e+01 +4.50000e+01 +4.50000e+01 0.010 1.000 0.000 +3 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 1.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +5 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 2.000 +6 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 4.000 +7 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +8 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 2.000 +18 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 2.000 +20 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +22 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +27 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +29 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +31 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 4.000 +40 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +46 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +55 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +58 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +59 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +61 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +62 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +64 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +70 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +72 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +79 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +83 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +87 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +90 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +96 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +101 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +106 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +108 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +109 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +112 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +114 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 2.000 +134 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +135 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +138 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +142 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +162 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +291 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +443 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +452 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +485 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +897 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1336 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +4 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +9 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +13 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +21 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +24 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +26 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +31 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +35 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 3.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +50 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +63 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +67 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +76 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +83 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +88 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +92 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +97 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +98 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +101 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +103 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +107 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +108 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +111 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +112 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +132 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +156 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +171 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +207 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +238 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +255 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +270 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +279 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +291 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +316 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +346 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +491 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +622 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +702 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +708 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +710 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +4.60000e+01 +4.60000e+01 +4.60000e+01 +4.60000e+01 0.010 1.000 0.000 +4 +4.70000e+01 +4.70000e+01 +4.70000e+01 +4.70000e+01 0.010 1.000 1.000 +6 +4.90000e+01 +4.90000e+01 +4.90000e+01 +4.90000e+01 0.010 1.000 4.000 +7 +5.00000e+01 +5.00000e+01 +5.00000e+01 +5.00000e+01 0.010 1.000 1.000 +10 +5.10000e+01 +5.10000e+01 +5.10000e+01 +5.10000e+01 0.010 1.000 1.000 +11 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 2.000 +18 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +22 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +24 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +25 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +26 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +27 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 3.000 +28 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 3.000 +33 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +34 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +36 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +42 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +43 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +47 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +56 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +60 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +68 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +73 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +88 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +91 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +93 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +102 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +103 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +106 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 3.000 +113 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +116 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +128 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +134 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +135 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +147 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +157 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +160 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +162 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 2.000 +190 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +240 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +267 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +287 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +296 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +304 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +327 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +361 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +434 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +452 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 3.000 +458 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +624 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 0.000 +2 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 3.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +8 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +11 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +13 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 2.000 +14 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +19 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +23 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 2.000 +24 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +33 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +39 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 3.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 3.000 +47 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +53 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +62 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +67 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +69 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +70 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +71 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +79 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +81 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +87 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +94 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +95 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +106 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +116 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +129 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +143 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +152 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +159 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +163 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +164 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +171 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +176 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +187 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +190 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +229 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +387 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +543 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 0.000 +3 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +4 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +5 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +9 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 2.000 +12 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 3.000 +17 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +18 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +19 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +24 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +28 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 3.000 +32 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 3.000 +36 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +40 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +42 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +43 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 2.000 +46 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +47 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +49 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 2.000 +50 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +58 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +63 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +64 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +67 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +72 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +85 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +92 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +98 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +103 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +105 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +131 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +164 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +169 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +216 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +232 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +238 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +257 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +280 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +355 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +427 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.20000e+01 +5.20000e+01 +5.20000e+01 +5.20000e+01 0.010 1.000 0.000 +4 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 1.000 +8 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +9 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +13 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +17 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 2.000 +19 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +21 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +22 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +30 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +33 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +38 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +43 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +47 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +55 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 2.000 +56 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +57 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 2.000 +60 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +63 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +66 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 2.000 +68 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +69 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +78 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 1.000 +81 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +93 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +102 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +105 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +120 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +144 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +148 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +170 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +172 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 3.000 +184 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 3.000 +186 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +195 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +205 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +253 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +289 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +348 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +369 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +391 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +434 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +678 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +2 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 3.000 +4 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 1.000 +7 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +11 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +14 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +16 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +23 +6.20000e+01 +6.20000e+01 +6.20000e+01 +6.20000e+01 0.010 1.000 1.000 +47 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 1.000 +60 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +63 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +66 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +73 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +74 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 2.000 +76 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +79 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 3.000 +80 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +91 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +108 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +110 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 2.000 +117 +7.90000e+01 +7.90000e+01 +7.90000e+01 +7.90000e+01 0.010 1.000 1.000 +122 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 3.000 +124 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 1.000 +138 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 2.000 +155 +8.40000e+01 +8.40000e+01 +8.40000e+01 +8.40000e+01 0.010 1.000 1.000 +158 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 1.000 +159 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +165 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +168 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +177 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 2.000 +190 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +215 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +229 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +238 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +249 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +269 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +290 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +364 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +577 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +1103 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "mutation_rate" "lambda" "l" +1 +5.30000e+01 +5.30000e+01 +5.30000e+01 +5.30000e+01 0.010 1.000 0.000 +9 +5.40000e+01 +5.40000e+01 +5.40000e+01 +5.40000e+01 0.010 1.000 1.000 +18 +5.50000e+01 +5.50000e+01 +5.50000e+01 +5.50000e+01 0.010 1.000 1.000 +20 +5.60000e+01 +5.60000e+01 +5.60000e+01 +5.60000e+01 0.010 1.000 1.000 +22 +5.70000e+01 +5.70000e+01 +5.70000e+01 +5.70000e+01 0.010 1.000 3.000 +23 +5.80000e+01 +5.80000e+01 +5.80000e+01 +5.80000e+01 0.010 1.000 1.000 +29 +5.90000e+01 +5.90000e+01 +5.90000e+01 +5.90000e+01 0.010 1.000 1.000 +30 +6.00000e+01 +6.00000e+01 +6.00000e+01 +6.00000e+01 0.010 1.000 1.000 +35 +6.10000e+01 +6.10000e+01 +6.10000e+01 +6.10000e+01 0.010 1.000 1.000 +37 +6.30000e+01 +6.30000e+01 +6.30000e+01 +6.30000e+01 0.010 1.000 2.000 +40 +6.40000e+01 +6.40000e+01 +6.40000e+01 +6.40000e+01 0.010 1.000 1.000 +43 +6.50000e+01 +6.50000e+01 +6.50000e+01 +6.50000e+01 0.010 1.000 1.000 +46 +6.60000e+01 +6.60000e+01 +6.60000e+01 +6.60000e+01 0.010 1.000 1.000 +51 +6.70000e+01 +6.70000e+01 +6.70000e+01 +6.70000e+01 0.010 1.000 1.000 +64 +6.80000e+01 +6.80000e+01 +6.80000e+01 +6.80000e+01 0.010 1.000 1.000 +68 +6.90000e+01 +6.90000e+01 +6.90000e+01 +6.90000e+01 0.010 1.000 1.000 +71 +7.00000e+01 +7.00000e+01 +7.00000e+01 +7.00000e+01 0.010 1.000 1.000 +78 +7.10000e+01 +7.10000e+01 +7.10000e+01 +7.10000e+01 0.010 1.000 1.000 +82 +7.20000e+01 +7.20000e+01 +7.20000e+01 +7.20000e+01 0.010 1.000 1.000 +84 +7.30000e+01 +7.30000e+01 +7.30000e+01 +7.30000e+01 0.010 1.000 1.000 +91 +7.40000e+01 +7.40000e+01 +7.40000e+01 +7.40000e+01 0.010 1.000 1.000 +97 +7.50000e+01 +7.50000e+01 +7.50000e+01 +7.50000e+01 0.010 1.000 1.000 +99 +7.60000e+01 +7.60000e+01 +7.60000e+01 +7.60000e+01 0.010 1.000 1.000 +101 +7.70000e+01 +7.70000e+01 +7.70000e+01 +7.70000e+01 0.010 1.000 1.000 +102 +7.80000e+01 +7.80000e+01 +7.80000e+01 +7.80000e+01 0.010 1.000 1.000 +109 +8.00000e+01 +8.00000e+01 +8.00000e+01 +8.00000e+01 0.010 1.000 2.000 +115 +8.10000e+01 +8.10000e+01 +8.10000e+01 +8.10000e+01 0.010 1.000 3.000 +118 +8.20000e+01 +8.20000e+01 +8.20000e+01 +8.20000e+01 0.010 1.000 1.000 +121 +8.30000e+01 +8.30000e+01 +8.30000e+01 +8.30000e+01 0.010 1.000 1.000 +124 +8.50000e+01 +8.50000e+01 +8.50000e+01 +8.50000e+01 0.010 1.000 2.000 +129 +8.60000e+01 +8.60000e+01 +8.60000e+01 +8.60000e+01 0.010 1.000 1.000 +159 +8.70000e+01 +8.70000e+01 +8.70000e+01 +8.70000e+01 0.010 1.000 1.000 +169 +8.80000e+01 +8.80000e+01 +8.80000e+01 +8.80000e+01 0.010 1.000 1.000 +173 +8.90000e+01 +8.90000e+01 +8.90000e+01 +8.90000e+01 0.010 1.000 1.000 +176 +9.00000e+01 +9.00000e+01 +9.00000e+01 +9.00000e+01 0.010 1.000 1.000 +192 +9.10000e+01 +9.10000e+01 +9.10000e+01 +9.10000e+01 0.010 1.000 1.000 +202 +9.20000e+01 +9.20000e+01 +9.20000e+01 +9.20000e+01 0.010 1.000 1.000 +288 +9.30000e+01 +9.30000e+01 +9.30000e+01 +9.30000e+01 0.010 1.000 1.000 +302 +9.40000e+01 +9.40000e+01 +9.40000e+01 +9.40000e+01 0.010 1.000 1.000 +306 +9.50000e+01 +9.50000e+01 +9.50000e+01 +9.50000e+01 0.010 1.000 1.000 +307 +9.60000e+01 +9.60000e+01 +9.60000e+01 +9.60000e+01 0.010 1.000 1.000 +329 +9.70000e+01 +9.70000e+01 +9.70000e+01 +9.70000e+01 0.010 1.000 1.000 +353 +9.80000e+01 +9.80000e+01 +9.80000e+01 +9.80000e+01 0.010 1.000 1.000 +391 +9.90000e+01 +9.90000e+01 +9.90000e+01 +9.90000e+01 0.010 1.000 1.000 +686 +1.00000e+02 +1.00000e+02 +1.00000e+02 +1.00000e+02 0.010 1.000 1.000 diff --git a/inst/shiny-server/LICENSE.txt b/inst/shiny-server/LICENSE.txt index d0795340..1b5b71d5 100644 --- a/inst/shiny-server/LICENSE.txt +++ b/inst/shiny-server/LICENSE.txt @@ -1,29 +1,29 @@ -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/inst/shiny-server/global.R b/inst/shiny-server/global.R index ef645e7b..1bf460a1 100644 --- a/inst/shiny-server/global.R +++ b/inst/shiny-server/global.R @@ -1,262 +1,262 @@ -suppressMessages(library(IOHanalyzer)) -suppressMessages(library(shiny)) -suppressMessages(library(shinyjs)) -suppressMessages(library(reshape2)) -suppressMessages(library(magrittr)) -suppressMessages(library(data.table)) -suppressMessages(library(dplyr)) -suppressMessages(library(plotly)) -suppressMessages(library(shinydashboard)) -suppressMessages(library(xtable)) -suppressMessages(library(colourpicker)) -suppressMessages(library(bsplus)) -suppressMessages(library(DT)) -suppressMessages(library(knitr)) -suppressMessages(library(kableExtra)) - -# global options -options(datatable.print.nrows = 20) -options(width = 80) -options(shiny.maxRequestSize = 500 * 1024 ^ 2) # maximal upload file size - -# for customized 'plotlyOutput' function ----- -widget_html <- function(name, package, id, style, class, inline = FALSE, ...) { - # attempt to lookup custom html function for widget - fn <- tryCatch(get(paste0(name, "_html"), - asNamespace(package), - inherits = FALSE), - error = function(e) NULL) - - # call the custom function if we have one, otherwise create a div - if (is.function(fn)) { - fn(id = id, style = style, class = class, ...) - } else if (inline) { - tags$span(id = id, style = style, class = class) - } else { - tags$div(id = id, style = style, class = class) - } -} - -checkShinyVersion <- function(error = TRUE) { - x <- utils::packageDescription('htmlwidgets', fields = 'Enhances') - r <- '^.*?shiny \\(>= ([0-9.]+)\\).*$' - if (is.na(x) || length(grep(r, x)) == 0 || system.file(package = 'shiny') == '') - return() - v <- gsub(r, '\\1', x) - f <- if (error) stop else packageStartupMessage - if (utils::packageVersion('shiny') < v) - f("Please upgrade the 'shiny' package to (at least) version ", v) -} - -widget_dependencies <- function(name, package){ - htmlwidgets::getDependency(name, package) -} - -plotlyOutput.IOHanalyzer <- function(outputId, width = '100%', aspect_ratio = 16/10) { - padding_bottom <- paste0(100 / aspect_ratio, '%') - reportSize <- TRUE - inline <- FALSE - - checkShinyVersion() - html <- htmltools::tagList( - widget_html('plotly', 'plotly', id = outputId, - class = paste0('plotly', " html-widget html-widget-output", - if (reportSize) - " shiny-report-size"), - style = sprintf("width:%s; height: 0; padding-bottom:%s; %s", - htmltools::validateCssUnit(width), - htmltools::validateCssUnit(padding_bottom), - if (inline) - "display: inline-block;" - else ""), - width = width, height = 0) - ) - dependencies <- widget_dependencies('plotly', 'plotly') - htmltools::attachDependencies(html, dependencies) -} - -# markers for plotly -symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", - "triangle-up-open", "triangle-down-open") - -# ploting settings for UI --------------------- -plotly_height <- "auto" -plotly_width <- "auto" -plotly_height2 <- "auto" -plotly_width2 <- "auto" - -IOHprofiler <- 'IOHprofiler' -COCO <- 'COCO' -TWO_COL <- 'TWO_COL' -AUTOMATIC <- 'AUTOMATIC' -BIBOJ_COCO <- 'BIBOJ_COCO' -NEVERGRAD <- 'NEVERGRAD' - -# directory where rds-data is stored -get_repo_location <- function() { - repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) - if (!is.null(getOption("IOHprofiler.repo_dir"))) { - repo_dir <- getOption("IOHprofiler.repo_dir") - } - if (file.exists(repo_dir)) repo_dir else '' -} - -print_html <- function(s, widget_id = 'process_data_promt') - shinyjs::html(widget_id, s, add = TRUE) - -# download file names: csv, image --------------------- -AUC_ECDF_aggr_name <- parse(text = "paste0('AUC_ECDF_MULTI.', input$RTECDF.AUC.Table.Format)") -overview_single_name <- parse(text = "paste0('Overview-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$Overview.Single.Format)") -overview_all_name <- parse(text = "paste0('Overview-All-', '.', input$Overview.All.Format)") -RT_csv_name <- parse(text = "paste0('RTStats-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RTSummary.Statistics.Format)") -RT_overview_name <- parse(text = "paste0('RTOverview-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RTSummary.Overview.Format)") -RTSample_csv_name <- parse(text = "paste0('RTSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RTSummary.Sample.Format)") -FV_csv_name <- parse(text = "paste0('FVStats-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FCESummary.Statistics.Format)") -FV_overview_name <- parse(text = "paste0('FVOverview-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FCESummary.Overview.Format)") -FVSample_csv_name <- parse(text = "paste0('FVSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FCESummary.Sample.FileFormat)") -FV_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Summary.Format)") -FV_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Sample.FileFormat)") -RT_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Summary.Format)") -RT_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Sample.FileFormat)") -ERT_multi_func_name <- parse(text = "paste0('MultiERT-', paste0(input$Overall.Dim, 'D'), - '.', input$ERTPlot.Aggr.TableFormat)") -ERT_multi_dim_name <- parse(text = "paste0('MultiERT-', paste0('F', input$Overall.Funcid), - '.', input$ERTPlot.Aggr_Dim.TableFormat)") -FCE_multi_func_name <- parse(text = "paste0('MultiFCE-', paste0(input$Overall.Dim, 'D'), - '.', input$FCEPlot.Aggr.TableFormat)") -RT_Glicko2_table_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.TableFormat)") -RT_Glicko2_figure_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.Format)") - -RT_DSC_table_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.TableFormat)") -RT_DSC_figure_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.Format)") -RT_DSC_figure_name_rank <- parse(text = "paste0('RT_DSC_PerformViz', '.', input$RT_Stats.DSC.Format_rank)") -RT_DSC_table_name_rank <- parse(text = "paste0('RT_DSC_Rank', '.', input$RT_Stats.DSC.TableFormat_rank)") - -FV_DSC_table_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.TableFormat)") -FV_DSC_figure_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.Format)") -FV_DSC_figure_name_rank <- parse(text = "paste0('FV_DSC_PerformViz', '.', input$FV_Stats.DSC.Format_rank)") -FV_DSC_table_name_rank <- parse(text = "paste0('FV_DSC_Rank', '.', input$FV_Stats.DSC.TableFormat_rank)") - -RT_Stats_table_name <- parse(text = "paste0('RT_Stat_Comp-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.TableFormat)") -RT_Stats_heatmap_name <- parse(text = "paste0('RT_Stat_Heatmap-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") -RT_Stats_network_name <- parse(text = "paste0('RT_Stat_Network-', paste0(input$Overall.Dim, 'D'), - paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") -RT_multifunc_ERT <- parse(text = "paste0('ERT_Table_Multi', '.', input$RT.MultiERT.Format)") -RT_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi', '.', input$RT.MultiSample.Format)") -FV_multifunc_FV <- parse(text = "paste0('FV_Table_Multi', '.', input$FV.MultiFV.Format)") -FV_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi_FV', '.', input$FV.MultiSample.Format)") -# max_samples <- 100 - -FIG_NAME_ERT_PER_FUN <- parse(text = "paste0('ERT-', Sys.Date(), '.', input$ERTPlot.Format)") -FIG_NAME_ERT_PER_FUN_MULTI <- parse(text = "paste0('ERT_Mult-', Sys.Date(), '.', input$ERTPlot.Multi.Format)") -FIG_NAME_ERT_AGGR <- parse(text = "paste0('ERT_Aggr-', Sys.Date(), '.', input$ERTPlot.Aggr.Format)") -FIG_NAME_ERT_AGGR_DIM <- parse(text = "paste0('ERT_Aggr_Dim-', Sys.Date(), '.', input$ERTPlot.Aggr_Dim.Format)") -FIG_NAME_RT_PMF <- parse(text = "paste0('RT_PMF-', Sys.Date(), '.', input$RTPMF.Bar.Format)") -FIG_NAME_RT_HIST <- parse(text = "paste0('RT_HIST-', Sys.Date(), '.', input$RTPMF.Hist.Format)") -FIG_NAME_RT_ECDF_AGGR <- parse(text = "paste0('RT_ECDF_AGGR-', Sys.Date(), '.', input$RTECDF.Multi.Format)") -FIG_NAME_RT_ECDF_MULT <- parse(text = "paste0('RT_ECDF_MULT-', Sys.Date(), '.', input$RTECDF.Aggr.Format)") -FIG_NAME_RT_AUC <- parse(text = "paste0('RT_AUC-', Sys.Date(), '.', input$RTECDF.AUC.Format)") - -FIG_NAME_FV_PER_FUN <- parse(text = "paste0('FV-', Sys.Date(), '.', input$FCEPlot.Format)") -FIG_NAME_FV_PER_FUN_MULTI <- parse(text = "paste0('FCE_Mult-', Sys.Date(), '.', input$FCEPlot.Multi.Format)") -FIG_NAME_FV_AGGR <- parse(text = "paste0('FCE_Aggr-', Sys.Date(), '.', input$FCEPlot.Aggr.Format)") -FIG_NAME_FV_PDF <- parse(text = "paste0('FV_PMF-', Sys.Date(), '.', input$FCEPDF.Bar.Format)") -FIG_NAME_FV_HIST <- parse(text = "paste0('FV_HIST-', Sys.Date(), '.', input$FCEPDF.Hist.Format)") -FIG_NAME_FV_ECDF_AGGR <- parse(text = "paste0('FV_ECDF_AGGR-', Sys.Date(), '.', input$FCEECDF.Mult.Format)") -FIG_NAME_FV_AUC <- parse(text = "paste0('FV_AUC-', Sys.Date(), '.', input$FCEECDF.AUC.Format)") - -FIG_NAME_RT_PAR_PER_FUN <- parse(text = "paste0('RT_PAR-', Sys.Date(), '.', input$RT_PAR.Plot.Format)") -FIG_NAME_FV_PAR_PER_FUN <- parse(text = "paste0('FV_PAR-', Sys.Date(), '.', input$FV_PAR.Plot.Format)") - -FIG_NAME_RT_SHAPLEY <- parse(text = "paste0('RT-Shapley-', Sys.Date(), '.', input$RTportfolio.Shapley.Format)") - -# ID of the control widget, whose current value should de always recorded and restored ---- -# those control widget are switched on and off -widget_id <- c('RTSummary.Statistics.Min', - 'RTSummary.Statistics.Max', - 'RTSummary.Statistics.Step', - 'RTSummary.Sample.Min', - 'RTSummary.Sample.Max', - 'RTSummary.Sample.Step', - 'RTECDF.Multi.Min', - 'RTECDF.Multi.Max', - 'RTECDF.Multi.Step', - 'RTECDF.Single.Target', - 'RTPMF.Bar.Target', - 'RTPMF.Hist.Target', - 'ERTPlot.Min', - 'ERTPlot.Max', - 'ERTPlot.Aggr.Targets', - 'RTECDF.AUC.Min', - 'RTECDF.AUC.Max', - 'RTECDF.AUC.Step', - 'FV_PAR.Plot.Min', - 'FV_PAR.Plot.Max', - 'FV_PAR.Summary.Min', - 'FV_PAR.Summary.Max', - 'FV_PAR.Summary.Step', - 'FV_PAR.Sample.Min', - 'FV_PAR.Sample.Max', - 'FV_PAR.Sample.Step', - 'RT_PAR.Plot.Min', - 'RT_PAR.Plot.Max', - 'RT_PAR.Summary.Min', - 'RT_PAR.Summary.Max', - 'RT_PAR.Summary.Step', - 'RT_PAR.Sample.Min', - 'RT_PAR.Sample.Max', - 'RT_PAR.Sample.Step', - 'FCESummary.Statistics.Min', - 'FCESummary.Statistics.Max', - 'FCESummary.Statistics.Step', - 'FCESummary.Sample.Min', - 'FCESummary.Sample.Max', - 'FCESummary.Sample.Step', - 'FCEPDF.Hist.Runtime', - 'FCEPDF.Bar.Runtime', - 'FCEPlot.Min', - 'FCEPlot.Max', - 'FCEECDF.Mult.Min', - 'FCEECDF.Mult.Max', - 'FCEECDF.Mult.Step', - 'FCEECDF.AUC.Min', - 'FCEECDF.AUC.Max', - 'FCEECDF.AUC.Step', - 'FCEECDF.Single.Target') - -eventExpr <- parse(text = paste0('{', paste(paste0('input$', widget_id), collapse = "\n"), '}')) - -# Supported formats table -supported_table_format <- c("csv", "tex", "md", "html") - -# token needed for mapbox, which is again needed for ocra... ------ -supported_fig_format <- c('pdf', 'png', 'eps', 'svg') -Sys.setenv('MAPBOX_TOKEN' = 'pk.eyJ1Ijoid2FuZ3JvbmluIiwiYSI6ImNqcmIzemhvMDBudnYzeWxoejh5c2Y5cXkifQ.9XGMWTDOsgi3-b5qG594kQ') - -sanity_check_id <- function(input) { - for (id in widget_id) { - tryCatch(eval(parse(text = paste0('input$', id))), - error = function(e) { - cat(paste('widget', id, 'does not exist!\n')) - }) - } -} - - - - - - +suppressMessages(library(IOHanalyzer)) +suppressMessages(library(shiny)) +suppressMessages(library(shinyjs)) +suppressMessages(library(reshape2)) +suppressMessages(library(magrittr)) +suppressMessages(library(data.table)) +suppressMessages(library(dplyr)) +suppressMessages(library(plotly)) +suppressMessages(library(shinydashboard)) +suppressMessages(library(xtable)) +suppressMessages(library(colourpicker)) +suppressMessages(library(bsplus)) +suppressMessages(library(DT)) +suppressMessages(library(knitr)) +suppressMessages(library(kableExtra)) + +# global options +options(datatable.print.nrows = 20) +options(width = 80) +options(shiny.maxRequestSize = 500 * 1024 ^ 2) # maximal upload file size + +# for customized 'plotlyOutput' function ----- +widget_html <- function(name, package, id, style, class, inline = FALSE, ...) { + # attempt to lookup custom html function for widget + fn <- tryCatch(get(paste0(name, "_html"), + asNamespace(package), + inherits = FALSE), + error = function(e) NULL) + + # call the custom function if we have one, otherwise create a div + if (is.function(fn)) { + fn(id = id, style = style, class = class, ...) + } else if (inline) { + tags$span(id = id, style = style, class = class) + } else { + tags$div(id = id, style = style, class = class) + } +} + +checkShinyVersion <- function(error = TRUE) { + x <- utils::packageDescription('htmlwidgets', fields = 'Enhances') + r <- '^.*?shiny \\(>= ([0-9.]+)\\).*$' + if (is.na(x) || length(grep(r, x)) == 0 || system.file(package = 'shiny') == '') + return() + v <- gsub(r, '\\1', x) + f <- if (error) stop else packageStartupMessage + if (utils::packageVersion('shiny') < v) + f("Please upgrade the 'shiny' package to (at least) version ", v) +} + +widget_dependencies <- function(name, package){ + htmlwidgets::getDependency(name, package) +} + +plotlyOutput.IOHanalyzer <- function(outputId, width = '100%', aspect_ratio = 16/10) { + padding_bottom <- paste0(100 / aspect_ratio, '%') + reportSize <- TRUE + inline <- FALSE + + checkShinyVersion() + html <- htmltools::tagList( + widget_html('plotly', 'plotly', id = outputId, + class = paste0('plotly', " html-widget html-widget-output", + if (reportSize) + " shiny-report-size"), + style = sprintf("width:%s; height: 0; padding-bottom:%s; %s", + htmltools::validateCssUnit(width), + htmltools::validateCssUnit(padding_bottom), + if (inline) + "display: inline-block;" + else ""), + width = width, height = 0) + ) + dependencies <- widget_dependencies('plotly', 'plotly') + htmltools::attachDependencies(html, dependencies) +} + +# markers for plotly +symbols <- c("circle-open", "diamond-open", "square-open", "cross-open", + "triangle-up-open", "triangle-down-open") + +# ploting settings for UI --------------------- +plotly_height <- "auto" +plotly_width <- "auto" +plotly_height2 <- "auto" +plotly_width2 <- "auto" + +IOHprofiler <- 'IOHprofiler' +COCO <- 'COCO' +TWO_COL <- 'TWO_COL' +AUTOMATIC <- 'AUTOMATIC' +BIBOJ_COCO <- 'BIBOJ_COCO' +NEVERGRAD <- 'NEVERGRAD' + +# directory where rds-data is stored +get_repo_location <- function() { + repo_dir <- paste0(file.path(Sys.getenv('HOME'), 'repository')) + if (!is.null(getOption("IOHprofiler.repo_dir"))) { + repo_dir <- getOption("IOHprofiler.repo_dir") + } + if (file.exists(repo_dir)) repo_dir else '' +} + +print_html <- function(s, widget_id = 'process_data_promt') + shinyjs::html(widget_id, s, add = TRUE) + +# download file names: csv, image --------------------- +AUC_ECDF_aggr_name <- parse(text = "paste0('AUC_ECDF_MULTI.', input$RTECDF.AUC.Table.Format)") +overview_single_name <- parse(text = "paste0('Overview-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$Overview.Single.Format)") +overview_all_name <- parse(text = "paste0('Overview-All-', '.', input$Overview.All.Format)") +RT_csv_name <- parse(text = "paste0('RTStats-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RTSummary.Statistics.Format)") +RT_overview_name <- parse(text = "paste0('RTOverview-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RTSummary.Overview.Format)") +RTSample_csv_name <- parse(text = "paste0('RTSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RTSummary.Sample.Format)") +FV_csv_name <- parse(text = "paste0('FVStats-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FCESummary.Statistics.Format)") +FV_overview_name <- parse(text = "paste0('FVOverview-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FCESummary.Overview.Format)") +FVSample_csv_name <- parse(text = "paste0('FVSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FCESummary.Sample.FileFormat)") +FV_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Summary.Format)") +FV_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$FV_PAR.Sample.FileFormat)") +RT_PAR_csv_name <- parse(text = "paste0('PARSummary-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Summary.Format)") +RT_PARSample_csv_name <- parse(text = "paste0('PARSample-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_PAR.Sample.FileFormat)") +ERT_multi_func_name <- parse(text = "paste0('MultiERT-', paste0(input$Overall.Dim, 'D'), + '.', input$ERTPlot.Aggr.TableFormat)") +ERT_multi_dim_name <- parse(text = "paste0('MultiERT-', paste0('F', input$Overall.Funcid), + '.', input$ERTPlot.Aggr_Dim.TableFormat)") +FCE_multi_func_name <- parse(text = "paste0('MultiFCE-', paste0(input$Overall.Dim, 'D'), + '.', input$FCEPlot.Aggr.TableFormat)") +RT_Glicko2_table_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.TableFormat)") +RT_Glicko2_figure_name <- parse(text = "paste0('RT_Glicko2', '.', input$RT_Stats.Glicko.Format)") + +RT_DSC_table_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.TableFormat)") +RT_DSC_figure_name <- parse(text = "paste0('RT_DSC', '.', input$RT_Stats.DSC.Format)") +RT_DSC_figure_name_rank <- parse(text = "paste0('RT_DSC_PerformViz', '.', input$RT_Stats.DSC.Format_rank)") +RT_DSC_table_name_rank <- parse(text = "paste0('RT_DSC_Rank', '.', input$RT_Stats.DSC.TableFormat_rank)") + +FV_DSC_table_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.TableFormat)") +FV_DSC_figure_name <- parse(text = "paste0('FV_DSC', '.', input$FV_Stats.DSC.Format)") +FV_DSC_figure_name_rank <- parse(text = "paste0('FV_DSC_PerformViz', '.', input$FV_Stats.DSC.Format_rank)") +FV_DSC_table_name_rank <- parse(text = "paste0('FV_DSC_Rank', '.', input$FV_Stats.DSC.TableFormat_rank)") + +RT_Stats_table_name <- parse(text = "paste0('RT_Stat_Comp-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.TableFormat)") +RT_Stats_heatmap_name <- parse(text = "paste0('RT_Stat_Heatmap-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") +RT_Stats_network_name <- parse(text = "paste0('RT_Stat_Network-', paste0(input$Overall.Dim, 'D'), + paste0('F', input$Overall.Funcid), '.', input$RT_Stats.Overview.Format)") +RT_multifunc_ERT <- parse(text = "paste0('ERT_Table_Multi', '.', input$RT.MultiERT.Format)") +RT_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi', '.', input$RT.MultiSample.Format)") +FV_multifunc_FV <- parse(text = "paste0('FV_Table_Multi', '.', input$FV.MultiFV.Format)") +FV_multifunc_sample <- parse(text = "paste0('Sample_Table_Multi_FV', '.', input$FV.MultiSample.Format)") +# max_samples <- 100 + +FIG_NAME_ERT_PER_FUN <- parse(text = "paste0('ERT-', Sys.Date(), '.', input$ERTPlot.Format)") +FIG_NAME_ERT_PER_FUN_MULTI <- parse(text = "paste0('ERT_Mult-', Sys.Date(), '.', input$ERTPlot.Multi.Format)") +FIG_NAME_ERT_AGGR <- parse(text = "paste0('ERT_Aggr-', Sys.Date(), '.', input$ERTPlot.Aggr.Format)") +FIG_NAME_ERT_AGGR_DIM <- parse(text = "paste0('ERT_Aggr_Dim-', Sys.Date(), '.', input$ERTPlot.Aggr_Dim.Format)") +FIG_NAME_RT_PMF <- parse(text = "paste0('RT_PMF-', Sys.Date(), '.', input$RTPMF.Bar.Format)") +FIG_NAME_RT_HIST <- parse(text = "paste0('RT_HIST-', Sys.Date(), '.', input$RTPMF.Hist.Format)") +FIG_NAME_RT_ECDF_AGGR <- parse(text = "paste0('RT_ECDF_AGGR-', Sys.Date(), '.', input$RTECDF.Multi.Format)") +FIG_NAME_RT_ECDF_MULT <- parse(text = "paste0('RT_ECDF_MULT-', Sys.Date(), '.', input$RTECDF.Aggr.Format)") +FIG_NAME_RT_AUC <- parse(text = "paste0('RT_AUC-', Sys.Date(), '.', input$RTECDF.AUC.Format)") + +FIG_NAME_FV_PER_FUN <- parse(text = "paste0('FV-', Sys.Date(), '.', input$FCEPlot.Format)") +FIG_NAME_FV_PER_FUN_MULTI <- parse(text = "paste0('FCE_Mult-', Sys.Date(), '.', input$FCEPlot.Multi.Format)") +FIG_NAME_FV_AGGR <- parse(text = "paste0('FCE_Aggr-', Sys.Date(), '.', input$FCEPlot.Aggr.Format)") +FIG_NAME_FV_PDF <- parse(text = "paste0('FV_PMF-', Sys.Date(), '.', input$FCEPDF.Bar.Format)") +FIG_NAME_FV_HIST <- parse(text = "paste0('FV_HIST-', Sys.Date(), '.', input$FCEPDF.Hist.Format)") +FIG_NAME_FV_ECDF_AGGR <- parse(text = "paste0('FV_ECDF_AGGR-', Sys.Date(), '.', input$FCEECDF.Mult.Format)") +FIG_NAME_FV_AUC <- parse(text = "paste0('FV_AUC-', Sys.Date(), '.', input$FCEECDF.AUC.Format)") + +FIG_NAME_RT_PAR_PER_FUN <- parse(text = "paste0('RT_PAR-', Sys.Date(), '.', input$RT_PAR.Plot.Format)") +FIG_NAME_FV_PAR_PER_FUN <- parse(text = "paste0('FV_PAR-', Sys.Date(), '.', input$FV_PAR.Plot.Format)") + +FIG_NAME_RT_SHAPLEY <- parse(text = "paste0('RT-Shapley-', Sys.Date(), '.', input$RTportfolio.Shapley.Format)") + +# ID of the control widget, whose current value should de always recorded and restored ---- +# those control widget are switched on and off +widget_id <- c('RTSummary.Statistics.Min', + 'RTSummary.Statistics.Max', + 'RTSummary.Statistics.Step', + 'RTSummary.Sample.Min', + 'RTSummary.Sample.Max', + 'RTSummary.Sample.Step', + 'RTECDF.Multi.Min', + 'RTECDF.Multi.Max', + 'RTECDF.Multi.Step', + 'RTECDF.Single.Target', + 'RTPMF.Bar.Target', + 'RTPMF.Hist.Target', + 'ERTPlot.Min', + 'ERTPlot.Max', + 'ERTPlot.Aggr.Targets', + 'RTECDF.AUC.Min', + 'RTECDF.AUC.Max', + 'RTECDF.AUC.Step', + 'FV_PAR.Plot.Min', + 'FV_PAR.Plot.Max', + 'FV_PAR.Summary.Min', + 'FV_PAR.Summary.Max', + 'FV_PAR.Summary.Step', + 'FV_PAR.Sample.Min', + 'FV_PAR.Sample.Max', + 'FV_PAR.Sample.Step', + 'RT_PAR.Plot.Min', + 'RT_PAR.Plot.Max', + 'RT_PAR.Summary.Min', + 'RT_PAR.Summary.Max', + 'RT_PAR.Summary.Step', + 'RT_PAR.Sample.Min', + 'RT_PAR.Sample.Max', + 'RT_PAR.Sample.Step', + 'FCESummary.Statistics.Min', + 'FCESummary.Statistics.Max', + 'FCESummary.Statistics.Step', + 'FCESummary.Sample.Min', + 'FCESummary.Sample.Max', + 'FCESummary.Sample.Step', + 'FCEPDF.Hist.Runtime', + 'FCEPDF.Bar.Runtime', + 'FCEPlot.Min', + 'FCEPlot.Max', + 'FCEECDF.Mult.Min', + 'FCEECDF.Mult.Max', + 'FCEECDF.Mult.Step', + 'FCEECDF.AUC.Min', + 'FCEECDF.AUC.Max', + 'FCEECDF.AUC.Step', + 'FCEECDF.Single.Target') + +eventExpr <- parse(text = paste0('{', paste(paste0('input$', widget_id), collapse = "\n"), '}')) + +# Supported formats table +supported_table_format <- c("csv", "tex", "md", "html") + +# token needed for mapbox, which is again needed for ocra... ------ +supported_fig_format <- c('pdf', 'png', 'eps', 'svg') +Sys.setenv('MAPBOX_TOKEN' = 'pk.eyJ1Ijoid2FuZ3JvbmluIiwiYSI6ImNqcmIzemhvMDBudnYzeWxoejh5c2Y5cXkifQ.9XGMWTDOsgi3-b5qG594kQ') + +sanity_check_id <- function(input) { + for (id in widget_id) { + tryCatch(eval(parse(text = paste0('input$', id))), + error = function(e) { + cat(paste('widget', id, 'does not exist!\n')) + }) + } +} + + + + + + diff --git a/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd index 64a818c8..cc4fbe0a 100644 --- a/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/FV_PAR_SUMMARY_TABLE.Rmd @@ -1,9 +1,9 @@ -This table summarizes for each algorithm and each runtime value chosen on the left: - - * runs: the number of runs where non-missing parameter values are observed, for each required runtime value, - * mean: the average value of the specified __parameter__ after the specified amount of function evaluations. - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values - -When not all runs used the specified runtime value, the statistics hold only for those runs that did. That is, the mean value is the mean of the applicable runs. Same for the quantiles. - - +This table summarizes for each algorithm and each runtime value chosen on the left: + + * runs: the number of runs where non-missing parameter values are observed, for each required runtime value, + * mean: the average value of the specified __parameter__ after the specified amount of function evaluations. + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values + +When not all runs used the specified runtime value, the statistics hold only for those runs that did. That is, the mean value is the mean of the applicable runs. Same for the quantiles. + + diff --git a/inst/shiny-server/markdown/INSTALL.Rmd b/inst/shiny-server/markdown/INSTALL.Rmd index d5f1fa86..7e6d6178 100644 --- a/inst/shiny-server/markdown/INSTALL.Rmd +++ b/inst/shiny-server/markdown/INSTALL.Rmd @@ -1,39 +1,39 @@ ---- -title: "Installation" -output: - word_document: default - pdf_document: default - html_document: default ---- - -To install __IOHProfiler__ locally, you need to install R environment first. Please download the windows version from (https://cran.r-project.org/bin/windows/base/R-3.5.1-win.exe). Please install the __R-3.5.1-win.exe__ file (the latest, as of 11 July, 2018): just like installing a normal windows software. - -For linux/MacOS users, please download the corresponding version of R. - -After installation, you should see a R console icon on your desktop. On windows 10, it is very likely that there is no such an icon on the desktop. But, it can be found in all programs (select the __Start__ button, and then scroll through the alphabetical list on the left). An example is shown in the following screenshot: - -![](../logo/screenshot.jpg) - -Please open the R console now. This is where we will run the post-processing application. -Then, some dependences need to be installed. Please run the following commands in the R console (copy and paste the following commands in the R console). -```{r wrapper=TRUE, eval=FALSE} -install.packages('shiny') -install.packages('shinyjs') -install.packages('shinydashboard') -install.packages('magrittr') -install.packages('dplyr') -install.packages('reshape2') -install.packages('data.table') -install.packages('markdown') -install.packages('devtools') -devtools::install_github("ropensci/plotly") -``` - -Now, extract the zip of containing the source code to your preferred location, e.g., your desktop. -After this step, please just type the following command in the R console: -```{r wrapper=TRUE, eval=FALSE} -shiny::runApp('path/to/the/code/folder') -``` -Please make sure that you replace the path __'path/to/the/code/folder'__ to the path where the source code is located on your computer. For example, if the source code folder is located on your desktop, this path should look like __C:\\\\Users\\(username)\\\\Desktop\\\\post-processing__. On Linux/MacOS, this path might look like __/home/(username)/post-processing__ .If there is no error in the installation, a webpage should automatically pop-up in your web browser! - - +--- +title: "Installation" +output: + word_document: default + pdf_document: default + html_document: default +--- + +To install __IOHProfiler__ locally, you need to install R environment first. Please download the windows version from (https://cran.r-project.org/bin/windows/base/R-3.5.1-win.exe). Please install the __R-3.5.1-win.exe__ file (the latest, as of 11 July, 2018): just like installing a normal windows software. + +For linux/MacOS users, please download the corresponding version of R. + +After installation, you should see a R console icon on your desktop. On windows 10, it is very likely that there is no such an icon on the desktop. But, it can be found in all programs (select the __Start__ button, and then scroll through the alphabetical list on the left). An example is shown in the following screenshot: + +![](../logo/screenshot.jpg) + +Please open the R console now. This is where we will run the post-processing application. +Then, some dependences need to be installed. Please run the following commands in the R console (copy and paste the following commands in the R console). +```{r wrapper=TRUE, eval=FALSE} +install.packages('shiny') +install.packages('shinyjs') +install.packages('shinydashboard') +install.packages('magrittr') +install.packages('dplyr') +install.packages('reshape2') +install.packages('data.table') +install.packages('markdown') +install.packages('devtools') +devtools::install_github("ropensci/plotly") +``` + +Now, extract the zip of containing the source code to your preferred location, e.g., your desktop. +After this step, please just type the following command in the R console: +```{r wrapper=TRUE, eval=FALSE} +shiny::runApp('path/to/the/code/folder') +``` +Please make sure that you replace the path __'path/to/the/code/folder'__ to the path where the source code is located on your computer. For example, if the source code folder is located on your desktop, this path should look like __C:\\\\Users\\(username)\\\\Desktop\\\\post-processing__. On Linux/MacOS, this path might look like __/home/(username)/post-processing__ .If there is no error in the installation, a webpage should automatically pop-up in your web browser! + + diff --git a/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd index a18490eb..fbf89f15 100644 --- a/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/PAR_SUMMARY_TABLE.Rmd @@ -1,9 +1,9 @@ -This table summarizes for each algorithm and each target value chosen on the left: - - * runs: the number of runs where non-missing parameter values are observed, for each required target value $f(x)$, - * mean: the average value of the specified __parameter__ when target value $f(x)$ is hit. - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values - -When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. - - +This table summarizes for each algorithm and each target value chosen on the left: + + * runs: the number of runs where non-missing parameter values are observed, for each required target value $f(x)$, + * mean: the average value of the specified __parameter__ when target value $f(x)$ is hit. + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these parameter values + +When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. + + diff --git a/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd b/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd index 05d6b4a9..4bbadda4 100644 --- a/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd +++ b/inst/shiny-server/markdown/RT_OVERVIEW_TABLE.Rmd @@ -1,10 +1,10 @@ -This table provides an overview on function values for all algorithms chosen on the left: - - * worst recorded: the worst $f(x)$ value ever recorded across _all iterations_, - * worst reached: the worst $f(x)$ value reached in the _last iterations_, - * best reached: the best $f(x)$ value reached in the _last iterations_, - * mean reached: the mean $f(x)$ value reached in the _last iterations_, - * median reached: the median $f(x)$ value reached in the _last iterations_, - * succ: the number of runs which successfully hit the best reached $f(x)$. - - +This table provides an overview on function values for all algorithms chosen on the left: + + * worst recorded: the worst $f(x)$ value ever recorded across _all iterations_, + * worst reached: the worst $f(x)$ value reached in the _last iterations_, + * best reached: the best $f(x)$ value reached in the _last iterations_, + * mean reached: the mean $f(x)$ value reached in the _last iterations_, + * median reached: the median $f(x)$ value reached in the _last iterations_, + * succ: the number of runs which successfully hit the best reached $f(x)$. + + diff --git a/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd index 4dd529a3..57b08a7c 100644 --- a/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/RT_SUMMARY_TABLE.Rmd @@ -1,9 +1,9 @@ -This table summarizes for each algorithm and each target value chosen on the left: - - * runs: the number of runs that have found at least one solution of the required target quality $f(x)$, - * mean: the average number of function evaluations needed to find a solution of function value at least $f(x)$ - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these first-hitting times - -When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. - - +This table summarizes for each algorithm and each target value chosen on the left: + + * runs: the number of runs that have found at least one solution of the required target quality $f(x)$, + * mean: the average number of function evaluations needed to find a solution of function value at least $f(x)$ + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of these first-hitting times + +When not all runs managed to find the target value, the statistics hold only for those runs that did. That is, the mean value is the mean of the successful runs. Same for the quantiles. An alternative version with simulated restarts is currently in preparation. + + diff --git a/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd b/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd index fac70ce9..5bdf95fa 100644 --- a/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_AUC.Rmd @@ -1,40 +1,40 @@ ---- -title: "ECDF_AUC" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$RTECDF.AUC.Min[name]) - fmax <- as.numeric(REG$RTECDF.AUC.Max[name]) - fstep <- as.numeric(REG$RTECDF.AUC.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) - if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(fvs) - fmax <- max(fvs) - fstep <- NULL - } -) -p <- Plot.RT.ECDF_AUC(dsl_sub, fmin, fmax, fstep) -save_plotly(p, paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "ECDF_AUC" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$RTECDF.AUC.Min[name]) + fmax <- as.numeric(REG$RTECDF.AUC.Max[name]) + fstep <- as.numeric(REG$RTECDF.AUC.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) + if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(fvs) + fmax <- max(fvs) + fstep <- NULL + } +) +p <- Plot.RT.ECDF_AUC(dsl_sub, fmin, fmax, fstep) +save_plotly(p, paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ECDF_AUC_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd b/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd index a82955f6..5aced466 100644 --- a/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_Aggregated.Rmd @@ -1,27 +1,27 @@ ---- -title: "ECDF_Aggregated" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsList <- subset(dsl, algId %in% algs) - -if (!input$RTECDF.Aggr.Func) - dsList <- subset(dsList, funcId == input$Overall.Funcid) -if (!input$RTECDF.Aggr.Dim) - dsList <- subset(dsList, DIM == input$Overall.Dim) -#TODO: Add support for the custom targets -p <- Plot.RT.ECDF_Multi_Func(dsList, - scale.xlog = input$RTECDF.Aggr.Logx) -save_plotly(p, 'ECDF_Aggr.pdf', 'pdf', width = 1000, height = 500) -``` -## Aggregated ECDF plot on `r if(input$RTECDF.Aggr.Func) paste("all functions ") ` `r if(input$RTECDF.Aggr.Dim) paste("all dimensions ") ` - -```{r aggregated_ecdf, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics('ECDF_Aggr.pdf') -``` +--- +title: "ECDF_Aggregated" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsList <- subset(dsl, algId %in% algs) + +if (!input$RTECDF.Aggr.Func) + dsList <- subset(dsList, funcId == input$Overall.Funcid) +if (!input$RTECDF.Aggr.Dim) + dsList <- subset(dsList, DIM == input$Overall.Dim) +#TODO: Add support for the custom targets +p <- Plot.RT.ECDF_Multi_Func(dsList, + scale.xlog = input$RTECDF.Aggr.Logx) +save_plotly(p, 'ECDF_Aggr.pdf', 'pdf', width = 1000, height = 500) +``` +## Aggregated ECDF plot on `r if(input$RTECDF.Aggr.Func) paste("all functions ") ` `r if(input$RTECDF.Aggr.Dim) paste("all dimensions ") ` + +```{r aggregated_ecdf, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics('ECDF_Aggr.pdf') +``` diff --git a/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd b/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd index bee167d6..bb4332be 100644 --- a/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_Single_Function.Rmd @@ -1,40 +1,40 @@ ---- -title: "ECDF_Single_Function" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$RTECDF.Multi.Min[name]) - fmax <- as.numeric(REG$RTECDF.Multi.Max[name]) - fstep <- as.numeric(REG$RTECDF.Multi.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) - if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(fvs) - fmax <- max(fvs) - fstep <- NULL - } -) -p <- Plot.RT.ECDF_Single_Func(dsl_sub, fmin, fmax, fstep, - scale.xlog = input$RTECDF.Multi.Logx) -save_plotly(p, paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "ECDF_Single_Function" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$RTECDF.Multi.Min[name]) + fmax <- as.numeric(REG$RTECDF.Multi.Max[name]) + fstep <- as.numeric(REG$RTECDF.Multi.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) + if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(fvs) + fmax <- max(fvs) + fstep <- NULL + } +) +p <- Plot.RT.ECDF_Single_Func(dsl_sub, fmin, fmax, fstep, + scale.xlog = input$RTECDF.Multi.Logx) +save_plotly(p, paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ECDF_single_function_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd b/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd index 09d51b1c..3c271788 100644 --- a/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd +++ b/inst/shiny-server/markdown/Report/ECDF_Single_Target.Rmd @@ -1,31 +1,31 @@ ---- -title: "ECDF_Single_Target" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$RTECDF.Single.Target[name]) - if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) -}, -error = function(e) { - ftarget <- median(fvs) - } -) -p <- Plot.RT.ECDF_Per_Target(dsl_sub, ftarget, scale.xlog = input$RTECDF.Single.Logx) -save_plotly(p, paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "ECDF_Single_Target" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$RTECDF.Single.Target[name]) + if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) +}, +error = function(e) { + ftarget <- median(fvs) + } +) +p <- Plot.RT.ECDF_Per_Target(dsl_sub, ftarget, scale.xlog = input$RTECDF.Single.Logx) +save_plotly(p, paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ECDF_single_target_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd b/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd index 9e9c3460..993088e9 100644 --- a/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd +++ b/inst/shiny-server/markdown/Report/FV_ECDF_AUC.Rmd @@ -1,40 +1,40 @@ ---- -title: "FV_ECDF_AUC" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$FCEECDF.AUC.Min[name]) - fmax <- as.numeric(REG$FCEECDF.AUC.Max[name]) - fstep <- as.numeric(REG$FCEECDF.AUC.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) - if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(fvs) - fmax <- max(fvs) - fstep <- NULL - } -) -p <- Plot.FV.ECDF_AUC(dsl_sub, fmin, fmax, fstep) -save_plotly(p, paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_ECDF_AUC" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$FCEECDF.AUC.Min[name]) + fmax <- as.numeric(REG$FCEECDF.AUC.Max[name]) + fstep <- as.numeric(REG$FCEECDF.AUC.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(fvs) + if (is.null(fmax) || is.na(fmax)) fmax <- max(fvs) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(fvs) + fmax <- max(fvs) + fstep <- NULL + } +) +p <- Plot.FV.ECDF_AUC(dsl_sub, fmin, fmax, fstep) +save_plotly(p, paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_ECDF_AUC_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd index 0d09f732..0f2ef0c9 100644 --- a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd +++ b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Function.Rmd @@ -1,42 +1,42 @@ ---- -title: "FV_ECDF_Single_Function" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - fmin <- as.numeric(REG$FCEECDF.Mult.Min[name]) - fmax <- as.numeric(REG$FCEECDF.Mult.Max[name]) - fstep <- as.numeric(REG$FCEECDF.Mult.Step[name]) - - if (is.null(fmin) || is.na(fmin)) fmin <- min(rts) - if (is.null(fmax) || is.na(fmax)) fmax <- max(rts) - if (is.null(fstep) || is.na(fstep)) fstep <- NULL - -}, -error = function(e) { - fmin <- min(rts) - fmax <- max(rts) - fstep <- NULL - } -) -p <- Plot.FV.ECDF_Single_Func(dsl_sub, rt_min = fmin, - rt_max = fmax, rt_step = fstep, - scale.xlog = input$FCEECDF.Mult.Logx, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_ECDF_Single_Function" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + fmin <- as.numeric(REG$FCEECDF.Mult.Min[name]) + fmax <- as.numeric(REG$FCEECDF.Mult.Max[name]) + fstep <- as.numeric(REG$FCEECDF.Mult.Step[name]) + + if (is.null(fmin) || is.na(fmin)) fmin <- min(rts) + if (is.null(fmax) || is.na(fmax)) fmax <- max(rts) + if (is.null(fstep) || is.na(fstep)) fstep <- NULL + +}, +error = function(e) { + fmin <- min(rts) + fmax <- max(rts) + fstep <- NULL + } +) +p <- Plot.FV.ECDF_Single_Func(dsl_sub, rt_min = fmin, + rt_max = fmax, rt_step = fstep, + scale.xlog = input$FCEECDF.Mult.Logx, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_ECDF_single_function_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd index 5198357b..aa0a1227 100644 --- a/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd +++ b/inst/shiny-server/markdown/Report/FV_ECDF_Single_Target.Rmd @@ -1,32 +1,32 @@ ---- -title: "FV_ECDF_Single_Target" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$FCEECDF.Single.Target[name]) - if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(rts) -}, -error = function(e) { - ftarget <- median(rts) - } -) -p <- Plot.FV.ECDF_Per_Target(dsl_sub, as.integer(ftarget), scale.xlog = input$FCEECDF.Single.Logx, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_ECDF_Single_Target" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$FCEECDF.Single.Target[name]) + if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(rts) +}, +error = function(e) { + ftarget <- median(rts) + } +) +p <- Plot.FV.ECDF_Per_Target(dsl_sub, as.integer(ftarget), scale.xlog = input$FCEECDF.Single.Logx, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ECDF plot of F`r paste(fid)` on `r paste(dim)`D at target `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_ECDF_single_target_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_Histogram.Rmd b/inst/shiny-server/markdown/Report/FV_Histogram.Rmd index 0693aae9..b6856928 100644 --- a/inst/shiny-server/markdown/Report/FV_Histogram.Rmd +++ b/inst/shiny-server/markdown/Report/FV_Histogram.Rmd @@ -1,32 +1,32 @@ ---- -title: "FV_Histogram" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - rttarget <- as.numeric(REG$FCEPDF.Hist.Runtime[name]) - if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) -}, -error = function(e) { - rttarget <- median(rts) - } -) -p <- Plot.FV.Histogram(dsl_sub, runtime = rttarget, plot_mode = input$FCEPDF.Hist.Mode, - use.equal.bins = input$FCEPDF.Hist.Equal) -save_plotly(p, paste0('FV_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Histogram of target values hit of F`r paste(fid)` on `r paste(dim)`D to at runtime `r paste(rttarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_Hist_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_Histogram" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + rttarget <- as.numeric(REG$FCEPDF.Hist.Runtime[name]) + if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) +}, +error = function(e) { + rttarget <- median(rts) + } +) +p <- Plot.FV.Histogram(dsl_sub, runtime = rttarget, plot_mode = input$FCEPDF.Hist.Mode, + use.equal.bins = input$FCEPDF.Hist.Equal) +save_plotly(p, paste0('FV_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Histogram of target values hit of F`r paste(fid)` on `r paste(dim)`D to at runtime `r paste(rttarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_Hist_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_PMF.Rmd b/inst/shiny-server/markdown/Report/FV_PMF.Rmd index 021f8f1f..6394965b 100644 --- a/inst/shiny-server/markdown/Report/FV_PMF.Rmd +++ b/inst/shiny-server/markdown/Report/FV_PMF.Rmd @@ -1,32 +1,32 @@ ---- -title: "FV_PMF" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -rts <- get_runtimes(dsl_sub) -tryCatch( - { - rttarget <- as.numeric(REG$FCEPDF.Bar.Runtime[name]) - if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) -}, -error = function(e) { - rttarget <- median(rts) - } -) -p <- Plot.FV.PDF(dsl_sub, rttarget, show.sample = input$FCEPDF.Bar.Samples, - scale.ylog = input$FCEPDF.Bar.Logy ) -save_plotly(p, paste0('FV_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Probability denisty function of target values hit of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(rttarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FV_PMF_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "FV_PMF" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +rts <- get_runtimes(dsl_sub) +tryCatch( + { + rttarget <- as.numeric(REG$FCEPDF.Bar.Runtime[name]) + if (is.null(rttarget) || is.na(rttarget)) rttarget <- median(rts) +}, +error = function(e) { + rttarget <- median(rts) + } +) +p <- Plot.FV.PDF(dsl_sub, rttarget, show.sample = input$FCEPDF.Bar.Samples, + scale.ylog = input$FCEPDF.Bar.Logy ) +save_plotly(p, paste0('FV_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Probability denisty function of target values hit of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(rttarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FV_PMF_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/FV_Rank.Rmd b/inst/shiny-server/markdown/Report/FV_Rank.Rmd index 605fb4dc..169e4ef1 100644 --- a/inst/shiny-server/markdown/Report/FV_Rank.Rmd +++ b/inst/shiny-server/markdown/Report/FV_Rank.Rmd @@ -1,23 +1,23 @@ ---- -title: "FV_Rank" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.FV.Aggregated(dsl_sub, plot_mode = input$FCEPlot.Aggr.Mode, - scale.ylog = input$FCEPlot.Aggr.Logy, - use_rank = input$FCEPlot.Aggr.Ranking, - aggr_on = ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM')) -save_plotly(p, paste0('FV_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Ranking plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('FV_Rank_', dim, 'D.pdf')) +--- +title: "FV_Rank" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.FV.Aggregated(dsl_sub, plot_mode = input$FCEPlot.Aggr.Mode, + scale.ylog = input$FCEPlot.Aggr.Logy, + use_rank = input$FCEPlot.Aggr.Ranking, + aggr_on = ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM')) +save_plotly(p, paste0('FV_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Ranking plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('FV_Rank_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Multi_ERT.Rmd b/inst/shiny-server/markdown/Report/Multi_ERT.Rmd index 14359ed1..c6c6b428 100644 --- a/inst/shiny-server/markdown/Report/Multi_ERT.Rmd +++ b/inst/shiny-server/markdown/Report/Multi_ERT.Rmd @@ -1,23 +1,23 @@ ---- -title: "Multi_ERT" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.RT.Multi_Func(dsl_sub, - scale.xlog = input$ERTPlot.Multi.Logx, - scale.ylog = input$ERTPlot.Multi.Logy, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('ERT_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) -``` -## ERT plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('ERT_multi_', dim, 'D.pdf')) +--- +title: "Multi_ERT" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.RT.Multi_Func(dsl_sub, + scale.xlog = input$ERTPlot.Multi.Logx, + scale.ylog = input$ERTPlot.Multi.Logy, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('ERT_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) +``` +## ERT plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('ERT_multi_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Multi_FCE.Rmd b/inst/shiny-server/markdown/Report/Multi_FCE.Rmd index 5c116e57..bb502a7f 100644 --- a/inst/shiny-server/markdown/Report/Multi_FCE.Rmd +++ b/inst/shiny-server/markdown/Report/Multi_FCE.Rmd @@ -1,22 +1,22 @@ ---- -title: "Multi_FCE" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.FV.Multi_Func(dsl_sub, - scale.xlog = input$FCEPlot.Multi.Logx, - scale.ylog = input$FCEPlot.Multi.Logy) -save_plotly(p, paste0('FCE_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) -``` -## ERT plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('FCE_multi_', dim, 'D.pdf')) +--- +title: "Multi_FCE" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.FV.Multi_Func(dsl_sub, + scale.xlog = input$FCEPlot.Multi.Logx, + scale.ylog = input$FCEPlot.Multi.Logy) +save_plotly(p, paste0('FCE_multi_', dim, 'D.pdf'), 'pdf', width = 1000, height = 1200) +``` +## ERT plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('FCE_multi_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Param_plot.Rmd b/inst/shiny-server/markdown/Report/Param_plot.Rmd index 2488e7d6..e1fc3617 100644 --- a/inst/shiny-server/markdown/Report/Param_plot.Rmd +++ b/inst/shiny-server/markdown/Report/Param_plot.Rmd @@ -1,39 +1,39 @@ ---- -title: "Param_plot" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -tryCatch( - { - fv_min <- as.numeric(REG$FV_PAR.Plot.Min[name]) - if (is.na(fv_min)) fv_min <- NULL - fv_max <- as.numeric(REG$FV_PAR.Plot.Max[name]) - if (is.na(fv_max)) fv_max <- NULL -}, -error = function(e) { - fv_min <- NULL - fv_max <- NULL - } -) -p <- Plot.Parameters(dsl_sub, fv_min, fv_max, - show.mean = (input$FV_PAR.Plot.show.mean == 'mean'), - show.median = (input$FV_PAR.Plot.show.mean == 'median'), - scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - show.CI = input$FV_PAR.Plot.CI, - par_name = input$FV_PAR.Plot.Params) -save_plotly(p, paste0('Param_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Plot of parameters of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('Param_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "Param_plot" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +tryCatch( + { + fv_min <- as.numeric(REG$FV_PAR.Plot.Min[name]) + if (is.na(fv_min)) fv_min <- NULL + fv_max <- as.numeric(REG$FV_PAR.Plot.Max[name]) + if (is.na(fv_max)) fv_max <- NULL +}, +error = function(e) { + fv_min <- NULL + fv_max <- NULL + } +) +p <- Plot.Parameters(dsl_sub, fv_min, fv_max, + show.mean = (input$FV_PAR.Plot.show.mean == 'mean'), + show.median = (input$FV_PAR.Plot.show.mean == 'median'), + scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + show.CI = input$FV_PAR.Plot.CI, + par_name = input$FV_PAR.Plot.Params) +save_plotly(p, paste0('Param_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Plot of parameters of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('Param_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/RT_Histogram.Rmd b/inst/shiny-server/markdown/Report/RT_Histogram.Rmd index 6bcb3dd3..5ebb70fd 100644 --- a/inst/shiny-server/markdown/Report/RT_Histogram.Rmd +++ b/inst/shiny-server/markdown/Report/RT_Histogram.Rmd @@ -1,32 +1,32 @@ ---- -title: "RT_Histogram" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$RTPMF.Hist.Target[name]) - if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) -}, -error = function(e) { - ftarget <- median(fvs) - } -) -p <- Plot.RT.Histogram(dsl_sub, ftarget = ftarget, plot_mode = input$RTPMF.Hist.Mode, - use.equal.bins = input$RTPMF.Hist.Equal) -save_plotly(p, paste0('RT_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Histogram of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('RT_Hist_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "RT_Histogram" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$RTPMF.Hist.Target[name]) + if (is.null(ftarget) || is.na(ftarget)) ftarget <- median(fvs) +}, +error = function(e) { + ftarget <- median(fvs) + } +) +p <- Plot.RT.Histogram(dsl_sub, ftarget = ftarget, plot_mode = input$RTPMF.Hist.Mode, + use.equal.bins = input$RTPMF.Hist.Equal) +save_plotly(p, paste0('RT_Hist_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Histogram of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('RT_Hist_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/RT_PMF.Rmd b/inst/shiny-server/markdown/Report/RT_PMF.Rmd index a7adb8d3..f69ec98f 100644 --- a/inst/shiny-server/markdown/Report/RT_PMF.Rmd +++ b/inst/shiny-server/markdown/Report/RT_PMF.Rmd @@ -1,32 +1,32 @@ ---- -title: "RT_PMF" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -fvs <- get_funvals(dsl_sub) -tryCatch( - { - ftarget <- as.numeric(REG$RTPMF.Bar.Target[name]) - if (is.na(ftarget)) ftarget <- median(fvs) -}, -error = function(e) { - ftarget <- median(fvs) - } -) -p <- Plot.RT.PMF(dsl_sub, ftarget = ftarget, show.sample = input$RTPMF.Bar.Sample, - scale.ylog = input$RTPMF.Bar.Logy) -save_plotly(p, paste0('RT_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Probability mass function of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('RT_PMF_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "RT_PMF" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +fvs <- get_funvals(dsl_sub) +tryCatch( + { + ftarget <- as.numeric(REG$RTPMF.Bar.Target[name]) + if (is.na(ftarget)) ftarget <- median(fvs) +}, +error = function(e) { + ftarget <- median(fvs) + } +) +p <- Plot.RT.PMF(dsl_sub, ftarget = ftarget, show.sample = input$RTPMF.Bar.Sample, + scale.ylog = input$RTPMF.Bar.Logy) +save_plotly(p, paste0('RT_PMF_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Probability mass function of hitting times of F`r paste(fid)` on `r paste(dim)`D at runtime `r paste(ftarget)` + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('RT_PMF_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/RT_Rank.Rmd b/inst/shiny-server/markdown/Report/RT_Rank.Rmd index 16fea2e5..8548af62 100644 --- a/inst/shiny-server/markdown/Report/RT_Rank.Rmd +++ b/inst/shiny-server/markdown/Report/RT_Rank.Rmd @@ -1,24 +1,24 @@ ---- -title: "RT_Rank" -author: "IOHprofiler" -date: "7/8/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) -p <- Plot.RT.Aggregated(dsl_sub, - plot_mode = input$ERTPlot.Aggr.Mode, - scale.ylog = input$ERTPlot.Aggr.Logy, - maximize = attr(dsl_sub[[1]],'maximization'), - use_rank = input$ERTPlot.Aggr.Ranking) -save_plotly(p, paste0('RT_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## Ranking plot of all functions in `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} -include_graphics(paste0('RT_Rank_', dim, 'D.pdf')) +--- +title: "RT_Rank" +author: "IOHprofiler" +date: "7/8/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && algId %in% algs) +p <- Plot.RT.Aggregated(dsl_sub, + plot_mode = input$ERTPlot.Aggr.Mode, + scale.ylog = input$ERTPlot.Aggr.Logy, + maximize = attr(dsl_sub[[1]],'maximization'), + use_rank = input$ERTPlot.Aggr.Ranking) +save_plotly(p, paste0('RT_Rank_', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## Ranking plot of all functions in `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=600, fig.pos='!hbt'} +include_graphics(paste0('RT_Rank_', dim, 'D.pdf')) ``` \ No newline at end of file diff --git a/inst/shiny-server/markdown/Report/Single_ERT.Rmd b/inst/shiny-server/markdown/Report/Single_ERT.Rmd index a0dbe480..9584ae97 100644 --- a/inst/shiny-server/markdown/Report/Single_ERT.Rmd +++ b/inst/shiny-server/markdown/Report/Single_ERT.Rmd @@ -1,40 +1,40 @@ ---- -title: "Single_ERT" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -tryCatch( - { - fv_min <- as.numeric(REG$ERTPlot.Min[name]) - if (is.na(fv_min)) fv_min <- NULL - fv_max <- as.numeric(REG$ERTPlot.Max[name]) - if (is.na(fv_max)) fv_max <- NULL -}, -error = function(e) { - fv_min <- NULL - fv_max <- NULL - } -) -p <- Plot.RT.Single_Func(dsl_sub, Fstart = fv_min, Fstop = fv_max, - show.CI = input$ERTPlot.show.CI, - show.ERT = input$ERTPlot.show.ERT, - show.mean = input$ERTPlot.show.mean, - show.median = input$ERTPlot.show.median, - scale.xlog = input$ERTPlot.semilogx, - scale.ylog = input$ERTPlot.semilogy, - scale.reverse = !attr(dsl_sub[[1]],'maximization')) -save_plotly(p, paste0('ERT_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ERT plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('ERT_single_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "Single_ERT" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +tryCatch( + { + fv_min <- as.numeric(REG$ERTPlot.Min[name]) + if (is.na(fv_min)) fv_min <- NULL + fv_max <- as.numeric(REG$ERTPlot.Max[name]) + if (is.na(fv_max)) fv_max <- NULL +}, +error = function(e) { + fv_min <- NULL + fv_max <- NULL + } +) +p <- Plot.RT.Single_Func(dsl_sub, Fstart = fv_min, Fstop = fv_max, + show.CI = input$ERTPlot.show.CI, + show.ERT = input$ERTPlot.show.ERT, + show.mean = input$ERTPlot.show.mean, + show.median = input$ERTPlot.show.median, + scale.xlog = input$ERTPlot.semilogx, + scale.ylog = input$ERTPlot.semilogy, + scale.reverse = !attr(dsl_sub[[1]],'maximization')) +save_plotly(p, paste0('ERT_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ERT plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('ERT_single_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/Single_FCE.Rmd b/inst/shiny-server/markdown/Report/Single_FCE.Rmd index a806b69a..6bad996b 100644 --- a/inst/shiny-server/markdown/Report/Single_FCE.Rmd +++ b/inst/shiny-server/markdown/Report/Single_FCE.Rmd @@ -1,35 +1,35 @@ ---- -title: "Single_ERT" -author: "IOHprofiler" -date: "7/5/2019" -output: pdf_document: - fig_caption: yes -graphics: yes ---- - -```{r, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) -name <- get_data_id(dsl_sub) -tryCatch( - { - rt_min <- as.numeric(REG$FCEPlot.Min[name]) - if (is.na(rt_min)) rt_min <- NULL - rt_max <- as.numeric(REG$FCEPlot.Max[name]) - if (is.na(rt_max)) rt_max <- NULL -}, -error = function(e) { - fv_min <- NULL - fv_max <- NULL - } -) -p <- Plot.FV.Single_Func(dsl_sub, RTstart = rt_min, RTstop = rt_max, show.CI = input$FCEPlot.show.CI, - show.mean = input$FCEPlot.show.mean, show.median = input$FCEPlot.show.median, - scale.xlog = isolate(input$FCEPlot.semilogx), scale.ylog = isolate(input$FCEPlot.semilogy)) -save_plotly(p, paste0('FCE_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) -``` -## ERT plot of F`r paste(fid)` on `r paste(dim)`D - -```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} -include_graphics(paste0('FCE_single_F', fid, '-', dim, 'D.pdf')) -``` +--- +title: "Single_ERT" +author: "IOHprofiler" +date: "7/5/2019" +output: pdf_document: + fig_caption: yes +graphics: yes +--- + +```{r, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% algs) +name <- get_data_id(dsl_sub) +tryCatch( + { + rt_min <- as.numeric(REG$FCEPlot.Min[name]) + if (is.na(rt_min)) rt_min <- NULL + rt_max <- as.numeric(REG$FCEPlot.Max[name]) + if (is.na(rt_max)) rt_max <- NULL +}, +error = function(e) { + fv_min <- NULL + fv_max <- NULL + } +) +p <- Plot.FV.Single_Func(dsl_sub, RTstart = rt_min, RTstop = rt_max, show.CI = input$FCEPlot.show.CI, + show.mean = input$FCEPlot.show.mean, show.median = input$FCEPlot.show.median, + scale.xlog = isolate(input$FCEPlot.semilogx), scale.ylog = isolate(input$FCEPlot.semilogy)) +save_plotly(p, paste0('FCE_single_F', fid, '-', dim, 'D.pdf'), 'pdf', width = 1000, height = 500) +``` +## ERT plot of F`r paste(fid)` on `r paste(dim)`D + +```{r {{name_figure}}, results='asis', fig.align='center', fig.cap="TEsting", fig.width=500, fig.height=300, fig.pos='!hbt'} +include_graphics(paste0('FCE_single_F', fid, '-', dim, 'D.pdf')) +``` diff --git a/inst/shiny-server/markdown/Report/Template_test.Rmd b/inst/shiny-server/markdown/Report/Template_test.Rmd index 628b0ffc..b9c78dc9 100644 --- a/inst/shiny-server/markdown/Report/Template_test.Rmd +++ b/inst/shiny-server/markdown/Report/Template_test.Rmd @@ -1,465 +1,465 @@ ---- -title: "Automatic report" -author: "IOHprofiler" -date: "6/28/2019" -output: - pdf_document: - keep_tex: true - fig_caption: yes -params: - input: NA - dsl: NA - figure_folder: NA - REG: NA -bibliography: bibliography.bib ---- - -```{r setup, include=FALSE} -knitr::opts_chunk$set(echo = FALSE) -input <- params$input -dsl <- params$dsl -figure_folder <- params$figure_folder -get_data_id <- function(dsList) { - if (is.null(dsList) | length(dsList) == 0) - return(NULL) - - paste(get_funcId(dsList), get_dim(dsList), sep = '-') -} -REG <- params$REG -``` - -```{asis, echo=input$Report.Introduction} -# Introduction - -This report has been automatically generated by the IOHanalyzer [@IOHprofiler]. - -``` - -# Fixed-target view - -```{r overview, results='asis', eval=input$Report.RT.Overview} -asis_output("### Overview of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.RT.Overview-DIM` ) { - for (fid in input$`Report.RT.Overview-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Overview_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Overview-Alg` ) - p <- kable(get_RT_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r statistics, results='asis', eval=input$Report.RT.Statistics} -asis_output("### Runtime statistics of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.RT.Statistics-DIM` ) { - for (fid in input$`Report.RT.Statistics-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Statistics_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Statistics-Alg`) - name <- get_data_id(dsl_sub) - fvs <- get_funvals(dsl_sub) - tryCatch( - { - fv_seq <- seq_FV(fvs, as.numeric(REG$RTSummary.Statistics.Min[[name]]), as.numeric(REG$RTSummary.Statistics.Max[[name]]), - by = as.numeric(REG$RTSummary.Statistics.Step[[name]])) - }, - error = function(e) { fv_seq <- seq_RT(fvs, length.out = 10)} - ) - p <- kable(get_RT_summary(dsl_sub, fv_seq), - caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), - format = 'latex') %>% - kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r single_erts, eval=input$Report.RT.Single_ERT, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.Single_ERT-DIM` ) { - for (fid in input$`Report.RT.Single_ERT-FuncId` ) { - name_figure <- paste0('ERT_single_F', fid, '-', dim, 'D') - algs = input$`Report.RT.Single_ERT-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_ERT.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.Single_ERT) paste(out, collapse='\n')` - -```{r single_erts_text, eval=input$Report.RT.Single_ERT, results='asis'} -asis_output(paste0("This section shows the ERTs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - - -```{r multi_erts, eval=input$Report.RT.Multi_ERT, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.Multi_ERT-DIM` ) { - name_figure <- paste0('ERT_multi_', dim, 'D') - algs = input$`Report.RT.Multi_ERT-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_ERT.Rmd")))) -} -``` - -`r if(input$Report.RT.Multi_ERT) paste(out, collapse='\n')` - -```{r ranking_erts, eval=input$Report.RT.Rank, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.Rank-DIM` ) { - name_figure <- paste0('RT_Rank_', dim, 'D') - algs = input$`Report.RT.Rank-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Rank.Rmd")))) -} -``` - -`r if(input$Report.RT.Rank) paste(out, collapse='\n')` - -```{r rt_hist, eval=input$Report.RT.Histogram, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.Histogram-DIM` ) { - for (fid in input$`Report.RT.Histogram-FuncId` ) { - name_figure <- paste0('RT_Hist_F', fid, '-', dim, 'D') - algs = input$`Report.RT.Histogram-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Histogram.Rmd")))) - } -} -``` - -`r if(input$Report.RT.Histogram) paste(out, collapse='\n')` - -```{r rt_pmf, eval=input$Report.RT.PMF, results='hide', include=F} -out <- NULL -for (dim in input$`Report.RT.PMF-DIM` ) { - for (fid in input$`Report.RT.PMF-FuncId` ) { - name_figure <- paste0('RT_PMF_F', fid, '-', dim, 'D') - algs = input$`Report.RT.PMF-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_PMF.Rmd")))) - } -} -``` - -`r if(input$Report.RT.PMF) paste(out, collapse='\n')` - -```{r ecdf_single_target, eval=input$Report.RT.ECDF_Single_Target, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.ECDF_Single_Target-DIM` ) { - for (fid in input$`Report.RT.ECDF_Single_Target-FuncId` ) { - name_figure <- paste0('ECDF_Single_Target', fid, '-', dim, 'D') - algs = input$`Report.RT.ECDF_Single_Target-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Target.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.ECDF_Single_Target) paste(out, collapse='\n')` - -```{r ecdf_single_target_text, eval=input$Report.RT.ECDF_Single_Target, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r ecdf_single_function, eval=input$Report.RT.ECDF_Single_Function, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.ECDF_Single_Function-DIM` ) { - for (fid in input$`Report.RT.ECDF_Single_Function-FuncId` ) { - name_figure <- paste0('ECDF_Single_Function', fid, '-', dim, 'D') - algs = input$`Report.RT.ECDF_Single_Function-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Function.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.ECDF_Single_Function) paste(out, collapse='\n')` - -```{r ecdf_single_function_text, eval=input$Report.RT.ECDF_Single_Function, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r ecdf_aggr, eval=input$Report.RT.ECDF_Aggregated, results='hide', include=F} -algs = input$`Report.RT.ECDF_Aggregated-Alg` -env = new.env() -out <- knit_child(text = knit_expand(file = "ECDF_Aggregated.Rmd")) - -``` - -`r if(input$Report.RT.ECDF_Aggregated) paste(out, collapse='\n')` - -```{r ecdf_aggr_text, eval=input$Report.RT.ECDF_Aggregated, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figure \\ref{fig:aggregated_ecdf}")) -``` - -```{r ecdf_auc, eval=input$Report.RT.ECDF_AUC, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.RT.ECDF_AUC-DIM` ) { - for (fid in input$`Report.RT.ECDF_AUC-FuncId` ) { - name_figure <- paste0('ECDF_AUC', fid, '-', dim, 'D') - algs = input$`Report.RT.ECDF_AUC-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_AUC.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.RT.ECDF_AUC) paste(out, collapse='\n')` - -```{r ecdf_auc_text, eval=input$Report.RT.ECDF_AUC, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -# Fixed-budget view - -```{r fv_overview, results='asis', eval=input$Report.FV.Overview} -asis_output("### Overview of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.FV.Overview-DIM` ) { - for (fid in input$`Report.FV.Overview-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Overview_FV_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Overview-Alg` ) - p <- kable(get_FV_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r fv_statistics, results='asis', eval=input$Report.FV.Statistics} -asis_output("### Runtime statistics of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.FV.Statistics-DIM` ) { - for (fid in input$`Report.FV.Statistics-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Statistics_FV_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Statistics-Alg`) - name <- get_data_id(dsl_sub) - rts <- get_runtimes(dsl_sub) - tryCatch( - { - rt_seq <- seq_RT(rts, as.numeric(REG$FCESummary.Statistics.Min[[name]]), as.numeric(REG$FCESummary.Statistics.Max[[name]]), - by = as.numeric(REG$FCESummary.Statistics.Step[[name]])) - }, - error = function(e) { rt_seq <- seq_RT(rts, length.out = 10)} - ) - p <- kable(get_FV_summary(dsl_sub, rt_seq), - caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), - format = 'latex') %>% - kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - - } -} -``` - -```{r single_fce, eval=input$Report.FV.Single_FCE, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.Single_FCE-DIM` ) { - for (fid in input$`Report.FV.Single_FCE-FuncId` ) { - name_figure <- paste0('FCE_single_F', fid, '-', dim, 'D') - algs = input$`Report.FV.Single_FCE-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_FCE.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.Single_FCE) paste(out, collapse='\n')` - -```{r single_fce_text, eval=input$Report.FV.Single_FCE, results='asis'} -asis_output(paste0("This section shows the FCEs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r multi_fces, eval=input$Report.FV.Multi_FCE, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.Multi_FCE-DIM` ) { - name_figure <- paste0('FCE_multi_', dim, 'D') - algs = input$`Report.FV.Multi_FCE-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_FCE.Rmd")))) -} -``` - -`r if(input$Report.FV.Multi_FCE) paste(out, collapse='\n')` - -```{r ranking_FVs, eval=input$Report.FV.Rank, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.Rank-DIM` ) { - name_figure <- paste0('FV_Rank_', dim, 'D') - algs = input$`Report.FV.Rank-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Rank.Rmd")))) -} -``` - -`r if(input$Report.FV.Rank) paste(out, collapse='\n')` - -```{r fv_hist, eval=input$Report.FV.Histogram, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.Histogram-DIM` ) { - for (fid in input$`Report.FV.Histogram-FuncId` ) { - name_figure <- paste0('FV_Hist_F', fid, '-', dim, 'D') - algs = input$`Report.FV.Histogram-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Histogram.Rmd")))) - } -} -``` - -`r if(input$Report.FV.Histogram) paste(out, collapse='\n')` - -```{r fv_pmf, eval=input$Report.FV.PMF, results='hide', include=F} -out <- NULL -for (dim in input$`Report.FV.PMF-DIM` ) { - for (fid in input$`Report.FV.PMF-FuncId` ) { - name_figure <- paste0('FV_PMF_F', fid, '-', dim, 'D') - algs = input$`Report.FV.PMF-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_PMF.Rmd")))) - } -} -``` - -`r if(input$Report.FV.PMF) paste(out, collapse='\n')` - -```{r fv_ecdf_single_target, eval=input$Report.FV.ECDF_Single_Target, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.ECDF_Single_Target-DIM` ) { - for (fid in input$`Report.FV.ECDF_Single_Target-FuncId` ) { - name_figure <- paste0('FV_ECDF_Single_Target', fid, '-', dim, 'D') - algs = input$`Report.FV.ECDF_Single_Target-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Target.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.ECDF_Single_Target) paste(out, collapse='\n')` - -```{r fv_ecdf_single_target_text, eval=input$Report.FV.ECDF_Single_Target, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r fv_ecdf_single_function, eval=input$Report.FV.ECDF_Single_Function, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.ECDF_Single_Function-DIM` ) { - for (fid in input$`Report.FV.ECDF_Single_Function-FuncId` ) { - name_figure <- paste0('FV_ECDF_Single_Function', fid, '-', dim, 'D') - algs = input$`Report.FV.ECDF_Single_Function-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Function.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.ECDF_Single_Function) paste(out, collapse='\n')` - -```{r fv_ecdf_single_function_text, eval=input$Report.FV.ECDF_Single_Function, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r fv_ecdf_auc, eval=input$Report.FV.ECDF_AUC, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.FV.ECDF_AUC-DIM` ) { - for (fid in input$`Report.FV.ECDF_AUC-FuncId` ) { - name_figure <- paste0('FV_ECDF_AUC', fid, '-', dim, 'D') - algs = input$`Report.FV.ECDF_AUC-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_AUC.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.FV.ECDF_AUC) paste(out, collapse='\n')` - -```{r fv_ecdf_auc_text, eval=input$Report.FV.ECDF_AUC, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -```{r param_statistics, results='asis', eval=input$Report.Param.Statistics} -asis_output("### Statistics of parameters of selected function / dimension pairs") -cat('\n') -for (dim in input$`Report.Param.Statistics-DIM` ) { - for (fid in input$`Report.Param.Statistics-FuncId` ) { - x <- paste0("### F", fid, " ", dim, "D \n") - cat(x) - cat('\n') - opts_current$set(label = paste0("Statistics_Param_F", fid, " ", dim, "D")) - dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.Param.Statistics-Alg`) - name <- get_data_id(dsl_sub) - fvs <- get_funvals(dsl_sub) - tryCatch( - { - fv_seq <- seq_FV(fvs, as.numeric(REG$PAR.Summary.Min[[name]]), as.numeric(REG$PAR.Summary.Max[[name]]), - by = as.numeric(REG$PAR.Summary.Step[[name]])) - }, - error = function(e) { fv_seq <- seq_FV(fvs, length.out = 10) } - ) - p <- kable(get_PAR_summary(dsl_sub, fv_seq), - caption = paste0("Statistics of parameters of selected algorithms on F", fid, " in ", dim, "D"), - format = 'latex') %>% - kable_styling(latex_options=c("scale_down", "hold_position")) - print(p) - cat('\n') - } -} -``` - -```{r param_plot, eval=input$Report.Param.Plot, results='hide', include=F} -out <- NULL -fig_names <- NULL -for (dim in input$`Report.Param.Plot-DIM` ) { - for (fid in input$`Report.Param.Plot-FuncId` ) { - name_figure <- paste0('Param_Plot', fid, '-', dim, 'D') - algs = input$`Report.Param.Plot-Alg` - env = new.env() - out <- invisible(c(out,knit_child(text = knit_expand(file = "Param_plot.Rmd")))) - fig_names <- c(fig_names, name_figure) - } -} -``` - -`r if(input$Report.Param.Plot) paste(out, collapse='\n')` - -```{r param_plot_text, eval=input$Report.Param.Plot, results='asis'} -asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) -``` - -# References +--- +title: "Automatic report" +author: "IOHprofiler" +date: "6/28/2019" +output: + pdf_document: + keep_tex: true + fig_caption: yes +params: + input: NA + dsl: NA + figure_folder: NA + REG: NA +bibliography: bibliography.bib +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = FALSE) +input <- params$input +dsl <- params$dsl +figure_folder <- params$figure_folder +get_data_id <- function(dsList) { + if (is.null(dsList) | length(dsList) == 0) + return(NULL) + + paste(get_funcId(dsList), get_dim(dsList), sep = '-') +} +REG <- params$REG +``` + +```{asis, echo=input$Report.Introduction} +# Introduction + +This report has been automatically generated by the IOHanalyzer [@IOHprofiler]. + +``` + +# Fixed-target view + +```{r overview, results='asis', eval=input$Report.RT.Overview} +asis_output("### Overview of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.RT.Overview-DIM` ) { + for (fid in input$`Report.RT.Overview-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Overview_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Overview-Alg` ) + p <- kable(get_RT_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r statistics, results='asis', eval=input$Report.RT.Statistics} +asis_output("### Runtime statistics of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.RT.Statistics-DIM` ) { + for (fid in input$`Report.RT.Statistics-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Statistics_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Statistics-Alg`) + name <- get_data_id(dsl_sub) + fvs <- get_funvals(dsl_sub) + tryCatch( + { + fv_seq <- seq_FV(fvs, as.numeric(REG$RTSummary.Statistics.Min[[name]]), as.numeric(REG$RTSummary.Statistics.Max[[name]]), + by = as.numeric(REG$RTSummary.Statistics.Step[[name]])) + }, + error = function(e) { fv_seq <- seq_RT(fvs, length.out = 10)} + ) + p <- kable(get_RT_summary(dsl_sub, fv_seq), + caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), + format = 'latex') %>% + kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r single_erts, eval=input$Report.RT.Single_ERT, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.Single_ERT-DIM` ) { + for (fid in input$`Report.RT.Single_ERT-FuncId` ) { + name_figure <- paste0('ERT_single_F', fid, '-', dim, 'D') + algs = input$`Report.RT.Single_ERT-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_ERT.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.Single_ERT) paste(out, collapse='\n')` + +```{r single_erts_text, eval=input$Report.RT.Single_ERT, results='asis'} +asis_output(paste0("This section shows the ERTs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + + +```{r multi_erts, eval=input$Report.RT.Multi_ERT, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.Multi_ERT-DIM` ) { + name_figure <- paste0('ERT_multi_', dim, 'D') + algs = input$`Report.RT.Multi_ERT-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_ERT.Rmd")))) +} +``` + +`r if(input$Report.RT.Multi_ERT) paste(out, collapse='\n')` + +```{r ranking_erts, eval=input$Report.RT.Rank, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.Rank-DIM` ) { + name_figure <- paste0('RT_Rank_', dim, 'D') + algs = input$`Report.RT.Rank-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Rank.Rmd")))) +} +``` + +`r if(input$Report.RT.Rank) paste(out, collapse='\n')` + +```{r rt_hist, eval=input$Report.RT.Histogram, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.Histogram-DIM` ) { + for (fid in input$`Report.RT.Histogram-FuncId` ) { + name_figure <- paste0('RT_Hist_F', fid, '-', dim, 'D') + algs = input$`Report.RT.Histogram-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Histogram.Rmd")))) + } +} +``` + +`r if(input$Report.RT.Histogram) paste(out, collapse='\n')` + +```{r rt_pmf, eval=input$Report.RT.PMF, results='hide', include=F} +out <- NULL +for (dim in input$`Report.RT.PMF-DIM` ) { + for (fid in input$`Report.RT.PMF-FuncId` ) { + name_figure <- paste0('RT_PMF_F', fid, '-', dim, 'D') + algs = input$`Report.RT.PMF-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_PMF.Rmd")))) + } +} +``` + +`r if(input$Report.RT.PMF) paste(out, collapse='\n')` + +```{r ecdf_single_target, eval=input$Report.RT.ECDF_Single_Target, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.ECDF_Single_Target-DIM` ) { + for (fid in input$`Report.RT.ECDF_Single_Target-FuncId` ) { + name_figure <- paste0('ECDF_Single_Target', fid, '-', dim, 'D') + algs = input$`Report.RT.ECDF_Single_Target-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Target.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.ECDF_Single_Target) paste(out, collapse='\n')` + +```{r ecdf_single_target_text, eval=input$Report.RT.ECDF_Single_Target, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r ecdf_single_function, eval=input$Report.RT.ECDF_Single_Function, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.ECDF_Single_Function-DIM` ) { + for (fid in input$`Report.RT.ECDF_Single_Function-FuncId` ) { + name_figure <- paste0('ECDF_Single_Function', fid, '-', dim, 'D') + algs = input$`Report.RT.ECDF_Single_Function-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Function.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.ECDF_Single_Function) paste(out, collapse='\n')` + +```{r ecdf_single_function_text, eval=input$Report.RT.ECDF_Single_Function, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r ecdf_aggr, eval=input$Report.RT.ECDF_Aggregated, results='hide', include=F} +algs = input$`Report.RT.ECDF_Aggregated-Alg` +env = new.env() +out <- knit_child(text = knit_expand(file = "ECDF_Aggregated.Rmd")) + +``` + +`r if(input$Report.RT.ECDF_Aggregated) paste(out, collapse='\n')` + +```{r ecdf_aggr_text, eval=input$Report.RT.ECDF_Aggregated, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figure \\ref{fig:aggregated_ecdf}")) +``` + +```{r ecdf_auc, eval=input$Report.RT.ECDF_AUC, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.RT.ECDF_AUC-DIM` ) { + for (fid in input$`Report.RT.ECDF_AUC-FuncId` ) { + name_figure <- paste0('ECDF_AUC', fid, '-', dim, 'D') + algs = input$`Report.RT.ECDF_AUC-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_AUC.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.RT.ECDF_AUC) paste(out, collapse='\n')` + +```{r ecdf_auc_text, eval=input$Report.RT.ECDF_AUC, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +# Fixed-budget view + +```{r fv_overview, results='asis', eval=input$Report.FV.Overview} +asis_output("### Overview of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.FV.Overview-DIM` ) { + for (fid in input$`Report.FV.Overview-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Overview_FV_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Overview-Alg` ) + p <- kable(get_FV_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r fv_statistics, results='asis', eval=input$Report.FV.Statistics} +asis_output("### Runtime statistics of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.FV.Statistics-DIM` ) { + for (fid in input$`Report.FV.Statistics-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Statistics_FV_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Statistics-Alg`) + name <- get_data_id(dsl_sub) + rts <- get_runtimes(dsl_sub) + tryCatch( + { + rt_seq <- seq_RT(rts, as.numeric(REG$FCESummary.Statistics.Min[[name]]), as.numeric(REG$FCESummary.Statistics.Max[[name]]), + by = as.numeric(REG$FCESummary.Statistics.Step[[name]])) + }, + error = function(e) { rt_seq <- seq_RT(rts, length.out = 10)} + ) + p <- kable(get_FV_summary(dsl_sub, rt_seq), + caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), + format = 'latex') %>% + kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + + } +} +``` + +```{r single_fce, eval=input$Report.FV.Single_FCE, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.Single_FCE-DIM` ) { + for (fid in input$`Report.FV.Single_FCE-FuncId` ) { + name_figure <- paste0('FCE_single_F', fid, '-', dim, 'D') + algs = input$`Report.FV.Single_FCE-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_FCE.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.Single_FCE) paste(out, collapse='\n')` + +```{r single_fce_text, eval=input$Report.FV.Single_FCE, results='asis'} +asis_output(paste0("This section shows the FCEs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r multi_fces, eval=input$Report.FV.Multi_FCE, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.Multi_FCE-DIM` ) { + name_figure <- paste0('FCE_multi_', dim, 'D') + algs = input$`Report.FV.Multi_FCE-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_FCE.Rmd")))) +} +``` + +`r if(input$Report.FV.Multi_FCE) paste(out, collapse='\n')` + +```{r ranking_FVs, eval=input$Report.FV.Rank, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.Rank-DIM` ) { + name_figure <- paste0('FV_Rank_', dim, 'D') + algs = input$`Report.FV.Rank-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Rank.Rmd")))) +} +``` + +`r if(input$Report.FV.Rank) paste(out, collapse='\n')` + +```{r fv_hist, eval=input$Report.FV.Histogram, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.Histogram-DIM` ) { + for (fid in input$`Report.FV.Histogram-FuncId` ) { + name_figure <- paste0('FV_Hist_F', fid, '-', dim, 'D') + algs = input$`Report.FV.Histogram-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Histogram.Rmd")))) + } +} +``` + +`r if(input$Report.FV.Histogram) paste(out, collapse='\n')` + +```{r fv_pmf, eval=input$Report.FV.PMF, results='hide', include=F} +out <- NULL +for (dim in input$`Report.FV.PMF-DIM` ) { + for (fid in input$`Report.FV.PMF-FuncId` ) { + name_figure <- paste0('FV_PMF_F', fid, '-', dim, 'D') + algs = input$`Report.FV.PMF-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_PMF.Rmd")))) + } +} +``` + +`r if(input$Report.FV.PMF) paste(out, collapse='\n')` + +```{r fv_ecdf_single_target, eval=input$Report.FV.ECDF_Single_Target, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.ECDF_Single_Target-DIM` ) { + for (fid in input$`Report.FV.ECDF_Single_Target-FuncId` ) { + name_figure <- paste0('FV_ECDF_Single_Target', fid, '-', dim, 'D') + algs = input$`Report.FV.ECDF_Single_Target-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Target.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.ECDF_Single_Target) paste(out, collapse='\n')` + +```{r fv_ecdf_single_target_text, eval=input$Report.FV.ECDF_Single_Target, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r fv_ecdf_single_function, eval=input$Report.FV.ECDF_Single_Function, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.ECDF_Single_Function-DIM` ) { + for (fid in input$`Report.FV.ECDF_Single_Function-FuncId` ) { + name_figure <- paste0('FV_ECDF_Single_Function', fid, '-', dim, 'D') + algs = input$`Report.FV.ECDF_Single_Function-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Function.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.ECDF_Single_Function) paste(out, collapse='\n')` + +```{r fv_ecdf_single_function_text, eval=input$Report.FV.ECDF_Single_Function, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r fv_ecdf_auc, eval=input$Report.FV.ECDF_AUC, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.FV.ECDF_AUC-DIM` ) { + for (fid in input$`Report.FV.ECDF_AUC-FuncId` ) { + name_figure <- paste0('FV_ECDF_AUC', fid, '-', dim, 'D') + algs = input$`Report.FV.ECDF_AUC-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_AUC.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.FV.ECDF_AUC) paste(out, collapse='\n')` + +```{r fv_ecdf_auc_text, eval=input$Report.FV.ECDF_AUC, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +```{r param_statistics, results='asis', eval=input$Report.Param.Statistics} +asis_output("### Statistics of parameters of selected function / dimension pairs") +cat('\n') +for (dim in input$`Report.Param.Statistics-DIM` ) { + for (fid in input$`Report.Param.Statistics-FuncId` ) { + x <- paste0("### F", fid, " ", dim, "D \n") + cat(x) + cat('\n') + opts_current$set(label = paste0("Statistics_Param_F", fid, " ", dim, "D")) + dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.Param.Statistics-Alg`) + name <- get_data_id(dsl_sub) + fvs <- get_funvals(dsl_sub) + tryCatch( + { + fv_seq <- seq_FV(fvs, as.numeric(REG$PAR.Summary.Min[[name]]), as.numeric(REG$PAR.Summary.Max[[name]]), + by = as.numeric(REG$PAR.Summary.Step[[name]])) + }, + error = function(e) { fv_seq <- seq_FV(fvs, length.out = 10) } + ) + p <- kable(get_PAR_summary(dsl_sub, fv_seq), + caption = paste0("Statistics of parameters of selected algorithms on F", fid, " in ", dim, "D"), + format = 'latex') %>% + kable_styling(latex_options=c("scale_down", "hold_position")) + print(p) + cat('\n') + } +} +``` + +```{r param_plot, eval=input$Report.Param.Plot, results='hide', include=F} +out <- NULL +fig_names <- NULL +for (dim in input$`Report.Param.Plot-DIM` ) { + for (fid in input$`Report.Param.Plot-FuncId` ) { + name_figure <- paste0('Param_Plot', fid, '-', dim, 'D') + algs = input$`Report.Param.Plot-Alg` + env = new.env() + out <- invisible(c(out,knit_child(text = knit_expand(file = "Param_plot.Rmd")))) + fig_names <- c(fig_names, name_figure) + } +} +``` + +`r if(input$Report.Param.Plot) paste(out, collapse='\n')` + +```{r param_plot_text, eval=input$Report.Param.Plot, results='asis'} +asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")}))) +``` + +# References diff --git a/inst/shiny-server/markdown/Report/bibliography.bib b/inst/shiny-server/markdown/Report/bibliography.bib index 5456cc1b..a3c40f98 100644 --- a/inst/shiny-server/markdown/Report/bibliography.bib +++ b/inst/shiny-server/markdown/Report/bibliography.bib @@ -1,11 +1,11 @@ -@ARTICLE{IOHprofiler, - author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, - title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, - journal = {arXiv e-prints:1810.05281}, - archivePrefix = "arXiv", - eprint = {1810.05281}, - year = 2018, - month = oct, - keywords = {Computer Science - Neural and Evolutionary Computing}, - url = {https://arxiv.org/abs/1810.05281} +@ARTICLE{IOHprofiler, + author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, + title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, + journal = {arXiv e-prints:1810.05281}, + archivePrefix = "arXiv", + eprint = {1810.05281}, + year = 2018, + month = oct, + keywords = {Computer Science - Neural and Evolutionary Computing}, + url = {https://arxiv.org/abs/1810.05281} } \ No newline at end of file diff --git a/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd b/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd index 19bbe555..ab79b4c6 100644 --- a/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd +++ b/inst/shiny-server/markdown/TAR_SUMMARY_TABLE.Rmd @@ -1,7 +1,7 @@ -This table summarizes for each algorithm and each __budget__ $B$ chosen on the left: - - * runs: the number of runs that have performed at least $B$ evaluations, - * mean: the average best-so-far function value obtain within a budget of $B$ evaluations, - * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of the best function values found within the first $B$ evaluations. - -When not all runs evaluated at least $B$ search points, the statistics hold for the subset of runs that did. Alternative statistics using simulated restarted algorithms are in preparation. +This table summarizes for each algorithm and each __budget__ $B$ chosen on the left: + + * runs: the number of runs that have performed at least $B$ evaluations, + * mean: the average best-so-far function value obtain within a budget of $B$ evaluations, + * median, $2\%, 5\%,\ldots,98\%$ : the quantiles of the best function values found within the first $B$ evaluations. + +When not all runs evaluated at least $B$ search points, the statistics hold for the subset of runs that did. Alternative statistics using simulated restarted algorithms are in preparation. diff --git a/inst/shiny-server/markdown/about.md b/inst/shiny-server/markdown/about.md index 3b5ff9c9..574923e3 100644 --- a/inst/shiny-server/markdown/about.md +++ b/inst/shiny-server/markdown/about.md @@ -1,97 +1,97 @@ -
- - - - -
-
- -# Iterative Optimization Heuristics Profiler - -The __performance analyzer__ for Iterative Optimization Heuristics (IOHs). -It provides: - -* a web-based interface to analyze and visualize the empirical performance of IOHs -* interactive plot -* statistical evaluation -* report generation -* `R` programming interfaces in the backend - -## Development Team - -* [Hao Wang](https://www.universiteitleiden.nl/en/staffmembers/hao-wang), Leiden Institute of Advanced Computer Science. -* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science. -* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science. -* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), CNRS and Sorbonne University. -* [Ofer Shir](https://www.migal.org.il/Ofer-Shir) Migal - The Galilee Research Institute, Tel-Hai College. -* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science. - -## Installation - -### Software dependency - -* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). -* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. -* [optional] `inkscape` required to support pdf and eps figure format in downloading. Please visit the Inskscape Wiki page [http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape](http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape) for the detail. - -## Host it online yourself? - -We provide docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. - -## Reference -When using IOHprofiler and parts thereof, please kindly cite this work as - -Carola Doerr, Hao Wang, Furong Ye, Sander van Rijn, Thomas Bäck: IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics, arXiv e-prints:1810.05281, 2018. - -```bibtex -@ARTICLE{IOHprofiler, - author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, - title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, - journal = {arXiv e-prints:1810.05281}, - archivePrefix = "arXiv", - eprint = {1810.05281}, - year = 2018, - month = oct, - keywords = {Computer Science - Neural and Evolutionary Computing}, - url = {https://arxiv.org/abs/1810.05281} -} -``` - -## Acknowledgements -This work was supported by the Chinese scholarship council (CSC No. 201706310143), a public grant as part of the -Investissement d’avenir project, reference ANR-11-LABX-0056-LMH, LabEx LMH, in a joint call with the Gaspard Monge -Program for optimization, operations research, and their interactions with data sciences, by Paris Ile-de-France Region, -and by COST Action CA15140 “Improving Applicability of Nature-Inspired Optimisation by Joining Theory and Practice (ImAppNIO)”. - -## License -This application is governed by the __BSD 3-Clause license__. - -BSD 3-Clause License - -Copyright (c) 2018, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +
+ + + + +
+
+ +# Iterative Optimization Heuristics Profiler + +The __performance analyzer__ for Iterative Optimization Heuristics (IOHs). +It provides: + +* a web-based interface to analyze and visualize the empirical performance of IOHs +* interactive plot +* statistical evaluation +* report generation +* `R` programming interfaces in the backend + +## Development Team + +* [Hao Wang](https://www.universiteitleiden.nl/en/staffmembers/hao-wang), Leiden Institute of Advanced Computer Science. +* [Diederick Vermetten](https://www.universiteitleiden.nl/en/staffmembers/diederick-vermetten), Leiden Institute of Advanced Computer Science. +* [Furong Ye](https://www.universiteitleiden.nl/en/staffmembers/furong-ye), Leiden Institute of Advanced Computer Science. +* [Carola Doerr](http://www-desir.lip6.fr/~doerr/), CNRS and Sorbonne University. +* [Ofer Shir](https://www.migal.org.il/Ofer-Shir) Migal - The Galilee Research Institute, Tel-Hai College. +* [Thomas Bäck](https://www.universiteitleiden.nl/en/staffmembers/thomas-back), Leiden Institute of Advanced Computer Science. + +## Installation + +### Software dependency + +* [mandatory] `R` As __IOHanalyzer__ is written as a `R` package, the `R` environment has to be installed first. The binary file and installation manual for R can be found here [https://cran.r-project.org/](https://cran.r-project.org/). +* [optional] `orca` required to download plotly figures. Please see [https://github.com/plotly/orca](https://github.com/plotly/orca) for the installation instruction. +* [optional] `inkscape` required to support pdf and eps figure format in downloading. Please visit the Inskscape Wiki page [http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape](http://wiki.inkscape.org/wiki/index.php/Installing_Inkscape) for the detail. + +## Host it online yourself? + +We provide docker file for deploying __IOHanalyzer__ on the server. Please see [https://github.com/IOHprofiler/IOHanalyzer-docker](https://github.com/IOHprofiler/IOHanalyzer-docker) for details. + +## Reference +When using IOHprofiler and parts thereof, please kindly cite this work as + +Carola Doerr, Hao Wang, Furong Ye, Sander van Rijn, Thomas Bäck: IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics, arXiv e-prints:1810.05281, 2018. + +```bibtex +@ARTICLE{IOHprofiler, + author = {Carola Doerr and Hao Wang and Furong Ye and Sander van Rijn and Thomas B{\"a}ck}, + title = {{IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics}}, + journal = {arXiv e-prints:1810.05281}, + archivePrefix = "arXiv", + eprint = {1810.05281}, + year = 2018, + month = oct, + keywords = {Computer Science - Neural and Evolutionary Computing}, + url = {https://arxiv.org/abs/1810.05281} +} +``` + +## Acknowledgements +This work was supported by the Chinese scholarship council (CSC No. 201706310143), a public grant as part of the +Investissement d’avenir project, reference ANR-11-LABX-0056-LMH, LabEx LMH, in a joint call with the Gaspard Monge +Program for optimization, operations research, and their interactions with data sciences, by Paris Ile-de-France Region, +and by COST Action CA15140 “Improving Applicability of Nature-Inspired Optimisation by Joining Theory and Practice (ImAppNIO)”. + +## License +This application is governed by the __BSD 3-Clause license__. + +BSD 3-Clause License + +Copyright (c) 2018, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/inst/shiny-server/markdown/dataformat.md b/inst/shiny-server/markdown/dataformat.md index de3b2355..38829ec3 100644 --- a/inst/shiny-server/markdown/dataformat.md +++ b/inst/shiny-server/markdown/dataformat.md @@ -1,105 +1,105 @@ -# Supported Data Format - -The user is encouraged to analyze your own benchmark data using **IOHanalyzer**. For this purpose, users have prepare their own data set according to the supported data formats. At the time of writing, three data formats are supported by **IOHanalyzer**: - -* **COCO** data format as regulated in [1]. -* **IOHprofiler** data format, which is motivated and modified from **COCO** data format. -* A **two-column** format that is simplified from **IOHprofiler** format. - -When loading the data in programming interface (and in the graphical user interface as well), it is not necessary to specified which format the user took as **IOHanalyzer** detects the format automatically. For all three data formats, data files are organized in the same manner within the file system. The structure of data files exemplified as follows: - - - -Generally, in the folder (e.g., `./` here) that contains the data set, the following files are mandatory for **IOHanalyzer**: - -* _Meta-data_ (e.g., `IOHprofiler_f1_i1.info`) summarizes the algorithmic performance for each problem instance, with naming the following naming convention: `IOHprofiler_f[:num:]_i[:num:].info` for problem $f1$ and instance $1$. Note that one meta-data file can consist of several dimensions. Please see the detail below. -* _Raw-data_ (e.g., `IOHprofiler_f1_DIM100_i1.dat`) are CSV-based files that contain performance information indexed by the running time. Raw-data files are named in the similar manner as with the meta-data, for example, `IOHprofiler_f1_DIM100_i1.dat` for problem $f1$, dimension $100$ and instance $1$. By default, the data files are organized in the group of test functions and stored in folders with naming convention: `data_[function ID]/`. It is important to note that those three data formats only differ in structure of the raw-data files. - -## Meta-data - -When benchmarking, it is common to specify a number of different dimensions, functions and instances, resulting in a quite large number of data files (e.g., `*.dat` files). It would make the data organization more structured if some meta data are provided. Here, the meta data are implemented in a format that is very similar to that in the well-known **COCO** environment. The meta data are indicated with suffix \verb|.info|. An small example is provided as follows: - -```{bash} -suite = 'PBO', funcId = 10, DIM = 100, algId = '(1+1) fGA' -% -data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, -1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, -1:1953125|5.49000e+02 -suite = 'PBO', funcId = 10, DIM = 625, algId = '(1+1) fGA' -% -data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, -1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, -1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, -1:1953125|5.49000e+02 -... -``` - -Note that, as this meta information is also used in **IOHanalyzer** when loading the data, it is **crucial** to give an attention to the format of the meta data, if the user would convert their own data sets. - -* **The first line** stores some meta-information of the experiment as (name, value) pairs. Note that, such pairs are separated by commas and three names, `funcId`, `DIM` and `algId` are _case-sensitive_ and _mandatory_. -* **The second line** always starts with a single `%`, indicating what follows this symbol should be the general comments from the user on this experiment. By default, it is left empty. -* **The third line** starts with the relative path to the actual data file, followed by the meta-information obtained on each instance, with the following format: $$\underbrace{1}_{\text{instance ID}}:\underbrace{1953125}_{\text{running time}}|\;\underbrace{5.59000e+02}_{\text{best-so-far f(x)}}$$ - -## Raw-data - -Despite different events are used for those four types of data file, those files take the same format, which is adapted from CSV format to accommodate multiple runs/instances. Typically, this format is illustrated by the example below (with dummy data records): - -```{bash} -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... -1 +2.95000e+02 +2.95000e+02 +2.95000e+02 +2.95000e+02 0.000000 ... -2 +2.96000e+02 +2.96000e+02 +2.96000e+02 +2.96000e+02 0.001600 ... -4 +3.07000e+02 +3.07000e+02 +3.07000e+02 +3.07000e+02 0.219200 ... -9 +3.11000e+02 +3.11000e+02 +3.11000e+02 +3.11000e+02 0.006400 ... -12 +3.12000e+02 +3.12000e+02 +3.12000e+02 +3.12000e+02 0.001600 ... -16 +3.16000e+02 +3.16000e+02 +3.16000e+02 +3.16000e+02 0.006400 ... -20 +3.17000e+02 +3.17000e+02 +3.17000e+02 +3.17000e+02 0.001600 ... -23 +3.28000e+02 +3.28000e+02 +3.28000e+02 +3.28000e+02 0.027200 ... -27 +3.39000e+02 +3.39000e+02 +3.39000e+02 +3.39000e+02 0.059200 ... -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... -1 +3.20000e+02 +3.20000e+02 +3.20000e+02 +3.20000e+02 1.000000 ... -24 +3.44000e+02 +3.44000e+02 +3.44000e+02 +3.44000e+02 2.000000 ... -60 +3.64000e+02 +3.64000e+02 +3.64000e+02 +3.64000e+02 3.000000 ... -"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... -... ... ... ... ... ... ... -``` - -Note that, - -1. **[mandatory]** each _separation line_ (line that starts with `"function evaluation"`) serves as a separator among different independent runs of the same algorithm. Therefore, it is clear that the data block between two separation lines corresponds to a single run a triplet of (dimension, function, instance). -2. **[mandatory]** `"function evaluation"` the current number of function evaluations. -3. **[mandatory]** `"best-so-far f(x)"` keeps track of the best function value observed since the beginning of one run. -4. **[optional]** `"current f(x)"` stands for the function value observed when the corresponding number of function evaluation is consumed. -5. **[optional]** The value stored under `"current af(x)+b"` and `"best af(x)+b"`, are so-called _transformed_ function values, obtained on each function instances that are generated by translating the orginal function in its domain and co-domain. -6. **[optional]** In addition, a parameter value (named `"parameter"`) is also tracked in this example and recording more parameter value is also facilitated (see below). - -## Two-Column Format - -The raw data file, in its simplest **two-column** format should resemble the following example: - -```{bash} -"function evaluation" "best-so-far f(x)" -1 +2.95000e+02 -2 +2.96000e+02 -4 +3.07000e+02 -23 +3.28000e+02 -27 +3.39000e+02 -"function evaluation" "best-so-far f(x)" -1 +3.20000e+02 -... ... -``` - -This format is regulated as follows: - -* The _double quotation_ (`"`) in the separation line shall always be kept and it cannot be replace with single quotation (`'`). -* The numbers in the record can either be written in the plain or scientific notation. -* To separate the column, _a single space or tab_ can be used (only one of them should be used consistently in a single data file). -* In the target-based tracking file (`*.dat`) each block of records (as divided by the separation line) **must** end with the last function evaluation. -* Each data line should contain a complete record. Incomplete data lines will be dropped when loading the data into **IOHanalyzer**. -* In case the quotation mark is needed in the parameter name, please use the single quotation (`'`). - -## Reference - -[1] Hansen N, Auger A, Finck S, Ros R (2009a). "Real-Parameter Black-Box OptimizationBenchmarking 2009: Experimental Setup." Research Report RR-6828, INRIA. URL +# Supported Data Format + +The user is encouraged to analyze your own benchmark data using **IOHanalyzer**. For this purpose, users have prepare their own data set according to the supported data formats. At the time of writing, three data formats are supported by **IOHanalyzer**: + +* **COCO** data format as regulated in [1]. +* **IOHprofiler** data format, which is motivated and modified from **COCO** data format. +* A **two-column** format that is simplified from **IOHprofiler** format. + +When loading the data in programming interface (and in the graphical user interface as well), it is not necessary to specified which format the user took as **IOHanalyzer** detects the format automatically. For all three data formats, data files are organized in the same manner within the file system. The structure of data files exemplified as follows: + + + +Generally, in the folder (e.g., `./` here) that contains the data set, the following files are mandatory for **IOHanalyzer**: + +* _Meta-data_ (e.g., `IOHprofiler_f1_i1.info`) summarizes the algorithmic performance for each problem instance, with naming the following naming convention: `IOHprofiler_f[:num:]_i[:num:].info` for problem $f1$ and instance $1$. Note that one meta-data file can consist of several dimensions. Please see the detail below. +* _Raw-data_ (e.g., `IOHprofiler_f1_DIM100_i1.dat`) are CSV-based files that contain performance information indexed by the running time. Raw-data files are named in the similar manner as with the meta-data, for example, `IOHprofiler_f1_DIM100_i1.dat` for problem $f1$, dimension $100$ and instance $1$. By default, the data files are organized in the group of test functions and stored in folders with naming convention: `data_[function ID]/`. It is important to note that those three data formats only differ in structure of the raw-data files. + +## Meta-data + +When benchmarking, it is common to specify a number of different dimensions, functions and instances, resulting in a quite large number of data files (e.g., `*.dat` files). It would make the data organization more structured if some meta data are provided. Here, the meta data are implemented in a format that is very similar to that in the well-known **COCO** environment. The meta data are indicated with suffix \verb|.info|. An small example is provided as follows: + +```{bash} +suite = 'PBO', funcId = 10, DIM = 100, algId = '(1+1) fGA' +% +data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, +1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, +1:1953125|5.49000e+02 +suite = 'PBO', funcId = 10, DIM = 625, algId = '(1+1) fGA' +% +data_f10/IOHprofiler_f10_DIM625.dat, 1:1953125|5.59000e+02, +1:1953125|5.59000e+02, 1:1953125|5.59000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.64000e+02, 1:1953125|5.54000e+02, +1:1953125|5.59000e+02, 1:1953125|5.49000e+02, 1:1953125|5.54000e+02, +1:1953125|5.49000e+02 +... +``` + +Note that, as this meta information is also used in **IOHanalyzer** when loading the data, it is **crucial** to give an attention to the format of the meta data, if the user would convert their own data sets. + +* **The first line** stores some meta-information of the experiment as (name, value) pairs. Note that, such pairs are separated by commas and three names, `funcId`, `DIM` and `algId` are _case-sensitive_ and _mandatory_. +* **The second line** always starts with a single `%`, indicating what follows this symbol should be the general comments from the user on this experiment. By default, it is left empty. +* **The third line** starts with the relative path to the actual data file, followed by the meta-information obtained on each instance, with the following format: $$\underbrace{1}_{\text{instance ID}}:\underbrace{1953125}_{\text{running time}}|\;\underbrace{5.59000e+02}_{\text{best-so-far f(x)}}$$ + +## Raw-data + +Despite different events are used for those four types of data file, those files take the same format, which is adapted from CSV format to accommodate multiple runs/instances. Typically, this format is illustrated by the example below (with dummy data records): + +```{bash} +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... +1 +2.95000e+02 +2.95000e+02 +2.95000e+02 +2.95000e+02 0.000000 ... +2 +2.96000e+02 +2.96000e+02 +2.96000e+02 +2.96000e+02 0.001600 ... +4 +3.07000e+02 +3.07000e+02 +3.07000e+02 +3.07000e+02 0.219200 ... +9 +3.11000e+02 +3.11000e+02 +3.11000e+02 +3.11000e+02 0.006400 ... +12 +3.12000e+02 +3.12000e+02 +3.12000e+02 +3.12000e+02 0.001600 ... +16 +3.16000e+02 +3.16000e+02 +3.16000e+02 +3.16000e+02 0.006400 ... +20 +3.17000e+02 +3.17000e+02 +3.17000e+02 +3.17000e+02 0.001600 ... +23 +3.28000e+02 +3.28000e+02 +3.28000e+02 +3.28000e+02 0.027200 ... +27 +3.39000e+02 +3.39000e+02 +3.39000e+02 +3.39000e+02 0.059200 ... +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... +1 +3.20000e+02 +3.20000e+02 +3.20000e+02 +3.20000e+02 1.000000 ... +24 +3.44000e+02 +3.44000e+02 +3.44000e+02 +3.44000e+02 2.000000 ... +60 +3.64000e+02 +3.64000e+02 +3.64000e+02 +3.64000e+02 3.000000 ... +"function evaluation" "current f(x)" "best-so-far f(x)" "current af(x)+b" "best af(x)+b" "parameter name" ... +... ... ... ... ... ... ... +``` + +Note that, + +1. **[mandatory]** each _separation line_ (line that starts with `"function evaluation"`) serves as a separator among different independent runs of the same algorithm. Therefore, it is clear that the data block between two separation lines corresponds to a single run a triplet of (dimension, function, instance). +2. **[mandatory]** `"function evaluation"` the current number of function evaluations. +3. **[mandatory]** `"best-so-far f(x)"` keeps track of the best function value observed since the beginning of one run. +4. **[optional]** `"current f(x)"` stands for the function value observed when the corresponding number of function evaluation is consumed. +5. **[optional]** The value stored under `"current af(x)+b"` and `"best af(x)+b"`, are so-called _transformed_ function values, obtained on each function instances that are generated by translating the orginal function in its domain and co-domain. +6. **[optional]** In addition, a parameter value (named `"parameter"`) is also tracked in this example and recording more parameter value is also facilitated (see below). + +## Two-Column Format + +The raw data file, in its simplest **two-column** format should resemble the following example: + +```{bash} +"function evaluation" "best-so-far f(x)" +1 +2.95000e+02 +2 +2.96000e+02 +4 +3.07000e+02 +23 +3.28000e+02 +27 +3.39000e+02 +"function evaluation" "best-so-far f(x)" +1 +3.20000e+02 +... ... +``` + +This format is regulated as follows: + +* The _double quotation_ (`"`) in the separation line shall always be kept and it cannot be replace with single quotation (`'`). +* The numbers in the record can either be written in the plain or scientific notation. +* To separate the column, _a single space or tab_ can be used (only one of them should be used consistently in a single data file). +* In the target-based tracking file (`*.dat`) each block of records (as divided by the separation line) **must** end with the last function evaluation. +* Each data line should contain a complete record. Incomplete data lines will be dropped when loading the data into **IOHanalyzer**. +* In case the quotation mark is needed in the parameter name, please use the single quotation (`'`). + +## Reference + +[1] Hansen N, Auger A, Finck S, Ros R (2009a). "Real-Parameter Black-Box OptimizationBenchmarking 2009: Experimental Setup." Research Report RR-6828, INRIA. URL diff --git a/inst/shiny-server/markdown/docs.Rmd b/inst/shiny-server/markdown/docs.Rmd index 3cfe8676..146cb229 100644 --- a/inst/shiny-server/markdown/docs.Rmd +++ b/inst/shiny-server/markdown/docs.Rmd @@ -1,26 +1,26 @@ ---- -title: "Untitled" -output: html_document ---- - -**Remark**: This is just a mockup, to show that we could put some important/concise documents here. - -## R Markdown - -This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see . - -When you click the **Knit** button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this: - -```{r cars} -summary(cars) -``` - -## Including Plots - -You can also embed plots, for example: - -```{r pressure, echo=FALSE} -plot(pressure) -``` - -Note that the `echo = FALSE` parameter was added to the code chunk to prevent printing of the R code that generated the plot. +--- +title: "Untitled" +output: html_document +--- + +**Remark**: This is just a mockup, to show that we could put some important/concise documents here. + +## R Markdown + +This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see . + +When you click the **Knit** button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this: + +```{r cars} +summary(cars) +``` + +## Including Plots + +You can also embed plots, for example: + +```{r pressure, echo=FALSE} +plot(pressure) +``` + +Note that the `echo = FALSE` parameter was added to the code chunk to prevent printing of the R code that generated the plot. diff --git a/inst/shiny-server/markdown/quantile.Rmd b/inst/shiny-server/markdown/quantile.Rmd index b8be0e1f..757e2c1d 100644 --- a/inst/shiny-server/markdown/quantile.Rmd +++ b/inst/shiny-server/markdown/quantile.Rmd @@ -1,9 +1,9 @@ ---- -title: "quantile" -output: html_document ---- - -**Remark**: all sample quantiles are defined as weighted averages of consecutive order statistics. Let $\{x_1, x_2, \ldots, x_N\}$ stands for i.i.d. runtime samples. Sample quantiles are defined as: -$$Q(p) = (1 - \alpha) x_{i:N}+ \alpha x_{i+1:N},$$ -**TODO** this is just a example showing we could explain some detials on the calculations. - +--- +title: "quantile" +output: html_document +--- + +**Remark**: all sample quantiles are defined as weighted averages of consecutive order statistics. Let $\{x_1, x_2, \ldots, x_N\}$ stands for i.i.d. runtime samples. Sample quantiles are defined as: +$$Q(p) = (1 - \alpha) x_{i:N}+ \alpha x_{i+1:N},$$ +**TODO** this is just a example showing we could explain some detials on the calculations. + diff --git a/inst/shiny-server/markdown/welcome.md b/inst/shiny-server/markdown/welcome.md index 2ec7b243..54f9cf72 100644 --- a/inst/shiny-server/markdown/welcome.md +++ b/inst/shiny-server/markdown/welcome.md @@ -1,45 +1,45 @@ -This the **analyzer** for the empirical performance of _Iterative Optimization Heuristics_ (IOHs). It provides _two perspectives_ to look at the empirical performance: - -* Fixed-Target running time: focuses on the distribution of **running time** (a.k.a. the number of function evaluations) at various given target values (thus called "fixed-target"), functions and dimensions. -* Fixed-Budget results: focuses on the distribution of the **best function value** at various given budget values (thus called "fixed-budget"), functions and dimensions. - -To get started, you could - -* **upload** your own dataset using the _Upload Data_ box on the bottom left. The supported format is specified in the data format tab. -* or **choose** one of the pre-loaded datasets in the _Load Data from Repository_ box on the bottom right and directly explore the analyzer. - -In details, the following functionalities are provided: - - - - - - - - - - - - - - - - - - - - - - - - -
Data SummaryExpected ValuesProbability Density/Mass FunctionEmpirical Cumulative Distribution (ECDF)
Fixed-Target running timeSummary of running timesExpected Running Time (ERT)Probability Mass Function of running timeECDF of running times
Fixed-Budget resultsSummary of function valuesExpected Target ValueProbability Density Function of target valuesECDF of targets times
-
- -For more information on the features of IOHanalyzer and how to use them, -as well as a full specification of the accepted data formats, please visit our wiki. +This the **analyzer** for the empirical performance of _Iterative Optimization Heuristics_ (IOHs). It provides _two perspectives_ to look at the empirical performance: + +* Fixed-Target running time: focuses on the distribution of **running time** (a.k.a. the number of function evaluations) at various given target values (thus called "fixed-target"), functions and dimensions. +* Fixed-Budget results: focuses on the distribution of the **best function value** at various given budget values (thus called "fixed-budget"), functions and dimensions. + +To get started, you could + +* **upload** your own dataset using the _Upload Data_ box on the bottom left. The supported format is specified in the data format tab. +* or **choose** one of the pre-loaded datasets in the _Load Data from Repository_ box on the bottom right and directly explore the analyzer. + +In details, the following functionalities are provided: + + + + + + + + + + + + + + + + + + + + + + + + +
Data SummaryExpected ValuesProbability Density/Mass FunctionEmpirical Cumulative Distribution (ECDF)
Fixed-Target running timeSummary of running timesExpected Running Time (ERT)Probability Mass Function of running timeECDF of running times
Fixed-Budget resultsSummary of function valuesExpected Target ValueProbability Density Function of target valuesECDF of targets times
+
+ +For more information on the features of IOHanalyzer and how to use them, +as well as a full specification of the accepted data formats, please visit our wiki. diff --git a/inst/shiny-server/server.R b/inst/shiny-server/server.R index 520a8ceb..d6a1ace2 100644 --- a/inst/shiny-server/server.R +++ b/inst/shiny-server/server.R @@ -1,65 +1,65 @@ - -repo_dir <- '' # repository directory -repo_data <- NULL # repository data -has_rendered_ERT_per_fct <- FALSE - -# Formatter for function values. Somewhat overkill with as.numeric, but this prevents unneeded precision -format_FV <- function(v) as.numeric(format(v, digits = getOption("IOHanalyzer.precision", 2), - nsmall = getOption("IOHanalyzer.precision", 2))) -format_RT <- function(v) as.integer(v) - -# directory where data are extracted from the zip file -exdir <- file.path(tempdir(), 'data') - -rand_strings <- function(n = 10) { - a <- do.call(paste0, replicate(5, sample(LETTERS, n, TRUE), FALSE)) - paste0(a, sprintf("%04d", sample(9999, n, TRUE)), sample(LETTERS, n, TRUE)) -} - -setTextInput <- function(session, id, name, alternative) { - v <- REG[[id]] - if (name %in% names(v)) - updateTextInput(session, id, value = v[[name]]) - else - updateTextInput(session, id, value = alternative) -} - -# register previous text inputs, which is used to restore those values -REG <- lapply(widget_id, function(x) list()) - -# TODO: maybe give this function a better name -# get the current 'id' of the selected data: funcID + DIM -get_data_id <- function(dsList) { - if (is.null(dsList) | length(dsList) == 0) - return(NULL) - - paste(get_funcId(dsList), get_dim(dsList), sep = '-') -} - -# Define server logic required to draw a histogram -shinyServer(function(input, output, session) { - - # clean up the temporarsy files on server when exiting - session$onSessionEnded(function() { - # close_connection() - unlink(exdir, recursive = T) - }) - - for (f in list.files('server', pattern = '.R', full.names = T)) { - source(f, local = TRUE) - } - - output$VersionBox <- renderValueBox({ - infoBox(title = "IOHanalyzer version:", value = paste0('v', packageVersion("IOHanalyzer")), icon = icon("code-branch"), - color = "blue", width = 12, fill = T, href = "https://github.com/IOHprofiler/IOHanalyzer", subtitle = "View source on Github" - )}) - output$WikiBox <- renderValueBox({ - infoBox(title = "Wiki Page", value = "For full documentation of IOHprofiler", icon = icon("wikipedia-w"), - color = "blue", width = 12, fill = T, href = "https://iohprofiler.github.io/" - )}) - output$ContactBox <- renderValueBox({ - infoBox(title = "Contact us by email:", value = "iohprofiler@liacs.leidenuniv.nl", icon = icon("envelope"), - color = "blue", width = 14, fill = T, href = "mailto:iohprofiler@liacs.leidenuniv.nl" - ) - }) -}) + +repo_dir <- '' # repository directory +repo_data <- NULL # repository data +has_rendered_ERT_per_fct <- FALSE + +# Formatter for function values. Somewhat overkill with as.numeric, but this prevents unneeded precision +format_FV <- function(v) as.numeric(format(v, digits = getOption("IOHanalyzer.precision", 2), + nsmall = getOption("IOHanalyzer.precision", 2))) +format_RT <- function(v) as.integer(v) + +# directory where data are extracted from the zip file +exdir <- file.path(tempdir(), 'data') + +rand_strings <- function(n = 10) { + a <- do.call(paste0, replicate(5, sample(LETTERS, n, TRUE), FALSE)) + paste0(a, sprintf("%04d", sample(9999, n, TRUE)), sample(LETTERS, n, TRUE)) +} + +setTextInput <- function(session, id, name, alternative) { + v <- REG[[id]] + if (name %in% names(v)) + updateTextInput(session, id, value = v[[name]]) + else + updateTextInput(session, id, value = alternative) +} + +# register previous text inputs, which is used to restore those values +REG <- lapply(widget_id, function(x) list()) + +# TODO: maybe give this function a better name +# get the current 'id' of the selected data: funcID + DIM +get_data_id <- function(dsList) { + if (is.null(dsList) | length(dsList) == 0) + return(NULL) + + paste(get_funcId(dsList), get_dim(dsList), sep = '-') +} + +# Define server logic required to draw a histogram +shinyServer(function(input, output, session) { + + # clean up the temporarsy files on server when exiting + session$onSessionEnded(function() { + # close_connection() + unlink(exdir, recursive = T) + }) + + for (f in list.files('server', pattern = '.R', full.names = T)) { + source(f, local = TRUE) + } + + output$VersionBox <- renderValueBox({ + infoBox(title = "IOHanalyzer version:", value = paste0('v', packageVersion("IOHanalyzer")), icon = icon("code-branch"), + color = "blue", width = 12, fill = T, href = "https://github.com/IOHprofiler/IOHanalyzer", subtitle = "View source on Github" + )}) + output$WikiBox <- renderValueBox({ + infoBox(title = "Wiki Page", value = "For full documentation of IOHprofiler", icon = icon("wikipedia-w"), + color = "blue", width = 12, fill = T, href = "https://iohprofiler.github.io/" + )}) + output$ContactBox <- renderValueBox({ + infoBox(title = "Contact us by email:", value = "iohprofiler@liacs.leidenuniv.nl", icon = icon("envelope"), + color = "blue", width = 14, fill = T, href = "mailto:iohprofiler@liacs.leidenuniv.nl" + ) + }) +}) diff --git a/inst/shiny-server/server/ERTPlot.R b/inst/shiny-server/server/ERTPlot.R index 5fb0d35e..eeead43f 100644 --- a/inst/shiny-server/server/ERTPlot.R +++ b/inst/shiny-server/server/ERTPlot.R @@ -1,195 +1,195 @@ -# The expected runtime plot --------------------- -# TODO: wrapp this as a separate function for DataSet class -# TODO: add very brief explanation on each function here - -output$ERT_PER_FUN <- renderPlotly({ - render_ert_per_fct() -}) - -get_data_ERT_PER_FUN <- reactive({ - req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) - selected_algs <- input$ERTPlot.Algs - data <- subset(DATA(), algId %in% input$ERTPlot.Algs) - fstart <- input$ERTPlot.Min %>% as.numeric - fstop <- input$ERTPlot.Max %>% as.numeric - budget <- input$ERTPlot.Additional.Budget %>% as.numeric - generate_data.Single_Function(data, fstart, fstop, input$ERTPlot.semilogx, - 'by_RT', include_opts = input$ERTPlot.inclueOpts, budget = budget) -}) - -render_ert_per_fct <- reactive({ - withProgress({ - y_attrs <- c() - if (input$ERTPlot.show.ERT) y_attrs <- c(y_attrs, 'ERT') - if (input$ERTPlot.show.mean) y_attrs <- c(y_attrs, 'mean') - if (input$ERTPlot.show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - else - p <- NULL - if (input$ERTPlot.show.CI) { - p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - if (input$ERTPlot.show.Quantiles) { - quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') - p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'median', - type = 'ribbon', legend_attr = 'algId', lower_attr = quantiles[[1]], - upper_attr = quantiles[[length(quantiles)]], p = p, - show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - if (input$ERTPlot.show.runs) { - fstart <- isolate(input$ERTPlot.Min %>% as.numeric) - fstop <- isolate(input$ERTPlot.Max %>% as.numeric) - data <- isolate(subset(DATA(), algId %in% input$ERTPlot.Algs)) - dt <- get_RT_sample(data, seq_FV(get_funvals(data), from = fstart, to = fstop, length.out = 50, - scale = ifelse(isolate(input$ERTPlot.semilogx), 'log', 'linear'))) - nr_runs <- ncol(dt) - 4 - for (i in seq_len(nr_runs)) { - p <- plot_general_data(dt, x_attr = 'target', y_attr = paste0('run.', i), type = 'line', - legend_attr = 'algId', p = p, show.legend = show_legend, - scale.ylog = input$ERTPlot.semilogy, - scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", - y_title = "Function Evaluations", - scale.reverse = !attr(DATA(), 'maximization')) - show_legend <- F - } - } - p - }, - message = "Creating plot" - ) -}) - -output$ERTPlot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_PER_FUN) - }, - content = function(file) { - save_plotly(render_ert_per_fct(), file) - }, - contentType = paste0('image/', input$ERTPlot.Format) -) - -# update_ert_per_fct_axis <- observe({ -# plotlyProxy("ERT_PER_FUN", session) %>% -# plotlyProxyInvoke( -# "restyle", -# list( -# xaxis = list(title = 'best-so-far-f(x)-value', -# type = ifelse(input$ERTPlot.semilogx, 'log', 'linear')), -# yaxis = list(title = 'function evaluations', -# type = ifelse(input$ERTPlot.semilogy, 'log', 'linear')) -# ) -# ) -# }) -# -# render_ert_per_fct <- reactive({ -# withProgress({ -# req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) -# selected_algs <- input$ERTPlot.Algs -# -# data <- subset(DATA(), algId %in% input$ERTPlot.Algs) -# req(length(data) > 0) -# -# fstart <- input$ERTPlot.Min %>% as.numeric -# fstop <- input$ERTPlot.Max %>% as.numeric -# -# Plot.RT.Single_Func(data, Fstart = fstart, Fstop = fstop, -# show.CI = input$ERTPlot.show.CI, -# show.ERT = input$ERTPlot.show.ERT, -# show.mean = input$ERTPlot.show.mean, -# show.median = input$ERTPlot.show.median, -# scale.xlog = input$ERTPlot.semilogx, -# scale.ylog = isolate(input$ERTPlot.semilogy), -# includeOpts = input$ERTPlot.inclueOpts, -# scale.reverse = !attr(data, 'maximization')) -# }, -# message = "Creating plot" -# ) -# }) - -output$ERTPlot.Multi.Plot <- renderPlotly( - render_ERTPlot_multi_plot() -) - -get_data_ERT_multi_func_bulk <- reactive({ - data <- subset(DATA_RAW(), - DIM == input$Overall.Dim) - if (length(get_algId(data)) < 20) { #Arbitrary limit for the time being - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, - which = 'by_RT') - })) - } - else - NULL -}) - -get_data_ERT_multi_func <- reactive({ - req(isolate(input$ERTPlot.Multi.Algs)) - input$ERTPlot.Multi.PlotButton - data <- subset(DATA_RAW(), - DIM == input$Overall.Dim) - if (length(get_algId(data)) < 20) { - get_data_ERT_multi_func_bulk()[algId %in% isolate(input$ERTPlot.Multi.Algs), ] - } - else { - selected_algs <- isolate(input$ERTPlot.Multi.Algs) - data <- subset(DATA_RAW(), - algId %in% selected_algs, - DIM == input$Overall.Dim) - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, - which = 'by_RT') - })) - } -}) - -render_ERTPlot_multi_plot <- reactive({ - req(isolate(input$ERTPlot.Multi.Algs)) - withProgress({ - dt <- get_data_ERT_multi_func() - if (is.null(dt)) - return(NULL) - plot_general_data(dt, x_attr = 'target', y_attr = 'ERT', - subplot_attr = 'funcId', type = 'line', scale.xlog = input$ERTPlot.Multi.Logx, - scale.ylog = input$ERTPlot.Multi.Logy, x_title = 'Best-so-far f(x)', - y_title = 'ERT', show.legend = T, - scale.reverse = !attr(DATA(), 'maximization')) - }, - message = "Creating plot") -}) - -output$ERTPlot.Multi.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_PER_FUN_MULTI) - }, - content = function(file) { - save_plotly(render_ERTPlot_multi_plot(), file) - }, - contentType = paste0('image/', input$ERTPlot.Multi.Format) -) - - - - +# The expected runtime plot --------------------- +# TODO: wrapp this as a separate function for DataSet class +# TODO: add very brief explanation on each function here + +output$ERT_PER_FUN <- renderPlotly({ + render_ert_per_fct() +}) + +get_data_ERT_PER_FUN <- reactive({ + req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) + selected_algs <- input$ERTPlot.Algs + data <- subset(DATA(), ID %in% input$ERTPlot.Algs) + fstart <- input$ERTPlot.Min %>% as.numeric + fstop <- input$ERTPlot.Max %>% as.numeric + budget <- input$ERTPlot.Additional.Budget %>% as.numeric + generate_data.Single_Function(data, fstart, fstop, input$ERTPlot.semilogx, + 'by_RT', include_opts = input$ERTPlot.inclueOpts, budget = budget) +}) + +render_ert_per_fct <- reactive({ + withProgress({ + y_attrs <- c() + if (input$ERTPlot.show.ERT) y_attrs <- c(y_attrs, 'ERT') + if (input$ERTPlot.show.mean) y_attrs <- c(y_attrs, 'mean') + if (input$ERTPlot.show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + else + p <- NULL + if (input$ERTPlot.show.CI) { + p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + if (input$ERTPlot.show.Quantiles) { + quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') + p <- plot_general_data(get_data_ERT_PER_FUN(), x_attr = 'target', y_attr = 'median', + type = 'ribbon', legend_attr = 'ID', lower_attr = quantiles[[1]], + upper_attr = quantiles[[length(quantiles)]], p = p, + show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + if (input$ERTPlot.show.runs) { + fstart <- isolate(input$ERTPlot.Min %>% as.numeric) + fstop <- isolate(input$ERTPlot.Max %>% as.numeric) + data <- isolate(subset(DATA(), ID %in% input$ERTPlot.Algs)) + dt <- get_RT_sample(data, seq_FV(get_funvals(data), from = fstart, to = fstop, length.out = 50, + scale = ifelse(isolate(input$ERTPlot.semilogx), 'log', 'linear'))) + nr_runs <- ncol(dt) - 4 + for (i in seq_len(nr_runs)) { + p <- plot_general_data(dt, x_attr = 'target', y_attr = paste0('run.', i), type = 'line', + legend_attr = 'ID', p = p, show.legend = show_legend, + scale.ylog = input$ERTPlot.semilogy, + scale.xlog = input$ERTPlot.semilogx, x_title = "Best-so-far f(x)-value", + y_title = "Function Evaluations", + scale.reverse = !attr(DATA(), 'maximization')) + show_legend <- F + } + } + p + }, + message = "Creating plot" + ) +}) + +output$ERTPlot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_PER_FUN) + }, + content = function(file) { + save_plotly(render_ert_per_fct(), file) + }, + contentType = paste0('image/', input$ERTPlot.Format) +) + +# update_ert_per_fct_axis <- observe({ +# plotlyProxy("ERT_PER_FUN", session) %>% +# plotlyProxyInvoke( +# "restyle", +# list( +# xaxis = list(title = 'best-so-far-f(x)-value', +# type = ifelse(input$ERTPlot.semilogx, 'log', 'linear')), +# yaxis = list(title = 'function evaluations', +# type = ifelse(input$ERTPlot.semilogy, 'log', 'linear')) +# ) +# ) +# }) +# +# render_ert_per_fct <- reactive({ +# withProgress({ +# req(input$ERTPlot.Min, input$ERTPlot.Max, length(DATA()) > 0) +# selected_algs <- input$ERTPlot.Algs +# +# data <- subset(DATA(), algId %in% input$ERTPlot.Algs) +# req(length(data) > 0) +# +# fstart <- input$ERTPlot.Min %>% as.numeric +# fstop <- input$ERTPlot.Max %>% as.numeric +# +# Plot.RT.Single_Func(data, Fstart = fstart, Fstop = fstop, +# show.CI = input$ERTPlot.show.CI, +# show.ERT = input$ERTPlot.show.ERT, +# show.mean = input$ERTPlot.show.mean, +# show.median = input$ERTPlot.show.median, +# scale.xlog = input$ERTPlot.semilogx, +# scale.ylog = isolate(input$ERTPlot.semilogy), +# includeOpts = input$ERTPlot.inclueOpts, +# scale.reverse = !attr(data, 'maximization')) +# }, +# message = "Creating plot" +# ) +# }) + +output$ERTPlot.Multi.Plot <- renderPlotly( + render_ERTPlot_multi_plot() +) + +get_data_ERT_multi_func_bulk <- reactive({ + data <- subset(DATA_RAW(), + DIM == input$Overall.Dim) + if (length(get_id(data)) < 20) { #Arbitrary limit for the time being + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, + which = 'by_RT') + })) + } + else + NULL +}) + +get_data_ERT_multi_func <- reactive({ + req(isolate(input$ERTPlot.Multi.Algs)) + input$ERTPlot.Multi.PlotButton + data <- subset(DATA_RAW(), + DIM == input$Overall.Dim) + if (length(get_id(data)) < 20) { + get_data_ERT_multi_func_bulk()[ID %in% isolate(input$ERTPlot.Multi.Algs), ] + } + else { + selected_algs <- isolate(input$ERTPlot.Multi.Algs) + data <- subset(DATA_RAW(), + ID %in% selected_algs, + DIM == input$Overall.Dim) + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$ERTPlot.Multi.Logx, + which = 'by_RT') + })) + } +}) + +render_ERTPlot_multi_plot <- reactive({ + req(isolate(input$ERTPlot.Multi.Algs)) + withProgress({ + dt <- get_data_ERT_multi_func() + if (is.null(dt)) + return(NULL) + plot_general_data(dt, x_attr = 'target', y_attr = 'ERT', + subplot_attr = 'funcId', type = 'line', scale.xlog = input$ERTPlot.Multi.Logx, + scale.ylog = input$ERTPlot.Multi.Logy, x_title = 'Best-so-far f(x)', + y_title = 'ERT', show.legend = T, + scale.reverse = !attr(DATA(), 'maximization')) + }, + message = "Creating plot") +}) + +output$ERTPlot.Multi.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_PER_FUN_MULTI) + }, + content = function(file) { + save_plotly(render_ERTPlot_multi_plot(), file) + }, + contentType = paste0('image/', input$ERTPlot.Multi.Format) +) + + + + diff --git a/inst/shiny-server/server/ERT_aggr_dim_plot.R b/inst/shiny-server/server/ERT_aggr_dim_plot.R index f2b9bebe..765ce438 100644 --- a/inst/shiny-server/server/ERT_aggr_dim_plot.R +++ b/inst/shiny-server/server/ERT_aggr_dim_plot.R @@ -1,133 +1,133 @@ -### Generate ERT table -get_data_ERT_aggr_dim <- reactive({ - input$ERTPlot.Aggr_Dim.Refresh - req(length(DATA_RAW()) > 0) - data <- ERTPlot.Aggr_Dim.data() - if (is.null(data)) return(NULL) - aggr_on <- 'DIM' - targets <- isolate(ERTPlot.Aggr_Dim.Targets_obj) - dt <- generate_data.Aggr(data, aggr_on, targets) - dt -}) - -### Plot the data -render_ERTPlot_aggr_plot_dim <- reactive({ - withProgress({ - data <- get_data_ERT_aggr_dim() - req(data) - y_attr <- if (input$ERTPlot.Aggr_Dim.Ranking) 'rank' else 'value' - y_title <- if (input$ERTPlot.Aggr_Dim.Ranking) 'Rank' else 'ERT' - reverse_scale <- input$ERTPlot.Aggr_Dim.Mode == 'radar' - - plot_general_data(data, type = input$ERTPlot.Aggr_Dim.Mode, x_attr = 'DIM', - y_attr = y_attr, x_title = "DIM", y_title = y_title, show.legend = T, - scale.ylog = input$ERTPlot.Aggr_Dim.Logy, scale.xlog = T, - scale.reverse = reverse_scale) - }, - message = "Creating plot") -}) - -### Gather relevant datasetlist -ERTPlot.Aggr_Dim.data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$ERTPlot.Aggr_Dim.Algs)) - if (length(data) == 0) return(NULL) - data <- subset(data, funcId == input$Overall.Funcid) - - if (length(unique(get_dim(data))) <= 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple dimensions for the selected function") - return(NULL) - } - - if (length(unique(get_algId(data))) <= 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected function") - return(NULL) - } - data -} - -### format table for display -ert_multi_dim <- function(){ - dt <- get_data_ERT_aggr_dim() - dt <- dcast(dt, DIM~algId, value.var = 'value') - format(dt, digits = 4) -} - -### Table with default targets -default_targets_table_dim <- reactive({ - data <- ERTPlot.Aggr_Dim.data() - if (is.null(data)) return(NULL) - targets <- get_target_dt(data) - targets <- targets[, c('DIM','target')] -}) - - -### Target table object -ERTPlot.Aggr_Dim.Targets_obj <- NULL - -### Target table proxy -proxy_ERTPlot.Aggr_Dim.Targets <- dataTableProxy('ERTPlot.Aggr_Dim.Targets') - -### Target table print -output$ERTPlot.Aggr_Dim.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - ERTPlot.Aggr_Dim.Targets_obj <<- default_targets_table_dim() - ERTPlot.Aggr_Dim.Targets_obj -}, editable = TRUE, rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - - -### Target table edit -observeEvent(input$ERTPlot.Aggr_Dim.Targets_cell_edit, { - info <- input$ERTPlot.Aggr_Dim.Targets_cell_edit - i <- info$row - req(i > 0) - j <- info$col + 1 - v <- info$value - ERTPlot.Aggr_Dim.Targets_obj[i, j] <<- - DT::coerceValue(v, ERTPlot.Aggr_Dim.Targets_obj[['target']][[i]]) - replaceData(proxy, ERTPlot.Aggr_Dim.Targets_obj, resetPaging = FALSE, rownames = FALSE) -}) - -### Table output -output$ERTPlot.Aggr_Dim.ERTTable <- DT::renderDataTable({ - input$ERTPlot.Aggr_Dim.Refresh - req(length(DATA_RAW()) > 0) - - withProgress({ - ert_multi_dim() - }, - message = "Creating table") -}, editable = FALSE, rownames = TRUE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - - -### plot output -output$ERTPlot.Aggr_Dim.Plot <- renderPlotly( - render_ERTPlot_aggr_plot_dim() -) - -### Download table -output$ERTPlot.Aggr_Dim.DownloadTable <- downloadHandler( - filename = function() { - eval(ERT_multi_dim_name) - }, - content = function(file) { - df <- ert_multi_dim() - save_table(df, file) - } -) - -### Download plot -output$ERTPlot.Aggr_Dim.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_AGGR_DIM) - }, - content = function(file) { - save_plotly(render_ERTPlot_aggr_plot_dim(), file) - }, - contentType = paste0('image/', input$ERTPlot.Aggr_Dim.Format) -) +### Generate ERT table +get_data_ERT_aggr_dim <- reactive({ + input$ERTPlot.Aggr_Dim.Refresh + req(length(DATA_RAW()) > 0) + data <- ERTPlot.Aggr_Dim.data() + if (is.null(data)) return(NULL) + aggr_on <- 'DIM' + targets <- isolate(ERTPlot.Aggr_Dim.Targets_obj) + dt <- generate_data.Aggr(data, aggr_on, targets) + dt +}) + +### Plot the data +render_ERTPlot_aggr_plot_dim <- reactive({ + withProgress({ + data <- get_data_ERT_aggr_dim() + req(data) + y_attr <- if (input$ERTPlot.Aggr_Dim.Ranking) 'rank' else 'value' + y_title <- if (input$ERTPlot.Aggr_Dim.Ranking) 'Rank' else 'ERT' + reverse_scale <- input$ERTPlot.Aggr_Dim.Mode == 'radar' + + plot_general_data(data, type = input$ERTPlot.Aggr_Dim.Mode, x_attr = 'DIM', + y_attr = y_attr, x_title = "DIM", y_title = y_title, show.legend = T, + scale.ylog = input$ERTPlot.Aggr_Dim.Logy, scale.xlog = T, + scale.reverse = reverse_scale) + }, + message = "Creating plot") +}) + +### Gather relevant datasetlist +ERTPlot.Aggr_Dim.data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr_Dim.Algs)) + if (length(data) == 0) return(NULL) + data <- subset(data, funcId == input$Overall.Funcid) + + if (length(unique(get_dim(data))) <= 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple dimensions for the selected function") + return(NULL) + } + + if (length(unique(get_id(data))) <= 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected function") + return(NULL) + } + data +} + +### format table for display +ert_multi_dim <- function(){ + dt <- get_data_ERT_aggr_dim() + dt <- dcast(dt, DIM~ID, value.var = 'value') + format(dt, digits = 4) +} + +### Table with default targets +default_targets_table_dim <- reactive({ + data <- ERTPlot.Aggr_Dim.data() + if (is.null(data)) return(NULL) + targets <- get_target_dt(data, 'by_FV') + targets <- targets[, c('DIM','target')] +}) + + +### Target table object +ERTPlot.Aggr_Dim.Targets_obj <- NULL + +### Target table proxy +proxy_ERTPlot.Aggr_Dim.Targets <- dataTableProxy('ERTPlot.Aggr_Dim.Targets') + +### Target table print +output$ERTPlot.Aggr_Dim.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + ERTPlot.Aggr_Dim.Targets_obj <<- default_targets_table_dim() + ERTPlot.Aggr_Dim.Targets_obj +}, editable = TRUE, rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + + +### Target table edit +observeEvent(input$ERTPlot.Aggr_Dim.Targets_cell_edit, { + info <- input$ERTPlot.Aggr_Dim.Targets_cell_edit + i <- info$row + req(i > 0) + j <- info$col + 1 + v <- info$value + ERTPlot.Aggr_Dim.Targets_obj[i, j] <<- + DT::coerceValue(v, ERTPlot.Aggr_Dim.Targets_obj[['target']][[i]]) + replaceData(proxy, ERTPlot.Aggr_Dim.Targets_obj, resetPaging = FALSE, rownames = FALSE) +}) + +### Table output +output$ERTPlot.Aggr_Dim.ERTTable <- DT::renderDataTable({ + input$ERTPlot.Aggr_Dim.Refresh + req(length(DATA_RAW()) > 0) + + withProgress({ + ert_multi_dim() + }, + message = "Creating table") +}, editable = FALSE, rownames = TRUE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + + +### plot output +output$ERTPlot.Aggr_Dim.Plot <- renderPlotly( + render_ERTPlot_aggr_plot_dim() +) + +### Download table +output$ERTPlot.Aggr_Dim.DownloadTable <- downloadHandler( + filename = function() { + eval(ERT_multi_dim_name) + }, + content = function(file) { + df <- ert_multi_dim() + save_table(df, file) + } +) + +### Download plot +output$ERTPlot.Aggr_Dim.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_AGGR_DIM) + }, + content = function(file) { + save_plotly(render_ERTPlot_aggr_plot_dim(), file) + }, + contentType = paste0('image/', input$ERTPlot.Aggr_Dim.Format) +) diff --git a/inst/shiny-server/server/ERT_aggr_plot.R b/inst/shiny-server/server/ERT_aggr_plot.R index b6d82476..33185043 100644 --- a/inst/shiny-server/server/ERT_aggr_plot.R +++ b/inst/shiny-server/server/ERT_aggr_plot.R @@ -1,134 +1,134 @@ -### Generate ERT table -ERTPlot.Aggr.ERTs_obj <- reactive({ - input$ERTPlot.Aggr.Refresh - dsList <- isolate(ERTPlot.Aggr.data()) - if (is.null(dsList)) return(NULL) - aggr_on <- 'funcId' - targets <- isolate(ERTPlot.Aggr.Targets_obj) - dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) - dt -}) - -### Plot the data -render_ERTPlot_aggr_plot <- reactive({ - withProgress({ - y_attr <- if (input$ERTPlot.Aggr.Ranking) 'rank' else 'value' - y_title <- if (input$ERTPlot.Aggr.Ranking) 'Rank' else 'ERT' - reverse_scale <- input$ERTPlot.Aggr.Mode == 'radar' - dt <- ERTPlot.Aggr.ERTs_obj() - if (is.null(dt)) - return(NULL) - dt <- dt[funcId %in% isolate(input$ERTPlot.Aggr.Funcs), ] - plot_general_data(dt, type = input$ERTPlot.Aggr.Mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = input$ERTPlot.Aggr.Logy, scale.reverse = reverse_scale, - inf.action = 'jitter') - }, - message = "Creating plot") -}) - -### Gather relevant datasetlist -ERTPlot.Aggr.data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$ERTPlot.Aggr.Algs)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM == input$Overall.Dim) - - if (length(unique(get_funcId(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple functions for the selected dimension.") - return(NULL) - } - - if (length(unique(get_algId(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected dimension.") - return(NULL) - } - data -} - -### format table for display -ert_multi_function <- function() { - dt <- ERTPlot.Aggr.ERTs_obj() - dt <- dcast(dt, funcId~algId, value.var = 'value') - format(dt, digits = 4) -} - -### Table with default targets -default_targets_table <- reactive({ - data <- ERTPlot.Aggr.data() - if (is.null(data)) return(NULL) - targets <- get_target_dt(data) - targets <- targets[, c('funcId','target')] - # %>% melt(id.vars = 'funcId') %>% dcast(variable ~ funcId) - # targets <- targets[, 'variable' = NULL] - # rownames(targets) <- 'funcId' - # targets -}) - -### Target table object -ERTPlot.Aggr.Targets_obj <- NULL - -### Target table proxy -proxy_ERTPlot.Aggr.Targets <- dataTableProxy('ERTPlot.Aggr.Targets') - -### Target table print -output$ERTPlot.Aggr.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - ERTPlot.Aggr.Targets_obj <<- default_targets_table() - ERTPlot.Aggr.Targets_obj -}, editable = TRUE, rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -### Target table edit -observeEvent(input$ERTPlot.Aggr.Targets_cell_edit, { - info <- input$ERTPlot.Aggr.Targets_cell_edit - i <- info$row - req(i > 0) - j <- info$col + 1 - v <- info$value - ERTPlot.Aggr.Targets_obj[i, j] <<- - DT::coerceValue(v, ERTPlot.Aggr.Targets_obj[['target']][[i]]) - replaceData(proxy, ERTPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) -}) - -### Table output -output$ERTPlot.Aggr.ERTTable <- DT::renderDataTable({ - input$ERTPlot.Aggr.Refresh - req(length(DATA_RAW()) > 0) - - withProgress({ - dt <- ert_multi_function() - }, - message = "Creating table") - dt -}, editable = FALSE, rownames = TRUE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - -### plot output -output$ERTPlot.Aggr.Plot <- renderPlotly( - render_ERTPlot_aggr_plot() -) - -### Download table -output$ERTPlot.Aggr.DownloadTable <- downloadHandler( - filename = function() { - eval(ERT_multi_func_name) - }, - content = function(file) { - df <- ert_multi_function() - save_table(df, file) - } -) - -### Download plot -output$ERTPlot.Aggr.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_ERT_AGGR) - }, - content = function(file) { - save_plotly(render_ERTPlot_aggr_plot(), file) - }, - contentType = paste0('image/', input$ERTPlot.Aggr.Format) -) +### Generate ERT table +ERTPlot.Aggr.ERTs_obj <- reactive({ + input$ERTPlot.Aggr.Refresh + dsList <- isolate(ERTPlot.Aggr.data()) + if (is.null(dsList)) return(NULL) + aggr_on <- 'funcId' + targets <- isolate(ERTPlot.Aggr.Targets_obj) + dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets) + dt +}) + +### Plot the data +render_ERTPlot_aggr_plot <- reactive({ + withProgress({ + y_attr <- if (input$ERTPlot.Aggr.Ranking) 'rank' else 'value' + y_title <- if (input$ERTPlot.Aggr.Ranking) 'Rank' else 'ERT' + reverse_scale <- input$ERTPlot.Aggr.Mode == 'radar' + dt <- ERTPlot.Aggr.ERTs_obj() + if (is.null(dt)) + return(NULL) + dt <- dt[funcId %in% isolate(input$ERTPlot.Aggr.Funcs), ] + plot_general_data(dt, type = input$ERTPlot.Aggr.Mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = input$ERTPlot.Aggr.Logy, scale.reverse = reverse_scale, + inf.action = 'jitter') + }, + message = "Creating plot") +}) + +### Gather relevant datasetlist +ERTPlot.Aggr.data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$ERTPlot.Aggr.Algs)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM == input$Overall.Dim) + + if (length(unique(get_funcId(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple functions for the selected dimension.") + return(NULL) + } + + if (length(unique(get_id(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected dimension.") + return(NULL) + } + data +} + +### format table for display +ert_multi_function <- function() { + dt <- ERTPlot.Aggr.ERTs_obj() + dt <- dcast(dt, funcId~ID, value.var = 'value') + format(dt, digits = 4) +} + +### Table with default targets +default_targets_table <- reactive({ + data <- ERTPlot.Aggr.data() + if (is.null(data)) return(NULL) + targets <- get_target_dt(data) + targets <- targets[, c('funcId','target')] + # %>% melt(id.vars = 'funcId') %>% dcast(variable ~ funcId) + # targets <- targets[, 'variable' = NULL] + # rownames(targets) <- 'funcId' + # targets +}) + +### Target table object +ERTPlot.Aggr.Targets_obj <- NULL + +### Target table proxy +proxy_ERTPlot.Aggr.Targets <- dataTableProxy('ERTPlot.Aggr.Targets') + +### Target table print +output$ERTPlot.Aggr.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + ERTPlot.Aggr.Targets_obj <<- default_targets_table() + ERTPlot.Aggr.Targets_obj +}, editable = TRUE, rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +### Target table edit +observeEvent(input$ERTPlot.Aggr.Targets_cell_edit, { + info <- input$ERTPlot.Aggr.Targets_cell_edit + i <- info$row + req(i > 0) + j <- info$col + 1 + v <- info$value + ERTPlot.Aggr.Targets_obj[i, j] <<- + DT::coerceValue(v, ERTPlot.Aggr.Targets_obj[['target']][[i]]) + replaceData(proxy, ERTPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) +}) + +### Table output +output$ERTPlot.Aggr.ERTTable <- DT::renderDataTable({ + input$ERTPlot.Aggr.Refresh + req(length(DATA_RAW()) > 0) + + withProgress({ + dt <- ert_multi_function() + }, + message = "Creating table") + dt +}, editable = FALSE, rownames = TRUE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + +### plot output +output$ERTPlot.Aggr.Plot <- renderPlotly( + render_ERTPlot_aggr_plot() +) + +### Download table +output$ERTPlot.Aggr.DownloadTable <- downloadHandler( + filename = function() { + eval(ERT_multi_func_name) + }, + content = function(file) { + df <- ert_multi_function() + save_table(df, file) + } +) + +### Download plot +output$ERTPlot.Aggr.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_ERT_AGGR) + }, + content = function(file) { + save_plotly(render_ERTPlot_aggr_plot(), file) + }, + contentType = paste0('image/', input$ERTPlot.Aggr.Format) +) diff --git a/inst/shiny-server/server/ERT_summary.R b/inst/shiny-server/server/ERT_summary.R index 5fa30543..3f825687 100644 --- a/inst/shiny-server/server/ERT_summary.R +++ b/inst/shiny-server/server/ERT_summary.R @@ -1,135 +1,136 @@ -# Data summary for Fixed-Target Runtime (ERT) -------------- -runtime_summary_condensed <- reactive({ - data <- DATA() - req(length(data) > 0) - fall <- get_funvals(data) - df <- get_FV_overview(data, algorithm = input$RTSummary.Overview.Algid) - df$budget %<>% as.numeric - df$runs %<>% as.integer - df$funcId %<>% as.integer - df$DIM %<>% as.integer - df$succ %<>% as.integer - df$"worst recorded" <- format_FV(df$"worst recorded") - df$"worst reached" <- format_FV(df$"worst reached") - df$"mean reached" <- format_FV(df$"mean reached") - df$"median reached" <- format_FV(df$"median reached") - df$"best reached" <- format_FV(df$"best reached") - df -}) - -output$table_RT_overview <- DT::renderDataTable({ - req(input$RTSummary.Overview.Algid) - runtime_summary_condensed() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RTSummary.Overview.Download <- downloadHandler( - filename = function() { - eval(RT_overview_name) - }, - content = function(file) { - df <- runtime_summary_condensed() - df <- df[input[["table_RT_overview_rows_all"]]] - save_table(df, file) - } -) - -# Data summary for Fixed-Target Runtime (ERT) -------------- -runtime_summary <- reactive({ - req(input$RTSummary.Statistics.Min, - input$RTSummary.Statistics.Max, - input$RTSummary.Statistics.Step) - - fstart <- format_FV(input$RTSummary.Statistics.Min) - fstop <- format_FV(input$RTSummary.Statistics.Max) - fstep <- format_FV(input$RTSummary.Statistics.Step) - data <- DATA() - - if (!input$RTSummary.Statistics.Single) { - req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, fstep) - req(fseq) - } else { - fseq <- fstart - } - - df <- get_RT_summary(data, fseq, algorithm = input$RTSummary.Statistics.Algid) - df <- df[, c('DIM', 'funcId') := NULL] - df$target <- format_FV(df$target) %>% as.numeric - - # format the integers - probs <- getOption("IOHanalyzer.quantiles") - for (p in paste0(probs * 100, '%')) { - df[[p]] %<>% as.integer - } - - df$mean <- round(df$mean, digits = 2) - df$sd <- round(df$sd, digits = 2) - df$ERT <- round(df$ERT, digits = 2) - df$ps <- round(df$ps, digits = 2) - - df$target <- format_FV(df$target) - df -}) - -output$table_RT_summary <- DT::renderDataTable({ - runtime_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) - -output$RTSummary.Statistics.Download <- downloadHandler( - filename = function() { - eval(RT_csv_name) - }, - content = function(file) { - df <- runtime_summary() - df <- df[input[["table_RT_summary_rows_all"]]] - save_table(df, file) - } -) - -get_RT <- reactive({ - req(input$RTSummary.Sample.Min, - input$RTSummary.Sample.Max, - input$RTSummary.Sample.Step) - - fstart <- format_FV(input$RTSummary.Sample.Min) - fstop <- format_FV(input$RTSummary.Sample.Max) - fstep <- format_FV(input$RTSummary.Sample.Step) - data <- DATA() - - if (!input$RTSummary.Sample.Single) { - req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, fstep) - req(fseq) - } - else{ - fseq <- fstart - } - - df <- get_RT_sample(data, ftarget = fseq, algorithm = input$RTSummary.Sample.Algid, - output = input$RTSummary.Sample.DownloadFormat) - df$target <- format_FV(df$target) - df[is.na(df)] <- 'NA' - df -}) - -output$RTSummary.Sample.Download <- downloadHandler( - filename = function() { - eval(RTSample_csv_name) - }, - content = function(file) { - df <- get_RT() - df <- df[input[["table_RT_sample_rows_all"]]] - save_table(df, file) - } -) - -output$table_RT_sample <- DT::renderDataTable({ - df <- get_RT() - df[is.na(df)] <- 'NA' - df -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) +# Data summary for Fixed-Target Runtime (ERT) -------------- +runtime_summary_condensed <- reactive({ + data <- DATA() + req(length(data) > 0) + # fall <- get_funvals(data) + data <- subset(data, ID %in% input$RTSummary.Overview.ID) + df <- get_FV_overview(data) + df$budget %<>% as.numeric + df$runs %<>% as.integer + df$funcId %<>% as.integer + df$DIM %<>% as.integer + df$succ %<>% as.integer + df$"worst recorded" <- format_FV(df$"worst recorded") + df$"worst reached" <- format_FV(df$"worst reached") + df$"mean reached" <- format_FV(df$"mean reached") + df$"median reached" <- format_FV(df$"median reached") + df$"best reached" <- format_FV(df$"best reached") + df +}) + +output$table_RT_overview <- DT::renderDataTable({ + req(input$RTSummary.Overview.ID) + runtime_summary_condensed() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RTSummary.Overview.Download <- downloadHandler( + filename = function() { + eval(RT_overview_name) + }, + content = function(file) { + df <- runtime_summary_condensed() + df <- df[input[["table_RT_overview_rows_all"]]] + save_table(df, file) + } +) + +# Data summary for Fixed-Target Runtime (ERT) -------------- +runtime_summary <- reactive({ + req(input$RTSummary.Statistics.Min, + input$RTSummary.Statistics.Max, + input$RTSummary.Statistics.Step) + + fstart <- format_FV(input$RTSummary.Statistics.Min) + fstop <- format_FV(input$RTSummary.Statistics.Max) + fstep <- format_FV(input$RTSummary.Statistics.Step) + data <- DATA() + data <- subset(data, ID %in% input$RTSummary.Statistics.ID) + if (!input$RTSummary.Statistics.Single) { + req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, fstep) + req(fseq) + } else { + fseq <- fstart + } + + df <- get_RT_summary(data, fseq) + df <- df[, c('DIM', 'funcId') := NULL] + df$target <- format_FV(df$target) %>% as.numeric + + # format the integers + probs <- getOption("IOHanalyzer.quantiles") + for (p in paste0(probs * 100, '%')) { + df[[p]] %<>% as.integer + } + + df$mean <- round(df$mean, digits = 2) + df$sd <- round(df$sd, digits = 2) + df$ERT <- round(df$ERT, digits = 2) + df$ps <- round(df$ps, digits = 2) + + df$target <- format_FV(df$target) + df +}) + +output$table_RT_summary <- DT::renderDataTable({ + runtime_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) + +output$RTSummary.Statistics.Download <- downloadHandler( + filename = function() { + eval(RT_csv_name) + }, + content = function(file) { + df <- runtime_summary() + df <- df[input[["table_RT_summary_rows_all"]]] + save_table(df, file) + } +) + +get_RT <- reactive({ + req(input$RTSummary.Sample.Min, + input$RTSummary.Sample.Max, + input$RTSummary.Sample.Step) + + fstart <- format_FV(input$RTSummary.Sample.Min) + fstop <- format_FV(input$RTSummary.Sample.Max) + fstep <- format_FV(input$RTSummary.Sample.Step) + data <- DATA() + data <- subset(data, ID %in% input$RTSummary.Sample.ID) + if (!input$RTSummary.Sample.Single) { + req(fstart <= fstop, fstep <= fstop - fstart, length(data) > 0) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, fstep) + req(fseq) + } + else{ + fseq <- fstart + } + + df <- get_RT_sample(data, ftarget = fseq, + output = input$RTSummary.Sample.DownloadFormat) + df$target <- format_FV(df$target) + df[is.na(df)] <- 'NA' + df +}) + +output$RTSummary.Sample.Download <- downloadHandler( + filename = function() { + eval(RTSample_csv_name) + }, + content = function(file) { + df <- get_RT() + df <- df[input[["table_RT_sample_rows_all"]]] + save_table(df, file) + } +) + +output$table_RT_sample <- DT::renderDataTable({ + df <- get_RT() + df[is.na(df)] <- 'NA' + df +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 10, scrollX = T, server = T)) diff --git a/inst/shiny-server/server/FCEPDF.R b/inst/shiny-server/server/FCEPDF.R index eadb2523..57469cc3 100644 --- a/inst/shiny-server/server/FCEPDF.R +++ b/inst/shiny-server/server/FCEPDF.R @@ -1,70 +1,70 @@ -# empirical p.d.f. of the target value -get_data_FV_PMF <- reactive({ - ftarget <- input$FCEPDF.Bar.Runtime %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEPDF.Bar.Algs) - generate_data.PMF(data, ftarget, 'by_FV') -}) - -render_FV_PDF <- reactive({ - withProgress({ - plot_general_data(get_data_FV_PMF(), 'algId', 'f(x)', scale.ylog = input$FCEPDF.Bar.Logy, - x_title = "Algorithm", y_title = "Target Value") - # ftarget <- input$RTPMF.Bar.Target %>% as.numeric - # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) - # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, - # scale.ylog = input$RTPMF.Bar.Logy) - }, - message = "Creating plot") -}) - -output$FCEPDF.Bar.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PDF) - }, - content = function(file) { - save_plotly(render_FV_PDF(), file) - }, - contentType = paste0('image/', input$FCEPDF.Bar.Format) -) - -output$FCE_PDF <- renderPlotly({ - render_FV_PDF() -}) - -# historgram of the target values ----------- - - -get_data_FV_HIST <- reactive({ - ftarget <- input$FCEPDF.Hist.Runtime %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEPDF.Hist.Algs) - generate_data.hist(data, ftarget, input$FCEPDF.Hist.Equal, 'by_FV') -}) - - -render_FV_HIST <- reactive({ - req(input$FCEPDF.Hist.Runtime != "", length(DATA()) > 0) # require non-empty input - withProgress({ - subplot_attr <- if (input$FCEPDF.Hist.Mode == 'subplot') 'algId' else NULL - plot_general_data(get_data_FV_HIST(), 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Target Values", - y_title = "Runs") - # runtime <- input$FCEPDF.Hist.Runtime %>% as.numeric - # data <- subset(DATA(), algId %in% input$FCEPDF.Hist.Algs) - # Plot.FV.Histogram(data, runtime, plot_mode = input$FCEPDF.Hist.Mode, use.equal.bins = input$FCEPDF.Hist.Equal) - }, - message = "Creating plot") -}) - -output$FCEPDF.Hist.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_HIST) - }, - content = function(file) { - save_plotly(render_FV_HIST(), file) - }, - contentType = paste0('image/', input$FCEPDF.Hist.Format) -) - -output$FCE_HIST <- renderPlotly({ - render_FV_HIST() -}) +# empirical p.d.f. of the target value +get_data_FV_PMF <- reactive({ + ftarget <- input$FCEPDF.Bar.Runtime %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEPDF.Bar.Algs) + generate_data.PMF(data, ftarget, 'by_FV') +}) + +render_FV_PDF <- reactive({ + withProgress({ + plot_general_data(get_data_FV_PMF(), 'ID', 'f(x)', scale.ylog = input$FCEPDF.Bar.Logy, + x_title = "Algorithm", y_title = "Target Value") + # ftarget <- input$RTPMF.Bar.Target %>% as.numeric + # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) + # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, + # scale.ylog = input$RTPMF.Bar.Logy) + }, + message = "Creating plot") +}) + +output$FCEPDF.Bar.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PDF) + }, + content = function(file) { + save_plotly(render_FV_PDF(), file) + }, + contentType = paste0('image/', input$FCEPDF.Bar.Format) +) + +output$FCE_PDF <- renderPlotly({ + render_FV_PDF() +}) + +# historgram of the target values ----------- + + +get_data_FV_HIST <- reactive({ + ftarget <- input$FCEPDF.Hist.Runtime %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEPDF.Hist.Algs) + generate_data.hist(data, ftarget, input$FCEPDF.Hist.Equal, 'by_FV') +}) + + +render_FV_HIST <- reactive({ + req(input$FCEPDF.Hist.Runtime != "", length(DATA()) > 0) # require non-empty input + withProgress({ + subplot_attr <- if (input$FCEPDF.Hist.Mode == 'subplot') 'ID' else NULL + plot_general_data(get_data_FV_HIST(), 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Target Values", + y_title = "Runs") + # runtime <- input$FCEPDF.Hist.Runtime %>% as.numeric + # data <- subset(DATA(), algId %in% input$FCEPDF.Hist.Algs) + # Plot.FV.Histogram(data, runtime, plot_mode = input$FCEPDF.Hist.Mode, use.equal.bins = input$FCEPDF.Hist.Equal) + }, + message = "Creating plot") +}) + +output$FCEPDF.Hist.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_HIST) + }, + content = function(file) { + save_plotly(render_FV_HIST(), file) + }, + contentType = paste0('image/', input$FCEPDF.Hist.Format) +) + +output$FCE_HIST <- renderPlotly({ + render_FV_HIST() +}) diff --git a/inst/shiny-server/server/FCEPlot.R b/inst/shiny-server/server/FCEPlot.R index c641c4ec..b28ef15e 100644 --- a/inst/shiny-server/server/FCEPlot.R +++ b/inst/shiny-server/server/FCEPlot.R @@ -1,140 +1,140 @@ -# Expected Target Value Convergence -output$FCE_PER_FUN <- renderPlotly({ - req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) - render_FV_PER_FUN() -}) - -get_data_FCE_PER_FUN <- reactive({ - req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) - data <- subset(DATA(), algId %in% input$FCEPlot.Algs) - fstart <- input$FCEPlot.Min %>% as.numeric - fstop <- input$FCEPlot.Max %>% as.numeric - generate_data.Single_Function(data, fstart, fstop, input$FCEPlot.semilogx, - 'by_FV') -}) - -render_FV_PER_FUN <- reactive({ - withProgress({ - y_attrs <- c() - if (input$FCEPlot.show.mean) y_attrs <- c(y_attrs, 'mean') - if (input$FCEPlot.show.median) y_attrs <- c(y_attrs, 'median') - show_legend <- T - if (length(y_attrs) > 0) { - p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = y_attrs, - type = 'line', legend_attr = 'algId', show.legend = show_legend, - scale.ylog = isolate(input$FCEPlot.semilogy), - scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", - y_title = "Best-so-far f(x)-value") - show_legend <- F - } - else - p <- NULL - if (input$FCEPlot.show.CI) { - p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = 'lower', - upper_attr = 'upper', p = p, show.legend = show_legend) - show_legend <- F - } - - else if (input$FCEPlot.show.IQR) { - IOHanalyzer.quantiles.bk <- getOption("IOHanalyzer.quantiles") - options(IOHanalyzer.quantiles = c(0.25, 0.75)) - p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', - type = 'ribbon', legend_attr = 'algId', lower_attr = '25%', - upper_attr = '75%', p = p, show.legend = show_legend) - show_legend <- F - options(IOHanalyzer.quantiles = IOHanalyzer.quantiles.bk) - } - if (input$FCEPlot.show.runs) { - fstart <- isolate(input$FCEPlot.Min %>% as.numeric) - fstop <- isolate(input$FCEPlot.Max %>% as.numeric) - data <- isolate(subset(DATA(), algId %in% input$FCEPlot.Algs)) - dt <- get_FV_sample(data, seq_RT(c(fstart, fstop), from = fstart, to = fstop, length.out = 50, - scale = ifelse(isolate(input$FCEPlot.semilogx), 'log', 'linear'))) - nr_runs <- ncol(dt) - 4 - for (i in seq_len(nr_runs)) { - p <- plot_general_data(dt, x_attr = 'runtime', y_attr = paste0('run.', i), type = 'line', - legend_attr = 'algId', p = p, show.legend = show_legend, - scale.ylog = input$FCEPlot.semilogy, - scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", - y_title = "Best-so-far f(x)-value") - show_legend <- F - } - } - p - }, - message = "Creating plot" - ) -}) - -output$FCEPlot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PER_FUN) - }, - content = function(file) { - save_plotly(render_FV_PER_FUN(), file) - }, - contentType = paste0('image/', input$FCEPlot.Format) -) - - -update_fv_per_fct_axis <- observe({ - plotlyProxy("FCE_PER_FUN", session) %>% - plotlyProxyInvoke("relayout", list(yaxis = list(title = 'best-so-far-f(x)-value', type = ifelse(input$FCEPlot.semilogy, 'log', 'linear')))) -}) - - -output$FCEPlot.Multi.Plot <- renderPlotly( - render_FCEPlot_multi_plot() -) - -get_data_FCE_multi_func_bulk <- reactive({ - data <- subset(DATA_RAW(), - DIM == input$Overall.Dim) - if (length(get_algId(data)) < 20) { #Arbitrary limit for the time being - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, - which = 'by_FV') - })) - } - else - NULL -}) - -get_data_FCEPlot_multi <- reactive({ - req(isolate(input$FCEPlot.Multi.Algs)) - input$FCEPlot.Multi.PlotButton - if (length(get_algId(data)) < 20) { - get_data_FCE_multi_func_bulk()[algId %in% isolate(input$FCEPlot.Multi.Algs), ] - } - else { - data <- subset(DATA_RAW(), - algId %in% isolate(input$FCEPlot.Multi.Algs), - DIM == input$Overall.Dim) - rbindlist(lapply(get_funcId(data), function(fid) { - generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, - which = 'by_FV') - })) - } -}) - -render_FCEPlot_multi_plot <- reactive({ - withProgress({ - plot_general_data(get_data_FCEPlot_multi(), x_attr = 'runtime', y_attr = 'mean', - subplot_attr = 'funcId', type = 'line', scale.xlog = input$FCEPlot.Multi.Logx, - scale.ylog = input$FCEPlot.Multi.Logy, x_title = 'Runtime', - y_title = 'Best-so-far f(x)', show.legend = T) - }, - message = "Creating plot") -}) - -output$FCEPlot.Multi.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PER_FUN_MULTI) - }, - content = function(file) { - save_plotly(render_FCEPlot_multi_plot(), file) - }, - contentType = paste0('image/', input$FCEPlot.Multi.Format) -) - +# Expected Target Value Convergence +output$FCE_PER_FUN <- renderPlotly({ + req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) + render_FV_PER_FUN() +}) + +get_data_FCE_PER_FUN <- reactive({ + req(input$FCEPlot.Min, input$FCEPlot.Max, length(DATA()) > 0) + data <- subset(DATA(), ID %in% input$FCEPlot.Algs) + fstart <- input$FCEPlot.Min %>% as.numeric + fstop <- input$FCEPlot.Max %>% as.numeric + generate_data.Single_Function(data, fstart, fstop, input$FCEPlot.semilogx, + 'by_FV') +}) + +render_FV_PER_FUN <- reactive({ + withProgress({ + y_attrs <- c() + if (input$FCEPlot.show.mean) y_attrs <- c(y_attrs, 'mean') + if (input$FCEPlot.show.median) y_attrs <- c(y_attrs, 'median') + show_legend <- T + if (length(y_attrs) > 0) { + p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = y_attrs, + type = 'line', legend_attr = 'ID', show.legend = show_legend, + scale.ylog = isolate(input$FCEPlot.semilogy), + scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", + y_title = "Best-so-far f(x)-value") + show_legend <- F + } + else + p <- NULL + if (input$FCEPlot.show.CI) { + p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = 'lower', + upper_attr = 'upper', p = p, show.legend = show_legend) + show_legend <- F + } + + else if (input$FCEPlot.show.IQR) { + IOHanalyzer.quantiles.bk <- getOption("IOHanalyzer.quantiles") + options(IOHanalyzer.quantiles = c(0.25, 0.75)) + p <- plot_general_data(get_data_FCE_PER_FUN(), x_attr = 'runtime', y_attr = 'mean', + type = 'ribbon', legend_attr = 'ID', lower_attr = '25%', + upper_attr = '75%', p = p, show.legend = show_legend) + show_legend <- F + options(IOHanalyzer.quantiles = IOHanalyzer.quantiles.bk) + } + if (input$FCEPlot.show.runs) { + fstart <- isolate(input$FCEPlot.Min %>% as.numeric) + fstop <- isolate(input$FCEPlot.Max %>% as.numeric) + data <- isolate(subset(DATA(), ID %in% input$FCEPlot.Algs)) + dt <- get_FV_sample(data, seq_RT(c(fstart, fstop), from = fstart, to = fstop, length.out = 50, + scale = ifelse(isolate(input$FCEPlot.semilogx), 'log', 'linear'))) + nr_runs <- ncol(dt) - 4 + for (i in seq_len(nr_runs)) { + p <- plot_general_data(dt, x_attr = 'runtime', y_attr = paste0('run.', i), type = 'line', + legend_attr = 'ID', p = p, show.legend = show_legend, + scale.ylog = input$FCEPlot.semilogy, + scale.xlog = input$FCEPlot.semilogx, x_title = "Function Evaluations", + y_title = "Best-so-far f(x)-value") + show_legend <- F + } + } + p + }, + message = "Creating plot" + ) +}) + +output$FCEPlot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PER_FUN) + }, + content = function(file) { + save_plotly(render_FV_PER_FUN(), file) + }, + contentType = paste0('image/', input$FCEPlot.Format) +) + + +update_fv_per_fct_axis <- observe({ + plotlyProxy("FCE_PER_FUN", session) %>% + plotlyProxyInvoke("relayout", list(yaxis = list(title = 'best-so-far-f(x)-value', type = ifelse(input$FCEPlot.semilogy, 'log', 'linear')))) +}) + + +output$FCEPlot.Multi.Plot <- renderPlotly( + render_FCEPlot_multi_plot() +) + +get_data_FCE_multi_func_bulk <- reactive({ + data <- subset(DATA_RAW(), + DIM == input$Overall.Dim) + if (length(get_id(data)) < 20) { #Arbitrary limit for the time being + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, + which = 'by_FV') + })) + } + else + NULL +}) + +get_data_FCEPlot_multi <- reactive({ + req(isolate(input$FCEPlot.Multi.Algs)) + input$FCEPlot.Multi.PlotButton + if (length(get_id(data)) < 20) { + get_data_FCE_multi_func_bulk()[ID %in% isolate(input$FCEPlot.Multi.Algs), ] + } + else { + data <- subset(DATA_RAW(), + ID %in% isolate(input$FCEPlot.Multi.Algs), + DIM == input$Overall.Dim) + rbindlist(lapply(get_funcId(data), function(fid) { + generate_data.Single_Function(subset(data, funcId == fid), scale_log = input$FCEPlot.Multi.Logx, + which = 'by_FV') + })) + } +}) + +render_FCEPlot_multi_plot <- reactive({ + withProgress({ + plot_general_data(get_data_FCEPlot_multi(), x_attr = 'runtime', y_attr = 'mean', + subplot_attr = 'funcId', type = 'line', scale.xlog = input$FCEPlot.Multi.Logx, + scale.ylog = input$FCEPlot.Multi.Logy, x_title = 'Runtime', + y_title = 'Best-so-far f(x)', show.legend = T) + }, + message = "Creating plot") +}) + +output$FCEPlot.Multi.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PER_FUN_MULTI) + }, + content = function(file) { + save_plotly(render_FCEPlot_multi_plot(), file) + }, + contentType = paste0('image/', input$FCEPlot.Multi.Format) +) + diff --git a/inst/shiny-server/server/FCESummary.R b/inst/shiny-server/server/FCESummary.R index e7f4da1c..8527fe0a 100644 --- a/inst/shiny-server/server/FCESummary.R +++ b/inst/shiny-server/server/FCESummary.R @@ -1,145 +1,128 @@ -# TODO: rename 'FCE'... -# Data summary for Fixed-Budget target (FCE) -------------- -FCE_runtime_summary_condensed <- reactive({ - data <- DATA() - fall <- get_funvals(data) - df <- get_RT_overview(data, algorithm = input$FCESummary.Overview.Algid) - df$"runs" %<>% as.integer - df$"Budget" %<>% as.numeric - df$"miminal runtime" %<>% as.numeric - df$"maximal runtime" %<>% as.numeric - - df -}) - -output$table_FV_overview <- DT::renderDataTable({ - req(input$FCESummary.Overview.Algid) - FCE_runtime_summary_condensed() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FCESummary.Overview.Download <- downloadHandler( - filename = function() { - eval(FV_overview_name) - }, - content = function(file) { - df <- FCE_runtime_summary_condensed() - df <- df[input[["FCE_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -get_FCE_summary <- reactive({ - req(input$FCESummary.Statistics.Min, input$FCESummary.Statistics.Max, input$FCESummary.Statistics.Step, length(DATA()) > 0) - - rt_min <- input$FCESummary.Statistics.Min %>% as.numeric - rt_max <- input$FCESummary.Statistics.Max %>% as.numeric - rt_step <- input$FCESummary.Statistics.Step %>% as.numeric - - data <- DATA() - - - if (!input$FCESummary.Statistics.Single){ - req(rt_min <= rt_max, rt_step <= rt_max - rt_min) - rt <- get_runtimes(data) - rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) - req(rt_seq) - } - else{ - rt_seq <- rt_min - } - - df <- get_FV_summary(data, rt_seq, algorithm = input$FCESummary.Statistics.Algid)[ - , c('DIM', 'funcId') := NULL - ] - df$runs %<>% as.integer - # df$runs.MaxFunvals %<>% as.integer - df$median %<>% format(format = 'e', digits = 3) - df$mean %<>% format(format = 'e', digits = 3) - df$runtime %<>% as.numeric - - probs <- getOption("IOHanalyzer.quantiles") - - # format the integers - for (p in paste0(probs * 100, '%')) { - df[[p]] %<>% format(format = 'e', digits = 3) - } - df$sd <- round(df$sd, 2) - df -}) - -output$FCE_SUMMARY <- DT::renderDataTable({ - get_FCE_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FCESummary.Statistics.Download <- downloadHandler( - filename = function() { - eval(FV_csv_name) - }, - content = function(file) { - df <- get_FCE_summary() - df <- df[input[["FCE_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -get_FCE <- reactive({ - req(input$FCESummary.Sample.Min, input$FCESummary.Sample.Max, input$FCESummary.Sample.Step, length(DATA()) > 0) - rt_min <- input$FCESummary.Sample.Min %>% as.numeric - rt_max <- input$FCESummary.Sample.Max %>% as.numeric - rt_step <- input$FCESummary.Sample.Step %>% as.numeric - - data <- DATA() - - if (!input$FCESummary.Sample.Single){ - req(rt_min <= rt_max, rt_step <= rt_max - rt_min) - rt <- get_runtimes(data) - rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) - req(rt_seq) - } - else{ - rt_seq <- rt_min - } - - get_FV_sample(data, rt_seq, algorithm = input$FCESummary.Sample.Algid, - output = input$FCESummary.Sample.Format) - - # res <- list() - # n_runs_max <- sapply(data, function(x) length(attr(x, 'instance'))) %>% max - # - # for (i in seq_along(data)) { - # ds <- data[[i]] - # algId <- attr(ds, 'algId') - # if (input$FCESummary.Sample.Algid != 'all' && algId != input$FCESummary.Sample.Algid) - # next - # - # rt <- get_FV_sample(ds, rt_seq, output = input$FCESummary.Sample.Format) - # if (input$FCESummary.Sample.Format == 'wide') { - # # impute the missing records - # n <- ncol(rt) - 2 - # if (n < n_runs_max) - # rt %<>% cbind(., matrix(NA, nrow(.), n_runs_max - n)) - # } - # res[[i]] <- rt - # } - # do.call(rbind, res) -}) - -output$FCESummary.Sample.Download <- downloadHandler( - filename = function() { - eval(FVSample_csv_name) - }, - content = function(file) { - df <- get_FCE() - df <- df[input[["FCE_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -output$FCE_SAMPLE <- DT::renderDataTable({ - df <- get_FCE() - df[is.na(df)] <- 'NA' - df -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) +# TODO: rename 'FCE'... +# Data summary for Fixed-Budget target (FCE) -------------- +FCE_runtime_summary_condensed <- reactive({ + data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Overview.ID) + # fall <- get_funvals(data) + df <- get_RT_overview(data) + df$"runs" %<>% as.integer + df$"Budget" %<>% as.numeric + df$"miminal runtime" %<>% as.numeric + df$"maximal runtime" %<>% as.numeric + + df +}) + +output$table_FV_overview <- DT::renderDataTable({ + req(input$FCESummary.Overview.ID) + FCE_runtime_summary_condensed() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FCESummary.Overview.Download <- downloadHandler( + filename = function() { + eval(FV_overview_name) + }, + content = function(file) { + df <- FCE_runtime_summary_condensed() + df <- df[input[["FCE_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +get_FCE_summary <- reactive({ + req(input$FCESummary.Statistics.Min, input$FCESummary.Statistics.Max, input$FCESummary.Statistics.Step, length(DATA()) > 0) + + rt_min <- input$FCESummary.Statistics.Min %>% as.numeric + rt_max <- input$FCESummary.Statistics.Max %>% as.numeric + rt_step <- input$FCESummary.Statistics.Step %>% as.numeric + + data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Statistics.ID) + + + if (!input$FCESummary.Statistics.Single) { + req(rt_min <= rt_max, rt_step <= rt_max - rt_min) + rt <- get_runtimes(data) + rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) + req(rt_seq) + } + else{ + rt_seq <- rt_min + } + + df <- get_FV_summary(data, rt_seq)[ + , c('DIM', 'funcId') := NULL + ] + df$runs %<>% as.integer + # df$runs.MaxFunvals %<>% as.integer + df$median %<>% format(format = 'e', digits = 3) + df$mean %<>% format(format = 'e', digits = 3) + df$runtime %<>% as.numeric + + probs <- getOption("IOHanalyzer.quantiles") + + # format the integers + for (p in paste0(probs * 100, '%')) { + df[[p]] %<>% format(format = 'e', digits = 3) + } + df$sd <- round(df$sd, 2) + df +}) + +output$FCE_SUMMARY <- DT::renderDataTable({ + get_FCE_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FCESummary.Statistics.Download <- downloadHandler( + filename = function() { + eval(FV_csv_name) + }, + content = function(file) { + df <- get_FCE_summary() + df <- df[input[["FCE_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +get_FCE <- reactive({ + req(input$FCESummary.Sample.Min, input$FCESummary.Sample.Max, input$FCESummary.Sample.Step, length(DATA()) > 0) + rt_min <- input$FCESummary.Sample.Min %>% as.numeric + rt_max <- input$FCESummary.Sample.Max %>% as.numeric + rt_step <- input$FCESummary.Sample.Step %>% as.numeric + + data <- DATA() + data <- subset(data, ID %in% input$FCESummary.Sample.ID) + + if (!input$FCESummary.Sample.Single) { + req(rt_min <= rt_max, rt_step <= rt_max - rt_min) + rt <- get_runtimes(data) + rt_seq <- seq_RT(rt, rt_min, rt_max, by = rt_step) + req(rt_seq) + } + else{ + rt_seq <- rt_min + } + + get_FV_sample(data, rt_seq, + output = input$FCESummary.Sample.Format) +}) + +output$FCESummary.Sample.Download <- downloadHandler( + filename = function() { + eval(FVSample_csv_name) + }, + content = function(file) { + df <- get_FCE() + df <- df[input[["FCE_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +output$FCE_SAMPLE <- DT::renderDataTable({ + df <- get_FCE() + df[is.na(df)] <- 'NA' + df +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) diff --git a/inst/shiny-server/server/FCE_ECDF.R b/inst/shiny-server/server/FCE_ECDF.R index 1c4d267d..4f2a2449 100644 --- a/inst/shiny-server/server/FCE_ECDF.R +++ b/inst/shiny-server/server/FCE_ECDF.R @@ -1,121 +1,121 @@ -# The ECDF plots for the target value ---------------- -output$FCE_ECDF_PER_TARGET <- renderPlotly({ - render_ecdf_per_target() -}) - -get_data_FV_ECDF_Single <- reactive({ - req(input$FCEECDF.Single.Target) - ftargets <- as.numeric(format_FV(input$FCEECDF.Single.Target)) - data <- subset(DATA(), algId %in% input$FCEECDF.Single.Algs) - generate_data.ECDF(data, ftargets, input$FCEECDF.Single.Logx, which = 'by_FV') -}) - -render_ecdf_per_target <- reactive({ - withProgress({ - plot_general_data(get_data_FV_ECDF_Single(), 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of runs", scale.xlog = input$FCEECDF.Single.Logx, - show.legend = T, - scale.reverse = !attr(DATA()[[1]], 'maximization')) - }, - message = "Creating plot") -}) - -output$FCE_RT_GRID <- renderPrint({ - req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) - - rt_min <- input$FCEECDF.Mult.Min %>% as.numeric - rt_max <- input$FCEECDF.Mult.Max %>% as.numeric - rt_step <- input$FCEECDF.Mult.Step %>% as.numeric - - req(rt_min <= rt_max, rt_step <= rt_max - rt_min) - data <- DATA() - rt <- get_runtimes(data) - - seq_RT(rt, from = rt_min, to = rt_max, by = rt_step) %>% cat -}) - -get_data_FV_ECDF_AGGR <- reactive({ - req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) - fstart <- format_FV(input$FCEECDF.Mult.Min) %>% as.numeric - fstop <- format_FV(input$FCEECDF.Mult.Max) %>% as.numeric - fstep <- format_FV(input$FCEECDF.Mult.Step) %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEECDF.Mult.Algs) - targets <- seq_RT(get_funvals(data), fstart, fstop, fstep) - generate_data.ECDF(data, targets, input$FCEECDF.Mult.Logx, which = 'by_FV') -}) - -render_FV_ECDF_AGGR <- reactive({ - withProgress({ - # rt_min <- input$FCEECDF.Mult.Min %>% as.numeric - # rt_max <- input$FCEECDF.Mult.Max %>% as.numeric - # rt_step <- input$FCEECDF.Mult.Step %>% as.numeric - # data <- subset(DATA(), algId %in% input$FCEECDF.Mult.Algs) - # - # Plot.FV.ECDF_Single_Func(data,rt_min = rt_min, - # rt_max = rt_max, rt_step = rt_step, - # scale.xlog = input$FCEECDF.Mult.Logx, - # # show.per_target = input$FCEECDF.Mult.Targets, - # scale.reverse = !attr(DATA()[[1]],'maximization')) - plot_general_data(get_data_FV_ECDF_AGGR(), 'x', 'mean', 'line', - x_title = "Target Value", - y_title = "Proportion of (run, target) pairs", - scale.xlog = input$FCEECDF.Mult.Logx, - scale.reverse = !attr(DATA()[[1]], 'maximization'), show.legend = T) - }, - message = "Creating plot") -}) - -output$FCEECDF.Mult.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_ECDF_AGGR) - }, - content = function(file) { - save_plotly(render_FV_ECDF_AGGR(), file) - }, - contentType = paste0('image/', input$FCEECDF.Mult.Format) -) - -output$FCE_ECDF_AGGR <- renderPlotly({ - render_FV_ECDF_AGGR() -}) - -get_data_FV_AUC <- reactive({ - req(input$FCEECDF.AUC.Min, input$FCEECDF.AUC.Max, input$FCEECDF.AUC.Step, length(DATA()) > 0) - - rt_min <- input$FCEECDF.AUC.Min %>% as.numeric - rt_max <- input$FCEECDF.AUC.Max %>% as.numeric - rt_step <- input$FCEECDF.AUC.Step %>% as.numeric - data <- subset(DATA(), algId %in% input$FCEECDF.AUC.Algs) - targets <- seq_RT(get_runtimes(data), rt_min, rt_max, rt_step, length.out = 10) - generate_data.AUC(data, targets, which = 'by_FV') -}) - - -render_FV_AUC <- reactive({ - withProgress({ - # rt_min <- input$FCEECDF.AUC.Min %>% as.numeric - # rt_max <- input$FCEECDF.AUC.Max %>% as.numeric - # rt_step <- input$FCEECDF.AUC.Step %>% as.numeric - # data <- subset(DATA(), algId %in% input$FCEECDF.AUC.Algs) - # - # Plot.FV.ECDF_AUC(data, rt_min = rt_min, - # rt_max = rt_max, rt_step = rt_step) - plot_general_data(get_data_FV_AUC(), 'x', 'AUC', 'radar') - }, - message = "Creating plot") -}) - -output$FCEECDF.AUC.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_AUC) - }, - content = function(file) { - save_plotly(render_FV_AUC(), file) - }, - contentType = paste0('image/', input$FCEECDF.AUC.Format) -) - -output$FCE_AUC <- renderPlotly({ - render_FV_AUC() -}) +# The ECDF plots for the target value ---------------- +output$FCE_ECDF_PER_TARGET <- renderPlotly({ + render_ecdf_per_target() +}) + +get_data_FV_ECDF_Single <- reactive({ + req(input$FCEECDF.Single.Target) + ftargets <- as.numeric(format_FV(input$FCEECDF.Single.Target)) + data <- subset(DATA(), ID %in% input$FCEECDF.Single.Algs) + generate_data.ECDF(data, ftargets, input$FCEECDF.Single.Logx, which = 'by_FV') +}) + +render_ecdf_per_target <- reactive({ + withProgress({ + plot_general_data(get_data_FV_ECDF_Single(), 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of runs", scale.xlog = input$FCEECDF.Single.Logx, + show.legend = T, + scale.reverse = !attr(DATA()[[1]], 'maximization')) + }, + message = "Creating plot") +}) + +output$FCE_RT_GRID <- renderPrint({ + req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) + + rt_min <- input$FCEECDF.Mult.Min %>% as.numeric + rt_max <- input$FCEECDF.Mult.Max %>% as.numeric + rt_step <- input$FCEECDF.Mult.Step %>% as.numeric + + req(rt_min <= rt_max, rt_step <= rt_max - rt_min) + data <- DATA() + rt <- get_runtimes(data) + + seq_RT(rt, from = rt_min, to = rt_max, by = rt_step) %>% cat +}) + +get_data_FV_ECDF_AGGR <- reactive({ + req(input$FCEECDF.Mult.Min, input$FCEECDF.Mult.Max, input$FCEECDF.Mult.Step, length(DATA()) > 0) + fstart <- format_FV(input$FCEECDF.Mult.Min) %>% as.numeric + fstop <- format_FV(input$FCEECDF.Mult.Max) %>% as.numeric + fstep <- format_FV(input$FCEECDF.Mult.Step) %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEECDF.Mult.Algs) + targets <- seq_RT(get_funvals(data), fstart, fstop, fstep) + generate_data.ECDF(data, targets, input$FCEECDF.Mult.Logx, which = 'by_FV') +}) + +render_FV_ECDF_AGGR <- reactive({ + withProgress({ + # rt_min <- input$FCEECDF.Mult.Min %>% as.numeric + # rt_max <- input$FCEECDF.Mult.Max %>% as.numeric + # rt_step <- input$FCEECDF.Mult.Step %>% as.numeric + # data <- subset(DATA(), algId %in% input$FCEECDF.Mult.Algs) + # + # Plot.FV.ECDF_Single_Func(data,rt_min = rt_min, + # rt_max = rt_max, rt_step = rt_step, + # scale.xlog = input$FCEECDF.Mult.Logx, + # # show.per_target = input$FCEECDF.Mult.Targets, + # scale.reverse = !attr(DATA()[[1]],'maximization')) + plot_general_data(get_data_FV_ECDF_AGGR(), 'x', 'mean', 'line', + x_title = "Target Value", + y_title = "Proportion of (run, target) pairs", + scale.xlog = input$FCEECDF.Mult.Logx, + scale.reverse = !attr(DATA()[[1]], 'maximization'), show.legend = T) + }, + message = "Creating plot") +}) + +output$FCEECDF.Mult.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_ECDF_AGGR) + }, + content = function(file) { + save_plotly(render_FV_ECDF_AGGR(), file) + }, + contentType = paste0('image/', input$FCEECDF.Mult.Format) +) + +output$FCE_ECDF_AGGR <- renderPlotly({ + render_FV_ECDF_AGGR() +}) + +get_data_FV_AUC <- reactive({ + req(input$FCEECDF.AUC.Min, input$FCEECDF.AUC.Max, input$FCEECDF.AUC.Step, length(DATA()) > 0) + + rt_min <- input$FCEECDF.AUC.Min %>% as.numeric + rt_max <- input$FCEECDF.AUC.Max %>% as.numeric + rt_step <- input$FCEECDF.AUC.Step %>% as.numeric + data <- subset(DATA(), ID %in% input$FCEECDF.AUC.Algs) + targets <- seq_RT(get_runtimes(data), rt_min, rt_max, rt_step, length.out = 10) + generate_data.AUC(data, targets, which = 'by_FV') +}) + + +render_FV_AUC <- reactive({ + withProgress({ + # rt_min <- input$FCEECDF.AUC.Min %>% as.numeric + # rt_max <- input$FCEECDF.AUC.Max %>% as.numeric + # rt_step <- input$FCEECDF.AUC.Step %>% as.numeric + # data <- subset(DATA(), algId %in% input$FCEECDF.AUC.Algs) + # + # Plot.FV.ECDF_AUC(data, rt_min = rt_min, + # rt_max = rt_max, rt_step = rt_step) + plot_general_data(get_data_FV_AUC(), 'x', 'AUC', 'radar') + }, + message = "Creating plot") +}) + +output$FCEECDF.AUC.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_AUC) + }, + content = function(file) { + save_plotly(render_FV_AUC(), file) + }, + contentType = paste0('image/', input$FCEECDF.AUC.Format) +) + +output$FCE_AUC <- renderPlotly({ + render_FV_AUC() +}) diff --git a/inst/shiny-server/server/FCE_aggr_plot.R b/inst/shiny-server/server/FCE_aggr_plot.R index 6a7e280c..5e231676 100644 --- a/inst/shiny-server/server/FCE_aggr_plot.R +++ b/inst/shiny-server/server/FCE_aggr_plot.R @@ -1,147 +1,147 @@ -### Generate FCE table -FCEPlot.Aggr.FCEs_obj <- reactive({ - input$FCEPlot.Aggr.Refresh - dsList <- isolate(FCEPlot.Aggr.data()) - if (is.null(dsList)) return(NULL) - aggr_on <- ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM') - targets <- isolate(FCEPlot.Aggr.Targets_obj) - dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, - which = 'by_FV') - dt -}) - -### Plot the data -render_FCEPlot_aggr_plot <- reactive({ - withProgress({ - y_attr <- if (input$FCEPlot.Aggr.Ranking) 'rank' else 'value' - y_title <- if (input$FCEPlot.Aggr.Ranking) 'Rank' else 'Best-so-far f(x)' - reverse_scale <- input$FCEPlot.Aggr.Mode == 'radar' - dt <- FCEPlot.Aggr.FCEs_obj() - plot_general_data(dt, type = input$FCEPlot.Aggr.Mode, x_attr = 'funcId', - y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, - scale.ylog = input$FCEPlot.Aggr.Logy, scale.reverse = reverse_scale) - }, - message = "Creating plot") -}) - - -### Gather relevant datasetlist -FCEPlot.Aggr.data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$FCEPlot.Aggr.Algs)) - if (length(data) == 0) return(NULL) - if (input$FCEPlot.Aggr.Aggregator == 'Functions') { - data <- subset(data, DIM == input$Overall.Dim) - if (length(unique(get_funcId(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple functions for the selected dimension.") - return(NULL) - } - } - else{ - data <- subset(data, funcId == input$Overall.Funcid) - if (length(unique(get_dim(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple dimensions for the selected function") - return(NULL) - } - } - if (length(unique(get_algId(data))) == 1) { - shinyjs::alert("This plot is only available when the dataset contains multiple algorithms for the selected dimension.") - return(NULL) - } - data -} - - -### format table for display -FCE_multi_function <- function() { - dt <- FCEPlot.Aggr.FCEs_obj() - if (input$FCEPlot.Aggr.Aggregator == 'Functions') - dt <- dcast(dt, funcId~algId, value.var = 'value') - else - dt <- dcast(dt, DIM~algId, value.var = 'value') - dt -} - -### Table with default targets -default_runtimes_table <- reactive({ - data <- FCEPlot.Aggr.data() - if (is.null(data)) return(NULL) - targets <- get_target_dt(data, 'by_FV') - if (input$FCEPlot.Aggr.Aggregator == 'Functions') - targets <- targets[, c('funcId', 'target')] - else - targets <- targets[, c('DIM', 'target')] -}) - -### Target table object -FCEPlot.Aggr.Targets_obj <- NULL - -### Target table proxy -proxy_FCEPlot.Aggr.Targets <- dataTableProxy('FCEPlot.Aggr.Targets') - -### Target table print -output$FCEPlot.Aggr.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - FCEPlot.Aggr.Targets_obj <<- default_runtimes_table() - FCEPlot.Aggr.Targets_obj -}, editable = TRUE, rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - - - -### Target table edit -observeEvent(input$FCEPlot.Aggr.Targets_cell_edit, { - info <- input$FCEPlot.Aggr.Targets_cell_edit - i <- info$row - req(i > 0) - j <- info$col + 1 - v <- info$value - FCEPlot.Aggr.Targets_obj[i, j] <<- - DT::coerceValue(v, FCEPlot.Aggr.Targets_obj[['target']][[i]]) - replaceData(proxy, FCEPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) -}) - -### Table output -output$FCEPlot.Aggr.FCETable <- DT::renderDataTable({ - input$FCEPlot.Aggr.Refresh - req(length(DATA_RAW()) > 0) - - withProgress({ - dt <- FCE_multi_function() - }, - message = "Creating table") - dt -}, editable = FALSE, rownames = TRUE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - -### plot output -output$FCEPlot.Aggr.Plot <- renderPlotly( - render_FCEPlot_aggr_plot() -) - - - -### Download table -output$FCEPlot.Aggr.DownloadTable <- downloadHandler( - filename = function() { - eval(FCE_multi_func_name) - }, - content = function(file) { - df <- FCE_multi_function() - save_table(df, file) - } -) - -### Download plot -output$FCEPlot.Aggr.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_AGGR) - }, - content = function(file) { - save_plotly(render_FCEPlot_aggr_plot(), file) - }, - contentType = paste0('image/', input$FCEPlot.Aggr.Format) -) - - - +### Generate FCE table +FCEPlot.Aggr.FCEs_obj <- reactive({ + input$FCEPlot.Aggr.Refresh + dsList <- isolate(FCEPlot.Aggr.data()) + if (is.null(dsList)) return(NULL) + aggr_on <- ifelse(input$FCEPlot.Aggr.Aggregator == 'Functions', 'funcId', 'DIM') + targets <- isolate(FCEPlot.Aggr.Targets_obj) + dt <- generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets, + which = 'by_FV') + dt +}) + +### Plot the data +render_FCEPlot_aggr_plot <- reactive({ + withProgress({ + y_attr <- if (input$FCEPlot.Aggr.Ranking) 'rank' else 'value' + y_title <- if (input$FCEPlot.Aggr.Ranking) 'Rank' else 'Best-so-far f(x)' + reverse_scale <- input$FCEPlot.Aggr.Mode == 'radar' + dt <- FCEPlot.Aggr.FCEs_obj() + plot_general_data(dt, type = input$FCEPlot.Aggr.Mode, x_attr = 'funcId', + y_attr = y_attr, x_title = "FuncId", y_title = y_title, show.legend = T, + scale.ylog = input$FCEPlot.Aggr.Logy, scale.reverse = reverse_scale) + }, + message = "Creating plot") +}) + + +### Gather relevant datasetlist +FCEPlot.Aggr.data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$FCEPlot.Aggr.Algs)) + if (length(data) == 0) return(NULL) + if (input$FCEPlot.Aggr.Aggregator == 'Functions') { + data <- subset(data, DIM == input$Overall.Dim) + if (length(unique(get_funcId(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains multiple functions for the selected dimension.") + return(NULL) + } + } + else{ + data <- subset(data, funcId == input$Overall.Funcid) + if (length(unique(get_dim(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains multiple dimensions for the selected function") + return(NULL) + } + } + if (length(unique(get_id(data))) == 1) { + shinyjs::alert("This plot is only available when the dataset contains multiple IDs for the selected dimension.") + return(NULL) + } + data +} + + +### format table for display +FCE_multi_function <- function() { + dt <- FCEPlot.Aggr.FCEs_obj() + if (input$FCEPlot.Aggr.Aggregator == 'Functions') + dt <- dcast(dt, funcId~ID, value.var = 'value') + else + dt <- dcast(dt, DIM~ID, value.var = 'value') + dt +} + +### Table with default targets +default_runtimes_table <- reactive({ + data <- FCEPlot.Aggr.data() + if (is.null(data)) return(NULL) + targets <- get_target_dt(data, 'by_FV') + if (input$FCEPlot.Aggr.Aggregator == 'Functions') + targets <- targets[, c('funcId', 'target')] + else + targets <- targets[, c('DIM', 'target')] +}) + +### Target table object +FCEPlot.Aggr.Targets_obj <- NULL + +### Target table proxy +proxy_FCEPlot.Aggr.Targets <- dataTableProxy('FCEPlot.Aggr.Targets') + +### Target table print +output$FCEPlot.Aggr.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + FCEPlot.Aggr.Targets_obj <<- default_runtimes_table() + FCEPlot.Aggr.Targets_obj +}, editable = TRUE, rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + + + +### Target table edit +observeEvent(input$FCEPlot.Aggr.Targets_cell_edit, { + info <- input$FCEPlot.Aggr.Targets_cell_edit + i <- info$row + req(i > 0) + j <- info$col + 1 + v <- info$value + FCEPlot.Aggr.Targets_obj[i, j] <<- + DT::coerceValue(v, FCEPlot.Aggr.Targets_obj[['target']][[i]]) + replaceData(proxy, FCEPlot.Aggr.Targets_obj, resetPaging = FALSE, rownames = FALSE) +}) + +### Table output +output$FCEPlot.Aggr.FCETable <- DT::renderDataTable({ + input$FCEPlot.Aggr.Refresh + req(length(DATA_RAW()) > 0) + + withProgress({ + dt <- FCE_multi_function() + }, + message = "Creating table") + dt +}, editable = FALSE, rownames = TRUE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + +### plot output +output$FCEPlot.Aggr.Plot <- renderPlotly( + render_FCEPlot_aggr_plot() +) + + + +### Download table +output$FCEPlot.Aggr.DownloadTable <- downloadHandler( + filename = function() { + eval(FCE_multi_func_name) + }, + content = function(file) { + df <- FCE_multi_function() + save_table(df, file) + } +) + +### Download plot +output$FCEPlot.Aggr.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_AGGR) + }, + content = function(file) { + save_plotly(render_FCEPlot_aggr_plot(), file) + }, + contentType = paste0('image/', input$FCEPlot.Aggr.Format) +) + + + diff --git a/inst/shiny-server/server/FV_DSCinterface.R b/inst/shiny-server/server/FV_DSCinterface.R index 8009808f..13aa224b 100644 --- a/inst/shiny-server/server/FV_DSCinterface.R +++ b/inst/shiny-server/server/FV_DSCinterface.R @@ -1,225 +1,225 @@ -FV_DSC_rank_result <- reactive({ - req(input$FV_Stats.DSC.Create_rank) - - isolate({ - withProgress({ - data <- FV_DSC_data() - req(length(data) > 0) - # if (input$FV_Stats.DSC.Value_type != 'Raw') { - # shinyjs::alert("This feature is not yet implemented in the current version of IOHanalyzer.") - # } - test_type <- if (input$FV_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" - rank_res <- get_dsc_rank(data, FV_stats_DSC_targets_obj, which = 'by_FV', - alpha = input$FV_Stats.DSC.Alpha_rank, - test_type = test_type, - monte_carlo_iterations = input$FV_Stats.DSC.MCsamples_rank, - epsilon = input$FV_Stats.DSC.Epsilon_rank) - if (is.null(rank_res)) { - shinyjs::alert("There was an error getting the ranking data from DSCtool. Please - select different settings and try again") - return(NULL) - } - updateSelectInput(session, 'FV_Stats.DSC.Omni_options', choices = rank_res$valid_methods, - selected = rank_res$valid_methods[[1]]) - - - rank_res - }, message = "Getting DSC ranking data") - }) -}) - -FV_render_performviz <- reactive({ - rank_res <- FV_DSC_rank_result() - if (is.null(rank_res)) {return(NULL)} - Plot.Performviz(rank_res) -}) - -output$FV_Stats.DSC.PerformViz <- renderPlot({ - FV_render_performviz() -}) - -output$FV_Stats.DSC.Download_rank_table <- downloadHandler( - filename = function() { - eval(FV_DSC_table_name_rank) - }, - content = function(file) { - mlist <- FV_DSC_rank_result()$ranked_matrix - df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { - df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - function(x) { - list(algorithm = x$algorithm, rank = x$rank) - })) - df_temp[, problem := mlist[[problem_idx]]$problem] - })) - df <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') - save_table(df, file) - } -) - -output$FV_Stats.DSC.Download_rank <- downloadHandler( - filename = function() { - eval(FV_DSC_figure_name_rank) - }, - content = function(file) { - #Temporary settings, will replace plotting from complexheatmap to - #Some other plotly-compatible library later, and use save_plotly - suppressWarnings({ - pdf(file = file, width = 16, height = 9) - RT_render_performviz() - dev.off() - }) - }, - contentType = paste0('image/', input$FV_Stats.DSC.Format_rank) -) - -FV_DSC_omni_result <- reactive({ - input$FV_Stats.DSC.Create_omni - - isolate({ - withProgress({ - rank_res <- FV_DSC_rank_result() - req(rank_res) - omni_res <- get_dsc_omnibus(rank_res, method = input$FV_Stats.DSC.Omni_options, - alpha = input$FV_Stats.DSC.Alpha_omni) - - }, message = "Creating Comparison, this might take a while") - }) -}) - - -output$FV_Stats.DSC.Output_omni <- renderText({ - res_omni <- FV_DSC_omni_result() - req(res_omni) - paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") -}) - - -FV_DSC_posthoc_result <- reactive({ - input$FV_Stats.DSC.Create_posthoc - - isolate({ - withProgress({ - omni_res <- FV_DSC_omni_result() - req(omni_res) - data <- FV_DSC_data() - req(length(data) > 0) - df_posthoc <- rbindlist(lapply(get_algId(data), function(algname){ - posthoc_res <- get_dsc_posthoc(omni_res, length(get_algId(data)), - nrow(FV_stats_DSC_targets_obj), - alpha = input$FV_Stats.DSC.Alpha_posthoc, - base_algorithm = algname, - method = input$FV_Stats.DSC.Posthoc_test) - if (is.null(posthoc_res)) { return(NULL)} - values <- lapply(seq(4), function(method_idx) { - lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, - function(x) { - x$value - }) - }) - - algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, - function(x) { - x$algorithm - }) - df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], - "Holm" = values[[3]], "Hochberg" = values[[4]]) - - })) - as.data.table(format(df_posthoc, digits = 3)) - }, message = "Creating Comparison, this might take a while") - }) -}) - -output$FV_Stats.DSC.PosthocTable <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - FV_DSC_posthoc_result() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -FV_render_DSC_plot <- reactive({ - dt <- FV_DSC_posthoc_result() - if (is.null(dt) || nrow(dt) < 1) {return(NULL)} - # plot_general_data(dt[method == input$FV_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', - # legend_attr = 'algId') - dt <- dt[,c('baseline', 'compared alg', input$FV_Stats.DSC.Posthoc_method), with = F] - p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$FV_Stats.DSC.Posthoc_method) - storage.mode(p_matrix) <- "numeric" - y <- p_matrix <= input$FV_Stats.DSC.Alpha_posthoc - colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), - col = c('blue', 'blue', 'white', 'white', 'red', 'red') - ) - heatmap <- y - t(y) - - p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', - xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) - p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), - xaxis = list(tickangle = 45)) - p -}) - -output$FV_Stats.DSC.PosthocViz <- renderPlotly({ - FV_render_DSC_plot() -}) - -output$FV_Stats.DSC.DownloadTable <- downloadHandler( - filename = function() { - eval(FV_DSC_table_name) - }, - content = function(file) { - df <- FV_DSC_posthoc_result() - save_table(df, file) - } -) - -output$FV_Stats.DSC.Download <- downloadHandler( - filename = function() { - eval(FV_DSC_figure_name) - }, - content = function(file) { - save_plotly(FV_render_DSC_plot(), file) - }, - contentType = paste0('image/', input$FV_Stats.DSC.Format) -) - - -FV_DSC_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$FV_Stats.DSC.Algid)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$FV_Stats.DSC.Dim) - data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) - if (length(unique(get_algId(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") - return(NULL) - } - data -} - -FV_stats_DSC_targets <- reactive({ - data <- FV_DSC_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_FV") -}) - -FV_stats_DSC_targets_obj <- NULL - -proxy_FV_Stats.DSC.Targets <- dataTableProxy('FV_Stats.DSC.Targets') - -output$FV_Stats.DSC.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - FV_stats_DSC_targets_obj <<- FV_stats_DSC_targets() - FV_stats_DSC_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$FV_Stats.DSC.Targets_cell_edit, { - info <- input$FV_Stats.DSC.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- FV_DSC_data() - if (is.null(data)) return(NULL) - FV_stats_DSC_targets_obj$target[[i]] <<- v - replaceData(proxy, FV_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) +FV_DSC_rank_result <- reactive({ + req(input$FV_Stats.DSC.Create_rank) + + isolate({ + withProgress({ + data <- FV_DSC_data() + req(length(data) > 0) + # if (input$FV_Stats.DSC.Value_type != 'Raw') { + # shinyjs::alert("This feature is not yet implemented in the current version of IOHanalyzer.") + # } + test_type <- if (input$FV_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" + rank_res <- get_dsc_rank(data, FV_stats_DSC_targets_obj, which = 'by_FV', + alpha = input$FV_Stats.DSC.Alpha_rank, + test_type = test_type, + monte_carlo_iterations = input$FV_Stats.DSC.MCsamples_rank, + epsilon = input$FV_Stats.DSC.Epsilon_rank) + if (is.null(rank_res)) { + shinyjs::alert("There was an error getting the ranking data from DSCtool. Please + select different settings and try again") + return(NULL) + } + updateSelectInput(session, 'FV_Stats.DSC.Omni_options', choices = rank_res$valid_methods, + selected = rank_res$valid_methods[[1]]) + + + rank_res + }, message = "Getting DSC ranking data") + }) +}) + +FV_render_performviz <- reactive({ + rank_res <- FV_DSC_rank_result() + if (is.null(rank_res)) {return(NULL)} + Plot.Performviz(rank_res) +}) + +output$FV_Stats.DSC.PerformViz <- renderPlot({ + FV_render_performviz() +}) + +output$FV_Stats.DSC.Download_rank_table <- downloadHandler( + filename = function() { + eval(FV_DSC_table_name_rank) + }, + content = function(file) { + mlist <- FV_DSC_rank_result()$ranked_matrix + df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { + df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + function(x) { + list(algorithm = x$algorithm, rank = x$rank) + })) + df_temp[, problem := mlist[[problem_idx]]$problem] + })) + df <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') + save_table(df, file) + } +) + +output$FV_Stats.DSC.Download_rank <- downloadHandler( + filename = function() { + eval(FV_DSC_figure_name_rank) + }, + content = function(file) { + #Temporary settings, will replace plotting from complexheatmap to + #Some other plotly-compatible library later, and use save_plotly + suppressWarnings({ + pdf(file = file, width = 16, height = 9) + RT_render_performviz() + dev.off() + }) + }, + contentType = paste0('image/', input$FV_Stats.DSC.Format_rank) +) + +FV_DSC_omni_result <- reactive({ + input$FV_Stats.DSC.Create_omni + + isolate({ + withProgress({ + rank_res <- FV_DSC_rank_result() + req(rank_res) + omni_res <- get_dsc_omnibus(rank_res, method = input$FV_Stats.DSC.Omni_options, + alpha = input$FV_Stats.DSC.Alpha_omni) + + }, message = "Creating Comparison, this might take a while") + }) +}) + + +output$FV_Stats.DSC.Output_omni <- renderText({ + res_omni <- FV_DSC_omni_result() + req(res_omni) + paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") +}) + + +FV_DSC_posthoc_result <- reactive({ + input$FV_Stats.DSC.Create_posthoc + + isolate({ + withProgress({ + omni_res <- FV_DSC_omni_result() + req(omni_res) + data <- FV_DSC_data() + req(length(data) > 0) + df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ + posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), + nrow(FV_stats_DSC_targets_obj), + alpha = input$FV_Stats.DSC.Alpha_posthoc, + base_algorithm = algname, + method = input$FV_Stats.DSC.Posthoc_test) + if (is.null(posthoc_res)) { return(NULL)} + values <- lapply(seq(4), function(method_idx) { + lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, + function(x) { + x$value + }) + }) + + algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, + function(x) { + x$algorithm + }) + df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], + "Holm" = values[[3]], "Hochberg" = values[[4]]) + + })) + as.data.table(format(df_posthoc, digits = 3)) + }, message = "Creating Comparison, this might take a while") + }) +}) + +output$FV_Stats.DSC.PosthocTable <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + FV_DSC_posthoc_result() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +FV_render_DSC_plot <- reactive({ + dt <- FV_DSC_posthoc_result() + if (is.null(dt) || nrow(dt) < 1) {return(NULL)} + # plot_general_data(dt[method == input$FV_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', + # legend_attr = 'algId') + dt <- dt[,c('baseline', 'compared alg', input$FV_Stats.DSC.Posthoc_method), with = F] + p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$FV_Stats.DSC.Posthoc_method) + storage.mode(p_matrix) <- "numeric" + y <- p_matrix <= input$FV_Stats.DSC.Alpha_posthoc + colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), + col = c('blue', 'blue', 'white', 'white', 'red', 'red') + ) + heatmap <- y - t(y) + + p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', + xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) + p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), + xaxis = list(tickangle = 45)) + p +}) + +output$FV_Stats.DSC.PosthocViz <- renderPlotly({ + FV_render_DSC_plot() +}) + +output$FV_Stats.DSC.DownloadTable <- downloadHandler( + filename = function() { + eval(FV_DSC_table_name) + }, + content = function(file) { + df <- FV_DSC_posthoc_result() + save_table(df, file) + } +) + +output$FV_Stats.DSC.Download <- downloadHandler( + filename = function() { + eval(FV_DSC_figure_name) + }, + content = function(file) { + save_plotly(FV_render_DSC_plot(), file) + }, + contentType = paste0('image/', input$FV_Stats.DSC.Format) +) + + +FV_DSC_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.DSC.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$FV_Stats.DSC.Dim) + data <- subset(data, funcId %in% input$FV_Stats.DSC.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +FV_stats_DSC_targets <- reactive({ + data <- FV_DSC_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_FV") +}) + +FV_stats_DSC_targets_obj <- NULL + +proxy_FV_Stats.DSC.Targets <- dataTableProxy('FV_Stats.DSC.Targets') + +output$FV_Stats.DSC.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + FV_stats_DSC_targets_obj <<- FV_stats_DSC_targets() + FV_stats_DSC_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$FV_Stats.DSC.Targets_cell_edit, { + info <- input$FV_Stats.DSC.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- FV_DSC_data() + if (is.null(data)) return(NULL) + FV_stats_DSC_targets_obj$target[[i]] <<- v + replaceData(proxy, FV_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) }) \ No newline at end of file diff --git a/inst/shiny-server/server/FV_PAR.R b/inst/shiny-server/server/FV_PAR.R index cfb262f9..86878303 100644 --- a/inst/shiny-server/server/FV_PAR.R +++ b/inst/shiny-server/server/FV_PAR.R @@ -1,182 +1,182 @@ -# Expected Evolution of parameters in the algorithm -get_data_FV_PAR_PER_FUN <- reactive({ - data <- subset(DATA(), algId %in% input$FV_PAR.Plot.Algs) - generate_data.Parameters(data, scale_log = input$FV_PAR.Plot.Logx, which = 'by_RT') -}) - -render_FV_PAR_PER_FUN <- reactive({ - req(input$FV_PAR.Plot.Min, input$FV_PAR.Plot.Max) - withProgress({ - #TODO: use these parameters - rt_min <- as.numeric(input$FV_PAR.Plot.Min) - rt_max <- as.numeric(input$FV_PAR.Plot.Max) - - dt <- get_data_FV_PAR_PER_FUN() - dt <- dt[parId %in% input$FV_PAR.Plot.Params] - sub_attr <- if (length(input$FV_PAR.Plot.Params) > 1) 'parId' else NULL - - lower <- 'lower' - upper <- 'upper' - if (input$FV_PAR.Plot.CI == 'None') type <- 'line' - else { - type <- 'line+ribbon' - if (input$FV_PAR.Plot.CI == 'Outer Quantiles') { - quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') - lower <- quantiles[[1]] - upper <- quantiles[[length(quantiles)]] - } - } - if (is.null(sub_attr) && type == 'line+ribbon') { - p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'line', - subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = T) - p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'ribbon', - subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = F, p = p) - } - else { - p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, type, - subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, - scale.ylog = input$FV_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper) - } - p - }, - message = "Creating plot") -}) - -output$FV_PAR.Plot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_FV_PAR_PER_FUN) - }, - content = function(file) { - save_plotly(render_FV_PAR_PER_FUN(), file) - }, - contentType = paste0('image/', input$FV_PAR.Plot.Format) -) - -output$FV_PAR.Plot.Figure <- renderPlotly({ - render_FV_PAR_PER_FUN() -}) - -# TODO: add ks test for ECDF later -# output$ks <- renderPrint({ -# target <- input$target %>% as.numeric -# df.aligneds <- aligned() -# -# running_time <- list() -# for (i in seq_along(df.aligneds)) { -# df <- df.aligneds[[i]] -# v <- rownames(df) %>% as.numeric -# idx <- order(abs(target - v))[1] -# running_time[[i]] <- df[idx, ] %>% as.vector -# } -# algorithm1 <- running_time[[1]] -# algorithm2 <- running_time[[2]] -# a <- ks.test(algorithm1, algorithm2, alternative = 'less') -# print(a) -# }) -# - -fv_parameter_summary <- reactive({ - req(input$FV_PAR.Summary.Min, input$FV_PAR.Summary.Max, input$FV_PAR.Summary.Step) - - rtstart <- as.numeric(input$FV_PAR.Summary.Min) - rtstop <- as.numeric(input$FV_PAR.Summary.Max) - rtstep <- as.numeric(input$FV_PAR.Summary.Step) - data <- DATA() - - if (!input$FV_PAR.Summary.Single) { - req(rtstart <= rtstop, rtstep <= rtstop - rtstart) - rtall <- get_runtimes(data) - rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) - req(rtseq) - } - else - rtseq <- rtstart - - dt <- get_PAR_summary(data, rtseq, input$FV_PAR.Summary.Algid, - input$FV_PAR.Summary.Param, which = 'by_RT') - req(length(dt) != 0) - - dt$runs %<>% as.integer - dt$mean %<>% format(digits = 2, nsmall = 2) - dt$median %<>% format(digits = 2, nsmall = 2) - dt$sd %<>% format(digits = 2, nsmall = 2) - - probs <- getOption("IOHanalyzer.quantiles") - - # format the integers - for (p in paste0(probs * 100, '%')) { - dt[[p]] %<>% format(digits = 2, nsmall = 2) - } - dt -}) - -fv_parameter_sample <- reactive({ - req(input$FV_PAR.Sample.Algid, input$FV_PAR.Sample.Max, - input$FV_PAR.Sample.Step, input$FV_PAR.Sample.Min, - input$FV_PAR.Sample.Param) - - rtstart <- as.numeric(input$FV_PAR.Sample.Min) - rtstop <- as.numeric(input$FV_PAR.Sample.Max) - rtstep <- as.numeric(input$FV_PAR.Sample.Step) - data <- DATA() - - if (!input$FV_PAR.Sample.Single) { - req(rtstart <= rtstop, rtstep <= rtstop - rtstart) - rtall <- get_runtimes(data) - rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) - req(rtseq) - } - else - rtseq <- rtstart - - df <- get_PAR_sample(data, idxValue = rtseq, - algorithm = input$FV_PAR.Sample.Algid, - parId = input$FV_PAR.Sample.Param, - output = input$FV_PAR.Sample.Format, - which = 'by_RT') - - for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) - df[[p]] %<>% format(digits = 2, nsmall = 2) - - df -}) - -output$table_FV_PAR_SAMPLE <- DT::renderDataTable({ - dt <- fv_parameter_sample() - req(length(dt) != 0) - dt[is.na(dt)] <- 'NA' - dt -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$table_FV_PAR_summary <- DT::renderDataTable({ - fv_parameter_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) - -output$FV_PAR.Sample.Download <- downloadHandler( - filename = function() { - eval(FV_PARSample_csv_name) - }, - content = function(file) { - df <- parameter_sample() - df <- df[input[["table_FV_PAR_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -output$FV_PAR.Summary.Download <- downloadHandler( - filename = function() { - eval(FV_PAR_csv_name) - }, - content = function(file) { - df <- fv_parameter_summary() - df <- df[input[["table_FV_PAR_summary_rows_all"]]] - save_table(df, file) - } +# Expected Evolution of parameters in the algorithm +get_data_FV_PAR_PER_FUN <- reactive({ + data <- subset(DATA(), ID %in% input$FV_PAR.Plot.Algs) + generate_data.Parameters(data, scale_log = input$FV_PAR.Plot.Logx, which = 'by_RT') +}) + +render_FV_PAR_PER_FUN <- reactive({ + req(input$FV_PAR.Plot.Min, input$FV_PAR.Plot.Max) + withProgress({ + #TODO: use these parameters + rt_min <- as.numeric(input$FV_PAR.Plot.Min) + rt_max <- as.numeric(input$FV_PAR.Plot.Max) + + dt <- get_data_FV_PAR_PER_FUN() + dt <- dt[parId %in% input$FV_PAR.Plot.Params] + sub_attr <- if (length(input$FV_PAR.Plot.Params) > 1) 'parId' else NULL + + lower <- 'lower' + upper <- 'upper' + if (input$FV_PAR.Plot.CI == 'None') type <- 'line' + else { + type <- 'line+ribbon' + if (input$FV_PAR.Plot.CI == 'Outer Quantiles') { + quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') + lower <- quantiles[[1]] + upper <- quantiles[[length(quantiles)]] + } + } + if (is.null(sub_attr) && type == 'line+ribbon') { + p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'line', + subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = T) + p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, 'ribbon', + subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = F, p = p) + } + else { + p <- plot_general_data(dt, 'runtime', input$FV_PAR.Plot.show.mean, type, + subplot_attr = sub_attr, scale.xlog = input$FV_PAR.Plot.Logx, + scale.ylog = input$FV_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper) + } + p + }, + message = "Creating plot") +}) + +output$FV_PAR.Plot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_FV_PAR_PER_FUN) + }, + content = function(file) { + save_plotly(render_FV_PAR_PER_FUN(), file) + }, + contentType = paste0('image/', input$FV_PAR.Plot.Format) +) + +output$FV_PAR.Plot.Figure <- renderPlotly({ + render_FV_PAR_PER_FUN() +}) + +# TODO: add ks test for ECDF later +# output$ks <- renderPrint({ +# target <- input$target %>% as.numeric +# df.aligneds <- aligned() +# +# running_time <- list() +# for (i in seq_along(df.aligneds)) { +# df <- df.aligneds[[i]] +# v <- rownames(df) %>% as.numeric +# idx <- order(abs(target - v))[1] +# running_time[[i]] <- df[idx, ] %>% as.vector +# } +# algorithm1 <- running_time[[1]] +# algorithm2 <- running_time[[2]] +# a <- ks.test(algorithm1, algorithm2, alternative = 'less') +# print(a) +# }) +# + +fv_parameter_summary <- reactive({ + req(input$FV_PAR.Summary.Min, input$FV_PAR.Summary.Max, input$FV_PAR.Summary.Step) + + rtstart <- as.numeric(input$FV_PAR.Summary.Min) + rtstop <- as.numeric(input$FV_PAR.Summary.Max) + rtstep <- as.numeric(input$FV_PAR.Summary.Step) + data <- DATA() + data <- subset(data, ID %in% input$FV_PAR.Summary.ID) + + if (!input$FV_PAR.Summary.Single) { + req(rtstart <= rtstop, rtstep <= rtstop - rtstart) + rtall <- get_runtimes(data) + rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) + req(rtseq) + } + else + rtseq <- rtstart + + dt <- get_PAR_summary(data, rtseq, parId = input$FV_PAR.Summary.Param, which = 'by_RT') + req(length(dt) != 0) + + dt$runs %<>% as.integer + dt$mean %<>% format(digits = 2, nsmall = 2) + dt$median %<>% format(digits = 2, nsmall = 2) + dt$sd %<>% format(digits = 2, nsmall = 2) + + probs <- getOption("IOHanalyzer.quantiles") + + # format the integers + for (p in paste0(probs * 100, '%')) { + dt[[p]] %<>% format(digits = 2, nsmall = 2) + } + dt +}) + +fv_parameter_sample <- reactive({ + req(input$FV_PAR.Sample.ID, input$FV_PAR.Sample.Max, + input$FV_PAR.Sample.Step, input$FV_PAR.Sample.Min, + input$FV_PAR.Sample.Param) + + rtstart <- as.numeric(input$FV_PAR.Sample.Min) + rtstop <- as.numeric(input$FV_PAR.Sample.Max) + rtstep <- as.numeric(input$FV_PAR.Sample.Step) + data <- DATA() + data <- subset(data, ID %in% input$FV_PAR.Sample.ID) + + if (!input$FV_PAR.Sample.Single) { + req(rtstart <= rtstop, rtstep <= rtstop - rtstart) + rtall <- get_runtimes(data) + rtseq <- seq_RT(rtall, rtstart, rtstop, by = rtstep) + req(rtseq) + } + else + rtseq <- rtstart + + df <- get_PAR_sample(data, idxValue = rtseq, + parId = input$FV_PAR.Sample.Param, + output = input$FV_PAR.Sample.Format, + which = 'by_RT') + + for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) + df[[p]] %<>% format(digits = 2, nsmall = 2) + + df +}) + +output$table_FV_PAR_SAMPLE <- DT::renderDataTable({ + dt <- fv_parameter_sample() + req(length(dt) != 0) + dt[is.na(dt)] <- 'NA' + dt +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$table_FV_PAR_summary <- DT::renderDataTable({ + fv_parameter_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) + +output$FV_PAR.Sample.Download <- downloadHandler( + filename = function() { + eval(FV_PARSample_csv_name) + }, + content = function(file) { + df <- parameter_sample() + df <- df[input[["table_FV_PAR_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +output$FV_PAR.Summary.Download <- downloadHandler( + filename = function() { + eval(FV_PAR_csv_name) + }, + content = function(file) { + df <- fv_parameter_summary() + df <- df[input[["table_FV_PAR_summary_rows_all"]]] + save_table(df, file) + } ) \ No newline at end of file diff --git a/inst/shiny-server/server/RTPMF.R b/inst/shiny-server/server/RTPMF.R index 93bb973a..62d86126 100644 --- a/inst/shiny-server/server/RTPMF.R +++ b/inst/shiny-server/server/RTPMF.R @@ -1,73 +1,73 @@ -# empirical p.m.f. of the runtime -output$RT_PMF <- renderPlotly({ - render_RT_PMF() -}) - -output$RTPMF.Bar.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_PMF) - }, - content = function(file) { - save_plotly(render_RT_PMF(), file) - }, - contentType = paste0('image/', input$RTPMF.Bar.Format) -) - -get_data_RT_PMF <- reactive({ - ftarget <- input$RTPMF.Bar.Target %>% as.numeric - data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) - generate_data.PMF(data, ftarget, 'by_RT') -}) - -render_RT_PMF <- reactive({ - withProgress({ - plot_general_data(get_data_RT_PMF(), 'algId', 'RT', scale.ylog = input$RTPMF.Bar.Logy, - x_title = "Algorithm", y_title = "Function Evaluations") - # ftarget <- input$RTPMF.Bar.Target %>% as.numeric - # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) - # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, - # scale.ylog = input$RTPMF.Bar.Logy) - }, - message = "Creating plot") -}) - -# historgram of the running time -output$RT_HIST <- renderPlotly({ - req(input$RTPMF.Bar.Target) - render_RT_HIST() -}) - -output$RTPMF.Hist.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_HIST) - }, - content = function(file) { - save_plotly(render_RT_HIST(), file) - }, - contentType = paste0('image/', input$RTPMF.Hist.Format) -) - -get_data_RT_HIST <- reactive({ - ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric - data <- subset(DATA(), algId %in% input$RTPMF.Hist.Algs) - generate_data.hist(data, ftarget, input$RTPMF.Hist.Equal, 'by_RT') -}) - -render_RT_HIST <- reactive({ - req(input$RTPMF.Hist.Target) - - withProgress({ - # ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric - # - # # TODO: remove 'DataSetList' in the future - # data <- subset(DATA(), algId %in% input$RTPMF.Hist.Algs) - # - # Plot.RT.Histogram(data, ftarget, plot_mode = input$RTPMF.Hist.Mode, - # use.equal.bins = input$RTPMF.Hist.Equal) - subplot_attr <- if (input$RTPMF.Hist.Mode == 'subplot') 'algId' else NULL - plot_general_data(get_data_RT_HIST(), 'x', 'y', width = 'width', type = 'hist', - subplot_attr = subplot_attr, x_title = "Function Evaluations", - y_title = "Runs") - }, - message = "Creating plot") -}) +# empirical p.m.f. of the runtime +output$RT_PMF <- renderPlotly({ + render_RT_PMF() +}) + +output$RTPMF.Bar.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_PMF) + }, + content = function(file) { + save_plotly(render_RT_PMF(), file) + }, + contentType = paste0('image/', input$RTPMF.Bar.Format) +) + +get_data_RT_PMF <- reactive({ + ftarget <- input$RTPMF.Bar.Target %>% as.numeric + data <- subset(DATA(), ID %in% input$RTPMF.Bar.Algs) + generate_data.PMF(data, ftarget, 'by_RT') +}) + +render_RT_PMF <- reactive({ + withProgress({ + plot_general_data(get_data_RT_PMF(), 'ID', 'RT', scale.ylog = input$RTPMF.Bar.Logy, + x_title = "Algorithm", y_title = "Function Evaluations") + # ftarget <- input$RTPMF.Bar.Target %>% as.numeric + # data <- subset(DATA(), algId %in% input$RTPMF.Bar.Algs) + # Plot.RT.PMF(data, ftarget, show.sample = input$RTPMF.Bar.Sample, + # scale.ylog = input$RTPMF.Bar.Logy) + }, + message = "Creating plot") +}) + +# historgram of the running time +output$RT_HIST <- renderPlotly({ + req(input$RTPMF.Bar.Target) + render_RT_HIST() +}) + +output$RTPMF.Hist.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_HIST) + }, + content = function(file) { + save_plotly(render_RT_HIST(), file) + }, + contentType = paste0('image/', input$RTPMF.Hist.Format) +) + +get_data_RT_HIST <- reactive({ + ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric + data <- subset(DATA(), ID %in% input$RTPMF.Hist.Algs) + generate_data.hist(data, ftarget, input$RTPMF.Hist.Equal, 'by_RT') +}) + +render_RT_HIST <- reactive({ + req(input$RTPMF.Hist.Target) + + withProgress({ + # ftarget <- format_FV(input$RTPMF.Hist.Target) %>% as.numeric + # + # # TODO: remove 'DataSetList' in the future + # data <- subset(DATA(), algId %in% input$RTPMF.Hist.Algs) + # + # Plot.RT.Histogram(data, ftarget, plot_mode = input$RTPMF.Hist.Mode, + # use.equal.bins = input$RTPMF.Hist.Equal) + subplot_attr <- if (input$RTPMF.Hist.Mode == 'subplot') 'ID' else NULL + plot_general_data(get_data_RT_HIST(), 'x', 'y', width = 'width', type = 'hist', + subplot_attr = subplot_attr, x_title = "Function Evaluations", + y_title = "Runs") + }, + message = "Creating plot") +}) diff --git a/inst/shiny-server/server/RT_DSCinterface.R b/inst/shiny-server/server/RT_DSCinterface.R index dbb4c909..33a766a5 100644 --- a/inst/shiny-server/server/RT_DSCinterface.R +++ b/inst/shiny-server/server/RT_DSCinterface.R @@ -1,224 +1,224 @@ -RT_DSC_rank_result <- reactive({ - req(input$RT_Stats.DSC.Create_rank) - - isolate({ - withProgress({ - data <- RT_DSC_data() - req(length(data) > 0) - test_type <- if (input$RT_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" - rank_res <- get_dsc_rank(data, RT_stats_DSC_targets_obj, which = 'by_RT', - alpha = input$RT_Stats.DSC.Alpha_rank, - test_type = test_type, - monte_carlo_iterations = input$RT_Stats.DSC.MCsamples_rank, - epsilon = input$RT_Stats.DSC.Epsilon_rank, - na.correction = input$RT_Stats.DSC.Value_type) - if (is.null(rank_res)) { - shinyjs::alert("There was an error getting the ranking data from DSCtool. Please - select different settings and try again") - return(NULL) - } - updateSelectInput(session, 'RT_Stats.DSC.Omni_options', choices = rank_res$valid_methods, - selected = rank_res$valid_methods[[1]]) - - - rank_res - }, message = "Getting DSC ranking data") - }) -}) - -RT_render_performviz <- reactive({ - rank_res <- RT_DSC_rank_result() - if (is.null(rank_res)) {return(NULL)} - Plot.Performviz(rank_res) -}) - -output$RT_Stats.DSC.PerformViz <- renderPlot({ - RT_render_performviz() -}) - -output$RT_Stats.DSC.Download_rank_table <- downloadHandler( - filename = function() { - eval(RT_DSC_table_name_rank) - }, - content = function(file) { - mlist <- RT_DSC_rank_result()$ranked_matrix - df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { - df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, - function(x) { - list(algorithm = x$algorithm, rank = x$rank) - })) - df_temp[, problem := mlist[[problem_idx]]$problem] - })) - df2 <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') - save_table(df, file) - } -) - -output$RT_Stats.DSC.Download_rank <- downloadHandler( - filename = function() { - eval(RT_DSC_figure_name_rank) - }, - content = function(file) { - #Temporary settings, will replace plotting from complexheatmap to - #Some other plotly-compatible library later, and use save_plotly - suppressWarnings({ - pdf(file = file, width = 16, height = 9) - RT_render_performviz() - dev.off() - }) - }, - contentType = paste0('image/', input$RT_Stats.DSC.Format_rank) -) - - -RT_DSC_omni_result <- reactive({ - input$RT_Stats.DSC.Create_omni - - isolate({ - withProgress({ - rank_res <- RT_DSC_rank_result() - req(rank_res) - omni_res <- get_dsc_omnibus(rank_res, method = input$RT_Stats.DSC.Omni_options, - alpha = input$RT_Stats.DSC.Alpha_omni) - - }, message = "Creating Comparison, this might take a while") - }) -}) - - -output$RT_Stats.DSC.Output_omni <- renderText({ - res_omni <- RT_DSC_omni_result() - req(res_omni) - paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") -}) - - -RT_DSC_posthoc_result <- reactive({ - input$RT_Stats.DSC.Create_posthoc - - isolate({ - withProgress({ - omni_res <- RT_DSC_omni_result() - req(omni_res) - data <- RT_DSC_data() - req(length(data) > 0) - df_posthoc <- rbindlist(lapply(get_algId(data), function(algname){ - posthoc_res <- get_dsc_posthoc(omni_res, length(get_algId(data)), - nrow(RT_stats_DSC_targets_obj), - alpha = input$RT_Stats.DSC.Alpha_posthoc, - base_algorithm = algname, - method = input$RT_Stats.DSC.Posthoc_test) - if (is.null(posthoc_res)) { return(NULL)} - values <- lapply(seq(4), function(method_idx) { - lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, - function(x) { - x$value - }) - }) - - algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, - function(x) { - x$algorithm - }) - df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], - "Holm" = values[[3]], "Hochberg" = values[[4]]) - - })) - as.data.table(format(df_posthoc, digits = 3)) - }, message = "Creating Comparison, this might take a while") - }) -}) - -output$RT_Stats.DSC.PosthocTable <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - RT_DSC_posthoc_result() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -RT_render_DSC_plot <- reactive({ - dt <- RT_DSC_posthoc_result() - if (is.null(dt) || nrow(dt) < 1) {return(NULL)} - # plot_general_data(dt[method == input$RT_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', - # legend_attr = 'algId') - dt <- dt[,c('baseline', 'compared alg', input$RT_Stats.DSC.Posthoc_method), with = F] - p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$RT_Stats.DSC.Posthoc_method) - storage.mode(p_matrix) <- "numeric" - y <- p_matrix <= input$RT_Stats.DSC.Alpha_posthoc - colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), - col = c('blue', 'blue', 'white', 'white', 'red', 'red') - ) - heatmap <- y - t(y) - - p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', - xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) - p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), - xaxis = list(tickangle = 45)) - p -}) - -output$RT_Stats.DSC.PosthocViz <- renderPlotly({ - RT_render_DSC_plot() -}) - -output$RT_Stats.DSC.DownloadTable <- downloadHandler( - filename = function() { - eval(RT_DSC_table_name) - }, - content = function(file) { - df <- RT_DSC_posthoc_result() - save_table(df, file) - } -) - -output$RT_Stats.DSC.Download <- downloadHandler( - filename = function() { - eval(RT_DSC_figure_name) - }, - content = function(file) { - save_plotly(RT_render_DSC_plot(), file) - }, - contentType = paste0('image/', input$RT_Stats.DSC.Format) -) - - -RT_DSC_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$RT_Stats.DSC.Algid)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$RT_Stats.DSC.Dim) - data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) - if (length(unique(get_algId(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") - return(NULL) - } - data -} - -RT_stats_DSC_targets <- reactive({ - data <- RT_DSC_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_RT") -}) - -RT_stats_DSC_targets_obj <- NULL - -proxy_RT_Stats.DSC.Targets <- dataTableProxy('RT_Stats.DSC.Targets') - -output$RT_Stats.DSC.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - RT_stats_DSC_targets_obj <<- RT_stats_DSC_targets() - RT_stats_DSC_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$RT_Stats.DSC.Targets_cell_edit, { - info <- input$RT_Stats.DSC.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- RT_DSC_data() - if (is.null(data)) return(NULL) - RT_stats_DSC_targets_obj$target[[i]] <<- v - replaceData(proxy, RT_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) +RT_DSC_rank_result <- reactive({ + req(input$RT_Stats.DSC.Create_rank) + + isolate({ + withProgress({ + data <- RT_DSC_data() + req(length(data) > 0) + test_type <- if (input$RT_Stats.DSC.Test_rank == "Anderson-Darling") "AD" else "KS" + rank_res <- get_dsc_rank(data, RT_stats_DSC_targets_obj, which = 'by_RT', + alpha = input$RT_Stats.DSC.Alpha_rank, + test_type = test_type, + monte_carlo_iterations = input$RT_Stats.DSC.MCsamples_rank, + epsilon = input$RT_Stats.DSC.Epsilon_rank, + na.correction = input$RT_Stats.DSC.Value_type) + if (is.null(rank_res)) { + shinyjs::alert("There was an error getting the ranking data from DSCtool. Please + select different settings and try again") + return(NULL) + } + updateSelectInput(session, 'RT_Stats.DSC.Omni_options', choices = rank_res$valid_methods, + selected = rank_res$valid_methods[[1]]) + + + rank_res + }, message = "Getting DSC ranking data") + }) +}) + +RT_render_performviz <- reactive({ + rank_res <- RT_DSC_rank_result() + if (is.null(rank_res)) {return(NULL)} + Plot.Performviz(rank_res) +}) + +output$RT_Stats.DSC.PerformViz <- renderPlot({ + RT_render_performviz() +}) + +output$RT_Stats.DSC.Download_rank_table <- downloadHandler( + filename = function() { + eval(RT_DSC_table_name_rank) + }, + content = function(file) { + mlist <- RT_DSC_rank_result()$ranked_matrix + df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) { + df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result, + function(x) { + list(algorithm = x$algorithm, rank = x$rank) + })) + df_temp[, problem := mlist[[problem_idx]]$problem] + })) + df2 <- reshape2::acast(df, algorithm ~ problem, value.var = 'rank') + save_table(df, file) + } +) + +output$RT_Stats.DSC.Download_rank <- downloadHandler( + filename = function() { + eval(RT_DSC_figure_name_rank) + }, + content = function(file) { + #Temporary settings, will replace plotting from complexheatmap to + #Some other plotly-compatible library later, and use save_plotly + suppressWarnings({ + pdf(file = file, width = 16, height = 9) + RT_render_performviz() + dev.off() + }) + }, + contentType = paste0('image/', input$RT_Stats.DSC.Format_rank) +) + + +RT_DSC_omni_result <- reactive({ + input$RT_Stats.DSC.Create_omni + + isolate({ + withProgress({ + rank_res <- RT_DSC_rank_result() + req(rank_res) + omni_res <- get_dsc_omnibus(rank_res, method = input$RT_Stats.DSC.Omni_options, + alpha = input$RT_Stats.DSC.Alpha_omni) + + }, message = "Creating Comparison, this might take a while") + }) +}) + + +output$RT_Stats.DSC.Output_omni <- renderText({ + res_omni <- RT_DSC_omni_result() + req(res_omni) + paste0(res_omni$message, "(p-value: ", res_omni$p_value, ")") +}) + + +RT_DSC_posthoc_result <- reactive({ + input$RT_Stats.DSC.Create_posthoc + + isolate({ + withProgress({ + omni_res <- RT_DSC_omni_result() + req(omni_res) + data <- RT_DSC_data() + req(length(data) > 0) + df_posthoc <- rbindlist(lapply(get_id(data), function(algname){ + posthoc_res <- get_dsc_posthoc(omni_res, length(get_id(data)), + nrow(RT_stats_DSC_targets_obj), + alpha = input$RT_Stats.DSC.Alpha_posthoc, + base_algorithm = algname, + method = input$RT_Stats.DSC.Posthoc_test) + if (is.null(posthoc_res)) { return(NULL)} + values <- lapply(seq(4), function(method_idx) { + lapply(posthoc_res$adjusted_p_values[[method_idx]]$algorithms, + function(x) { + x$value + }) + }) + + algnames <- lapply(posthoc_res$adjusted_p_values[[1]]$algorithms, + function(x) { + x$algorithm + }) + df <- data.table("baseline" = algname, "compared alg" = algnames, z = values[[1]], "unadjusted P" = values[[2]], + "Holm" = values[[3]], "Hochberg" = values[[4]]) + + })) + as.data.table(format(df_posthoc, digits = 3)) + }, message = "Creating Comparison, this might take a while") + }) +}) + +output$RT_Stats.DSC.PosthocTable <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + RT_DSC_posthoc_result() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +RT_render_DSC_plot <- reactive({ + dt <- RT_DSC_posthoc_result() + if (is.null(dt) || nrow(dt) < 1) {return(NULL)} + # plot_general_data(dt[method == input$RT_Stats.DSC.method], x_attr = 'algId', y_attr = 'value', type = 'bar', + # legend_attr = 'algId') + dt <- dt[,c('baseline', 'compared alg', input$RT_Stats.DSC.Posthoc_method), with = F] + p_matrix <- acast(dt, baseline ~ `compared alg`, value.var = input$RT_Stats.DSC.Posthoc_method) + storage.mode(p_matrix) <- "numeric" + y <- p_matrix <= input$RT_Stats.DSC.Alpha_posthoc + colorScale <- data.frame(x = c(-1, -0.33, -0.33, 0.33, 0.33, 1), + col = c('blue', 'blue', 'white', 'white', 'red', 'red') + ) + heatmap <- y - t(y) + + p <- plot_ly(x = colnames(y), y = rownames(y), z = heatmap, type = 'heatmap', + xgap = 0.2, ygap = 0.2, colorscale = colorScale, showscale = F) + p %<>% layout(yaxis = list(autorange = 'reversed', scaleratio = 1), + xaxis = list(tickangle = 45)) + p +}) + +output$RT_Stats.DSC.PosthocViz <- renderPlotly({ + RT_render_DSC_plot() +}) + +output$RT_Stats.DSC.DownloadTable <- downloadHandler( + filename = function() { + eval(RT_DSC_table_name) + }, + content = function(file) { + df <- RT_DSC_posthoc_result() + save_table(df, file) + } +) + +output$RT_Stats.DSC.Download <- downloadHandler( + filename = function() { + eval(RT_DSC_figure_name) + }, + content = function(file) { + save_plotly(RT_render_DSC_plot(), file) + }, + contentType = paste0('image/', input$RT_Stats.DSC.Format) +) + + +RT_DSC_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.DSC.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$RT_Stats.DSC.Dim) + data <- subset(data, funcId %in% input$RT_Stats.DSC.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +RT_stats_DSC_targets <- reactive({ + data <- RT_DSC_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_RT") +}) + +RT_stats_DSC_targets_obj <- NULL + +proxy_RT_Stats.DSC.Targets <- dataTableProxy('RT_Stats.DSC.Targets') + +output$RT_Stats.DSC.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + RT_stats_DSC_targets_obj <<- RT_stats_DSC_targets() + RT_stats_DSC_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$RT_Stats.DSC.Targets_cell_edit, { + info <- input$RT_Stats.DSC.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- RT_DSC_data() + if (is.null(data)) return(NULL) + RT_stats_DSC_targets_obj$target[[i]] <<- v + replaceData(proxy, RT_stats_DSC_targets_obj, resetPaging = FALSE, rownames = FALSE) }) \ No newline at end of file diff --git a/inst/shiny-server/server/RT_ECDF.R b/inst/shiny-server/server/RT_ECDF.R index 97990789..74855e99 100644 --- a/inst/shiny-server/server/RT_ECDF.R +++ b/inst/shiny-server/server/RT_ECDF.R @@ -1,303 +1,303 @@ -output$RT_ECDF_MULT <- renderPlotly({ - req(length(DATA_RAW()) > 0) - render_RT_ECDF_MULT() -}) - -get_data_RT_ECDF_MULT <- reactive({ - req(input$RTECDF.Aggr.Func || input$RTECDF.Aggr.Dim) - input$RTECDF.Aggr.Refresh - dsList <- subset(DATA_RAW(), algId %in% input$RTECDF.Aggr.Algs) - - if (!input$RTECDF.Aggr.Func) - dsList <- subset(dsList, funcId == input$Overall.Funcid) - - if (!input$RTECDF.Aggr.Dim) - dsList <- subset(dsList, DIM == input$Overall.Dim) - - if (length(dsList) <= 1) { - shinyjs::alert("This is an invalid configuration for this plot. \n - Please ensure that the dataset contains multiple functions / - dimensions to aggregate over.") - return(NULL) - } - - isolate({ - targets <- RT_ECDF_MULTI_TABLE() - }) - generate_data.ECDF(dsList, targets, input$RTECDF.Aggr.Logx) -}) - -render_RT_ECDF_MULT <- reactive({ - withProgress({ - plot_general_data(get_data_RT_ECDF_MULT(), 'x', 'mean', 'line', - scale.xlog = input$RTECDF.Aggr.Logx, - scale.ylog = input$RTECDF.Aggr.Logy, - x_title = "Function Evaluations", - y_title = "Proportion of (run, target, ...) pairs", - show.legend = T) - }, - message = "Creating plot") -}) - -output$RTECDF.Aggr.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_ECDF_MULT) - }, - content = function(file) { - save_plotly(render_RT_ECDF_MULT(), file) - }, - contentType = paste0('image/', input$RTECDF.Aggr.Format) -) - -RT_ECDF_MULTI_TABLE <- reactiveVal(NULL) -trigger_renderDT <- reactiveVal(NULL) -proxy <- dataTableProxy('RT_GRID_GENERATED') - -observe({ - alg <- input$RTECDF.Aggr.Algs - funcid <- input$Overall.Funcid - data <- DATA_RAW() - dim <- input$Overall.Dim - - req(length(data) > 0) - req(alg) - req(funcid) - req(dim) - # req(data$suite != NEVERGRAD) - - dsList <- subset(data, algId %in% alg) - req(length(dsList) > 0) - - if (!input$RTECDF.Aggr.Func) - dsList <- subset(dsList, funcId == input$Overall.Funcid) - - if (!input$RTECDF.Aggr.Dim) - dsList <- subset(dsList, DIM == input$Overall.Dim) - - if (length(dsList) == 0) return(NULL) - - targets <- get_ECDF_targets(dsList, input$RTECDF.Aggr.Target_type, input$RTECDF.Aggr.Target_number) - - RT_ECDF_MULTI_TABLE(targets) # add values to `RT_ECDF_MULTI_TABLE` - trigger_renderDT(rnorm(1)) -}) - -output$RTECDF.AUC.Table.Download <- downloadHandler( - filename = function() { - eval(AUC_ECDF_aggr_name) - }, - content = function(file) { - save_table(auc_grid_table(), file) - } -) - -auc_grid_table <- reactive({ - dt_ecdf <- get_data_RT_ECDF_MULT() - generate_data.AUC(NULL, NULL, dt_ecdf = dt_ecdf) -}) - -output$AUC_GRID_GENERATED <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - isolate({auc_grid_table()}) - }, - editable = FALSE, - rownames = FALSE, - options = list( - pageLength = 10, - lengthMenu = c(5, 10, 25, -1), - scrollX = T, - server = T, - columnDefs = list( - list( - className = 'dt-right', targets = "_all" - ) - ) - ), - filter = 'top' -) - -output$RT_GRID_GENERATED <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - trigger_renderDT() - isolate({RT_ECDF_MULTI_TABLE()}) - }, - editable = TRUE, - rownames = FALSE, - options = list( - pageLength = 10, - lengthMenu = c(5, 10, 25, -1), - scrollX = T, - server = T, - columnDefs = list( - list( - className = 'dt-right', targets = "_all" - ) - ) - ), - filter = 'top' -) - -observeEvent(input$RT_GRID_GENERATED_cell_edit, { - info <- input$RT_GRID_GENERATED_cell_edit - i <- info$row - j <- info$col + 1 - v <- info$value - - suppressWarnings({ - df <- RT_ECDF_MULTI_TABLE() - set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) - RT_ECDF_MULTI_TABLE(df) - }) - replaceData(proxy, RT_ECDF_MULTI_TABLE(), - resetPaging = FALSE, rownames = FALSE) -}) - -observeEvent(input$RTECDF.Aggr.Table.Upload, { - if (!is.null(input$RTECDF.Aggr.Table.Upload)) { - df <- read.csv2(input$RTECDF.Aggr.Table.Upload$datapath, header = T) %>% - as.data.table - - if (ncol(df) == ncol(RT_ECDF_MULTI_TABLE())) { - RT_ECDF_MULTI_TABLE(df) - replaceData(proxy, RT_ECDF_MULTI_TABLE(), - resetPaging = FALSE, - rownames = FALSE) - } else { - RT_ECDF_MULTI_TABLE(df) - trigger_renderDT(rnorm(1)) - } - } else - NULL -}) - -output$RTECDF.Aggr.Table.Download <- downloadHandler( - filename = 'Example_ECDF_TARGETS.csv', - content = function(file) { - write.csv2(RT_ECDF_MULTI_TABLE(), file, row.names = F) - }, - contentType = "text/csv" -) - -# The ECDF plots for the runtime ---------------- - -get_data_RT_ECDF_Single <- reactive({ - ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) - data <- subset(DATA(), algId %in% input$RTECDF.Single.Algs) - generate_data.ECDF(data, ftargets, input$RTECDF.Single.Logx) -}) - -output$RT_ECDF <- renderPlotly({ - req(input$RTECDF.Single.Target) - plot_general_data(get_data_RT_ECDF_Single(), 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of runs", scale.xlog = input$RTECDF.Single.Logx, show.legend = T) - # ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) - # data <- subset(DATA(), algId %in% input$RTECDF.Single.Algs) - # Plot.RT.ECDF_Per_Target(data, ftargets, scale.xlog = input$RTECDF.Single.Logx) -}) - -output$AUC_GRID_GENERATED_SINGLE <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_Single()) -}) - -output$AUC_GRID_GENERATED_FUNC <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - input$RTECDF.Aggr.Refresh - generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_AGGR()) -}) - - -output$RT_GRID <- renderPrint({ - req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) - - fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric - fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric - fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric - - req(fstart <= fstop, fstep <= fstop - fstart) - data <- DATA() - fall <- get_funvals(data) - - cat(seq_FV(fall, fstart, fstop, by = fstep)) -}) - -output$RT_ECDF_AGGR <- renderPlotly({ - render_RT_ECDF_AGGR() -}) - -output$RTECDF.Multi.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_ECDF_AGGR) - }, - content = function(file) { - save_plotly(render_RT_ECDF_AGGR(), file) - }, - contentType = paste0('image/', input$RTECDF.Multi.Format) -) - -get_data_RT_ECDF_AGGR <- reactive({ - req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) - fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric - fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric - fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric - data <- subset(DATA(), algId %in% input$RTECDF.Multi.Algs) - targets <- seq_FV(get_funvals(data), fstart, fstop, fstep) - generate_data.ECDF(data, targets, input$RTECDF.Multi.Logx) -}) - -render_RT_ECDF_AGGR <- reactive({ - withProgress({ - - plot_general_data(get_data_RT_ECDF_AGGR(), 'x', 'mean', 'line', - x_title = "Function Evaluations", - y_title = "Proportion of (run, target) pairs", - scale.xlog = input$RTECDF.Multi.Logx, show.legend = T) - - # Plot.RT.ECDF_Single_Func( - # data, fstart, fstop, fstep, - # scale.xlog = input$RTECDF.Multi.Logx - # ) - }, - message = "Creating plot") -}) - -# # evaluation rake of all courses -# output$RT_AUC <- renderPlotly({ -# render_RT_AUC() -# }) -# -# output$RTECDF.AUC.Download <- downloadHandler( -# filename = function() { -# eval(FIG_NAME_RT_AUC) -# }, -# content = function(file) { -# save_plotly(render_RT_AUC(), file) -# }, -# contentType = paste0('image/', input$RTECDF.AUC.Format) -# ) -# -# get_data_RT_AUC <- reactive({ -# req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) -# -# fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric -# fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric -# fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric -# data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) -# targets <- seq_FV(get_funvals(data), fstart, fstop, fstep, length.out = 10) -# generate_data.AUC(data, targets) -# }) -# -# render_RT_AUC <- reactive({ -# # req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) -# # -# # fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric -# # fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric -# # fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric -# # data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) -# # -# # Plot.RT.ECDF_AUC( -# # data, fstart, fstop, fstep, fval_formatter = format_FV -# # ) -# plot_general_data(get_data_RT_AUC(), 'x', 'AUC', 'radar') -# }) +output$RT_ECDF_MULT <- renderPlotly({ + req(length(DATA_RAW()) > 0) + render_RT_ECDF_MULT() +}) + +get_data_RT_ECDF_MULT <- reactive({ + req(input$RTECDF.Aggr.Func || input$RTECDF.Aggr.Dim) + input$RTECDF.Aggr.Refresh + dsList <- subset(DATA_RAW(), ID %in% input$RTECDF.Aggr.Algs) + + if (!input$RTECDF.Aggr.Func) + dsList <- subset(dsList, funcId == input$Overall.Funcid) + + if (!input$RTECDF.Aggr.Dim) + dsList <- subset(dsList, DIM == input$Overall.Dim) + + if (length(dsList) <= 1) { + shinyjs::alert("This is an invalid configuration for this plot. \n + Please ensure that the dataset contains multiple functions / + dimensions to aggregate over.") + return(NULL) + } + + isolate({ + targets <- RT_ECDF_MULTI_TABLE() + }) + generate_data.ECDF(dsList, targets, input$RTECDF.Aggr.Logx) +}) + +render_RT_ECDF_MULT <- reactive({ + withProgress({ + plot_general_data(get_data_RT_ECDF_MULT(), 'x', 'mean', 'line', + scale.xlog = input$RTECDF.Aggr.Logx, + scale.ylog = input$RTECDF.Aggr.Logy, + x_title = "Function Evaluations", + y_title = "Proportion of (run, target, ...) pairs", + show.legend = T) + }, + message = "Creating plot") +}) + +output$RTECDF.Aggr.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_ECDF_MULT) + }, + content = function(file) { + save_plotly(render_RT_ECDF_MULT(), file) + }, + contentType = paste0('image/', input$RTECDF.Aggr.Format) +) + +RT_ECDF_MULTI_TABLE <- reactiveVal(NULL) +trigger_renderDT <- reactiveVal(NULL) +proxy <- dataTableProxy('RT_GRID_GENERATED') + +observe({ + alg <- input$RTECDF.Aggr.Algs + funcid <- input$Overall.Funcid + data <- DATA_RAW() + dim <- input$Overall.Dim + + req(length(data) > 0) + req(alg) + req(funcid) + req(dim) + # req(data$suite != NEVERGRAD) + + dsList <- subset(data, ID %in% alg) + req(length(dsList) > 0) + + if (!input$RTECDF.Aggr.Func) + dsList <- subset(dsList, funcId == input$Overall.Funcid) + + if (!input$RTECDF.Aggr.Dim) + dsList <- subset(dsList, DIM == input$Overall.Dim) + + if (length(dsList) == 0) return(NULL) + + targets <- get_ECDF_targets(dsList, input$RTECDF.Aggr.Target_type, input$RTECDF.Aggr.Target_number) + + RT_ECDF_MULTI_TABLE(targets) # add values to `RT_ECDF_MULTI_TABLE` + trigger_renderDT(rnorm(1)) +}) + +output$RTECDF.AUC.Table.Download <- downloadHandler( + filename = function() { + eval(AUC_ECDF_aggr_name) + }, + content = function(file) { + save_table(auc_grid_table(), file) + } +) + +auc_grid_table <- reactive({ + dt_ecdf <- get_data_RT_ECDF_MULT() + generate_data.AUC(NULL, NULL, dt_ecdf = dt_ecdf) +}) + +output$AUC_GRID_GENERATED <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + isolate({auc_grid_table()}) + }, + editable = FALSE, + rownames = FALSE, + options = list( + pageLength = 10, + lengthMenu = c(5, 10, 25, -1), + scrollX = T, + server = T, + columnDefs = list( + list( + className = 'dt-right', targets = "_all" + ) + ) + ), + filter = 'top' +) + +output$RT_GRID_GENERATED <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + trigger_renderDT() + isolate({RT_ECDF_MULTI_TABLE()}) + }, + editable = TRUE, + rownames = FALSE, + options = list( + pageLength = 10, + lengthMenu = c(5, 10, 25, -1), + scrollX = T, + server = T, + columnDefs = list( + list( + className = 'dt-right', targets = "_all" + ) + ) + ), + filter = 'top' +) + +observeEvent(input$RT_GRID_GENERATED_cell_edit, { + info <- input$RT_GRID_GENERATED_cell_edit + i <- info$row + j <- info$col + 1 + v <- info$value + + suppressWarnings({ + df <- RT_ECDF_MULTI_TABLE() + set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) + RT_ECDF_MULTI_TABLE(df) + }) + replaceData(proxy, RT_ECDF_MULTI_TABLE(), + resetPaging = FALSE, rownames = FALSE) +}) + +observeEvent(input$RTECDF.Aggr.Table.Upload, { + if (!is.null(input$RTECDF.Aggr.Table.Upload)) { + df <- read.csv2(input$RTECDF.Aggr.Table.Upload$datapath, header = T) %>% + as.data.table + + if (ncol(df) == ncol(RT_ECDF_MULTI_TABLE())) { + RT_ECDF_MULTI_TABLE(df) + replaceData(proxy, RT_ECDF_MULTI_TABLE(), + resetPaging = FALSE, + rownames = FALSE) + } else { + RT_ECDF_MULTI_TABLE(df) + trigger_renderDT(rnorm(1)) + } + } else + NULL +}) + +output$RTECDF.Aggr.Table.Download <- downloadHandler( + filename = 'Example_ECDF_TARGETS.csv', + content = function(file) { + write.csv2(RT_ECDF_MULTI_TABLE(), file, row.names = F) + }, + contentType = "text/csv" +) + +# The ECDF plots for the runtime ---------------- + +get_data_RT_ECDF_Single <- reactive({ + ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) + data <- subset(DATA(), ID %in% input$RTECDF.Single.Algs) + generate_data.ECDF(data, ftargets, input$RTECDF.Single.Logx) +}) + +output$RT_ECDF <- renderPlotly({ + req(input$RTECDF.Single.Target) + plot_general_data(get_data_RT_ECDF_Single(), 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of runs", scale.xlog = input$RTECDF.Single.Logx, show.legend = T) + # ftargets <- as.numeric(format_FV(input$RTECDF.Single.Target)) + # data <- subset(DATA(), algId %in% input$RTECDF.Single.Algs) + # Plot.RT.ECDF_Per_Target(data, ftargets, scale.xlog = input$RTECDF.Single.Logx) +}) + +output$AUC_GRID_GENERATED_SINGLE <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_Single()) +}) + +output$AUC_GRID_GENERATED_FUNC <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + input$RTECDF.Aggr.Refresh + generate_data.AUC(NULL, NULL, dt_ecdf = get_data_RT_ECDF_AGGR()) +}) + + +output$RT_GRID <- renderPrint({ + req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) + + fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric + fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric + fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric + + req(fstart <= fstop, fstep <= fstop - fstart) + data <- DATA() + fall <- get_funvals(data) + + cat(seq_FV(fall, fstart, fstop, by = fstep)) +}) + +output$RT_ECDF_AGGR <- renderPlotly({ + render_RT_ECDF_AGGR() +}) + +output$RTECDF.Multi.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_ECDF_AGGR) + }, + content = function(file) { + save_plotly(render_RT_ECDF_AGGR(), file) + }, + contentType = paste0('image/', input$RTECDF.Multi.Format) +) + +get_data_RT_ECDF_AGGR <- reactive({ + req(input$RTECDF.Multi.Min, input$RTECDF.Multi.Max, input$RTECDF.Multi.Step) + fstart <- format_FV(input$RTECDF.Multi.Min) %>% as.numeric + fstop <- format_FV(input$RTECDF.Multi.Max) %>% as.numeric + fstep <- format_FV(input$RTECDF.Multi.Step) %>% as.numeric + data <- subset(DATA(), ID %in% input$RTECDF.Multi.Algs) + targets <- seq_FV(get_funvals(data), fstart, fstop, fstep) + generate_data.ECDF(data, targets, input$RTECDF.Multi.Logx) +}) + +render_RT_ECDF_AGGR <- reactive({ + withProgress({ + + plot_general_data(get_data_RT_ECDF_AGGR(), 'x', 'mean', 'line', + x_title = "Function Evaluations", + y_title = "Proportion of (run, target) pairs", + scale.xlog = input$RTECDF.Multi.Logx, show.legend = T) + + # Plot.RT.ECDF_Single_Func( + # data, fstart, fstop, fstep, + # scale.xlog = input$RTECDF.Multi.Logx + # ) + }, + message = "Creating plot") +}) + +# # evaluation rake of all courses +# output$RT_AUC <- renderPlotly({ +# render_RT_AUC() +# }) +# +# output$RTECDF.AUC.Download <- downloadHandler( +# filename = function() { +# eval(FIG_NAME_RT_AUC) +# }, +# content = function(file) { +# save_plotly(render_RT_AUC(), file) +# }, +# contentType = paste0('image/', input$RTECDF.AUC.Format) +# ) +# +# get_data_RT_AUC <- reactive({ +# req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) +# +# fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric +# fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric +# fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric +# data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) +# targets <- seq_FV(get_funvals(data), fstart, fstop, fstep, length.out = 10) +# generate_data.AUC(data, targets) +# }) +# +# render_RT_AUC <- reactive({ +# # req(input$RTECDF.AUC.Min, input$RTECDF.AUC.Max, input$RTECDF.AUC.Step) +# # +# # fstart <- format_FV(input$RTECDF.AUC.Min) %>% as.numeric +# # fstop <- format_FV(input$RTECDF.AUC.Max) %>% as.numeric +# # fstep <- format_FV(input$RTECDF.AUC.Step) %>% as.numeric +# # data <- subset(DATA(), algId %in% input$RTECDF.AUC.Algs) +# # +# # Plot.RT.ECDF_AUC( +# # data, fstart, fstop, fstep, fval_formatter = format_FV +# # ) +# plot_general_data(get_data_RT_AUC(), 'x', 'AUC', 'radar') +# }) diff --git a/inst/shiny-server/server/RT_Par.R b/inst/shiny-server/server/RT_Par.R index b92d2aef..232b4f66 100644 --- a/inst/shiny-server/server/RT_Par.R +++ b/inst/shiny-server/server/RT_Par.R @@ -1,180 +1,181 @@ - -get_data_RT_PAR_PER_FUN <- reactive({ - data <- subset(DATA(), algId %in% input$RT_PAR.Plot.Algs) - generate_data.Parameters(data, scale_log = input$RT_PAR.Plot.Logx, which = 'by_FV') -}) - -# Expected Evolution of parameters in the algorithm -render_RT_PAR_PER_FUN <- reactive({ - req(input$RT_PAR.Plot.Min, input$RT_PAR.Plot.Max) - withProgress({ - f_min <- as.numeric(format_FV(input$RT_PAR.Plot.Min)) - f_max <- as.numeric(format_FV(input$RT_PAR.Plot.Max)) - - dt <- get_data_RT_PAR_PER_FUN() - dt <- dt[parId %in% input$RT_PAR.Plot.Params] - sub_attr <- if (length(input$RT_PAR.Plot.Params) > 1) 'parId' else NULL - - lower <- 'lower' - upper <- 'upper' - if (input$RT_PAR.Plot.CI == 'None') type <- 'line' - else { - type <- 'line+ribbon' - if (input$RT_PAR.Plot.CI == 'Outer Quantiles') { - quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') - lower <- quantiles[[1]] - upper <- quantiles[[length(quantiles)]] - } - } - if (is.null(sub_attr) && type == 'line+ribbon') { - p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'line', - subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, - scale.ylog = input$RT_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = T) - p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'ribbon', - subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, - scale.ylog = input$RT_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = F, p = p) - } - else { - p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, type, - subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, - scale.ylog = input$RT_PAR.Plot.Logy, - lower_attr = lower, upper_attr = upper, show.legend = T) - } - p - }, - message = "Creating plot") -}) - -output$RT_PAR.Plot.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_PAR_PER_FUN) - }, - content = function(file) { - save_plotly(render_RT_PAR_PER_FUN(), file) - }, - contentType = paste0('image/', input$RT_PAR.Plot.Format) -) - -output$RT_PAR.Plot.Figure <- renderPlotly({ - render_RT_PAR_PER_FUN() -}) - -# TODO: add ks test for ECDF later -# output$ks <- renderPrint({ -# target <- input$target %>% as.numeric -# df.aligneds <- aligned() -# -# running_time <- list() -# for (i in seq_along(df.aligneds)) { -# df <- df.aligneds[[i]] -# v <- rownames(df) %>% as.numeric -# idx <- order(abs(target - v))[1] -# running_time[[i]] <- df[idx, ] %>% as.vector -# } -# algorithm1 <- running_time[[1]] -# algorithm2 <- running_time[[2]] -# a <- ks.test(algorithm1, algorithm2, alternative = 'less') -# print(a) -# }) -# - -rt_parameter_summary <- reactive({ - req(input$RT_PAR.Summary.Min, input$RT_PAR.Summary.Max, input$RT_PAR.Summary.Step) - - fstart <- format_FV(input$RT_PAR.Summary.Min) %>% as.numeric - fstop <- format_FV(input$RT_PAR.Summary.Max) %>% as.numeric - fstep <- format_FV(input$RT_PAR.Summary.Step) %>% as.numeric - data <- DATA() - - if (!input$RT_PAR.Summary.Single) { - req(fstart <= fstop, fstep <= fstop - fstart) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, by = fstep) - req(fseq) - } - else - fseq <- fstart - - dt <- get_PAR_summary(data, fseq, input$RT_PAR.Summary.Algid, input$RT_PAR.Summary.Param) - req(length(dt) != 0) - - dt$runs %<>% as.integer - dt$mean %<>% format(digits = 2, nsmall = 2) - dt$median %<>% format(digits = 2, nsmall = 2) - dt$sd %<>% format(digits = 2, nsmall = 2) - - probs <- getOption("IOHanalyzer.quantiles") - - # format the integers - for (p in paste0(probs * 100, '%')) { - dt[[p]] %<>% format(digits = 2, nsmall = 2) - } - dt -}) - -rt_parameter_sample <- reactive({ - req(input$RT_PAR.Sample.Algid, input$RT_PAR.Sample.Max, - input$RT_PAR.Sample.Step, input$RT_PAR.Sample.Min, - input$RT_PAR.Sample.Param) - - fstart <- format_FV(input$RT_PAR.Sample.Min) %>% as.numeric - fstop <- format_FV(input$RT_PAR.Sample.Max) %>% as.numeric - fstep <- format_FV(input$RT_PAR.Sample.Step) %>% as.numeric - data <- DATA() - - if (!input$RT_PAR.Sample.Single) { - req(fstart <= fstop, fstep <= fstop - fstart) - fall <- get_funvals(data) - fseq <- seq_FV(fall, fstart, fstop, by = fstep) - req(fseq) - } - else - fseq <- fstart - - df <- get_PAR_sample(data, idxValue = fseq, - algorithm = input$RT_PAR.Sample.Algid, - parId = input$RT_PAR.Sample.Param, - output = input$RT_PAR.Sample.Format) - - for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) - df[[p]] %<>% format(digits = 2, nsmall = 2) - - df -}) - -output$table_RT_PAR_SAMPLE <- DT::renderDataTable({ - dt <- rt_parameter_sample() - req(length(dt) != 0) - dt[is.na(dt)] <- 'NA' - dt -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$table_RT_PAR_summary <- DT::renderDataTable({ - rt_parameter_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) - -output$RT_PAR.Sample.Download <- downloadHandler( - filename = function() { - eval(RT_PARSample_csv_name) - }, - content = function(file) { - df <- parameter_sample() - df <- df[input[["table_RT_PAR_SAMPLE_rows_all"]]] - save_table(df, file) - } -) - -output$RT_PAR.Summary.Download <- downloadHandler( - filename = function() { - eval(RT_PAR_csv_name) - }, - content = function(file) { - df <- rt_parameter_summary() - df <- df[input[["table_RT_PAR_summary_rows_all"]]] - save_table(df, file) - } + +get_data_RT_PAR_PER_FUN <- reactive({ + data <- subset(DATA(), ID %in% input$RT_PAR.Plot.Algs) + generate_data.Parameters(data, scale_log = input$RT_PAR.Plot.Logx, which = 'by_FV') +}) + +# Expected Evolution of parameters in the algorithm +render_RT_PAR_PER_FUN <- reactive({ + req(input$RT_PAR.Plot.Min, input$RT_PAR.Plot.Max) + withProgress({ + f_min <- as.numeric(format_FV(input$RT_PAR.Plot.Min)) + f_max <- as.numeric(format_FV(input$RT_PAR.Plot.Max)) + + dt <- get_data_RT_PAR_PER_FUN() + dt <- dt[parId %in% input$RT_PAR.Plot.Params] + sub_attr <- if (length(input$RT_PAR.Plot.Params) > 1) 'parId' else NULL + + lower <- 'lower' + upper <- 'upper' + if (input$RT_PAR.Plot.CI == 'None') type <- 'line' + else { + type <- 'line+ribbon' + if (input$RT_PAR.Plot.CI == 'Outer Quantiles') { + quantiles <- paste0(getOption("IOHanalyzer.quantiles", c(0.2, 0.98)) * 100, '%') + lower <- quantiles[[1]] + upper <- quantiles[[length(quantiles)]] + } + } + if (is.null(sub_attr) && type == 'line+ribbon') { + p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'line', + subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, + scale.ylog = input$RT_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = T) + p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, 'ribbon', + subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, + scale.ylog = input$RT_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = F, p = p) + } + else { + p <- plot_general_data(dt, 'target', input$RT_PAR.Plot.show.mean, type, + subplot_attr = sub_attr, scale.xlog = input$RT_PAR.Plot.Logx, + scale.ylog = input$RT_PAR.Plot.Logy, + lower_attr = lower, upper_attr = upper, show.legend = T) + } + p + }, + message = "Creating plot") +}) + +output$RT_PAR.Plot.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_PAR_PER_FUN) + }, + content = function(file) { + save_plotly(render_RT_PAR_PER_FUN(), file) + }, + contentType = paste0('image/', input$RT_PAR.Plot.Format) +) + +output$RT_PAR.Plot.Figure <- renderPlotly({ + render_RT_PAR_PER_FUN() +}) + +# TODO: add ks test for ECDF later +# output$ks <- renderPrint({ +# target <- input$target %>% as.numeric +# df.aligneds <- aligned() +# +# running_time <- list() +# for (i in seq_along(df.aligneds)) { +# df <- df.aligneds[[i]] +# v <- rownames(df) %>% as.numeric +# idx <- order(abs(target - v))[1] +# running_time[[i]] <- df[idx, ] %>% as.vector +# } +# algorithm1 <- running_time[[1]] +# algorithm2 <- running_time[[2]] +# a <- ks.test(algorithm1, algorithm2, alternative = 'less') +# print(a) +# }) +# + +rt_parameter_summary <- reactive({ + req(input$RT_PAR.Summary.Min, input$RT_PAR.Summary.Max, input$RT_PAR.Summary.Step) + + fstart <- format_FV(input$RT_PAR.Summary.Min) %>% as.numeric + fstop <- format_FV(input$RT_PAR.Summary.Max) %>% as.numeric + fstep <- format_FV(input$RT_PAR.Summary.Step) %>% as.numeric + data <- DATA() + data <- subset(data, ID %in% input$RT_PAR.Summary.ID) + + if (!input$RT_PAR.Summary.Single) { + req(fstart <= fstop, fstep <= fstop - fstart) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, by = fstep) + req(fseq) + } + else + fseq <- fstart + + dt <- get_PAR_summary(data, fseq, parId = input$RT_PAR.Summary.Param) + req(length(dt) != 0) + + dt$runs %<>% as.integer + dt$mean %<>% format(digits = 2, nsmall = 2) + dt$median %<>% format(digits = 2, nsmall = 2) + dt$sd %<>% format(digits = 2, nsmall = 2) + + probs <- getOption("IOHanalyzer.quantiles") + + # format the integers + for (p in paste0(probs * 100, '%')) { + dt[[p]] %<>% format(digits = 2, nsmall = 2) + } + dt +}) + +rt_parameter_sample <- reactive({ + req(input$RT_PAR.Sample.ID, input$RT_PAR.Sample.Max, + input$RT_PAR.Sample.Step, input$RT_PAR.Sample.Min, + input$RT_PAR.Sample.Param) + + fstart <- format_FV(input$RT_PAR.Sample.Min) %>% as.numeric + fstop <- format_FV(input$RT_PAR.Sample.Max) %>% as.numeric + fstep <- format_FV(input$RT_PAR.Sample.Step) %>% as.numeric + data <- DATA() + data <- subset(data, ID %in% input$RT_PAR.Sample.ID) + + if (!input$RT_PAR.Sample.Single) { + req(fstart <= fstop, fstep <= fstop - fstart) + fall <- get_funvals(data) + fseq <- seq_FV(fall, fstart, fstop, by = fstep) + req(fseq) + } + else + fseq <- fstart + + df <- get_PAR_sample(data, idxValue = fseq, + parId = input$RT_PAR.Sample.Param, + output = input$RT_PAR.Sample.Format) + + for (p in paste0('run.', seq(ncol(data[[1]]$FV)))) + df[[p]] %<>% format(digits = 2, nsmall = 2) + + df +}) + +output$table_RT_PAR_SAMPLE <- DT::renderDataTable({ + dt <- rt_parameter_sample() + req(length(dt) != 0) + dt[is.na(dt)] <- 'NA' + dt +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$table_RT_PAR_summary <- DT::renderDataTable({ + rt_parameter_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T, digits = 2)) + +output$RT_PAR.Sample.Download <- downloadHandler( + filename = function() { + eval(RT_PARSample_csv_name) + }, + content = function(file) { + df <- parameter_sample() + df <- df[input[["table_RT_PAR_SAMPLE_rows_all"]]] + save_table(df, file) + } +) + +output$RT_PAR.Summary.Download <- downloadHandler( + filename = function() { + eval(RT_PAR_csv_name) + }, + content = function(file) { + df <- rt_parameter_summary() + df <- df[input[["table_RT_PAR_summary_rows_all"]]] + save_table(df, file) + } ) \ No newline at end of file diff --git a/inst/shiny-server/server/Shapley_computations.R b/inst/shiny-server/server/Shapley_computations.R index c4766685..9602ca14 100644 --- a/inst/shiny-server/server/Shapley_computations.R +++ b/inst/shiny-server/server/Shapley_computations.R @@ -1,123 +1,123 @@ -output$RT_SHAPLEY <- renderPlotly({ - req(length(DATA_RAW()) > 0) - render_RT_SHAPLEY() -}) - -get_data_RT_SHAPLEY <- reactive({ - input$RTportfolio.Shapley.Refresh - dsList <- subset(DATA_RAW(), algId %in% input$RTportfolio.Shapley.Algs & - funcId %in% input$RTportfolio.Shapley.Funcs & - DIM %in% input$RTportfolio.Shapley.Dims) - - - if (length(get_algId(dsList)) <= 1) { - shinyjs::alert("This is an invalid configuration for this plot. \n - Please ensure that the dataset contains multiple algorithms.") - return(NULL) - } - - isolate({ - targets <- RT_SHAPLEY_TARGETS_TABLE() - }) - get_shapley_values(dsList, targets, input$RTportfolio.Shapley.Logx, input$RTportfolio.Shapley.Groupsize, - input$RTportfolio.Shapley.Permsize) -}) - -render_RT_SHAPLEY <- reactive({ - withProgress({ - dt <- get_data_RT_SHAPLEY() - plot_general_data(dt, x_attr = 'algId', y_attr = 'shapley', type = 'bar', - legend_attr = 'algId', show.legend = T) - }, - message = "Creating plot") -}) - -output$RTportfolio.Shapley.Download <- downloadHandler( - filename = function() { - eval(FIG_NAME_RT_SHAPLEY) - }, - content = function(file) { - save_plotly(render_RT_SHAPLEY(), file) - }, - contentType = paste0('image/', input$RTportfolio.Shapley.Format) -) - -RT_SHAPLEY_TARGETS_TABLE <- reactiveVal(NULL) -trigger_renderDT_shap <- reactiveVal(NULL) -proxy <- dataTableProxy('RT_SHAPLEY_TARGETS_GENERATED') - -observe({ - req(length(DATA_RAW()) > 0) - dsList <- subset(DATA_RAW(), algId %in% input$RTportfolio.Shapley.Algs & - funcId %in% input$RTportfolio.Shapley.Funcs & - DIM %in% input$RTportfolio.Shapley.Dims) - req(length(dsList) > 0) - - targets <- get_ECDF_targets(dsList, input$RTportfolio.Shapley.Target_type, input$RTportfolio.Shapley.Target_number) - - RT_SHAPLEY_TARGETS_TABLE(targets) # add values to `RT_SHAPLEY_TARGETS_TABLE` - trigger_renderDT_shap(rnorm(1)) -}) - - -output$RT_SHAPLEY_TARGETS_GENERATED <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - trigger_renderDT_shap() - isolate({RT_SHAPLEY_TARGETS_TABLE()}) -}, -editable = TRUE, -rownames = FALSE, -options = list( - pageLength = 10, - lengthMenu = c(5, 10, 25, -1), - scrollX = T, - server = T, - columnDefs = list( - list( - className = 'dt-right', targets = "_all" - ) - ) -), -filter = 'top' -) - -observeEvent(input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit, { - info <- input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit - i <- info$row - j <- info$col + 1 - v <- info$value - - suppressWarnings({ - df <- RT_SHAPLEY_TARGETS_TABLE() - set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) - RT_SHAPLEY_TARGETS_TABLE(df) - }) - replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), - resetPaging = FALSE, rownames = FALSE) -}) - -observeEvent(input$RTportfolio.Shapley.Table.Upload, { - if (!is.null(input$RTportfolio.Shapley.Table.Upload)) { - df <- read.csv2(input$RTportfolio.Shapley.Table.Upload$datapath, header = T, row.names = F) %>% - as.data.table - - if (ncol(df) == ncol(RT_SHAPLEY_TARGETS_TABLE())) { - RT_SHAPLEY_TARGETS_TABLE(df) - replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), - resetPaging = FALSE, - rownames = FALSE) - } else { - RT_SHAPLEY_TARGETS_TABLE(df) - trigger_renderDT_shap(rnorm(1)) - } - } else - NULL -}) - -output$RTportfolio.Shapley.Table.Download <- downloadHandler( - filename = 'Example_ECDF_TARGETS.csv', - content = function(file) { - write.csv2(RT_SHAPLEY_TARGETS_TABLE(), file, row.names = F) - }, - contentType = "text/csv" +output$RT_SHAPLEY <- renderPlotly({ + req(length(DATA_RAW()) > 0) + render_RT_SHAPLEY() +}) + +get_data_RT_SHAPLEY <- reactive({ + input$RTportfolio.Shapley.Refresh + dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & + funcId %in% input$RTportfolio.Shapley.Funcs & + DIM %in% input$RTportfolio.Shapley.Dims) + + + if (length(get_id(dsList)) <= 1) { + shinyjs::alert("This is an invalid configuration for this plot. \n + Please ensure that the dataset contains multiple IDs.") + return(NULL) + } + + isolate({ + targets <- RT_SHAPLEY_TARGETS_TABLE() + }) + get_shapley_values(dsList, targets, input$RTportfolio.Shapley.Logx, input$RTportfolio.Shapley.Groupsize, + input$RTportfolio.Shapley.Permsize) +}) + +render_RT_SHAPLEY <- reactive({ + withProgress({ + dt <- get_data_RT_SHAPLEY() + plot_general_data(dt, x_attr = 'ID', y_attr = 'shapley', type = 'bar', + legend_attr = 'ID', show.legend = T) + }, + message = "Creating plot") +}) + +output$RTportfolio.Shapley.Download <- downloadHandler( + filename = function() { + eval(FIG_NAME_RT_SHAPLEY) + }, + content = function(file) { + save_plotly(render_RT_SHAPLEY(), file) + }, + contentType = paste0('image/', input$RTportfolio.Shapley.Format) +) + +RT_SHAPLEY_TARGETS_TABLE <- reactiveVal(NULL) +trigger_renderDT_shap <- reactiveVal(NULL) +proxy <- dataTableProxy('RT_SHAPLEY_TARGETS_GENERATED') + +observe({ + req(length(DATA_RAW()) > 0) + dsList <- subset(DATA_RAW(), ID %in% input$RTportfolio.Shapley.Algs & + funcId %in% input$RTportfolio.Shapley.Funcs & + DIM %in% input$RTportfolio.Shapley.Dims) + req(length(dsList) > 0) + + targets <- get_ECDF_targets(dsList, input$RTportfolio.Shapley.Target_type, input$RTportfolio.Shapley.Target_number) + + RT_SHAPLEY_TARGETS_TABLE(targets) # add values to `RT_SHAPLEY_TARGETS_TABLE` + trigger_renderDT_shap(rnorm(1)) +}) + + +output$RT_SHAPLEY_TARGETS_GENERATED <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + trigger_renderDT_shap() + isolate({RT_SHAPLEY_TARGETS_TABLE()}) +}, +editable = TRUE, +rownames = FALSE, +options = list( + pageLength = 10, + lengthMenu = c(5, 10, 25, -1), + scrollX = T, + server = T, + columnDefs = list( + list( + className = 'dt-right', targets = "_all" + ) + ) +), +filter = 'top' +) + +observeEvent(input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit, { + info <- input$RT_SHAPLEY_TARGETS_GENERATED_cell_edit + i <- info$row + j <- info$col + 1 + v <- info$value + + suppressWarnings({ + df <- RT_SHAPLEY_TARGETS_TABLE() + set(df, i, j, coerceValue(v, as.numeric(df[i, ..j]))) + RT_SHAPLEY_TARGETS_TABLE(df) + }) + replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), + resetPaging = FALSE, rownames = FALSE) +}) + +observeEvent(input$RTportfolio.Shapley.Table.Upload, { + if (!is.null(input$RTportfolio.Shapley.Table.Upload)) { + df <- read.csv2(input$RTportfolio.Shapley.Table.Upload$datapath, header = T, row.names = F) %>% + as.data.table + + if (ncol(df) == ncol(RT_SHAPLEY_TARGETS_TABLE())) { + RT_SHAPLEY_TARGETS_TABLE(df) + replaceData(proxy, RT_SHAPLEY_TARGETS_TABLE(), + resetPaging = FALSE, + rownames = FALSE) + } else { + RT_SHAPLEY_TARGETS_TABLE(df) + trigger_renderDT_shap(rnorm(1)) + } + } else + NULL +}) + +output$RTportfolio.Shapley.Table.Download <- downloadHandler( + filename = 'Example_ECDF_TARGETS.csv', + content = function(file) { + write.csv2(RT_SHAPLEY_TARGETS_TABLE(), file, row.names = F) + }, + contentType = "text/csv" ) \ No newline at end of file diff --git a/inst/shiny-server/server/fv_stat_tests.R b/inst/shiny-server/server/fv_stat_tests.R index 0da88088..f04ad7ff 100644 --- a/inst/shiny-server/server/fv_stat_tests.R +++ b/inst/shiny-server/server/fv_stat_tests.R @@ -1,161 +1,161 @@ -fv_render_heatmap <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$FV_Stats.Overview.Algid) - Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), - bootstrap.size = 0, which = 'by_RT') - }, - message = "Creating plot") -}) - - -output$FV_Stats.Overview.Heatmap <- renderPlotly( - fv_render_heatmap() -) - -fv_create_stats_table <- reactive({ - req(length(DATA()) > 0) - req(length(get_algId(DATA())) > 1) - data <- subset(DATA(), algId %in% input$FV_Stats.Overview.Algid) - target <- as.numeric(input$FV_Stats.Overview.Target) - df <- pairwise.test(data, target, bootstrap.size = 0, which = 'by_RT') - df <- format(df, digits = 3) - df -}) - -output$FV_Stats.Overview.Pmatrix <- DT::renderDataTable({ - fv_create_stats_table() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FV_Stats.Overview.Graph <- renderPlot({ - fv_render_graph() -}) - -fv_render_graph <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$FV_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$FV_Stats.Overview.Algid) - Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), - bootstrap.size = 0, which = 'by_RT') - }, - message = "Creating plot") -}) - - -output$FV_Stats.Overview.DownloadTable <- downloadHandler( - filename = function() { - eval(FV_Stats_table_name) - }, - content = function(file) { - df <- create_stats_table() - save_table(df, file) - } -) - -output$FV_Stats.Overview.DownloadHeatmap <- downloadHandler( - filename = function() { - eval(FV_Stats_heatmap_name) - }, - content = function(file) { - save_plotly(render_heatmap(), file) - }, - contentType = paste0('image/', input$FV_Stats.Overview.Format) -) - -fv_data_table_glicko2 <- reactive({ - input$FV_Stats.Glicko.Create - isolate({ - withProgress({ - data <- FV_glicko_data() - req(length(data) > 0 && length(get_algId(data)) > 0) - nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) - df <- glicko2_ranking(data, nr_games, which = 'by_RT', - target_dt = FV_stats_glicko_targets_obj)$ratings - format(df, digits = 3) - }, message = "Creating Ranking, this might take a while") - }) -}) - -output$FV_Stats.Glicko.Dataframe <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - fv_data_table_glicko2() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -fv_render_glico2_plot <- reactive({ - isolate({ - data <- FV_glicko_data() - nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) - }) - Plot.Stats.Glicko2_Candlestick(data, nr_games, fv_data_table_glicko2(), which = 'by_RT', - target_dt = FV_stats_glicko_targets_obj) -}) - -output$FV_Stats.Glicko.Candlestick <- renderPlotly({ - fv_render_glico2_plot() -}) - -output$FV_Stats.Glicko.DownloadTable <- downloadHandler( - filename = function() { - eval(FV_Glicko2_table_name) - }, - content = function(file) { - df <- fv_data_table_glicko2() - save_table(df, file) - } -) - -output$FV_Stats.Glicko.Download <- downloadHandler( - filename = function() { - eval(FV_Glicko2_figure_name) - }, - content = function(file) { - save_plotly(fv_render_glico2_plot(), file) - }, - contentType = paste0('image/', input$FV_Stats.Glicko.Format) -) - - -FV_glicko_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$FV_Stats.Glicko.Algid)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$FV_Stats.Glicko.Dim) - data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) - if (length(unique(get_algId(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") - return(NULL) - } - data -} - -FV_stats_glicko_targets <- reactive({ - data <- FV_glicko_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_RT") -}) - -FV_stats_glicko_targets_obj <- NULL - -proxy_FV_Stats.Glicko.Targets <- dataTableProxy('FV_Stats.Glicko.Targets') - -output$FV_Stats.Glicko.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - FV_stats_glicko_targets_obj <<- FV_stats_glicko_targets() - FV_stats_glicko_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$FV_Stats.Glicko.Targets_cell_edit, { - info <- input$FV_Stats.Glicko.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- FV_glicko_data() - if (is.null(data)) return(NULL) - FV_stats_glicko_targets_obj$target[[i]] <<- v - replaceData(proxy, FV_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) +fv_render_heatmap <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$FV_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) + Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), + bootstrap.size = 0, which = 'by_RT') + }, + message = "Creating plot") +}) + + +output$FV_Stats.Overview.Heatmap <- renderPlotly( + fv_render_heatmap() +) + +fv_create_stats_table <- reactive({ + req(length(DATA()) > 0) + req(length(get_id(DATA())) > 1) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) + target <- as.numeric(input$FV_Stats.Overview.Target) + df <- pairwise.test(data, target, bootstrap.size = 0, which = 'by_RT') + df <- format(df, digits = 3) + df +}) + +output$FV_Stats.Overview.Pmatrix <- DT::renderDataTable({ + fv_create_stats_table() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FV_Stats.Overview.Graph <- renderPlot({ + fv_render_graph() +}) + +fv_render_graph <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$FV_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$FV_Stats.Overview.ID) + Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$FV_Stats.Overview.Alpha), + bootstrap.size = 0, which = 'by_RT') + }, + message = "Creating plot") +}) + + +output$FV_Stats.Overview.DownloadTable <- downloadHandler( + filename = function() { + eval(FV_Stats_table_name) + }, + content = function(file) { + df <- create_stats_table() + save_table(df, file) + } +) + +output$FV_Stats.Overview.DownloadHeatmap <- downloadHandler( + filename = function() { + eval(FV_Stats_heatmap_name) + }, + content = function(file) { + save_plotly(render_heatmap(), file) + }, + contentType = paste0('image/', input$FV_Stats.Overview.Format) +) + +fv_data_table_glicko2 <- reactive({ + input$FV_Stats.Glicko.Create + isolate({ + withProgress({ + data <- FV_glicko_data() + req(length(data) > 0 && length(get_id(data)) > 0) + nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) + df <- glicko2_ranking(data, nr_games, which = 'by_RT', + target_dt = FV_stats_glicko_targets_obj)$ratings + format(df, digits = 3) + }, message = "Creating Ranking, this might take a while") + }) +}) + +output$FV_Stats.Glicko.Dataframe <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + fv_data_table_glicko2() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +fv_render_glico2_plot <- reactive({ + isolate({ + data <- FV_glicko_data() + nr_games <- as.numeric(input$FV_Stats.Glicko.Nrgames) + }) + Plot.Stats.Glicko2_Candlestick(data, nr_games, fv_data_table_glicko2(), which = 'by_RT', + target_dt = FV_stats_glicko_targets_obj) +}) + +output$FV_Stats.Glicko.Candlestick <- renderPlotly({ + fv_render_glico2_plot() +}) + +output$FV_Stats.Glicko.DownloadTable <- downloadHandler( + filename = function() { + eval(FV_Glicko2_table_name) + }, + content = function(file) { + df <- fv_data_table_glicko2() + save_table(df, file) + } +) + +output$FV_Stats.Glicko.Download <- downloadHandler( + filename = function() { + eval(FV_Glicko2_figure_name) + }, + content = function(file) { + save_plotly(fv_render_glico2_plot(), file) + }, + contentType = paste0('image/', input$FV_Stats.Glicko.Format) +) + + +FV_glicko_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$FV_Stats.Glicko.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$FV_Stats.Glicko.Dim) + data <- subset(data, funcId %in% input$FV_Stats.Glicko.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +FV_stats_glicko_targets <- reactive({ + data <- FV_glicko_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_RT") +}) + +FV_stats_glicko_targets_obj <- NULL + +proxy_FV_Stats.Glicko.Targets <- dataTableProxy('FV_Stats.Glicko.Targets') + +output$FV_Stats.Glicko.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + FV_stats_glicko_targets_obj <<- FV_stats_glicko_targets() + FV_stats_glicko_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$FV_Stats.Glicko.Targets_cell_edit, { + info <- input$FV_Stats.Glicko.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- FV_glicko_data() + if (is.null(data)) return(NULL) + FV_stats_glicko_targets_obj$target[[i]] <<- v + replaceData(proxy, FV_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) }) \ No newline at end of file diff --git a/inst/shiny-server/server/general_overview.R b/inst/shiny-server/server/general_overview.R index 89a1cd60..7c78d2d6 100644 --- a/inst/shiny-server/server/general_overview.R +++ b/inst/shiny-server/server/general_overview.R @@ -1,62 +1,62 @@ -overview_table_single <- reactive({ - data <- DATA() - req(length(data) > 0) - df <- get_overview(subset(data, algId %in% input$Overview.Single.Algid)) - - df$budget %<>% as.numeric - df$runs %<>% as.integer - df$funcId %<>% as.integer - df$DIM %<>% as.integer - df$succ %<>% as.integer - df$"worst recorded" <- format_FV(df$"worst recorded") - df$"worst reached" <- format_FV(df$"worst reached") - df$"mean reached" <- format_FV(df$"mean reached") - df$"median reached" <- format_FV(df$"median reached") - df$"best reached" <- format_FV(df$"best reached") - df$"max evals used" %<>% as.numeric - df -}) - -output$Overview.Single.Table <- DT::renderDataTable({ - req(input$Overview.Single.Algid) - overview_table_single() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$Overview.Single.Download <- downloadHandler( - filename = function() { - eval(overview_single_name) - }, - content = function(file) { - df <- overview_table_single() - df <- df[input[["Overview.Single.Table_rows_all"]]] - save_table(df, file) - } -) - -overview_table_all <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - df <- get_overview(data) - df$"worst recorded" <- format_FV(df$"worst recorded") - df$"worst reached" <- format_FV(df$"worst reached") - df$"mean reached" <- format_FV(df$"mean reached") - df$"best reached" <- format_FV(df$"best reached") - df -}) - -output$Overview.All.Table <- DT::renderDataTable({ - overview_table_all() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$Overview.All.Download <- downloadHandler( - filename = function() { - eval(overview_all_name) - }, - content = function(file) { - df <- overview_table_all() - df <- df[input[["Overview.All.Table_rows_all"]]] - save_table(df, file) - } +overview_table_single <- reactive({ + data <- DATA() + req(length(data) > 0) + df <- get_overview(subset(data, ID %in% input$Overview.Single.ID)) + + df$budget %<>% as.numeric + df$runs %<>% as.integer + df$funcId %<>% as.integer + df$DIM %<>% as.integer + df$succ %<>% as.integer + df$"worst recorded" <- format_FV(df$"worst recorded") + df$"worst reached" <- format_FV(df$"worst reached") + df$"mean reached" <- format_FV(df$"mean reached") + df$"median reached" <- format_FV(df$"median reached") + df$"best reached" <- format_FV(df$"best reached") + df$"max evals used" %<>% as.numeric + df +}) + +output$Overview.Single.Table <- DT::renderDataTable({ + req(input$Overview.Single.ID) + overview_table_single() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$Overview.Single.Download <- downloadHandler( + filename = function() { + eval(overview_single_name) + }, + content = function(file) { + df <- overview_table_single() + df <- df[input[["Overview.Single.Table_rows_all"]]] + save_table(df, file) + } +) + +overview_table_all <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + df <- get_overview(data) + df$"worst recorded" <- format_FV(df$"worst recorded") + df$"worst reached" <- format_FV(df$"worst reached") + df$"mean reached" <- format_FV(df$"mean reached") + df$"best reached" <- format_FV(df$"best reached") + df +}) + +output$Overview.All.Table <- DT::renderDataTable({ + overview_table_all() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$Overview.All.Download <- downloadHandler( + filename = function() { + eval(overview_all_name) + }, + content = function(file) { + df <- overview_table_all() + df <- df[input[["Overview.All.Table_rows_all"]]] + save_table(df, file) + } ) \ No newline at end of file diff --git a/inst/shiny-server/server/parrallel_coord.R b/inst/shiny-server/server/parrallel_coord.R index 71a5d0ac..de990d9e 100644 --- a/inst/shiny-server/server/parrallel_coord.R +++ b/inst/shiny-server/server/parrallel_coord.R @@ -1,39 +1,39 @@ -render_parallel_coord <- reactive({ - data <- DATA() - req(attr(data, 'suite') == "SOS") - data <- subset(data, algId == input$ParCoordPlot.Algs) - withProgress({ - p <- IOH_plot_ly_default("Parallel coordinate plot", "Coordinate", "Value") - - pos <- rbindlist(lapply(data, function(ds) { - final_pos_dt <- rbindlist(lapply(ds$PAR$final_position, function(pos) - {dt <- as.data.table(t(pos))})) - colnames(final_pos_dt) <- as.character(seq_len(ncol(final_pos_dt))) - final_pos_dt[, algId := attr(ds, 'algId')] - final_pos_dt[, funcId := attr(ds, 'funcId')] - final_pos_dt[, fval_log := log10(attr(ds, 'finalFV'))] - })) - - pos2 <- pos %>% melt(id.vars = c('fval_log', 'funcId', 'algId')) - p %>% add_trace(data = pos2, x = ~variable, y = ~value, type = 'scatter', - mode = 'markers', marker = list(color = ~fval_log, colorscale = 'Viridis', - colorbar = list(title = 'Log best f(x)'), - symbol = 'cross'), - legendgroup = ~algId, opacity = 0.9, showlegend = F) %>% - layout(yaxis = list(range = c(-0.02,1.02)), xaxis = list(range = c(-0.5, get_dim(dsl) + 0.5))) - }) -}) - -output$Parallel_Coord_Plot <- renderPlotly({ - render_parallel_coord() -}) - -output$ParCoordPlot.Download <- downloadHandler( - filename = function() { - paste0('Par_coord_', input$ParCoordPlot.Algs, '.', input$ParCoordPlot.Format ) - }, - content = function(file) { - save_plotly(render_parallel_coord(), file) - }, - contentType = paste0('image/', input$ParCoordPlot.Format) +render_parallel_coord <- reactive({ + data <- DATA() + req(attr(data, 'suite') == "SOS") + data <- subset(data, ID == input$ParCoordPlot.Algs) + withProgress({ + p <- IOH_plot_ly_default("Parallel coordinate plot", "Coordinate", "Value") + + pos <- rbindlist(lapply(data, function(ds) { + final_pos_dt <- rbindlist(lapply(ds$PAR$final_position, function(pos) + {dt <- as.data.table(t(pos))})) + colnames(final_pos_dt) <- as.character(seq_len(ncol(final_pos_dt))) + final_pos_dt[, ID := attr(ds, 'ID')] + final_pos_dt[, funcId := attr(ds, 'funcId')] + final_pos_dt[, fval_log := log10(attr(ds, 'finalFV'))] + })) + + pos2 <- pos %>% melt(id.vars = c('fval_log', 'funcId', 'ID')) + p %>% add_trace(data = pos2, x = ~variable, y = ~value, type = 'scatter', + mode = 'markers', marker = list(color = ~fval_log, colorscale = 'Viridis', + colorbar = list(title = 'Log best f(x)'), + symbol = 'cross'), + legendgroup = ~ID, opacity = 0.9, showlegend = F) %>% + layout(yaxis = list(range = c(-0.02,1.02)), xaxis = list(range = c(-0.5, get_dim(dsl) + 0.5))) + }) +}) + +output$Parallel_Coord_Plot <- renderPlotly({ + render_parallel_coord() +}) + +output$ParCoordPlot.Download <- downloadHandler( + filename = function() { + paste0('Par_coord_', input$ParCoordPlot.Algs, '.', input$ParCoordPlot.Format ) + }, + content = function(file) { + save_plotly(render_parallel_coord(), file) + }, + contentType = paste0('image/', input$ParCoordPlot.Format) ) \ No newline at end of file diff --git a/inst/shiny-server/server/report_generation.R b/inst/shiny-server/server/report_generation.R index 41b105aa..15051706 100644 --- a/inst/shiny-server/server/report_generation.R +++ b/inst/shiny-server/server/report_generation.R @@ -1,40 +1,40 @@ -output$Report.Generate <- downloadHandler( - filename = "report.pdf", - content = function(file) { - generate_report(file) - }, - contentType = "pdf" -) - -output$Report.Generate.Tar <- downloadHandler( - filename = "report.tar", - content = function(file) { - get_tarbal_report(file) - }, - contentType = "tar" -) - -generate_report <- function(file){ - dir_loc <- tempdir() - dir.create(file.path(dir_loc, "Report"), showWarnings = F) - file.copy("markdown/Report/", file.path(dir_loc), overwrite = TRUE, recursive = TRUE) - tempReport <- file.path(file.path(dir_loc, "Report"), "Template_test.Rmd") - params <- list(input = input, dsl = DATA_RAW(), figure_folder = dir_loc, REG = REG) - rmarkdown::render(tempReport, output_file = file, - params = params, - envir = new.env(parent = globalenv())) -} - -get_tarbal_report <- function(file){ - dir_loc <- tempdir() - if(length(list.files(dir_loc, pattern = ".tex")) == 0){ - generate_report(tempfile(fileext = "pdf")) - } - dir.create(file.path(dir_loc, "Report_files"), showWarnings = F) - file.copy(list.files(dir_loc, pattern = ".tex", full.names = T), file.path(dir_loc, "Report_files")) - file.copy(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T), file.path(dir_loc, "Report_files")) - tar(tarfile = file, files = file.path(dir_loc, "Report_files")) - file.remove(list.files(file.path(dir_loc, "Report_files"), full.names = T)) - file.remove(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T)) - file.remove(file.path(dir_loc, list.files(dir_loc, pattern = ".tex"))) +output$Report.Generate <- downloadHandler( + filename = "report.pdf", + content = function(file) { + generate_report(file) + }, + contentType = "pdf" +) + +output$Report.Generate.Tar <- downloadHandler( + filename = "report.tar", + content = function(file) { + get_tarbal_report(file) + }, + contentType = "tar" +) + +generate_report <- function(file){ + dir_loc <- tempdir() + dir.create(file.path(dir_loc, "Report"), showWarnings = F) + file.copy("markdown/Report/", file.path(dir_loc), overwrite = TRUE, recursive = TRUE) + tempReport <- file.path(file.path(dir_loc, "Report"), "Template_test.Rmd") + params <- list(input = input, dsl = DATA_RAW(), figure_folder = dir_loc, REG = REG) + rmarkdown::render(tempReport, output_file = file, + params = params, + envir = new.env(parent = globalenv())) +} + +get_tarbal_report <- function(file){ + dir_loc <- tempdir() + if(length(list.files(dir_loc, pattern = ".tex")) == 0){ + generate_report(tempfile(fileext = "pdf")) + } + dir.create(file.path(dir_loc, "Report_files"), showWarnings = F) + file.copy(list.files(dir_loc, pattern = ".tex", full.names = T), file.path(dir_loc, "Report_files")) + file.copy(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T), file.path(dir_loc, "Report_files")) + tar(tarfile = file, files = file.path(dir_loc, "Report_files")) + file.remove(list.files(file.path(dir_loc, "Report_files"), full.names = T)) + file.remove(list.files(file.path(dir_loc, "Report"), pattern = ".pdf", full.names = T)) + file.remove(file.path(dir_loc, list.files(dir_loc, pattern = ".tex"))) } \ No newline at end of file diff --git a/inst/shiny-server/server/settings_page.R b/inst/shiny-server/server/settings_page.R index 7cc884c5..9686c15b 100644 --- a/inst/shiny-server/server/settings_page.R +++ b/inst/shiny-server/server/settings_page.R @@ -1,203 +1,231 @@ -observe({ - if (input$Settings.Color.Scheme != "Custom") { - set_color_scheme(input$Settings.Color.Scheme, get_algId(DATA()), NULL) - } -}) - -output$Settings.Color.Example <- downloadHandler( - filename = function() { - "Example_Colorfile" - }, - content = function(file) { - write.csv2(get_color_scheme_dt(), file, row.names = F) - } -) - -output$Settings.Color.Plot <- renderPlotly({ - plot_color_example() -}) - -plot_color_example <- function(){ - curr_settings <- c(input$Settings.Color.Bg, - input$Settings.Color.Grid, - input$Settings.Color.Tick, - input$Settings.Legend.Location, - input$Settings.Font.Title, - input$Settings.Font.Legend, - input$Settings.Font.Label, - input$Settings.Font.Tick, - input$Settings.Color.Linewidth, - input$Settings.Color.Markersize, - input$IOHanalyzer.custom_legend_x, - input$IOHanalyzer.custom_legend_y - ) - if (any(is.null(curr_settings))) return(NULL) - if (length(DATA_RAW()) > 0) { - algnames <- get_algId(DATA_RAW()) - } - else algnames <- c("Alg 1", "Alg 2", "Alg 3", "Alg 4", "Alg 5") - colors <- get_color_scheme(algnames) - schemename <- input$Settings.Color.Scheme - if (schemename == "Custom" && !is.null(input$Settings.Color.Upload)) { - schemename <- paste0(schemename, ": ", input$Settings.Color.Upload$datapath) - } - - x <- c(rep(1, length(algnames)),rep(2, length(algnames))) - y <- seq_len(length(algnames)) - dt <- data.table(algId = rep(algnames, 2), x, y) - - p <- plot_general_data(dt, 'x', 'y', 'line', show.legend = T, - x_title = 'X-title', y_title = 'Y-title', plot_title = 'Plot Title') - - p -} - -selected_color_congfig <- observe({ - if (!is.null(input$Settings.Color.Upload)) { - datapath <- input$Settings.Color.Upload$datapath - tryCatch( - expr = { - set_color_scheme("Custom", path = datapath) - }, - error = function(e) { - shinyjs::alert("File could not be read, please upload a file in the same format as the example.") - } - ) - } -}) - -observe({ - input$Settings.General.Probs %>% - strsplit(.,',') %>% - .[[1]] %>% - as.numeric %>% - options("IOHanalyzer.quantiles" = .) -}) - -observe({ - options("IOHanalyzer.max_samples" = input$Settings.General.Max_samples) -}) - -observe({ - options("IOHanalyzer.backend" = input$Settings.General.Backend) -}) - -observe({ - options("IOHanalyzer.bgcolor" = input$Settings.Color.Bg) -}) - -observe({ - options("IOHanalyzer.gridcolor" = input$Settings.Color.Grid) -}) - -observe({ - options("IOHanalyzer.tickcolor" = input$Settings.Color.Tick) -}) - -observe({ - options("IOHanalyzer.linewidth" = input$Settings.Color.Linewidth) -}) - -observe({ - options("IOHanalyzer.markersize" = input$Settings.Color.Markersize) -}) - -observe({ - options("IOHanalyzer.figure_width" = input$Settings.Download.Width) -}) - -observe({ - options("IOHanalyzer.figure_height" = input$Settings.Download.Height) -}) - -observe({ - options("IOHanalyzer.custom_legend_x" = input$Settings.Legend.LocationX) -}) - -observe({ - options("IOHanalyzer.custom_legend_y" = input$Settings.Legend.LocationY) -}) - -observe({ - legend_loc <- input$Settings.Legend.Location - if (legend_loc == "Outside, right") legend_loc_str <- "outside_right" - else if (legend_loc == "Inside, right") legend_loc_str <- "inside_right" - else if (legend_loc == "Inside, left") legend_loc_str <- "inside_left" - else if (legend_loc == "Below") legend_loc_str <- "below" - else if (legend_loc == "Custom") legend_loc_str <- "custom" - options("IOHanalyzer.legend_location" = legend_loc_str) -}) - -observe({ - options("IOHanalyzer.tick_fontsize" = input$Settings.Font.Tick) -}) - -observe({ - options("IOHanalyzer.legend_fontsize" = input$Settings.Font.Legend) -}) - -observe({ - options("IOHanalyzer.title_fontsize" = input$Settings.Font.Title) -}) - -observe({ - options("IOHanalyzer.label_fontsize" = input$Settings.Font.Label) -}) - -observe({ - options("IOHanalyzer.precision" = input$Settings.General.Precision) -}) - -observe({ - setting_preset <- input$Settings.Download.Preset - if (setting_preset == "Default") { - updateNumericInput(session, 'Settings.Download.Width', value = 1000) - updateNumericInput(session, 'Settings.Download.Height', value = 1000) - updateNumericInput(session, 'Settings.Font.Tick', value = 12) - updateNumericInput(session, 'Settings.Font.Legend', value = 13) - updateNumericInput(session, 'Settings.Font.Title', value = 16) - updateNumericInput(session, 'Settings.Font.Label', value = 16) - } - else if (setting_preset == "Paper-1col") { - updateNumericInput(session, 'Settings.Download.Width', value = 700) - updateNumericInput(session, 'Settings.Download.Height', value = 400) - updateNumericInput(session, 'Settings.Font.Tick', value = 9) - updateNumericInput(session, 'Settings.Font.Legend', value = 10) - updateNumericInput(session, 'Settings.Font.Title', value = 13) - updateNumericInput(session, 'Settings.Font.Label', value = 13) - } - else if (setting_preset == "Paper-2col") { - updateNumericInput(session, 'Settings.Download.Width', value = 900) - updateNumericInput(session, 'Settings.Download.Height', value = 600) - updateNumericInput(session, 'Settings.Font.Tick', value = 11) - updateNumericInput(session, 'Settings.Font.Legend', value = 12) - updateNumericInput(session, 'Settings.Font.Title', value = 16) - updateNumericInput(session, 'Settings.Font.Label', value = 15) - } -}) - -output$Settings.Download <- downloadHandler( - filename = "IOHanalyzer_settings.rds", - content = function(file){ - curr_opts <- options() - IOH_opts <- curr_opts[grep(names(curr_opts), pattern = "IOH")] - saveRDS(IOH_opts, file) - }, - contentType = "rds" -) - -observe({ - if (!is.null(input$Settings.Upload)) { - file <- input$Settings.Upload$datapath - IOH_opts <- readRDS(file) - options(IOH_opts[grep(names(IOH_opts), pattern = "IOH")]) #Ensure no other options get changed by the user - } -}) - -output$Settings.Plot.Download <- downloadHandler( - filename = "Sample_plot.pdf", - content = function(file) { - save_plotly(plot_color_example(), file) - }, - contentType = 'image/pdf' +observe({ + if (input$Settings.Color.Scheme != "Custom") { + set_color_scheme(input$Settings.Color.Scheme, get_id(DATA()), NULL) + } +}) + +output$Settings.Color.Example <- downloadHandler( + filename = function() { + "Example_Colorfile" + }, + content = function(file) { + write.csv2(get_color_scheme_dt(), file, row.names = F) + } +) + +output$Settings.Color.Plot <- renderPlotly({ + plot_color_example() +}) + +plot_color_example <- function(){ + curr_settings <- c(input$Settings.Color.Bg, + input$Settings.Color.Grid, + input$Settings.Color.Tick, + input$Settings.Legend.Location, + input$Settings.Font.Title, + input$Settings.Font.Legend, + input$Settings.Font.Label, + input$Settings.Font.Tick, + input$Settings.Color.Linewidth, + input$Settings.Color.Markersize, + input$IOHanalyzer.custom_legend_x, + input$IOHanalyzer.custom_legend_y + ) + if (any(is.null(curr_settings))) return(NULL) + if (length(DATA_RAW()) > 0) { + algnames <- get_id(DATA_RAW()) + } + else algnames <- c("Alg 1", "Alg 2", "Alg 3", "Alg 4", "Alg 5") + colors <- get_color_scheme(algnames) + schemename <- input$Settings.Color.Scheme + if (schemename == "Custom" && !is.null(input$Settings.Color.Upload)) { + schemename <- paste0(schemename, ": ", input$Settings.Color.Upload$datapath) + } + + x <- c(rep(1, length(algnames)),rep(2, length(algnames))) + y <- seq_len(length(algnames)) + dt <- data.table(ID = rep(algnames, 2), x, y) + + p <- plot_general_data(dt, 'x', 'y', 'line', show.legend = T, + x_title = 'X-title', y_title = 'Y-title', plot_title = 'Plot Title') + + p +} + +selected_color_congfig <- observe({ + if (!is.null(input$Settings.Color.Upload)) { + datapath <- input$Settings.Color.Upload$datapath + tryCatch( + expr = { + set_color_scheme("Custom", path = datapath) + }, + error = function(e) { + shinyjs::alert("File could not be read, please upload a file in the same format as the example.") + } + ) + } +}) + +observe({ + input$Settings.General.Probs %>% + strsplit(.,',') %>% + .[[1]] %>% + as.numeric %>% + options("IOHanalyzer.quantiles" = .) +}) + +observe({ + options("IOHanalyzer.max_samples" = input$Settings.General.Max_samples) +}) + +observe({ + options("IOHanalyzer.backend" = input$Settings.General.Backend) +}) + +observe({ + options("IOHanalyzer.bgcolor" = input$Settings.Color.Bg) +}) + +observe({ + options("IOHanalyzer.gridcolor" = input$Settings.Color.Grid) +}) + +observe({ + options("IOHanalyzer.tickcolor" = input$Settings.Color.Tick) +}) + +observe({ + options("IOHanalyzer.linewidth" = input$Settings.Color.Linewidth) +}) + +observe({ + options("IOHanalyzer.markersize" = input$Settings.Color.Markersize) +}) + +observe({ + options("IOHanalyzer.figure_width" = input$Settings.Download.Width) +}) + +observe({ + options("IOHanalyzer.figure_height" = input$Settings.Download.Height) +}) + +observe({ + options("IOHanalyzer.custom_legend_x" = input$Settings.Legend.LocationX) +}) + +observe({ + options("IOHanalyzer.custom_legend_y" = input$Settings.Legend.LocationY) +}) + +observe({ + legend_loc <- input$Settings.Legend.Location + if (legend_loc == "Outside, right") legend_loc_str <- "outside_right" + else if (legend_loc == "Inside, right") legend_loc_str <- "inside_right" + else if (legend_loc == "Inside, left") legend_loc_str <- "inside_left" + else if (legend_loc == "Below") legend_loc_str <- "below" + else if (legend_loc == "Custom") legend_loc_str <- "custom" + options("IOHanalyzer.legend_location" = legend_loc_str) +}) + +observe({ + options("IOHanalyzer.tick_fontsize" = input$Settings.Font.Tick) +}) + +observe({ + options("IOHanalyzer.legend_fontsize" = input$Settings.Font.Legend) +}) + +observe({ + options("IOHanalyzer.title_fontsize" = input$Settings.Font.Title) +}) + +observe({ + options("IOHanalyzer.label_fontsize" = input$Settings.Font.Label) +}) + +observe({ + options("IOHanalyzer.precision" = input$Settings.General.Precision) +}) + +observe({ + req(input$Settings.ID.Variables) + withProgress({ + id_vars <- input$Settings.ID.Variables + if (!setequal(id_vars,getOption('IOHanalyzer.ID_vars', c('algId')))) { + options("IOHanalyzer.ID_vars" = input$Settings.ID.Variables) + DataList$data <- change_id(DataList$data, input$Settings.ID.Variables) + } + if ('algId' %in% id_vars) + shinyjs::hide(id = "overall_algid_box") + else + shinyjs::show(id = "overall_algid_box") +}, message = "Processing IDs") +}) + +observe({ + if (input$Settings.Use_Funcname) { + shinyjs::show(id = "overall_funcname_box") + shinyjs::hide(id = "overall_funcid_box") + options('IOHanalyzer.function_representation' = 'funcname') + } + else { + shinyjs::hide(id = "overall_funcname_box") + shinyjs::show(id = "overall_funcid_box") + options('IOHanalyzer.function_representation' = 'funcId') + } +}) + +observe({ + setting_preset <- input$Settings.Download.Preset + if (setting_preset == "Default") { + updateNumericInput(session, 'Settings.Download.Width', value = 1000) + updateNumericInput(session, 'Settings.Download.Height', value = 1000) + updateNumericInput(session, 'Settings.Font.Tick', value = 12) + updateNumericInput(session, 'Settings.Font.Legend', value = 13) + updateNumericInput(session, 'Settings.Font.Title', value = 16) + updateNumericInput(session, 'Settings.Font.Label', value = 16) + } + else if (setting_preset == "Paper-1col") { + updateNumericInput(session, 'Settings.Download.Width', value = 700) + updateNumericInput(session, 'Settings.Download.Height', value = 400) + updateNumericInput(session, 'Settings.Font.Tick', value = 9) + updateNumericInput(session, 'Settings.Font.Legend', value = 10) + updateNumericInput(session, 'Settings.Font.Title', value = 13) + updateNumericInput(session, 'Settings.Font.Label', value = 13) + } + else if (setting_preset == "Paper-2col") { + updateNumericInput(session, 'Settings.Download.Width', value = 900) + updateNumericInput(session, 'Settings.Download.Height', value = 600) + updateNumericInput(session, 'Settings.Font.Tick', value = 11) + updateNumericInput(session, 'Settings.Font.Legend', value = 12) + updateNumericInput(session, 'Settings.Font.Title', value = 16) + updateNumericInput(session, 'Settings.Font.Label', value = 15) + } +}) + +output$Settings.Download <- downloadHandler( + filename = "IOHanalyzer_settings.rds", + content = function(file){ + curr_opts <- options() + IOH_opts <- curr_opts[grep(names(curr_opts), pattern = "IOH")] + saveRDS(IOH_opts, file) + }, + contentType = "rds" +) + +observe({ + if (!is.null(input$Settings.Upload)) { + file <- input$Settings.Upload$datapath + IOH_opts <- readRDS(file) + options(IOH_opts[grep(names(IOH_opts), pattern = "IOH")]) #Ensure no other options get changed by the user + } +}) + +output$Settings.Plot.Download <- downloadHandler( + filename = "Sample_plot.pdf", + content = function(file) { + save_plotly(plot_color_example(), file) + }, + contentType = 'image/pdf' ) \ No newline at end of file diff --git a/inst/shiny-server/server/statistical_significance.R b/inst/shiny-server/server/statistical_significance.R index ed6c6dca..18c16835 100644 --- a/inst/shiny-server/server/statistical_significance.R +++ b/inst/shiny-server/server/statistical_significance.R @@ -1,159 +1,159 @@ -render_heatmap <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$RT_Stats.Overview.Algid) - Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), - bootstrap.size = input$RT_Stats.Overview.Samples) - }, - message = "Creating plot") -}) - - -output$RT_Stats.Overview.Heatmap <- renderPlotly( - render_heatmap() -) - -create_stats_table <- reactive({ - req(length(DATA()) > 0) - req(length(get_algId(DATA())) > 1) - data <- subset(DATA(), algId %in% input$RT_Stats.Overview.Algid) - target <- as.numeric(input$RT_Stats.Overview.Target) - df <- pairwise.test(data, target, bootstrap.size = input$RT_Stats.Overview.Samples) - df <- format(df, digits = 3) - df -}) - -output$RT_Stats.Overview.Pmatrix <- DT::renderDataTable({ - create_stats_table() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RT_Stats.Overview.Graph <- renderPlot({ - render_graph() -}) - -render_graph <- reactive({ - req(length(DATA()) > 0) - withProgress({ - target <- as.numeric(input$RT_Stats.Overview.Target) - data <- subset(DATA(), algId %in% input$RT_Stats.Overview.Algid) - Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), - bootstrap.size = input$RT_Stats.Overview.Samples) - }, - message = "Creating plot") -}) - - -output$RT_Stats.Overview.DownloadTable <- downloadHandler( - filename = function() { - eval(RT_Stats_table_name) - }, - content = function(file) { - df <- create_stats_table() - save_table(df, file) - } -) - -output$RT_Stats.Overview.DownloadHeatmap <- downloadHandler( - filename = function() { - eval(RT_Stats_heatmap_name) - }, - content = function(file) { - save_plotly(render_heatmap(), file) - }, - contentType = paste0('image/', input$RT_Stats.Overview.Format) -) - -data_table_glicko2 <- reactive({ - input$RT_Stats.Glicko.Create - isolate({ - withProgress({ - data <- RT_glicko_data() - nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) - df <- glicko2_ranking(data, nr_games, target_dt = RT_stats_glicko_targets_obj, which = 'by_FV')$ratings - format(df, digits = 3) - }, message = "Creating Ranking, this might take a while") - }) -}) - -output$RT_Stats.Glicko.Dataframe <- DT::renderDataTable({ - - req(length(DATA_RAW()) > 0) - data_table_glicko2() -}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -render_glico2_plot <- reactive({ - isolate({ - data <- RT_glicko_data() - nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) - }) - Plot.Stats.Glicko2_Candlestick(data, nr_games, data_table_glicko2(), which = 'by_FV', - target_dt = RT_stats_glicko_targets_obj) -}) - -output$RT_Stats.Glicko.Candlestick <- renderPlotly({ - render_glico2_plot() -}) - -output$RT_Stats.Glicko.DownloadTable <- downloadHandler( - filename = function() { - eval(RT_Glicko2_table_name) - }, - content = function(file) { - df <- data_table_glicko2() - save_table(df, file) - } -) - -output$RT_Stats.Glicko.Download <- downloadHandler( - filename = function() { - eval(RT_Glicko2_figure_name) - }, - content = function(file) { - save_plotly(render_glico2_plot(), file) - }, - contentType = paste0('image/', input$RT_Stats.Glicko.Format) -) - - -RT_glicko_data <- function() { - data <- subset(DATA_RAW(), algId %in% isolate(input$RT_Stats.Glicko.Algid)) - if (length(data) == 0) return(NULL) - data <- subset(data, DIM %in% input$RT_Stats.Glicko.Dim) - data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) - if (length(unique(get_algId(data))) < 2) { - shinyjs::alert("This plot is only available when the dataset contains - multiple algorithms for the selected functions and dimensions.") - return(NULL) - } - data -} - -RT_stats_glicko_targets <- reactive({ - data <- RT_glicko_data() - if (is.null(data)) return(NULL) - get_target_dt(data, "by_FV") -}) - -RT_stats_glicko_targets_obj <- NULL - -proxy_RT_Stats.Glicko.Targets <- dataTableProxy('RT_Stats.Glicko.Targets') - -output$RT_Stats.Glicko.Targets <- DT::renderDataTable({ - req(length(DATA_RAW()) > 0) - RT_stats_glicko_targets_obj <<- RT_stats_glicko_targets() - RT_stats_glicko_targets_obj -}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, -options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) - - -observeEvent(input$RT_Stats.Glicko.Targets_cell_edit, { - info <- input$RT_Stats.Glicko.Targets_cell_edit - i <- info$row - j <- info$col - v <- info$value - data <- RT_glicko_data() - if (is.null(data)) return(NULL) - RT_stats_glicko_targets_obj$target[[i]] <<- v - replaceData(proxy, RT_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) -}) +render_heatmap <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$RT_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) + Plot.Stats.Significance_Heatmap(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), + bootstrap.size = input$RT_Stats.Overview.Samples) + }, + message = "Creating plot") +}) + + +output$RT_Stats.Overview.Heatmap <- renderPlotly( + render_heatmap() +) + +create_stats_table <- reactive({ + req(length(DATA()) > 0) + req(length(get_id(DATA())) > 1) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) + target <- as.numeric(input$RT_Stats.Overview.Target) + df <- pairwise.test(data, target, bootstrap.size = input$RT_Stats.Overview.Samples) + df <- format(df, digits = 3) + df +}) + +output$RT_Stats.Overview.Pmatrix <- DT::renderDataTable({ + create_stats_table() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RT_Stats.Overview.Graph <- renderPlot({ + render_graph() +}) + +render_graph <- reactive({ + req(length(DATA()) > 0) + withProgress({ + target <- as.numeric(input$RT_Stats.Overview.Target) + data <- subset(DATA(), ID %in% input$RT_Stats.Overview.ID) + Plot.Stats.Significance_Graph(data, target, alpha = as.numeric(input$RT_Stats.Overview.Alpha), + bootstrap.size = input$RT_Stats.Overview.Samples) + }, + message = "Creating plot") +}) + + +output$RT_Stats.Overview.DownloadTable <- downloadHandler( + filename = function() { + eval(RT_Stats_table_name) + }, + content = function(file) { + df <- create_stats_table() + save_table(df, file) + } +) + +output$RT_Stats.Overview.DownloadHeatmap <- downloadHandler( + filename = function() { + eval(RT_Stats_heatmap_name) + }, + content = function(file) { + save_plotly(render_heatmap(), file) + }, + contentType = paste0('image/', input$RT_Stats.Overview.Format) +) + +data_table_glicko2 <- reactive({ + input$RT_Stats.Glicko.Create + isolate({ + withProgress({ + data <- RT_glicko_data() + nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) + df <- glicko2_ranking(data, nr_games, target_dt = RT_stats_glicko_targets_obj, which = 'by_FV')$ratings + format(df, digits = 3) + }, message = "Creating Ranking, this might take a while") + }) +}) + +output$RT_Stats.Glicko.Dataframe <- DT::renderDataTable({ + + req(length(DATA_RAW()) > 0) + data_table_glicko2() +}, options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +render_glico2_plot <- reactive({ + isolate({ + data <- RT_glicko_data() + nr_games <- as.numeric(input$RT_Stats.Glicko.Nrgames) + }) + Plot.Stats.Glicko2_Candlestick(data, nr_games, data_table_glicko2(), which = 'by_FV', + target_dt = RT_stats_glicko_targets_obj) +}) + +output$RT_Stats.Glicko.Candlestick <- renderPlotly({ + render_glico2_plot() +}) + +output$RT_Stats.Glicko.DownloadTable <- downloadHandler( + filename = function() { + eval(RT_Glicko2_table_name) + }, + content = function(file) { + df <- data_table_glicko2() + save_table(df, file) + } +) + +output$RT_Stats.Glicko.Download <- downloadHandler( + filename = function() { + eval(RT_Glicko2_figure_name) + }, + content = function(file) { + save_plotly(render_glico2_plot(), file) + }, + contentType = paste0('image/', input$RT_Stats.Glicko.Format) +) + + +RT_glicko_data <- function() { + data <- subset(DATA_RAW(), ID %in% isolate(input$RT_Stats.Glicko.ID)) + if (length(data) == 0) return(NULL) + data <- subset(data, DIM %in% input$RT_Stats.Glicko.Dim) + data <- subset(data, funcId %in% input$RT_Stats.Glicko.Funcid) + if (length(unique(get_id(data))) < 2) { + shinyjs::alert("This plot is only available when the dataset contains + multiple IDs for the selected functions and dimensions.") + return(NULL) + } + data +} + +RT_stats_glicko_targets <- reactive({ + data <- RT_glicko_data() + if (is.null(data)) return(NULL) + get_target_dt(data, "by_FV") +}) + +RT_stats_glicko_targets_obj <- NULL + +proxy_RT_Stats.Glicko.Targets <- dataTableProxy('RT_Stats.Glicko.Targets') + +output$RT_Stats.Glicko.Targets <- DT::renderDataTable({ + req(length(DATA_RAW()) > 0) + RT_stats_glicko_targets_obj <<- RT_stats_glicko_targets() + RT_stats_glicko_targets_obj +}, editable = list(target = 'cell', disable = list(columns = c(0,1))), rownames = FALSE, +options = list(pageLength = 5, lengthMenu = c(5, 10, 25, -1), scrollX = T, server = T)) + + +observeEvent(input$RT_Stats.Glicko.Targets_cell_edit, { + info <- input$RT_Stats.Glicko.Targets_cell_edit + i <- info$row + j <- info$col + v <- info$value + data <- RT_glicko_data() + if (is.null(data)) return(NULL) + RT_stats_glicko_targets_obj$target[[i]] <<- v + replaceData(proxy, RT_stats_glicko_targets_obj, resetPaging = FALSE, rownames = FALSE) +}) diff --git a/inst/shiny-server/server/tables_multi_func.R b/inst/shiny-server/server/tables_multi_func.R index bdb5627f..bc949590 100644 --- a/inst/shiny-server/server/tables_multi_func.R +++ b/inst/shiny-server/server/tables_multi_func.R @@ -1,102 +1,102 @@ -multi_function_data_summary <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - data <- subset(data, algId %in% input$RT.MultiERT.AlgId & - funcId %in% input$RT.MultiERT.FuncId & - DIM %in% input$RT.MultiERT.DIM) - - get_RT_summary(data, as.numeric(input$RT.MultiERT.Target)) -}) - -output$RT.MultiERT.Table <- DT::renderDataTable({ - req(input$RT.MultiERT.AlgId) - multi_function_data_summary() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RT.MultiERT.Download <- downloadHandler( - filename = function() { - eval(RT_multifunc_ERT) - }, - content = function(file) { - save_table(multi_function_data_summary(), file) - } -) - -multi_function_data_sample <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - data <- subset(data, algId %in% input$RT.Multisample.AlgId & - funcId %in% input$RT.Multisample.FuncId & - DIM %in% input$RT.Multisample.DIM) - - get_RT_sample(data, as.numeric(input$RT.Multisample.Target), output = input$RT.Multisample.mode) -}) - -output$RT.Multisample.Table <- DT::renderDataTable({ - req(input$RT.Multisample.AlgId) - multi_function_data_sample() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$RT.Multisample.Download <- downloadHandler( - filename = function() { - eval(RT_multifunc_sample) - }, - content = function(file) { - save_table(multi_function_data_sample(), file) - } -) - - -multi_function_data_summary_FV <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - req(as.numeric(input$FV.MultiFV.Target) > 0) - data <- subset(data, algId %in% input$FV.MultiFV.AlgId & - funcId %in% input$FV.MultiFV.FuncId & - DIM %in% input$FV.MultiFV.DIM) - - get_FV_summary(data, as.numeric(input$FV.MultiFV.Target)) -}) - -output$FV.MultiFV.Table <- DT::renderDataTable({ - req(input$FV.MultiFV.AlgId) - multi_function_data_summary_FV() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FV.MultiFV.Download <- downloadHandler( - filename = function() { - eval(FV_multifunc_FV) - }, - content = function(file) { - save_table(multi_function_data_summary_FV(), file) - } -) - -multi_function_data_sample_FV <- reactive({ - data <- DATA_RAW() - req(length(data) > 0) - data <- subset(data, algId %in% input$FV.Multisample.AlgId & - funcId %in% input$FV.Multisample.FuncId & - DIM %in% input$FV.Multisample.DIM) - - get_FV_sample(data, as.numeric(input$FV.Multisample.Target), output = input$FV.Multisample.mode) -}) - -output$FV.Multisample.Table <- DT::renderDataTable({ - req(input$FV.Multisample.AlgId) - multi_function_data_sample_FV() -}, filter = list(position = 'top', clear = FALSE), -options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) - -output$FV.Multisample.Download <- downloadHandler( - filename = function() { - eval(FV_multifunc_sample) - }, - content = function(file) { - save_table(multi_function_data_sample_FV(), file) - } -) - +multi_function_data_summary <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + data <- subset(data, ID %in% input$RT.MultiERT.ID & + funcId %in% input$RT.MultiERT.FuncId & + DIM %in% input$RT.MultiERT.DIM) + + get_RT_summary(data, as.numeric(input$RT.MultiERT.Target)) +}) + +output$RT.MultiERT.Table <- DT::renderDataTable({ + req(input$RT.MultiERT.ID) + multi_function_data_summary() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RT.MultiERT.Download <- downloadHandler( + filename = function() { + eval(RT_multifunc_ERT) + }, + content = function(file) { + save_table(multi_function_data_summary(), file) + } +) + +multi_function_data_sample <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + data <- subset(data, ID %in% input$RT.Multisample.ID & + funcId %in% input$RT.Multisample.FuncId & + DIM %in% input$RT.Multisample.DIM) + + get_RT_sample(data, as.numeric(input$RT.Multisample.Target), output = input$RT.Multisample.mode) +}) + +output$RT.Multisample.Table <- DT::renderDataTable({ + req(input$RT.Multisample.ID) + multi_function_data_sample() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$RT.Multisample.Download <- downloadHandler( + filename = function() { + eval(RT_multifunc_sample) + }, + content = function(file) { + save_table(multi_function_data_sample(), file) + } +) + + +multi_function_data_summary_FV <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + req(as.numeric(input$FV.MultiFV.Target) > 0) + data <- subset(data, ID %in% input$FV.MultiFV.ID & + funcId %in% input$FV.MultiFV.FuncId & + DIM %in% input$FV.MultiFV.DIM) + + get_FV_summary(data, as.numeric(input$FV.MultiFV.Target)) +}) + +output$FV.MultiFV.Table <- DT::renderDataTable({ + req(input$FV.MultiFV.ID) + multi_function_data_summary_FV() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FV.MultiFV.Download <- downloadHandler( + filename = function() { + eval(FV_multifunc_FV) + }, + content = function(file) { + save_table(multi_function_data_summary_FV(), file) + } +) + +multi_function_data_sample_FV <- reactive({ + data <- DATA_RAW() + req(length(data) > 0) + data <- subset(data, ID %in% input$FV.Multisample.ID & + funcId %in% input$FV.Multisample.FuncId & + DIM %in% input$FV.Multisample.DIM) + + get_FV_sample(data, as.numeric(input$FV.Multisample.Target), output = input$FV.Multisample.mode) +}) + +output$FV.Multisample.Table <- DT::renderDataTable({ + req(input$FV.Multisample.ID) + multi_function_data_sample_FV() +}, filter = list(position = 'top', clear = FALSE), +options = list(dom = 'lrtip', pageLength = 15, scrollX = T, server = T)) + +output$FV.Multisample.Download <- downloadHandler( + filename = function() { + eval(FV_multifunc_sample) + }, + content = function(file) { + save_table(multi_function_data_sample_FV(), file) + } +) + diff --git a/inst/shiny-server/server/upload.R b/inst/shiny-server/server/upload.R index 4f817040..328a4725 100644 --- a/inst/shiny-server/server/upload.R +++ b/inst/shiny-server/server/upload.R @@ -1,682 +1,734 @@ -# shared reactive variables -folderList <- reactiveValues(data = list()) -DataList <- reactiveValues(data = DataSetList()) - -observe({ - repo_dir <- get_repo_location() - dirs <- list.dirs(repo_dir, full.names = F) - if (length(dirs) == 0) { - shinyjs::alert("No repository directory found. To make use of the IOHProfiler-repository, - please create a folder called 'repository' in your home directory - and make sure it contains at least one '.rds'-file of a DataSetList-object, - such as the ones provided on the IOHProfiler github-page.") - } - else - updateSelectInput(session, 'repository.type', choices = dirs, selected = dirs[[1]]) -}) - -# set up list of datasets (scan the repository, looking for .rds files) -observe({ - req(input$repository.type) - repo_dir <- get_repo_location() - dir <- file.path(repo_dir, input$repository.type) - - rds_files <- list.files(dir, pattern = '.rds$') %>% sub('\\.rds$', '', .) - if (length(rds_files) != 0) { - updateSelectInput(session, 'repository.dataset', choices = rds_files, selected = rds_files[[1]]) - } else {# TODO: the alert msg should be updated - shinyjs::alert("No repository file found. To make use of the IOHProfiler-repository, - please create a folder called 'repository' in your home directory - and make sure it contains at least one '.rds'-file of a DataSetList-object, - such as the ones provided on the IOHProfiler github-page.") - updateSelectInput(session, 'repository.dataset', choices = NULL, selected = NULL) - } -}) - -# load repository that is selected -observeEvent(input$repository.dataset, { - req(input$repository.dataset) - repo_dir <- get_repo_location() - algs <- c() - dims <- c() - funcs <- c() - - for (f in input$repository.dataset) { - rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) - if (file.exists(paste0(rds_file, '_info'))) { - info <- readRDS(paste0(rds_file, '_info')) - } - else { - dsl <- readRDS(rds_file) - info = list(algId = get_algId(dsl), funcId = get_funcId(dsl), DIM = get_dim(dsl)) - } - algs <- c(algs, info$algId) - dims <- c(dims, info$DIM) - funcs <- c(funcs, info$funcId) - } - - algs <- unique(algs) - dims <- unique(dims) - funcs <- unique(funcs) - - updateSelectInput(session, 'repository.algId', choices = algs, selected = algs) - updateSelectInput(session, 'repository.dim', choices = dims, selected = dims) - updateSelectInput(session, 'repository.funcId', choices = funcs, selected = funcs) - shinyjs::enable('repository.load_button') -}) - -# add the data from repository -observeEvent(input$repository.load_button, { - data <- DataSetList() - repo_dir <- get_repo_location() - for (f in input$repository.dataset) { - rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) - data <- c(data, readRDS(rds_file)) - } - data <- subset(data, funcId %in% input$repository.funcId) - data <- subset(data, DIM %in% input$repository.dim) - data <- subset(data, algId %in% input$repository.algId) - - if (length(DataList$data) > 0 && attr(data, 'maximization') != attr(DataList$data, 'maximization')) { - shinyjs::alert(paste0("Attempting to add data from a different optimization type to the currently", - " loaded data.\nPlease either remove the currently loaded data or", - " choose a different dataset to load.")) - return(NULL) - } - - DataList$data <- c(DataList$data, data) - update_menu_visibility(attr(DataList$data, 'suite')) - # set_format_func(attr(DataList$data, 'suite')) - algids <- get_algId(DataList$data) - if (!all(algids %in% get_color_scheme_dt()[['algnames']])) { - set_color_scheme("Default", algids) - } -}) - -# decompress zip files recursively and return the root directory of extracted files -unzip_fct_recursive <- function(zipfile, exdir, print_fun = print, alert_fun = print, depth = 0) { - filetype <- basename(zipfile) %>% - strsplit('\\.') %>% `[[`(1) %>% - rev %>% - `[`(1) - folders <- list() - - if (filetype == 'zip') - unzip_fct <- unzip - else if (filetype %in% c('bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')) - unzip_fct <- untar - - files <- unzip_fct(zipfile, list = FALSE, exdir = file.path(exdir, rand_strings(1))) - if (length(files) == 0) { - alert_fun("An error occured while unzipping the provided files.\n - Please ensure no archives are corrupted and the filenames are - in base-64.") - return(NULL) - } - print_fun(paste0('

Succesfully unzipped ', basename(zipfile), '.
')) - - folders <- grep('*.info|csv|txt', files, value = T) %>% - dirname %>% - unique %>% - grep('__MACOSX', ., value = T, invert = T) %>% # to get rid of __MACOSX folder on MAC.. - c(folders) - - zip_files <- grep('.*zip|bz2|bz|gz|tar|tgz|tar\\.gz|xz', files, value = T, perl = T) %>% - grep('__MACOSX', ., value = T, invert = T) - - if (depth <= 3) { # only allow for 4 levels of recursions - for (zipfile in zip_files) { - .folders <- unzip_fct_recursive(zipfile, dirname(zipfile), alert_fun, - print_fun, depth + 1) - folders <- c(folders, .folders) - } - } - unique(folders) -} - -# upload the compressed the data file and uncompress them -selected_folders <- reactive({ - if (is.null(input$upload.add_zip)) return(NULL) - - tryCatch({ - datapath <- input$upload.add_zip$datapath - folders <- c() - - for (i in seq(datapath)) { - filetype <- basename(datapath[i]) %>% - strsplit('\\.') %>% `[[`(1) %>% - rev %>% - `[`(1) - - print_html(paste0('

Handling ', filetype, '-data.
')) - if (filetype == 'csv' || filetype == 'rds') { - # add the data path to the folders list direct - folders <- c(folders, datapath[[i]]) - next - } - - .folders <- unzip_fct_recursive(datapath[i], exdir, print_html) %>% unique - folders %<>% c(.folders) - } - unique(folders) - }, error = function(e) shinyjs::alert(paste0("The following error occured when processing the uploaded data: ", e)) - ) -}) - -# load, process the data folders, and update DataSetList -observeEvent(selected_folders(), { - withProgress({ - folders <- selected_folders() - req(length(folders) != 0) - - format_selected <- input$upload.data_format - maximization <- input$upload.maximization - - if (maximization == "AUTOMATIC") maximization <- NULL - - if (length(folderList$data) == 0) - folder_new <- folders - else - folder_new <- setdiff(folders, intersect(folderList$data, folders)) - - req(length(folder_new) != 0) - format_detected <- lapply(folder_new, check_format) %>% unique - - if (length(format_detected) > 1) - print_html(paste('

more than one format:
', - format_detected, - 'is detected in the uploaded data set... skip the uploaded data')) - else - format_detected <- format_detected[[1]] - - print_html(paste0('

Data processing of source type:', format_detected, '
')) - - for (folder in folder_new) { - indexFiles <- scan_index_file(folder) - - if (length(indexFiles) == 0 && !(format_detected %in% c(NEVERGRAD, "SOS", "RDS"))) - print_html(paste('

No .info-files detected in the - uploaded folder, while they were expected:

', folder)) - else { - folderList$data <- c(folderList$data, folder) - - if (format_detected == "RDS") { - new_data <- readRDS(folder) - if (!("DataSetList" %in% class(new_data))) - DataSetList() - } - else { - # read the data set and handle potential errors - new_data <- tryCatch( - DataSetList(folder, print_fun = print_html, - maximization = maximization, - format = format_detected, - subsampling = input$upload.subsampling), - error = function(e) { - print_html(paste('

The following error happened - when processing the data set:

')) - print_html(paste('

', e, '

')) - DataSetList() - } - ) - } - tryCatch( - DataList$data <- c(DataList$data, new_data), - error = function(e) { - print_html(paste('

The following error happened', - 'when adding the uploaded data set:

')) - print_html(paste('

', e, - '\nRemoving the old data.

')) - DataList$data <- new_data - } - ) - shinyjs::html("upload_data_promt", - sprintf('%d: %s\n', length(folderList$data), folder), - add = TRUE) - } - } - - DataList$data <- clean_DataSetList(DataList$data) - if (is.null(DataList$data)) { - shinyjs::alert("An error occurred when processing the uploaded data. - Please ensure the data is not corrupted.") - return(NULL) - } - - update_menu_visibility(attr(DataList$data, 'suite')) - # set_format_func(attr(DataList$data, 'suite')) - algids <- get_algId(DataList$data) - if (!all(algids %in% get_color_scheme_dt()[['algnames']])) { - set_color_scheme("Default", algids) - } - }, message = "Processing data, this might take some time") -}) - -update_menu_visibility <- function(suite){ - if (all(suite == NEVERGRAD)) { - #TODO: Better way of doing this such that these pages are not even populated with data instead of just being hidden - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "#shiny-tab-ERT")) - } - else{ - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "#shiny-tab-ERT")) - } - if (all(suite == "PBO")) { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "FCE_ECDF")) - } - else { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "FCE_ECDF")) - } - if (any(suite == "SOS")) { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "Positions")) - } - else { - session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "Positions")) - } -} - -observeEvent(input$Upload.Add_to_repo, { - data <- DATA_RAW() - repo_dir <- get_repo_location() - if (!file.exists(file.path(repo_dir, "public_repo"))) return(NULL) - nr_skipped <- 0 - public_dir <- file.path(repo_dir, "public_repo") - for (algname in get_algId(data)) { - filename <- file.path(public_dir, paste0(algname, '.rds')) - if (file.exists(filename)) { - nr_skipped <- nr_skipped + 1 - next - } - dsl <- subset(data, algId == algname) - saveRDS(dsl, file = filename) - } - nr_success <- length(get_algId(data)) - nr_skipped - shinyjs::alert(paste0("Successfully added ", nr_success, " algorithms to the public repository.", - "A total of ", nr_skipped, " algorithms were not uploaded because an algorithm", - "of the same name already exists, and overwriting data in the public repository is not yet", - "supported.")) -}) - -# remove all uploaded data set -observeEvent(input$upload.remove_data, { - if (length(DataList$data) != 0) { - DataList$data <- DataSetList() # NOTE: this must be a `DataSetList` object - unlink(folderList$data, T) - folderList$data <- list() - - updateSelectInput(session, 'Overall.Dim', choices = c(), selected = '') - updateSelectInput(session, 'Overall.Funcid', choices = c(), selected = '') - - print_html('

all data are removed!

') - print_html('', 'upload_data_promt') - } -}) - -# show the detailed information on DataSetList -output$data_info <- renderDataTable({ - datasets <- DataList$data - if (length(datasets) != 0) - summary(datasets) - else - data.frame() -}, options = list(pageLength = 10, scrollX = F, autoWidth = TRUE, - columnDefs = list(list(width = '20px', targets = c(0, 1))))) - -# update the list of dimensionality, funcId and algId and parameter list -observe({ - data <- DataList$data - if (length(data) == 0) - return() - - # TODO: create reactive values for them - algIds_ <- get_algId(data) - algIds <- c(algIds_, 'all') - parIds_ <- get_parId(data) - parIds <- c(parIds_, 'all') - funcIds <- get_funcId(data) - DIMs <- get_dim(data) - - selected_ds <- data[[1]] - selected_f <- attr(selected_ds,'funcId') - selected_dim <- attr(selected_ds, 'DIM') - selected_alg <- attr(selected_ds, 'algId') - - updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) - updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) - - updateSelectInput(session, 'Overview.Single.Algid', choices = algIds_, selected = algIds_) - - # updateSelectInput(session, 'Report.RT.Overview-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Overview-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Overview-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.Statistics-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Statistics-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.Single_ERT-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Single_ERT-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Single_ERT-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.Multi_ERT-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Multi_ERT-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.Rank-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Rank-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.Histogram-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.Histogram-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.Histogram-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.PMF-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.PMF-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.PMF-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.ECDF_Aggregated-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.RT.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.RT.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.RT.ECDF_AUC-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.Overview-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Overview-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Overview-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.Statistics-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Statistics-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.Single_FCE-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Single_FCE-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Single_FCE-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.Multi_FCE-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Multi_FCE-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.Rank-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Rank-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.Histogram-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.Histogram-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.Histogram-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.PMF-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.PMF-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.PMF-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.ECDF_Aggregated-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.FV.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.FV.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.FV.ECDF_AUC-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.Param.Plot-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.Param.Plot-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.Param.Plot-Alg', choices = algIds_, selected = algIds_) - # - # updateSelectInput(session, 'Report.Param.Statistics-FuncId', choices = funcIds, selected = selected_f) - # updateSelectInput(session, 'Report.Param.Statistics-DIM', choices = DIMs, selected = selected_dim) - # updateSelectInput(session, 'Report.Param.Statistics-Alg', choices = algIds_, selected = algIds_) - - updateSelectInput(session, 'RTportfolio.Shapley.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTportfolio.Shapley.Funcs', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RTportfolio.Shapley.Dims', choices = DIMs, selected = DIMs) - updateNumericInput(session, 'RTportfolio.Shapley.Permsize', min = 2, max = length(algIds_), - value = min(10, length(algIds_))) - - - updateSelectInput(session, 'RT.MultiERT.AlgId', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT.MultiERT.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RT.MultiERT.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'RT.Multisample.AlgId', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT.Multisample.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RT.Multisample.DIM', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'FV.MultiFV.AlgId', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV.MultiFV.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'FV.MultiFV.DIM', choices = DIMs, selected = selected_dim) - updateSelectInput(session, 'FV.Multisample.AlgId', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV.Multisample.FuncId', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'FV.Multisample.DIM', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'RT_Stats.Glicko.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) - updateSelectInput(session, 'RT_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'RT_Stats.DSC.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'RT_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - - updateSelectInput(session, 'FV_Stats.DSC.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) - updateSelectInput(session, 'FV_Stats.DSC.Dim', choices = DIMs, selected = DIMs) - - updateSelectInput(session, 'RT_Stats.Overview.Algid', choices = algIds_, selected = algIds_) - - updateSelectInput(session, 'FV_Stats.Glicko.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) - updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) - - updateSelectInput(session, 'FV_Stats.Overview.Algid', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTSummary.Statistics.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RTSummary.Overview.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FCESummary.Overview.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RTSummary.Sample.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCESummary.Statistics.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FCESummary.Sample.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Summary.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Sample.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Summary.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Sample.Algid', choices = algIds, selected = 'all') - updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = algIds_, selected = selected_alg) - updateSelectInput(session, 'ERTPlot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ERTPlot.Aggr_Dim.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPlot.Multi.Algs', choices = algIds_, selected = selected_alg) - updateSelectInput(session, 'FCEPlot.Aggr.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPlot.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPDF.Bar.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEPDF.Hist.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTPMF.Bar.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTPMF.Hist.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FV_PAR.Summary.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'FV_PAR.Sample.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Summary.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RT_PAR.Sample.Param', choices = parIds, selected = 'all') - updateSelectInput(session, 'RTECDF.Single.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTECDF.Aggr.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTECDF.AUC.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'RTECDF.Multi.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEECDF.Single.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEECDF.Mult.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'FCEECDF.AUC.Algs', choices = algIds_, selected = algIds_) - updateSelectInput(session, 'ParCoordPlot.Algs', choices = algIds_, selected = algIds_[[1]]) - updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) - updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) -}) - -# update (filter) according to users selection DataSets -DATA <- reactive({ - dim <- input$Overall.Dim - id <- input$Overall.Funcid - req(dim, id) - - if (length(DataList$data) == 0) return(NULL) - - d <- subset(DataList$data, DIM == dim, funcId == id) - if (length(d) == 0 && dim != "" && id != "") { - showNotification("There is no data available for this (dimension,function)-pair") - } - d -}) - -# TODO: give a different name for DATA and DATA_RAW -DATA_RAW <- reactive({ - DataList$data -}) - -MAX_ERTS_FUNC <- reactive({ - dim <- input$Overall.Dim - data <- subset(DataList$data, DIM == dim) - max_ERTs(data, aggr_on = 'funcId', maximize = attr(data, 'maximization')) -}) - -MAX_ERTS_DIM <- reactive({ - func <- input$Overall.Funcid - data <- subset(DataList$data, funcId == func) - max_ERTs(data, aggr_on = 'DIM', maximize = attr(data, 'maximization')) -}) - -MEAN_FVALS_FUNC <- reactive({ - dim <- input$Overall.Dim - data <- subset(DataList$data, DIM == dim) - mean_FVs(data, aggr_on = 'funcId') -}) - -MEAN_FVALS_DIM <- reactive({ - func <- input$Overall.Funcid - data <- subset(DataList$data, funcId == func) - mean_FVs(data, aggr_on = 'DIM') -}) - -# TODO: make this urgely snippet look better... -# register the TextInput and restore them when switching funcID and DIM -observeEvent(eval(eventExpr), { - data <- DATA() - name <- get_data_id(data) - - if (is.null(name)) - return() - - for (id in widget_id) { - REG[[id]][[name]] <<- input[[id]] - } -}) - -# update the values for the grid of target values -observe({ - data <- DATA() - v <- get_funvals(data) - name <- get_data_id(data) - req(v) - - # choose proper scaling for the function value - # v <- trans_funeval(v) # Do the scaling in seq_FV instead - q <- quantile(v, probs = c(.25, .5, .75), names = F) - - # TODO: we need to fix this for the general case!!! - length.out <- 10 - fseq <- seq_FV(v, length.out = length.out) - start <- fseq[1] - stop <- fseq[length.out] - - #TODO: Make more general - if (abs(2 * fseq[3] - fseq[2] - fseq[4]) < 1e-12) #arbitrary precision - step <- fseq[3] - fseq[2] - else - step <- log10(fseq[3]) - log10(fseq[2]) - - setTextInput(session, 'RTSummary.Statistics.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTSummary.Statistics.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTSummary.Statistics.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RTSummary.Sample.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTSummary.Sample.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTSummary.Sample.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RTECDF.Multi.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTECDF.Multi.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTECDF.Multi.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RTPMF.Bar.Target', name, alternative = format_FV(median(v))) - setTextInput(session, 'RTPMF.Hist.Target', name, alternative = format_FV(median(v))) - setTextInput(session, 'ERTPlot.Min', name, alternative = format_FV(start)) - setTextInput(session, 'ERTPlot.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'ERTPlot.Aggr.Targets', name, alternative = "") - setTextInput(session, 'FCEPlot.Aggr.Targets', name, alternative = "") - setTextInput(session, 'RTECDF.Single.Target', name, alternative = format_FV(q[2])) - setTextInput(session, 'RTECDF.AUC.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RTECDF.AUC.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RTECDF.AUC.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RT_PAR.Plot.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RT_PAR.Plot.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RT_PAR.Summary.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RT_PAR.Summary.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RT_PAR.Summary.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RT_PAR.Sample.Min', name, alternative = format_FV(start)) - setTextInput(session, 'RT_PAR.Sample.Max', name, alternative = format_FV(stop)) - setTextInput(session, 'RT_PAR.Sample.Step', name, alternative = format_FV(step)) - setTextInput(session, 'RT_Stats.Overview.Target', name, alternative = format_FV(stop)) - setTextInput(session, 'RT.Multisample.Target', name, alternative = format_FV(median(v))) - setTextInput(session, 'RT.MultiERT.Target', name, alternative = format_FV(median(v))) -}) - -# update the values for the grid of running times -observe({ - data <- DATA() - v <- get_runtimes(data) - name <- get_data_id(data) - - if (length(v) == 0) return() - - # TODO: this part should be made generic!!! - q <- quantile(v, probs = c(.25, .5, .75), names = F, type = 3) - - grid <- seq(min(v), max(v), length.out = 10) - step <- max(1, min(grid[-1] - grid[-length(grid)])) - - start <- min(v) - stop <- max(v) - setTextInput(session, 'FV.Multisample.Target', name, alternative = max(v)) - setTextInput(session, 'FV.MultiFV.Target', name, alternative = max(v)) - setTextInput(session, 'FCESummary.Statistics.Min', name, alternative = min(v)) - setTextInput(session, 'FCESummary.Statistics.Max', name, alternative = max(v)) - setTextInput(session, 'FCESummary.Statistics.Step', name, alternative = step) - setTextInput(session, 'FCESummary.Sample.Min', name, alternative = min(v)) - setTextInput(session, 'FCESummary.Sample.Max', name, alternative = max(v)) - setTextInput(session, 'FCESummary.Sample.Step', name, alternative = step) - setTextInput(session, 'FCEPDF.Hist.Runtime', name, alternative = median(v)) - setTextInput(session, 'FCEPDF.Bar.Runtime', name, alternative = median(v)) - setTextInput(session, 'FCEPlot.Min', name, alternative = start) - setTextInput(session, 'FCEPlot.Max', name, alternative = stop) - setTextInput(session, 'FCEECDF.Mult.Min', name, alternative = min(v)) - setTextInput(session, 'FCEECDF.Mult.Max', name, alternative = max(v)) - setTextInput(session, 'FCEECDF.Mult.Step', name, alternative = step) - setTextInput(session, 'FCEECDF.AUC.Min', name, alternative = min(v)) - setTextInput(session, 'FCEECDF.AUC.Max', name, alternative = max(v)) - setTextInput(session, 'FCEECDF.AUC.Step', name, alternative = step) - setTextInput(session, 'FV_PAR.Plot.Min', name, alternative = min(v)) - setTextInput(session, 'FV_PAR.Plot.Max', name, alternative = max(v)) - setTextInput(session, 'FV_PAR.Summary.Min', name, alternative = min(v)) - setTextInput(session, 'FV_PAR.Summary.Max', name, alternative = max(v)) - setTextInput(session, 'FV_PAR.Summary.Step', name, alternative = step) - setTextInput(session, 'FV_PAR.Sample.Min', name, alternative = min(v)) - setTextInput(session, 'FV_PAR.Sample.Max', name, alternative = max(v)) - setTextInput(session, 'FV_PAR.Sample.Step', name, alternative = step) - setTextInput(session, 'FV_Stats.Overview.Target', name, alternative = max(v)) - #TODO: remove q and replace by single number - setTextInput(session, 'FCEECDF.Single.Target', name, alternative = q[2]) -}) - -output$upload.Download_processed <- downloadHandler( - filename = "DataSetList.rds", - content = function(file) { - saveRDS(DATA_RAW(), file = file) - } -) - +# shared reactive variables +folderList <- reactiveValues(data = list()) +DataList <- reactiveValues(data = DataSetList()) + +observe({ + repo_dir <- get_repo_location() + dirs <- list.dirs(repo_dir, full.names = F) + if (length(dirs) == 0) { + shinyjs::alert("No repository directory found. To make use of the IOHProfiler-repository, + please create a folder called 'repository' in your home directory + and make sure it contains at least one '.rds'-file of a DataSetList-object, + such as the ones provided on the IOHProfiler github-page.") + } + else + updateSelectInput(session, 'repository.type', choices = dirs, selected = dirs[[1]]) +}) + +# set up list of datasets (scan the repository, looking for .rds files) +observe({ + req(input$repository.type) + repo_dir <- get_repo_location() + dir <- file.path(repo_dir, input$repository.type) + + rds_files <- list.files(dir, pattern = '.rds$') %>% sub('\\.rds$', '', .) + if (length(rds_files) != 0) { + updateSelectInput(session, 'repository.dataset', choices = rds_files, selected = rds_files[[1]]) + } else {# TODO: the alert msg should be updated + shinyjs::alert("No repository file found. To make use of the IOHProfiler-repository, + please create a folder called 'repository' in your home directory + and make sure it contains at least one '.rds'-file of a DataSetList-object, + such as the ones provided on the IOHProfiler github-page.") + updateSelectInput(session, 'repository.dataset', choices = NULL, selected = NULL) + } +}) + +# load repository that is selected +observeEvent(input$repository.dataset, { + req(input$repository.dataset) + repo_dir <- get_repo_location() + algs <- c() + dims <- c() + funcs <- c() + + for (f in input$repository.dataset) { + rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) + if (file.exists(paste0(rds_file, '_info'))) { + info <- readRDS(paste0(rds_file, '_info')) + } + else { + dsl <- readRDS(rds_file) + info = list(algId = get_algId(dsl), funcId = get_funcId(dsl), DIM = get_dim(dsl)) + } + algs <- c(algs, info$algId) + dims <- c(dims, info$DIM) + funcs <- c(funcs, info$funcId) + } + + algs <- unique(algs) + dims <- unique(dims) + funcs <- unique(funcs) + + updateSelectInput(session, 'repository.ID', choices = algs, selected = algs) + updateSelectInput(session, 'repository.dim', choices = dims, selected = dims) + updateSelectInput(session, 'repository.funcId', choices = funcs, selected = funcs) + shinyjs::enable('repository.load_button') +}) + +# add the data from repository +observeEvent(input$repository.load_button, { + data <- DataSetList() + repo_dir <- get_repo_location() + for (f in input$repository.dataset) { + rds_file <- file.path(repo_dir, input$repository.type, paste0(f, ".rds")) + data <- c(data, readRDS(rds_file)) + } + data <- subset(data, funcId %in% input$repository.funcId) + data <- subset(data, DIM %in% input$repository.dim) + data <- subset(data, algId %in% input$repository.ID) + + if (length(DataList$data) > 0 && attr(data, 'maximization') != attr(DataList$data, 'maximization')) { + shinyjs::alert(paste0("Attempting to add data from a different optimization type to the currently", + " loaded data.\nPlease either remove the currently loaded data or", + " choose a different dataset to load.")) + return(NULL) + } + + DataList$data <- c(DataList$data, data) + DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) + update_menu_visibility(attr(DataList$data, 'suite')) + # set_format_func(attr(DataList$data, 'suite')) + IDs <- get_id(DataList$data) + if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { + set_color_scheme("Default", IDs) + } +}) + +# decompress zip files recursively and return the root directory of extracted files +unzip_fct_recursive <- function(zipfile, exdir, print_fun = print, alert_fun = print, depth = 0) { + filetype <- basename(zipfile) %>% + strsplit('\\.') %>% `[[`(1) %>% + rev %>% + `[`(1) + folders <- list() + + if (filetype == 'zip') + unzip_fct <- unzip + else if (filetype %in% c('bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')) + unzip_fct <- untar + + files <- unzip_fct(zipfile, list = FALSE, exdir = file.path(exdir, rand_strings(1))) + if (length(files) == 0) { + alert_fun("An error occured while unzipping the provided files.\n + Please ensure no archives are corrupted and the filenames are + in base-64.") + return(NULL) + } + print_fun(paste0('

Succesfully unzipped ', basename(zipfile), '.
')) + + folders <- grep('*.info|csv|txt', files, value = T) %>% + dirname %>% + unique %>% + grep('__MACOSX', ., value = T, invert = T) %>% # to get rid of __MACOSX folder on MAC.. + c(folders) + + zip_files <- grep('.*zip|bz2|bz|gz|tar|tgz|tar\\.gz|xz', files, value = T, perl = T) %>% + grep('__MACOSX', ., value = T, invert = T) + + if (depth <= 3) { # only allow for 4 levels of recursions + for (zipfile in zip_files) { + .folders <- unzip_fct_recursive(zipfile, dirname(zipfile), alert_fun, + print_fun, depth + 1) + folders <- c(folders, .folders) + } + } + unique(folders) +} + +# upload the compressed the data file and uncompress them +selected_folders <- reactive({ + if (is.null(input$upload.add_zip)) return(NULL) + + tryCatch({ + datapath <- input$upload.add_zip$datapath + folders <- c() + + for (i in seq(datapath)) { + filetype <- basename(datapath[i]) %>% + strsplit('\\.') %>% `[[`(1) %>% + rev %>% + `[`(1) + + print_html(paste0('

Handling ', filetype, '-data.
')) + if (filetype == 'csv' || filetype == 'rds') { + # add the data path to the folders list direct + folders <- c(folders, datapath[[i]]) + next + } + + .folders <- unzip_fct_recursive(datapath[i], exdir, print_html) %>% unique + folders %<>% c(.folders) + } + unique(folders) + }, error = function(e) shinyjs::alert(paste0("The following error occured when processing the uploaded data: ", e)) + ) +}) + +# load, process the data folders, and update DataSetList +observeEvent(selected_folders(), { + withProgress({ + folders <- selected_folders() + req(length(folders) != 0) + + format_selected <- input$upload.data_format + maximization <- input$upload.maximization + + if (maximization == "AUTOMATIC") maximization <- NULL + + if (length(folderList$data) == 0) + folder_new <- folders + else + folder_new <- setdiff(folders, intersect(folderList$data, folders)) + + req(length(folder_new) != 0) + format_detected <- lapply(folder_new, check_format) %>% unique + + if (length(format_detected) > 1) + print_html(paste('

more than one format:
', + format_detected, + 'is detected in the uploaded data set... skip the uploaded data')) + else + format_detected <- format_detected[[1]] + + print_html(paste0('

Data processing of source type:', format_detected, '
')) + + for (folder in folder_new) { + indexFiles <- scan_index_file(folder) + + if (length(indexFiles) == 0 && !(format_detected %in% c(NEVERGRAD, "SOS", "RDS"))) + print_html(paste('

No .info-files detected in the + uploaded folder, while they were expected:

', folder)) + else { + folderList$data <- c(folderList$data, folder) + + if (format_detected == "RDS") { + new_data <- readRDS(folder) + if (!("DataSetList" %in% class(new_data))) + DataSetList() + } + else { + # read the data set and handle potential errors + new_data <- tryCatch( + DataSetList(folder, print_fun = print_html, + maximization = maximization, + format = format_detected, + subsampling = input$upload.subsampling), + error = function(e) { + print_html(paste('

The following error happened + when processing the data set:

')) + print_html(paste('

', e, '

')) + DataSetList() + } + ) + } + tryCatch( + DataList$data <- c(DataList$data, new_data), + error = function(e) { + print_html(paste('

The following error happened', + 'when adding the uploaded data set:

')) + print_html(paste('

', e, + '\nRemoving the old data.

')) + DataList$data <- new_data + } + ) + shinyjs::html("upload_data_promt", + sprintf('%d: %s\n', length(folderList$data), folder), + add = TRUE) + } + } + + DataList$data <- clean_DataSetList(DataList$data) + DataList$data <- change_id(DataList$data, getOption("IOHanalyzer.ID_vars", c("algId"))) + if (is.null(DataList$data)) { + shinyjs::alert("An error occurred when processing the uploaded data. + Please ensure the data is not corrupted.") + return(NULL) + } + + update_menu_visibility(attr(DataList$data, 'suite')) + # set_format_func(attr(DataList$data, 'suite')) + IDs <- get_id(DataList$data) + if (!all(IDs %in% get_color_scheme_dt()[['ids']])) { + set_color_scheme("Default", IDs) + } + }, message = "Processing data, this might take some time") +}) + +update_menu_visibility <- function(suite){ + if (all(suite == NEVERGRAD)) { + #TODO: Better way of doing this such that these pages are not even populated with data instead of just being hidden + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "#shiny-tab-ERT")) + } + else{ + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "#shiny-tab-ERT")) + } + if (all(suite == "PBO")) { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "FCE_ECDF")) + } + else { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "FCE_ECDF")) + } + if (any(suite == "SOS")) { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "show", tabName = "Positions")) + } + else { + session$sendCustomMessage(type = "manipulateMenuItem", message = list(action = "hide", tabName = "Positions")) + } + if (!is.null(get_funcName(DataList$data))) + shinyjs::enable("Settings.Use_Funcname") + else + shinyjs::disable("Settings.Use_Funcname") +} + +observeEvent(input$Upload.Add_to_repo, { + data <- DATA_RAW() + repo_dir <- get_repo_location() + if (!file.exists(file.path(repo_dir, "public_repo"))) return(NULL) + nr_skipped <- 0 + public_dir <- file.path(repo_dir, "public_repo") + for (algname in get_algId(data)) { + filename <- file.path(public_dir, paste0(algname, '.rds')) + if (file.exists(filename)) { + nr_skipped <- nr_skipped + 1 + next + } + dsl <- subset(data, algId == algname) + saveRDS(dsl, file = filename) + } + nr_success <- length(get_algId(data)) - nr_skipped + shinyjs::alert(paste0("Successfully added ", nr_success, " algorithms to the public repository.", + "A total of ", nr_skipped, " algorithms were not uploaded because an algorithm", + "of the same name already exists, and overwriting data in the public repository is not yet", + "supported.")) +}) + +# remove all uploaded data set +observeEvent(input$upload.remove_data, { + if (length(DataList$data) != 0) { + DataList$data <- DataSetList() # NOTE: this must be a `DataSetList` object + unlink(folderList$data, T) + folderList$data <- list() + + updateSelectInput(session, 'Overall.Dim', choices = c(), selected = '') + updateSelectInput(session, 'Overall.Funcid', choices = c(), selected = '') + updateSelectInput(session, 'Overall.Funcname', choices = c(), selected = '') + updateSelectInput(session, 'Overall.ID', choices = c(), selected = '') + + print_html('

all data are removed!

') + print_html('', 'upload_data_promt') + } +}) + +# show the detailed information on DataSetList +output$data_info <- renderDataTable({ + datasets <- DataList$data + if (length(datasets) != 0) + summary(datasets) + else + data.frame() +}, options = list(pageLength = 10, scrollX = F, autoWidth = TRUE, + columnDefs = list(list(width = '20px', targets = c(0, 1))))) + +# update the list of dimensionality, funcId and algId and parameter list +observe({ + data <- DataList$data + if (length(data) == 0) + return() + + # TODO: create reactive values for them + IDs <- get_id(data) + # algIds <- c(IDs, 'all') + parIds_ <- get_parId(data) + parIds <- c(parIds_, 'all') + funcIds <- get_funcId(data) + DIMs <- get_dim(data) + algIds <- get_algId(data) + + selected_ds <- data[[1]] + selected_f <- attr(selected_ds,'funcId') + selected_dim <- attr(selected_ds, 'DIM') + selected_alg <- attr(selected_ds, 'algId') + selected_ID <- get_id(selected_ds) + + updateSelectInput(session, 'Overall.Dim', choices = DIMs, selected = selected_dim) + updateSelectInput(session, 'Overall.Funcid', choices = funcIds, selected = selected_f) + if (input$Settings.Use_Funcname) + updateSelectInput(session, 'Overall.Funcname', choices = get_funcName(data), selected = get_funcName(data[1])) + + if ('algId' %in% input$Settings.ID.Variables) + updateSelectInput(session, 'Overall.ID', choices = NULL, selected = NULL) + else + updateSelectInput(session, 'Overall.ID', choices = algIds, selected = selected_alg) + + updateSelectInput(session, 'ERTPlot.Aggr.Funcs', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'Overview.Single.ID', choices = IDs, selected = IDs) + + # updateSelectInput(session, 'Report.RT.Overview-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Overview-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Overview-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Statistics-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Statistics-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Statistics-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Single_ERT-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Single_ERT-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Single_ERT-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Multi_ERT-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Multi_ERT-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Rank-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Rank-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.Histogram-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.Histogram-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.Histogram-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.PMF-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.PMF-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.PMF-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.RT.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.RT.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.RT.ECDF_AUC-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Overview-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Overview-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Overview-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Statistics-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Statistics-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Statistics-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Single_FCE-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Single_FCE-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Single_FCE-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Multi_FCE-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Multi_FCE-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Rank-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Rank-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.Histogram-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.Histogram-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.Histogram-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.PMF-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.PMF-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.PMF-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Target-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.ECDF_Single_Function-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_Aggregated-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.FV.ECDF_AUC-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.FV.ECDF_AUC-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.FV.ECDF_AUC-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.Param.Plot-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.Param.Plot-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.Param.Plot-Alg', choices = IDs, selected = IDs) + # + # updateSelectInput(session, 'Report.Param.Statistics-FuncId', choices = funcIds, selected = selected_f) + # updateSelectInput(session, 'Report.Param.Statistics-DIM', choices = DIMs, selected = selected_dim) + # updateSelectInput(session, 'Report.Param.Statistics-Alg', choices = IDs, selected = IDs) + + updateSelectInput(session, 'RTportfolio.Shapley.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTportfolio.Shapley.Funcs', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RTportfolio.Shapley.Dims', choices = DIMs, selected = DIMs) + updateNumericInput(session, 'RTportfolio.Shapley.Permsize', min = 2, max = length(IDs), + value = min(10, length(IDs))) + + + updateSelectInput(session, 'RT.MultiERT.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT.MultiERT.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RT.MultiERT.DIM', choices = DIMs, selected = selected_dim) + updateSelectInput(session, 'RT.Multisample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT.Multisample.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RT.Multisample.DIM', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'FV.MultiFV.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV.MultiFV.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'FV.MultiFV.DIM', choices = DIMs, selected = selected_dim) + updateSelectInput(session, 'FV.Multisample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV.Multisample.FuncId', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'FV.Multisample.DIM', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'RT_Stats.Glicko.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) + updateSelectInput(session, 'RT_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'RT_Stats.DSC.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'RT_Stats.DSC.Dim', choices = DIMs, selected = DIMs) + + updateSelectInput(session, 'FV_Stats.DSC.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.DSC.Funcid', choices = funcIds, selected = funcIds) + updateSelectInput(session, 'FV_Stats.DSC.Dim', choices = DIMs, selected = DIMs) + + updateSelectInput(session, 'RT_Stats.Overview.ID', choices = IDs, selected = IDs) + + updateSelectInput(session, 'FV_Stats.Glicko.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_Stats.Glicko.Funcid', choices = funcIds, selected = selected_f) + updateSelectInput(session, 'FV_Stats.Glicko.Dim', choices = DIMs, selected = selected_dim) + + updateSelectInput(session, 'FV_Stats.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Statistics.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Overview.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTSummary.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Plot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Plot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Statistics.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCESummary.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Summary.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Summary.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'RT_PAR.Sample.ID', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Multi.Algs', choices = IDs, selected = selected_ID) + updateSelectInput(session, 'ERTPlot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ERTPlot.Aggr_Dim.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPlot.Multi.Algs', choices = IDs, selected = selected_ID) + updateSelectInput(session, 'FCEPlot.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPlot.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPDF.Bar.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEPDF.Hist.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTPMF.Bar.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTPMF.Hist.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FV_PAR.Summary.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'FV_PAR.Sample.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'RT_PAR.Summary.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'RT_PAR.Sample.Param', choices = parIds, selected = 'all') + updateSelectInput(session, 'RTECDF.Single.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.Aggr.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.AUC.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'RTECDF.Multi.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.Single.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.Mult.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'FCEECDF.AUC.Algs', choices = IDs, selected = IDs) + updateSelectInput(session, 'ParCoordPlot.Algs', choices = IDs, selected = IDs[[1]]) + updateSelectInput(session, 'FV_PAR.Plot.Params', choices = parIds_, selected = parIds_) + updateSelectInput(session, 'RT_PAR.Plot.Params', choices = parIds_, selected = parIds_) + + attr_choices <- get_static_attributes(data) + invalid_choices <- c('funcId', 'DIM', 'ID') + updateSelectInput(session, 'Settings.ID.Variables', choices = attr_choices[!attr_choices %in% invalid_choices], + selected = attr(data, 'ID_attributes')) +}) + + + +# update (filter) according to users selection DataSets +DATA <- reactive({ + dim <- input$Overall.Dim + req(dim) + + d <- subset(DataList$data, DIM == dim) + + if (!'algId' %in% input$Settings.ID.Variables) { + algid <- input$Overall.ID + if (!is.null(algid)) d <- subset(d, algId == algid) + } + + if (input$Settings.Use_Funcname) { + fname <- input$Overall.Funcname + if (!is.null(fname)) d <- subset(d, funcname == fname) + } + else { + fid <- input$Overall.Funcid + if (!is.null(fid)) d <- subset(d, funcId == fid) + } + + if (length(DataList$data) == 0) return(NULL) + + if (length(d) == 0 && dim != "") { + showNotification("There is no data available for this (dimension,function)-pair") + } + d +}) + +# TODO: give a different name for DATA and DATA_RAW +DATA_RAW <- reactive({ + DataList$data +}) + +# This observe statement tries to match funcid and funcname seletions so funcId can still be used internally. +# TODO: think of a better solution to ensure this matching doesn't break. +observe({ + req(length(DATA_RAW()) > 0) + fname <- input$Overall.Funcname + req(fname) + req(getOption('IOHanalyzer.function_representation', 'funcId') == 'funcname') + dsl_sub <- subset(DATA_RAW(), funcName == fname) + fids <- get_funcId(dsl_sub) + if (length(fids) == 1) { + updateSelectInput(session, 'Overall.Funcid', selected = fids) + } +}) + +MAX_ERTS_FUNC <- reactive({ + dim <- input$Overall.Dim + data <- subset(DataList$data, DIM == dim) + max_ERTs(data, aggr_on = 'funcId', maximize = attr(data, 'maximization')) +}) + +MAX_ERTS_DIM <- reactive({ + func <- input$Overall.Funcid + data <- subset(DataList$data, funcId == func) + max_ERTs(data, aggr_on = 'DIM', maximize = attr(data, 'maximization')) +}) + +MEAN_FVALS_FUNC <- reactive({ + dim <- input$Overall.Dim + data <- subset(DataList$data, DIM == dim) + mean_FVs(data, aggr_on = 'funcId') +}) + +MEAN_FVALS_DIM <- reactive({ + func <- input$Overall.Funcid + data <- subset(DataList$data, funcId == func) + mean_FVs(data, aggr_on = 'DIM') +}) + +# TODO: make this urgely snippet look better... +# register the TextInput and restore them when switching funcID and DIM +observeEvent(eval(eventExpr), { + data <- DATA() + name <- get_data_id(data) + + if (is.null(name)) + return() + + for (id in widget_id) { + REG[[id]][[name]] <<- input[[id]] + } +}) + +# update the values for the grid of target values +observe({ + data <- DATA() + v <- get_funvals(data) + name <- get_data_id(data) + req(v) + + # choose proper scaling for the function value + # v <- trans_funeval(v) # Do the scaling in seq_FV instead + q <- quantile(v, probs = c(.25, .5, .75), names = F) + + # TODO: we need to fix this for the general case!!! + length.out <- 10 + fseq <- seq_FV(v, length.out = length.out) + start <- fseq[1] + stop <- fseq[length.out] + + #TODO: Make more general + if (abs(2 * fseq[3] - fseq[2] - fseq[4]) < 1e-12) #arbitrary precision + step <- fseq[3] - fseq[2] + else + step <- log10(fseq[3]) - log10(fseq[2]) + + setTextInput(session, 'RTSummary.Statistics.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTSummary.Statistics.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTSummary.Statistics.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RTSummary.Sample.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTSummary.Sample.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTSummary.Sample.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RTECDF.Multi.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTECDF.Multi.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTECDF.Multi.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RTPMF.Bar.Target', name, alternative = format_FV(median(v))) + setTextInput(session, 'RTPMF.Hist.Target', name, alternative = format_FV(median(v))) + setTextInput(session, 'ERTPlot.Min', name, alternative = format_FV(start)) + setTextInput(session, 'ERTPlot.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'ERTPlot.Aggr.Targets', name, alternative = "") + setTextInput(session, 'FCEPlot.Aggr.Targets', name, alternative = "") + setTextInput(session, 'RTECDF.Single.Target', name, alternative = format_FV(q[2])) + setTextInput(session, 'RTECDF.AUC.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RTECDF.AUC.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RTECDF.AUC.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RT_PAR.Plot.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RT_PAR.Plot.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RT_PAR.Summary.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RT_PAR.Summary.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RT_PAR.Summary.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RT_PAR.Sample.Min', name, alternative = format_FV(start)) + setTextInput(session, 'RT_PAR.Sample.Max', name, alternative = format_FV(stop)) + setTextInput(session, 'RT_PAR.Sample.Step', name, alternative = format_FV(step)) + setTextInput(session, 'RT_Stats.Overview.Target', name, alternative = format_FV(stop)) + setTextInput(session, 'RT.Multisample.Target', name, alternative = format_FV(median(v))) + setTextInput(session, 'RT.MultiERT.Target', name, alternative = format_FV(median(v))) +}) + +# update the values for the grid of running times +observe({ + data <- DATA() + v <- get_runtimes(data) + name <- get_data_id(data) + + if (length(v) == 0) return() + + # TODO: this part should be made generic!!! + q <- quantile(v, probs = c(.25, .5, .75), names = F, type = 3) + + grid <- seq(min(v), max(v), length.out = 10) + step <- max(1, min(grid[-1] - grid[-length(grid)])) + + start <- min(v) + stop <- max(v) + setTextInput(session, 'FV.Multisample.Target', name, alternative = max(v)) + setTextInput(session, 'FV.MultiFV.Target', name, alternative = max(v)) + setTextInput(session, 'FCESummary.Statistics.Min', name, alternative = min(v)) + setTextInput(session, 'FCESummary.Statistics.Max', name, alternative = max(v)) + setTextInput(session, 'FCESummary.Statistics.Step', name, alternative = step) + setTextInput(session, 'FCESummary.Sample.Min', name, alternative = min(v)) + setTextInput(session, 'FCESummary.Sample.Max', name, alternative = max(v)) + setTextInput(session, 'FCESummary.Sample.Step', name, alternative = step) + setTextInput(session, 'FCEPDF.Hist.Runtime', name, alternative = median(v)) + setTextInput(session, 'FCEPDF.Bar.Runtime', name, alternative = median(v)) + setTextInput(session, 'FCEPlot.Min', name, alternative = start) + setTextInput(session, 'FCEPlot.Max', name, alternative = stop) + setTextInput(session, 'FCEECDF.Mult.Min', name, alternative = min(v)) + setTextInput(session, 'FCEECDF.Mult.Max', name, alternative = max(v)) + setTextInput(session, 'FCEECDF.Mult.Step', name, alternative = step) + setTextInput(session, 'FCEECDF.AUC.Min', name, alternative = min(v)) + setTextInput(session, 'FCEECDF.AUC.Max', name, alternative = max(v)) + setTextInput(session, 'FCEECDF.AUC.Step', name, alternative = step) + setTextInput(session, 'FV_PAR.Plot.Min', name, alternative = min(v)) + setTextInput(session, 'FV_PAR.Plot.Max', name, alternative = max(v)) + setTextInput(session, 'FV_PAR.Summary.Min', name, alternative = min(v)) + setTextInput(session, 'FV_PAR.Summary.Max', name, alternative = max(v)) + setTextInput(session, 'FV_PAR.Summary.Step', name, alternative = step) + setTextInput(session, 'FV_PAR.Sample.Min', name, alternative = min(v)) + setTextInput(session, 'FV_PAR.Sample.Max', name, alternative = max(v)) + setTextInput(session, 'FV_PAR.Sample.Step', name, alternative = step) + setTextInput(session, 'FV_Stats.Overview.Target', name, alternative = max(v)) + #TODO: remove q and replace by single number + setTextInput(session, 'FCEECDF.Single.Target', name, alternative = q[2]) +}) + +output$upload.Download_processed <- downloadHandler( + filename = "DataSetList.rds", + content = function(file) { + saveRDS(DATA_RAW(), file = file) + } +) + diff --git a/inst/shiny-server/theme.R b/inst/shiny-server/theme.R index c9d17cab..9ededc31 100644 --- a/inst/shiny-server/theme.R +++ b/inst/shiny-server/theme.R @@ -1,101 +1,101 @@ -theme_grey_light <- shinyDashboardThemeDIY( - - ### general - appFontFamily = "Arial" - ,appFontColor = "rgb(45,45,45)" - ,primaryFontColor = "rgb(255,255,255)" - ,infoFontColor = "rgb(15,15,15)" - ,successFontColor = "rgb(15,15,15)" - ,warningFontColor = "rgb(15,15,15)" - ,dangerFontColor = "rgb(15,15,15)" - ,bodyBackColor = "rgb(240,240,240)" - - ### header - ,logoBackColor = "rgb(0,17,88)" - - ,headerButtonBackColor = "rgb(0,17,88)" - ,headerButtonIconColor = "rgb(220,220,220)" - ,headerButtonBackColorHover = "rgb(0,17,88)" - ,headerButtonIconColorHover = "rgb(60,60,60)" - - ,headerBackColor = "rgb(0,17,88)" - ,headerBoxShadowColor = "#dfdfdf" - ,headerBoxShadowSize = "3px 5px 5px" - - ### sidebar - ,sidebarBackColor = "rgb(255,255,255)" - ,sidebarPadding = 0 - - ,sidebarMenuBackColor = "transparent" - ,sidebarMenuPadding = 0 - ,sidebarMenuBorderRadius = 0 - - ,sidebarShadowRadius = "3px 5px 5px" - ,sidebarShadowColor = "#dfdfdf" - - ,sidebarUserTextColor = "rgb(115,115,115)" - - ,sidebarSearchBackColor = "rgb(240,240,240)" - ,sidebarSearchIconColor = "rgb(100,100,100)" - ,sidebarSearchBorderColor = "rgb(220,220,220)" - - ,sidebarTabTextColor = "rgb(100,100,100)" - ,sidebarTabTextSize = 14 - ,sidebarTabBorderStyle = "none" - ,sidebarTabBorderColor = "none" - ,sidebarTabBorderWidth = 0 - - ,sidebarTabBackColorSelected = "rgb(230,230,230)" - ,sidebarTabTextColorSelected = "rgb(0,0,0)" - ,sidebarTabRadiusSelected = "0px" - - ,sidebarTabBackColorHover = "rgb(245,245,245)" - ,sidebarTabTextColorHover = "rgb(0,0,0)" - ,sidebarTabBorderStyleHover = "none solid none none" - ,sidebarTabBorderColorHover = "rgb(200,200,200)" - ,sidebarTabBorderWidthHover = 4 - ,sidebarTabRadiusHover = "0px" - - ,boxBackColor = "rgb(248,248,248)" - ,boxBorderRadius = 5 - ,boxShadowSize = "none" - ,boxShadowColor = "" - ,boxTitleSize = 18 - ,boxDefaultColor = "rgb(225,225,225)" - ,boxPrimaryColor = "rgb(0,17,88)" - ,boxInfoColor = "rgb(180,180,180)" - ,boxSuccessColor = "rgb(112,173,71)" - ,boxWarningColor = "rgb(237,125,49)" - ,boxDangerColor = "rgb(232,76,34)" - - ,tabBoxTabColor = "rgb(0,17,88)" - ,tabBoxTabTextSize = 14 - ,tabBoxTabTextColor = "rgb(0,0,0)" - ,tabBoxTabTextColorSelected = "rgb(0,0,0)" - ,tabBoxBackColor = "rgb(248,248,248)" - ,tabBoxHighlightColor = "rgb(200,200,200)" - ,tabBoxBorderRadius = 5 - - ### inputs - ,buttonBackColor = "rgb(215,215,215)" - ,buttonTextColor = "rgb(45,45,45)" - ,buttonBorderColor = "rgb(150,150,150)" - ,buttonBorderRadius = 5 - - ,buttonBackColorHover = "rgb(190,190,190)" - ,buttonTextColorHover = "rgb(0,0,0)" - ,buttonBorderColorHover = "rgb(150,150,150)" - - ,textboxBackColor = "rgb(255,255,255)" - ,textboxBorderColor = "rgb(118,118,118)" - ,textboxBorderRadius = 5 - ,textboxBackColorSelect = "rgb(245,245,245)" - ,textboxBorderColorSelect = "rgb(108,108,108)" - - ### tables - ,tableBackColor = "rgb(248,248,248)" - ,tableBorderColor = "rgb(238,238,238)" - ,tableBorderTopSize = 1 - ,tableBorderRowSize = 1 - +theme_grey_light <- shinyDashboardThemeDIY( + + ### general + appFontFamily = "Arial" + ,appFontColor = "rgb(45,45,45)" + ,primaryFontColor = "rgb(255,255,255)" + ,infoFontColor = "rgb(15,15,15)" + ,successFontColor = "rgb(15,15,15)" + ,warningFontColor = "rgb(15,15,15)" + ,dangerFontColor = "rgb(15,15,15)" + ,bodyBackColor = "rgb(240,240,240)" + + ### header + ,logoBackColor = "rgb(0,17,88)" + + ,headerButtonBackColor = "rgb(0,17,88)" + ,headerButtonIconColor = "rgb(220,220,220)" + ,headerButtonBackColorHover = "rgb(0,17,88)" + ,headerButtonIconColorHover = "rgb(60,60,60)" + + ,headerBackColor = "rgb(0,17,88)" + ,headerBoxShadowColor = "#dfdfdf" + ,headerBoxShadowSize = "3px 5px 5px" + + ### sidebar + ,sidebarBackColor = "rgb(255,255,255)" + ,sidebarPadding = 0 + + ,sidebarMenuBackColor = "transparent" + ,sidebarMenuPadding = 0 + ,sidebarMenuBorderRadius = 0 + + ,sidebarShadowRadius = "3px 5px 5px" + ,sidebarShadowColor = "#dfdfdf" + + ,sidebarUserTextColor = "rgb(115,115,115)" + + ,sidebarSearchBackColor = "rgb(240,240,240)" + ,sidebarSearchIconColor = "rgb(100,100,100)" + ,sidebarSearchBorderColor = "rgb(220,220,220)" + + ,sidebarTabTextColor = "rgb(100,100,100)" + ,sidebarTabTextSize = 14 + ,sidebarTabBorderStyle = "none" + ,sidebarTabBorderColor = "none" + ,sidebarTabBorderWidth = 0 + + ,sidebarTabBackColorSelected = "rgb(230,230,230)" + ,sidebarTabTextColorSelected = "rgb(0,0,0)" + ,sidebarTabRadiusSelected = "0px" + + ,sidebarTabBackColorHover = "rgb(245,245,245)" + ,sidebarTabTextColorHover = "rgb(0,0,0)" + ,sidebarTabBorderStyleHover = "none solid none none" + ,sidebarTabBorderColorHover = "rgb(200,200,200)" + ,sidebarTabBorderWidthHover = 4 + ,sidebarTabRadiusHover = "0px" + + ,boxBackColor = "rgb(248,248,248)" + ,boxBorderRadius = 5 + ,boxShadowSize = "none" + ,boxShadowColor = "" + ,boxTitleSize = 18 + ,boxDefaultColor = "rgb(225,225,225)" + ,boxPrimaryColor = "rgb(0,17,88)" + ,boxInfoColor = "rgb(180,180,180)" + ,boxSuccessColor = "rgb(112,173,71)" + ,boxWarningColor = "rgb(237,125,49)" + ,boxDangerColor = "rgb(232,76,34)" + + ,tabBoxTabColor = "rgb(0,17,88)" + ,tabBoxTabTextSize = 14 + ,tabBoxTabTextColor = "rgb(0,0,0)" + ,tabBoxTabTextColorSelected = "rgb(0,0,0)" + ,tabBoxBackColor = "rgb(248,248,248)" + ,tabBoxHighlightColor = "rgb(200,200,200)" + ,tabBoxBorderRadius = 5 + + ### inputs + ,buttonBackColor = "rgb(215,215,215)" + ,buttonTextColor = "rgb(45,45,45)" + ,buttonBorderColor = "rgb(150,150,150)" + ,buttonBorderRadius = 5 + + ,buttonBackColorHover = "rgb(190,190,190)" + ,buttonTextColorHover = "rgb(0,0,0)" + ,buttonBorderColorHover = "rgb(150,150,150)" + + ,textboxBackColor = "rgb(255,255,255)" + ,textboxBorderColor = "rgb(118,118,118)" + ,textboxBorderRadius = 5 + ,textboxBackColorSelect = "rgb(245,245,245)" + ,textboxBorderColorSelect = "rgb(108,108,108)" + + ### tables + ,tableBackColor = "rgb(248,248,248)" + ,tableBorderColor = "rgb(238,238,238)" + ,tableBorderTopSize = 1 + ,tableBorderRowSize = 1 + ) \ No newline at end of file diff --git a/inst/shiny-server/ui.R b/inst/shiny-server/ui.R index ad64ee12..154bf62d 100644 --- a/inst/shiny-server/ui.R +++ b/inst/shiny-server/ui.R @@ -1,622 +1,622 @@ -for (f in list.files('ui', pattern = '.R', full.names = T)) { - source(f) -} - -# NOTE: this is copied from `shinydashboard` -#' Assert that a tag has specified properties -#' @param tag A tag object. -#' @param type The type of a tag, like "div", "a", "span". -#' @param class An HTML class. -#' @param allowUI If TRUE (the default), allow dynamic outputs generated by -#' \code{\link[shiny]{uiOutput}} or \code{\link[shiny]{htmlOutput}}. When a -#' dynamic output is provided, \code{tagAssert} won't try to validate the the -#' contents. -#' @keywords internal -tagAssert <- function(tag, type = NULL, class = NULL, allowUI = TRUE) { - if (!inherits(tag, "shiny.tag")) { - print(tag) - stop("Expected an object with class 'shiny.tag'.") - } - - # Skip dynamic output elements - if (allowUI && - (hasCssClass(tag, "shiny-html-output") || - hasCssClass(tag, "shinydashboard-menu-output"))) { - return() - } - - if (!is.null(type) && tag$name != type) { - stop("Expected tag to be of type ", type) - } - - if (!is.null(class)) { - if (is.null(tag$attribs$class)) { - stop("Expected tag to have class '", class, "'") - - } else { - tagClasses <- strsplit(tag$attribs$class, " ")[[1]] - if (!(class %in% tagClasses)) { - stop("Expected tag to have class '", class, "'") - } - } - } -} - -# Modified dashboard header from the original source for incorporating two `select input` widgets -.dashboardHeader <- function(..., title = NULL, titleWidth = NULL, disable = FALSE, .list = NULL) { - items <- c(list(...), .list) - lapply(items, tagAssert, type = "li", class = "dropdown") - - titleWidth <- validateCssUnit(titleWidth) - - # Set up custom CSS for custom width. - custom_css <- NULL - if (!is.null(titleWidth)) { - # This CSS is derived from the header-related instances of '230px' (the - # default sidebar width) from inst/AdminLTE/AdminLTE.css. One change is that - # instead making changes to the global settings, we've put them in a media - # query (min-width: 768px), so that it won't override other media queries - # (like max-width: 767px) that work for narrower screens. - custom_css <- tags$head( - tags$style( - HTML( - gsub( - "_WIDTH_", titleWidth, fixed = TRUE, ' - @media (min-width: 768px) { - .main-header > .navbar { - margin-left: _WIDTH_; - } - .main-header .logo { - width: _WIDTH_; - } - }' - ) - ) - ) - ) - } - - tags$header( - class = "main-header", - id = 'header', - custom_css, - style = if (disable) "display: none;", - span(class = "logo", title), - tags$nav( - class = "navbar navbar-static-top", role = "navigation", - # Embed hidden icon so that we get the font-awesome dependency - span(shiny::icon("bars"), style = "display:none;"), - # Sidebar toggle button - a( - href="#", class="sidebar-toggle", `data-toggle`="offcanvas", - role="button", span(class="sr-only", "Toggle navigation") - ), - - # select inputs for dimension and function/problem ID - HTML(' -
-
- - - - - -
- Dimension: - - - Problem ID: - -
' - ), - - div( - class = "navbar-custom-menu", - tags$ul(class = "nav navbar-nav", items) - ) - ) - ) -} - -header <- .dashboardHeader(title = HTML('
IOHanalyzer
')) - -# The side bar layout --------------------------------------------- -sidebar <- dashboardSidebar( - useShinyjs(), - sidebar_menu(), - hr() - # DIM_fID_panel() -) - -body <- dashboardBody( - tags$style( - HTML(' - .popover-title {color:black;} - .popover-content {color:black;} - .main-sidebar {z-index:auto;} - .fa-exclamation-triangle {color:#E87722} - .sticky { - position: fixed; - top: 0; - width: 100%; - } - - .sticky2 { - position: fixed; - } - - .table { - border-collapse: collapse; - width: 100%; - } - .table td, tr { - padding: 0px; - margin: 0px; - vertical-align: middle; - height: 45px; - } - .table th {height: 0px;}' - ) - ), - - # to show text on the header (heading banner) - tags$head( - tags$style( - HTML(' - .myClass { - font-size: 20px; - line-height: 50px; - text-align: left; - font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; - padding: 0 15px; - overflow: hidden; - color: white; - }' - ) - ) - ), - - tags$head( - tags$style( - HTML( - '.box-title { - font-size: 20px; - line-height: 50px; - text-align: left; - font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; - padding: 0 15px; - overflow: hidden; - color: white; - }' - ) - ) - ), - - tags$head( - tags$style( - HTML( - "label { font-size:120%; }" - ) - ) - ), - - tags$script( - HTML(' - $(document).ready(function() { - $("header").find("nav").append(\'Performance Evaluation for Iterative - Optimization Heuristics\'); - })' - ) - ), - - tags$script( - "Shiny.addCustomMessageHandler('background-color', - function(color) { - document.body.style.backgroundColor = color; - document.body.innerText = color; - });" - ), - - tags$script( - HTML(' - window.setInterval(function() { - var elem = document.getElementById("process_data_promt"); - if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; - }, 20);' - ) - ), - - tags$head( - tags$script( - HTML(" - Shiny.addCustomMessageHandler('manipulateMenuItem', function(message){ - var aNodeList = document.getElementsByTagName('a'); - - for (var i = 0; i < aNodeList.length; i++) { - if(aNodeList[i].getAttribute('data-value') == message.tabName || aNodeList[i].getAttribute('href') == message.tabName) { - if(message.action == 'hide'){ - aNodeList[i].setAttribute('style', 'display: none;'); - } else { - aNodeList[i].setAttribute('style', 'display: block;'); - }; - }; - } - }); - ") - ) - ), - - # make the data uploading prompt always scroll to the bottom - tags$script( - HTML(' - window.setInterval(function() { - var elem = document.getElementById("upload_data_promt"); - if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; - }, 20);' - ) - ), - - # render the header and the side bar 'sticky' - tags$script( - HTML( - '// When the user scrolls the page, execute myFunction - window.onscroll = function() {myFunction()}; - - // Get the header - var header = document.getElementById("header"); - - // Get the side bar - var sideBar = document.getElementById("sidebarCollapsed"); - sideBar.classList.add("sticky2"); - - // Get the offset position of the navbar - var sticky = header.offsetTop; - - // Add the sticky class to the header when you reach its scroll position. - // Remove "sticky" when you leave the scroll position - function myFunction() { - if (window.pageYOffset > sticky) { - header.classList.add("sticky"); - } else { - header.classList.remove("sticky"); - } - }' - ) - ), - - if (suppressWarnings(require("dashboardthemes", quietly = T))) { - shinyDashboardThemes( - theme = "grey_light" - ) - }, - - # load MathJax - # TODO: download MathJax and its license and include it in our package - HTML(""), - use_bs_tooltip(), - use_bs_popover(), - - # tabitems ---------------------- - tabItems( - tabItem(tabName = 'about', includeMarkdown('markdown/about.md')), - # tabItem(tabName = 'dataformat', includeMarkdown('markdown/dataformat.md')), - - # data uploading functionalities ----------------- - tabItem( - tabName = 'upload', - fluidRow( - welcome_bar(width = 12) - ), - - fluidRow( - column( - width = 6, - upload_box(collapsible = F) - ), - column( - width = 6, - repository_box(collapsible = F) - ) - ), - - fluidRow( - column( - width = 6, - upload_prompt_box(collapsible = F) - ), - column( - width = 6, - data_list_box(collapsible = F) - ) - ) - ), - - # General data overview ---------------------- - tabItem( - tabName = 'overview', - fluidRow( - column( - width = 12, - general_overview_box_all(collapsed = F), - general_overview_box_single(collapsed = F) - ) - ) - ), - - # RT (RunTime): Data Summary ----------------- - tabItem( - tabName = 'ERT_data', - fluidRow( - column( - width = 12, - rt_overview_box(collapsed = F), - rt_stats_box(collapsed = F), - rt_sample_box() - ) - ) - ), - - # RT: Expected Convergence Curve --------------------------------------------- - tabItem( - tabName = 'ERT_convergence_single', - fluidRow( - column( - width = 12, - ERT_box(collapsed = F), - ERT_comparison_box_dim() - ) - ) - ), - - tabItem( - tabName = 'ERT_convergence_aggr', - fluidRow( - column( - width = 12, - ERT_comparison_box(collapsed = F), - ERT_agg_box(height = '800px') - ) - ) - ), - - # RT: histograms, violin plots ------------------------------------------ - tabItem( - tabName = 'RT_PMF', - fluidRow( - column( - width = 12, - rt_histogram_box(collapsed = F), - rt_pmf_box() - ) - ) - ), - - # RT ECDF ------------------------------------------ - tabItem( - tabName = 'RT_ECDF_single', - fluidRow( - column( - width = 12, - rt_ecdf_single_target_box(collapsed = F), - rt_ecdf_agg_targets_box() - # rt_ecdf_auc_box() - ) - ) - ), - - tabItem( - tabName = 'RT_ECDF_aggr', - fluidRow( - column( - width = 12, - rt_ecdf_agg_fct_box(collapsed = F) - ) - ) - ), - - # Parameter tab ------- - tabItem( - tabName = 'RT_PARAMETER', - fluidRow( - column( - width = 12, - rt_par_summary_box(collapsed = F), - rt_par_expected_value_box(), - rt_par_sample_box() - ) - ) - ), - - tabItem( - tabName = 'RT_Statistics_single', - fluidRow( - column( - width = 12, - rt_heatmap_box() - ) - ) - ), - tabItem(tabName = 'RT_DSC', - fluidRow( - column( - width = 12, - rt_dsc_box_rank(), - rt_dsc_box_omni(), - rt_dsc_box_posthoc() - ) - ) - ), - tabItem(tabName = 'RT_Statistics_aggr', - fluidRow( - column( - width = 12, - rt_glicko2_box(collapsed = F) - ) - ) - - ), - tabItem(tabName = 'RT_table_multi', - fluidRow( - column( - width = 12, - multi_function_ert_box(collapsed = F), - multi_function_sample_box(collapsed = T) - ) - ) - - ), - tabItem(tabName = 'RT_portfolio', - fluidRow( - column( - width = 12, - rt_shapleys_box(collapsed = F) - ) - ) - - ), - # FCE: Data Summary ----------------- - tabItem( - tabName = 'FCE_DATA', - fluidRow( - column( - width = 12, - fv_overview_box(collapsed = F), - fv_stats_box(collapsed = F), - fv_sample_box() - ) - ) - ), - - # FCE: Expected Convergence Curve --------------------------------------------- - tabItem( - tabName = 'FCE_convergence_single', - fluidRow( - column( - width = 12, - fv_per_fct_box(collapsed = F) - ) - ) - ), - - tabItem( - tabName = 'FCE_convergence_aggr', - fluidRow( - column( - width = 12, - fv_agg_box(collapsed = F), - fv_comparison_box() - ) - ) - ), - # FCE: historgrams, p.d.f. -------- - tabItem( - tabName = 'FCE_PDF', - fluidRow( - column( - width = 12, - fv_histgram_box(collapsed = F), - fv_pdf_box() - ) - ) - ), - - # FCE: empirical c.d.f. ------------------------------------------ - tabItem( - tabName = 'FCE_ECDF', - fluidRow( - column( - width = 12, - fv_ecdf_single_budget_box(collapsed = F), - fv_ecdf_agg_budgets_box(), - fv_ecdf_auc_box() - ) - ) - ), - - # Parameter tab ------- - tabItem( - tabName = 'FCE_PARAMETER', - fluidRow( - column( - width = 12, - fv_par_expected_value_box(collapsed = F), - fv_par_summary_box(), - fv_par_sample_box() - ) - ) - ), - - tabItem( - tabName = 'FCE_Statistics_single', - fluidRow( - column( - width = 12, - fv_heatmap_box(collapsed = F) - ) - ) - ), - tabItem(tabName = 'FV_table_multi', - fluidRow( - column( - width = 12, - multi_function_fv_box(collapsed = F), - multi_function_sample_box_fv(collapsed = T) - ) - ) - - ), - - tabItem(tabName = 'FCE_DSC', - fluidRow( - column( - width = 12, - fv_dsc_box_rank(), - fv_dsc_box_omni(), - fv_dsc_box_posthoc() - ) - ) - ), - - tabItem(tabName = 'FCE_Statistics_aggr', - fluidRow( - column( - width = 12, - fv_glicko2_box(collapsed = F) - ) - ) - ), - tabItem(tabName = 'Positions', - fluidRow( - column( - width = 12, - Par_coord_box() - ) - ) - ), - - tabItem( - tabName = 'Settings', - fluidRow( - column( - width = 12, - color_settings_box(), - general_settings_box() - ) - ) - ) - # tabItem(tabName = 'Report', - # fluidRow( - # column( - # width = 12, - # main_report_box() - # ) - # ) - # ) - ) -) - -# ----------------------------------------------------------- +for (f in list.files('ui', pattern = '.R', full.names = T)) { + source(f) +} + +# NOTE: this is copied from `shinydashboard` +#' Assert that a tag has specified properties +#' @param tag A tag object. +#' @param type The type of a tag, like "div", "a", "span". +#' @param class An HTML class. +#' @param allowUI If TRUE (the default), allow dynamic outputs generated by +#' \code{\link[shiny]{uiOutput}} or \code{\link[shiny]{htmlOutput}}. When a +#' dynamic output is provided, \code{tagAssert} won't try to validate the the +#' contents. +#' @keywords internal +tagAssert <- function(tag, type = NULL, class = NULL, allowUI = TRUE) { + if (!inherits(tag, "shiny.tag")) { + print(tag) + stop("Expected an object with class 'shiny.tag'.") + } + + # Skip dynamic output elements + if (allowUI && + (hasCssClass(tag, "shiny-html-output") || + hasCssClass(tag, "shinydashboard-menu-output"))) { + return() + } + + if (!is.null(type) && tag$name != type) { + stop("Expected tag to be of type ", type) + } + + if (!is.null(class)) { + if (is.null(tag$attribs$class)) { + stop("Expected tag to have class '", class, "'") + + } else { + tagClasses <- strsplit(tag$attribs$class, " ")[[1]] + if (!(class %in% tagClasses)) { + stop("Expected tag to have class '", class, "'") + } + } + } +} + +# Modified dashboard header from the original source for incorporating two `select input` widgets +.dashboardHeader <- function(..., title = NULL, titleWidth = NULL, disable = FALSE, .list = NULL) { + items <- c(list(...), .list) + lapply(items, tagAssert, type = "li", class = "dropdown") + + titleWidth <- validateCssUnit(titleWidth) + + # Set up custom CSS for custom width. + custom_css <- NULL + if (!is.null(titleWidth)) { + # This CSS is derived from the header-related instances of '230px' (the + # default sidebar width) from inst/AdminLTE/AdminLTE.css. One change is that + # instead making changes to the global settings, we've put them in a media + # query (min-width: 768px), so that it won't override other media queries + # (like max-width: 767px) that work for narrower screens. + custom_css <- tags$head( + tags$style( + HTML( + gsub( + "_WIDTH_", titleWidth, fixed = TRUE, ' + @media (min-width: 768px) { + .main-header > .navbar { + margin-left: _WIDTH_; + } + .main-header .logo { + width: _WIDTH_; + } + }' + ) + ) + ) + ) + } + + tags$header( + class = "main-header", + id = 'header', + custom_css, + style = if (disable) "display: none;", + span(class = "logo", title), + tags$nav( + class = "navbar navbar-static-top", role = "navigation", + # Embed hidden icon so that we get the font-awesome dependency + span(shiny::icon("bars"), style = "display:none;"), + # Sidebar toggle button + a( + href="#", class="sidebar-toggle", `data-toggle`="offcanvas", + role="button", span(class="sr-only", "Toggle navigation") + ), + + # select inputs for dimension and function/problem ID + # HTML(' + #
+ #
+ # + # + # + # + # + #
+ # Dimension: + # + # + # Problem ID: + # + #
' + # ), + + div( + class = "navbar-custom-menu", + tags$ul(class = "nav navbar-nav", items) + ) + ) + ) +} + +header <- .dashboardHeader(title = HTML('
IOHanalyzer
')) + +# The side bar layout --------------------------------------------- +sidebar <- dashboardSidebar( + useShinyjs(), + sidebar_menu(), + hr(), + DIM_fID_panel() +) + +body <- dashboardBody( + tags$style( + HTML(' + .popover-title {color:black;} + .popover-content {color:black;} + .main-sidebar {z-index:auto;} + .fa-exclamation-triangle {color:#E87722} + .sticky { + position: fixed; + top: 0; + width: 100%; + } + + .sticky2 { + position: fixed; + } + + .table { + border-collapse: collapse; + width: 100%; + } + .table td, tr { + padding: 0px; + margin: 0px; + vertical-align: middle; + height: 45px; + } + .table th {height: 0px;}' + ) + ), + + # to show text on the header (heading banner) + tags$head( + tags$style( + HTML(' + .myClass { + font-size: 20px; + line-height: 50px; + text-align: left; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + padding: 0 15px; + overflow: hidden; + color: white; + }' + ) + ) + ), + + tags$head( + tags$style( + HTML( + '.box-title { + font-size: 20px; + line-height: 50px; + text-align: left; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + padding: 0 15px; + overflow: hidden; + color: white; + }' + ) + ) + ), + + tags$head( + tags$style( + HTML( + "label { font-size:120%; }" + ) + ) + ), + + tags$script( + HTML(' + $(document).ready(function() { + $("header").find("nav").append(\'Performance Evaluation for Iterative + Optimization Heuristics\'); + })' + ) + ), + + tags$script( + "Shiny.addCustomMessageHandler('background-color', + function(color) { + document.body.style.backgroundColor = color; + document.body.innerText = color; + });" + ), + + tags$script( + HTML(' + window.setInterval(function() { + var elem = document.getElementById("process_data_promt"); + if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; + }, 20);' + ) + ), + + tags$head( + tags$script( + HTML(" + Shiny.addCustomMessageHandler('manipulateMenuItem', function(message){ + var aNodeList = document.getElementsByTagName('a'); + + for (var i = 0; i < aNodeList.length; i++) { + if(aNodeList[i].getAttribute('data-value') == message.tabName || aNodeList[i].getAttribute('href') == message.tabName) { + if(message.action == 'hide'){ + aNodeList[i].setAttribute('style', 'display: none;'); + } else { + aNodeList[i].setAttribute('style', 'display: block;'); + }; + }; + } + }); + ") + ) + ), + + # make the data uploading prompt always scroll to the bottom + tags$script( + HTML(' + window.setInterval(function() { + var elem = document.getElementById("upload_data_promt"); + if (typeof elem !== "undefined" && elem !== null) elem.scrollTop = elem.scrollHeight; + }, 20);' + ) + ), + + # render the header and the side bar 'sticky' + tags$script( + HTML( + '// When the user scrolls the page, execute myFunction + window.onscroll = function() {myFunction()}; + + // Get the header + var header = document.getElementById("header"); + + // Get the side bar + var sideBar = document.getElementById("sidebarCollapsed"); + sideBar.classList.add("sticky2"); + + // Get the offset position of the navbar + var sticky = header.offsetTop; + + // Add the sticky class to the header when you reach its scroll position. + // Remove "sticky" when you leave the scroll position + function myFunction() { + if (window.pageYOffset > sticky) { + header.classList.add("sticky"); + } else { + header.classList.remove("sticky"); + } + }' + ) + ), + + if (suppressWarnings(require("dashboardthemes", quietly = T))) { + shinyDashboardThemes( + theme = "grey_light" + ) + }, + + # load MathJax + # TODO: download MathJax and its license and include it in our package + HTML(""), + use_bs_tooltip(), + use_bs_popover(), + + # tabitems ---------------------- + tabItems( + tabItem(tabName = 'about', includeMarkdown('markdown/about.md')), + # tabItem(tabName = 'dataformat', includeMarkdown('markdown/dataformat.md')), + + # data uploading functionalities ----------------- + tabItem( + tabName = 'upload', + fluidRow( + welcome_bar(width = 12) + ), + + fluidRow( + column( + width = 6, + upload_box(collapsible = F) + ), + column( + width = 6, + repository_box(collapsible = F) + ) + ), + + fluidRow( + column( + width = 6, + upload_prompt_box(collapsible = F) + ), + column( + width = 6, + data_list_box(collapsible = F) + ) + ) + ), + + # General data overview ---------------------- + tabItem( + tabName = 'overview', + fluidRow( + column( + width = 12, + general_overview_box_all(collapsed = F), + general_overview_box_single(collapsed = F) + ) + ) + ), + + # RT (RunTime): Data Summary ----------------- + tabItem( + tabName = 'ERT_data', + fluidRow( + column( + width = 12, + rt_overview_box(collapsed = F), + rt_stats_box(collapsed = F), + rt_sample_box() + ) + ) + ), + + # RT: Expected Convergence Curve --------------------------------------------- + tabItem( + tabName = 'ERT_convergence_single', + fluidRow( + column( + width = 12, + ERT_box(collapsed = F), + ERT_comparison_box_dim() + ) + ) + ), + + tabItem( + tabName = 'ERT_convergence_aggr', + fluidRow( + column( + width = 12, + ERT_agg_box(collapsed = F), + ERT_comparison_box(collapsed = T) + ) + ) + ), + + # RT: histograms, violin plots ------------------------------------------ + tabItem( + tabName = 'RT_PMF', + fluidRow( + column( + width = 12, + rt_histogram_box(collapsed = F), + rt_pmf_box() + ) + ) + ), + + # RT ECDF ------------------------------------------ + tabItem( + tabName = 'RT_ECDF_single', + fluidRow( + column( + width = 12, + rt_ecdf_single_target_box(collapsed = F), + rt_ecdf_agg_targets_box() + # rt_ecdf_auc_box() + ) + ) + ), + + tabItem( + tabName = 'RT_ECDF_aggr', + fluidRow( + column( + width = 12, + rt_ecdf_agg_fct_box(collapsed = F) + ) + ) + ), + + # Parameter tab ------- + tabItem( + tabName = 'RT_PARAMETER', + fluidRow( + column( + width = 12, + rt_par_summary_box(collapsed = F), + rt_par_expected_value_box(), + rt_par_sample_box() + ) + ) + ), + + tabItem( + tabName = 'RT_Statistics_single', + fluidRow( + column( + width = 12, + rt_heatmap_box() + ) + ) + ), + tabItem(tabName = 'RT_DSC', + fluidRow( + column( + width = 12, + rt_dsc_box_rank(), + rt_dsc_box_omni(), + rt_dsc_box_posthoc() + ) + ) + ), + tabItem(tabName = 'RT_Statistics_aggr', + fluidRow( + column( + width = 12, + rt_glicko2_box(collapsed = F) + ) + ) + + ), + tabItem(tabName = 'RT_table_multi', + fluidRow( + column( + width = 12, + multi_function_ert_box(collapsed = F), + multi_function_sample_box(collapsed = T) + ) + ) + + ), + tabItem(tabName = 'RT_portfolio', + fluidRow( + column( + width = 12, + rt_shapleys_box(collapsed = F) + ) + ) + + ), + # FCE: Data Summary ----------------- + tabItem( + tabName = 'FCE_DATA', + fluidRow( + column( + width = 12, + fv_overview_box(collapsed = F), + fv_stats_box(collapsed = F), + fv_sample_box() + ) + ) + ), + + # FCE: Expected Convergence Curve --------------------------------------------- + tabItem( + tabName = 'FCE_convergence_single', + fluidRow( + column( + width = 12, + fv_per_fct_box(collapsed = F) + ) + ) + ), + + tabItem( + tabName = 'FCE_convergence_aggr', + fluidRow( + column( + width = 12, + fv_agg_box(collapsed = F), + fv_comparison_box() + ) + ) + ), + # FCE: historgrams, p.d.f. -------- + tabItem( + tabName = 'FCE_PDF', + fluidRow( + column( + width = 12, + fv_histgram_box(collapsed = F), + fv_pdf_box() + ) + ) + ), + + # FCE: empirical c.d.f. ------------------------------------------ + tabItem( + tabName = 'FCE_ECDF', + fluidRow( + column( + width = 12, + fv_ecdf_single_budget_box(collapsed = F), + fv_ecdf_agg_budgets_box(), + fv_ecdf_auc_box() + ) + ) + ), + + # Parameter tab ------- + tabItem( + tabName = 'FCE_PARAMETER', + fluidRow( + column( + width = 12, + fv_par_expected_value_box(collapsed = F), + fv_par_summary_box(), + fv_par_sample_box() + ) + ) + ), + + tabItem( + tabName = 'FCE_Statistics_single', + fluidRow( + column( + width = 12, + fv_heatmap_box(collapsed = F) + ) + ) + ), + tabItem(tabName = 'FV_table_multi', + fluidRow( + column( + width = 12, + multi_function_fv_box(collapsed = F), + multi_function_sample_box_fv(collapsed = T) + ) + ) + + ), + + tabItem(tabName = 'FCE_DSC', + fluidRow( + column( + width = 12, + fv_dsc_box_rank(), + fv_dsc_box_omni(), + fv_dsc_box_posthoc() + ) + ) + ), + + tabItem(tabName = 'FCE_Statistics_aggr', + fluidRow( + column( + width = 12, + fv_glicko2_box(collapsed = F) + ) + ) + ), + tabItem(tabName = 'Positions', + fluidRow( + column( + width = 12, + Par_coord_box() + ) + ) + ), + + tabItem( + tabName = 'Settings', + fluidRow( + column( + width = 12, + color_settings_box(), + general_settings_box() + ) + ) + ) + # tabItem(tabName = 'Report', + # fluidRow( + # column( + # width = 12, + # main_report_box() + # ) + # ) + # ) + ) +) + +# ----------------------------------------------------------- dashboardPage(title = 'IOHanalyzer', header, sidebar, body) \ No newline at end of file diff --git a/inst/shiny-server/ui/DIM_fID_panel.R b/inst/shiny-server/ui/DIM_fID_panel.R index 10d9e963..b9bbf9be 100644 --- a/inst/shiny-server/ui/DIM_fID_panel.R +++ b/inst/shiny-server/ui/DIM_fID_panel.R @@ -1,31 +1,54 @@ -DIM_fID_panel <- function() { - conditionalPanel( - "input.tabs!='Report' && input.tabs!='upload' && input.tabs!='readme' && input.tabs!='about' && input.tabs!='Settings'", - column(12, offset = 0, - div( - # style = "padding: 0px 0px; margin-top:-5em; margin:0%", - fluidRow( - column( - width = 11, - selectInput( - "Overall.Funcid", - label = HTML('

Select the function ID

'), - choices = NULL, selected = NULL) - ) - ) - ), - div( - # style = "padding: 0px 0px; margin-top:-50em;", - fluidRow( - column( - width = 11, - selectInput( - "Overall.Dim", - label = HTML('

Select the dimension

'), - choices = NULL, selected = NULL) - ) - ) - ) - ) - ) -} +DIM_fID_panel <- function() { + conditionalPanel( + "input.tabs!='Report' && input.tabs!='upload' && input.tabs!='readme' && input.tabs!='about' && input.tabs!='Settings'", + column(12, offset = 0, + div(id = 'overall_funcid_box', + # style = "padding: 0px 0px; margin-top:-5em; margin:0%", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Funcid", + label = HTML('

Function ID:

'), + choices = NULL, selected = NULL) + ) + ) + ), + div(id = 'overall_funcname_box', + # style = "padding: 0px 0px; margin-top:-5em; margin:0%", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Funcname", + label = HTML('

Function Name:

'), + choices = NULL, selected = NULL) + ) + ) + ), + div( + # style = "padding: 0px 0px; margin-top:-50em;", + fluidRow( + column( + width = 11, + selectInput( + "Overall.Dim", + label = HTML('

Dimension:

'), + choices = NULL, selected = NULL) + ) + ) + ), + div(id = 'overall_algid_box', + fluidRow( + column( + width = 11, + selectInput( + "Overall.ID", + label = HTML('

Algorithm:

'), + choices = NULL, selected = NULL) + ) + ) + )#, + ) + ) +} diff --git a/inst/shiny-server/ui/DSC_interface.R b/inst/shiny-server/ui/DSC_interface.R index 494aaf67..d5c21e5a 100644 --- a/inst/shiny-server/ui/DSC_interface.R +++ b/inst/shiny-server/ui/DSC_interface.R @@ -1,281 +1,281 @@ -fv_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Ranking per Function

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.DSC.Algid', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.DSC.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - hr(), - numericInput('FV_Stats.DSC.Alpha_rank', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - numericInput('FV_Stats.DSC.Epsilon_rank', - label = "Threshold for practical significance (pDSC)", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Practical Significance", content = Pdsc_info, - placement = "auto" - ) - ), - numericInput('FV_Stats.DSC.MCsamples_rank', - label = "Number of monte carlo samples to use in the DSC procdure", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Monte Carlo Samples", content = Pdsc_mc_info, - placement = "auto" - ) - ), - selectInput("FV_Stats.DSC.Test_rank", "Test Type", - choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), - selected = "Anderson-Darling"), - actionButton('FV_Stats.DSC.Create_rank', 'Create Ranking'), - hr(), - selectInput('FV_Stats.DSC.Format_rank', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FV_Stats.DSC.Download_rank', label = 'Download the figure'), - selectInput('FV_Stats.DSC.TableFormat_rank', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') - - ), - - mainPanel( - width = 9, - HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based - framework for statistical comparison of stochastic optimization algorithms.' by - T. Eftimov et al. - This is the first of 3 parts of the process: the per-function ranking procedure. - The two other processes are the omnibus test and the post-hoc processing, - which are shown in the two boxes below this one. - Note that both of these are impacted by the settings selected for this - ranking procedure!"), - HTML_P("The chosen budget values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("FV_Stats.DSC.Targets"), - hr(), - HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques - as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and - Understand the Performance of Single-objective Optimization - Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, - from a single plot, which algorithms are most suited for a given problem, - the influence of each problem on the overall algorithm performance and - similarities among both algorithms and problems."), - plotOutput("FV_Stats.DSC.PerformViz", height = "800px") - ) - ) -} - -fv_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Omnibus Test

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.DSC.Omni_options', "Select which statistical test to use", - choices = NULL, selected = NULL), - numericInput('FV_Stats.DSC.Alpha_omni', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('FV_Stats.DSC.Create_omni', 'Perform Omnibus Test') - ), - - mainPanel( - width = 9, - HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. - Note that this test has to be performed before doing the post-hoc comparison!'), - hr(), - textOutput('FV_Stats.DSC.Output_omni') - ) - ) -} - -fv_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Posthoc comparison

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = - c('friedman', 'friedman-aligned-rank'), - selected = 'friedman', multiple = F), - selectInput('FV_Stats.DSC.Posthoc_method', 'Post-hoc P-value Correction Method', choices = - c('Holm', 'Hochberg', 'unadjusted P'), - selected = 'Holm', multiple = F), - numericInput('FV_Stats.DSC.Alpha_posthoc', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('FV_Stats.DSC.Create_posthoc', 'Create Comparison'), - hr(), - selectInput('FV_Stats.DSC.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FV_Stats.DSC.Download', label = 'Download the figure'), - hr(), - selectInput('FV_Stats.DSC.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.DSC.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P("The results of the post-hoc comparison are:"), - plotlyOutput.IOHanalyzer("FV_Stats.DSC.PosthocViz"), - DT::dataTableOutput('FV_Stats.DSC.PosthocTable') - ) - ) -} - -rt_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Ranking per Function

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.DSC.Algid', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.DSC.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - hr(), - selectInput('RT_Stats.DSC.Value_type', "Select which type of hitting times to use", - choices = c('PAR-10', 'ERT', 'Remove-na', 'PAR-1')), - numericInput('RT_Stats.DSC.Alpha_rank', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - numericInput('RT_Stats.DSC.Epsilon_rank', - label = "Threshold for practical significance (pDSC)", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Practical Significance", content = Pdsc_info, - placement = "auto" - ) - ), - numericInput('RT_Stats.DSC.MCsamples_rank', - label = "Number of monte carlo samples to use in the DSC procdure", - value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Monte Carlo Samples", content = Pdsc_mc_info, - placement = "auto" - ) - ), - selectInput("RT_Stats.DSC.Test_rank", "Test Type", - choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), - selected = "Anderson-Darling"), - actionButton('RT_Stats.DSC.Create_rank', 'Create Ranking'), - hr(), - selectInput('RT_Stats.DSC.Format_rank', label = 'Select the figure format', - choices = c("pdf"), selected = 'pdf'), - - downloadButton('RT_Stats.DSC.Download_rank', label = 'Download the figure'), - selectInput('RT_Stats.DSC.TableFormat_rank', label = 'Select the table format', - choices = c('csv','tex'), selected = 'csv'), - downloadButton('RT_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') - - ), - - mainPanel( - width = 9, - HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based - framework for statistical comparison of stochastic optimization algorithms.' by - T. Eftimov et al. - This is the first of 3 parts of the process: the per-function ranking procedure. - The two other processes are the omnibus test and the post-hoc processing, - which are shown in the two boxes below this one. - Note that both of these are impacted by the settings selected for this - ranking procedure!"), - HTML_P("The chosen budget values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("RT_Stats.DSC.Targets"), - hr(), - HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques - as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and - Understand the Performance of Single-objective Optimization - Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, - from a single plot, which algorithms are most suited for a given problem, - the influence of each problem on the overall algorithm performance and - similarities among both algorithms and problems."), - plotOutput("RT_Stats.DSC.PerformViz", height = "800px") - ) - ) -} - -rt_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Omnibus Test

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.DSC.Omni_options', "Select which statistical test to use", - choices = NULL, selected = NULL), - numericInput('RT_Stats.DSC.Alpha_omni', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('RT_Stats.DSC.Create_omni', 'Perform Omnibus Test') - ), - - mainPanel( - width = 9, - HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. - Note that this test has to be performed before doing the post-hoc comparison!'), - hr(), - textOutput('RT_Stats.DSC.Output_omni') - ) - ) -} - -rt_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Deep Statistical Comparison (DSC) - analysis - Posthoc comparison

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = - c('friedman', 'friedman-aligned-rank'), - selected = 'friedman', multiple = F), - selectInput('RT_Stats.DSC.Posthoc_method', 'Post-hoc Method', choices = - c('Holm', 'Hochberg', 'unadjusted P'), - selected = 'Holm', multiple = F), - numericInput('RT_Stats.DSC.Alpha_posthoc', - label = "Threshold for statistical significance", - value = 0.05, min = 0, max = 0.5), - actionButton('RT_Stats.DSC.Create_posthoc', 'Create Comparison'), - hr(), - selectInput('RT_Stats.DSC.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RT_Stats.DSC.Download', label = 'Download the figure'), - hr(), - selectInput('RT_Stats.DSC.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('RT_Stats.DSC.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P("The results of the post-hoc comparison are:"), - plotlyOutput.IOHanalyzer("RT_Stats.DSC.PosthocViz"), - DT::dataTableOutput('RT_Stats.DSC.PosthocTable') - ) - ) +fv_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Ranking per Function

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.DSC.ID', 'IDs to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.DSC.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + hr(), + numericInput('FV_Stats.DSC.Alpha_rank', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + numericInput('FV_Stats.DSC.Epsilon_rank', + label = "Threshold for practical significance (pDSC)", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Practical Significance", content = Pdsc_info, + placement = "auto" + ) + ), + numericInput('FV_Stats.DSC.MCsamples_rank', + label = "Number of monte carlo samples to use in the DSC procdure", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Monte Carlo Samples", content = Pdsc_mc_info, + placement = "auto" + ) + ), + selectInput("FV_Stats.DSC.Test_rank", "Test Type", + choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), + selected = "Anderson-Darling"), + actionButton('FV_Stats.DSC.Create_rank', 'Create Ranking'), + hr(), + selectInput('FV_Stats.DSC.Format_rank', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FV_Stats.DSC.Download_rank', label = 'Download the figure'), + selectInput('FV_Stats.DSC.TableFormat_rank', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') + + ), + + mainPanel( + width = 9, + HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based + framework for statistical comparison of stochastic optimization algorithms.' by + T. Eftimov et al. + This is the first of 3 parts of the process: the per-function ranking procedure. + The two other processes are the omnibus test and the post-hoc processing, + which are shown in the two boxes below this one. + Note that both of these are impacted by the settings selected for this + ranking procedure!"), + HTML_P("The chosen budget values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("FV_Stats.DSC.Targets"), + hr(), + HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques + as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and + Understand the Performance of Single-objective Optimization + Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, + from a single plot, which algorithms are most suited for a given problem, + the influence of each problem on the overall algorithm performance and + similarities among both algorithms and problems."), + plotOutput("FV_Stats.DSC.PerformViz", height = "800px") + ) + ) +} + +fv_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Omnibus Test

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.DSC.Omni_options', "Select which statistical test to use", + choices = NULL, selected = NULL), + numericInput('FV_Stats.DSC.Alpha_omni', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('FV_Stats.DSC.Create_omni', 'Perform Omnibus Test') + ), + + mainPanel( + width = 9, + HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. + Note that this test has to be performed before doing the post-hoc comparison!'), + hr(), + textOutput('FV_Stats.DSC.Output_omni') + ) + ) +} + +fv_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Posthoc comparison

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = + c('friedman', 'friedman-aligned-rank'), + selected = 'friedman', multiple = F), + selectInput('FV_Stats.DSC.Posthoc_method', 'Post-hoc P-value Correction Method', choices = + c('Holm', 'Hochberg', 'unadjusted P'), + selected = 'Holm', multiple = F), + numericInput('FV_Stats.DSC.Alpha_posthoc', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('FV_Stats.DSC.Create_posthoc', 'Create Comparison'), + hr(), + selectInput('FV_Stats.DSC.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FV_Stats.DSC.Download', label = 'Download the figure'), + hr(), + selectInput('FV_Stats.DSC.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.DSC.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P("The results of the post-hoc comparison are:"), + plotlyOutput.IOHanalyzer("FV_Stats.DSC.PosthocViz"), + DT::dataTableOutput('FV_Stats.DSC.PosthocTable') + ) + ) +} + +rt_dsc_box_rank <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Ranking per Function

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.DSC.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.DSC.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.DSC.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + hr(), + selectInput('RT_Stats.DSC.Value_type', "Select which type of hitting times to use", + choices = c('PAR-10', 'ERT', 'Remove-na', 'PAR-1')), + numericInput('RT_Stats.DSC.Alpha_rank', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + numericInput('RT_Stats.DSC.Epsilon_rank', + label = "Threshold for practical significance (pDSC)", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Practical Significance", content = Pdsc_info, + placement = "auto" + ) + ), + numericInput('RT_Stats.DSC.MCsamples_rank', + label = "Number of monte carlo samples to use in the DSC procdure", + value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Monte Carlo Samples", content = Pdsc_mc_info, + placement = "auto" + ) + ), + selectInput("RT_Stats.DSC.Test_rank", "Test Type", + choices = c("Anderson-Darling", "Kolmogorov-Smirnov"), + selected = "Anderson-Darling"), + actionButton('RT_Stats.DSC.Create_rank', 'Create Ranking'), + hr(), + selectInput('RT_Stats.DSC.Format_rank', label = 'Select the figure format', + choices = c("pdf"), selected = 'pdf'), + + downloadButton('RT_Stats.DSC.Download_rank', label = 'Download the figure'), + selectInput('RT_Stats.DSC.TableFormat_rank', label = 'Select the table format', + choices = c('csv','tex'), selected = 'csv'), + downloadButton('RT_Stats.DSC.Download_rank_table', label = 'Download the raw ranking data') + + ), + + mainPanel( + width = 9, + HTML_P("The DSC comparison is described in the paper: 'DSCTool: A web-service-based + framework for statistical comparison of stochastic optimization algorithms.' by + T. Eftimov et al. + This is the first of 3 parts of the process: the per-function ranking procedure. + The two other processes are the omnibus test and the post-hoc processing, + which are shown in the two boxes below this one. + Note that both of these are impacted by the settings selected for this + ranking procedure!"), + HTML_P("The chosen budget values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("RT_Stats.DSC.Targets"), + hr(), + HTML_P("The results of the ranking are shown in the following plot, using the visualization techniques + as described in the paper: 'PerformViz: A Machine Learning Approach to Visualize and + Understand the Performance of Single-objective Optimization + Algorithms' By T. Eftimov et al. Performviz allows one to clearly see, + from a single plot, which algorithms are most suited for a given problem, + the influence of each problem on the overall algorithm performance and + similarities among both algorithms and problems."), + plotOutput("RT_Stats.DSC.PerformViz", height = "800px") + ) + ) +} + +rt_dsc_box_omni <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Omnibus Test

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.DSC.Omni_options', "Select which statistical test to use", + choices = NULL, selected = NULL), + numericInput('RT_Stats.DSC.Alpha_omni', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('RT_Stats.DSC.Create_omni', 'Perform Omnibus Test') + ), + + mainPanel( + width = 9, + HTML_P('This is the result of the omnibus test on the data from the ranking procedure above. + Note that this test has to be performed before doing the post-hoc comparison!'), + hr(), + textOutput('RT_Stats.DSC.Output_omni') + ) + ) +} + +rt_dsc_box_posthoc <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Deep Statistical Comparison (DSC) + analysis - Posthoc comparison

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.DSC.Posthoc_test', 'Post-hoc Test', choices = + c('friedman', 'friedman-aligned-rank'), + selected = 'friedman', multiple = F), + selectInput('RT_Stats.DSC.Posthoc_method', 'Post-hoc Method', choices = + c('Holm', 'Hochberg', 'unadjusted P'), + selected = 'Holm', multiple = F), + numericInput('RT_Stats.DSC.Alpha_posthoc', + label = "Threshold for statistical significance", + value = 0.05, min = 0, max = 0.5), + actionButton('RT_Stats.DSC.Create_posthoc', 'Create Comparison'), + hr(), + selectInput('RT_Stats.DSC.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RT_Stats.DSC.Download', label = 'Download the figure'), + hr(), + selectInput('RT_Stats.DSC.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('RT_Stats.DSC.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P("The results of the post-hoc comparison are:"), + plotlyOutput.IOHanalyzer("RT_Stats.DSC.PosthocViz"), + DT::dataTableOutput('RT_Stats.DSC.PosthocTable') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/ERT_agg_box.R b/inst/shiny-server/ui/ERT_agg_box.R index 558ccb61..26e0d62e 100644 --- a/inst/shiny-server/ui/ERT_agg_box.R +++ b/inst/shiny-server/ui/ERT_agg_box.R @@ -1,46 +1,46 @@ -ERT_agg_box <- function(width = 12, height = '600px', collapsible = T, - collapsed = T) { - box( - title = HTML('

Expected Runtime (ERT): all functions

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('ERTPlot.Multi.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput('ERTPlot.Multi.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('ERTPlot.Multi.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - - actionButton('ERTPlot.Multi.PlotButton', label = 'Refresh the figure'), - hr(), - selectInput('ERTPlot.Multi.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Multi.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('The ERT is shown against the target - values for all functions in the selected dimension.'), - plotlyOutput.IOHanalyzer('ERTPlot.Multi.Plot', aspect_ratio = 1) - ) - ) - ) - ) -} +ERT_agg_box <- function(width = 12, height = '600px', collapsible = T, + collapsed = T) { + box( + title = HTML('

Expected Runtime (ERT): all functions

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('ERTPlot.Multi.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput('ERTPlot.Multi.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('ERTPlot.Multi.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + + actionButton('ERTPlot.Multi.PlotButton', label = 'Refresh the figure'), + hr(), + selectInput('ERTPlot.Multi.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Multi.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('The ERT is shown against the target + values for all functions in the selected dimension.'), + plotlyOutput.IOHanalyzer('ERTPlot.Multi.Plot', aspect_ratio = 1) + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/ERT_box.R b/inst/shiny-server/ui/ERT_box.R index 819fe04f..8c0ec0e2 100644 --- a/inst/shiny-server/ui/ERT_box.R +++ b/inst/shiny-server/ui/ERT_box.R @@ -1,133 +1,133 @@ -ERT_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Runtime (ERT): single function

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - - selectInput('ERTPlot.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - - HTML('

Range of the displayed target values

'), - - textInput('ERTPlot.Min', - label = F_MIN_LABEL, - value = '') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Interpolation", content = "The points on this plot are all interpollated where needed - to create solid lines. This means that for functions with limited domains (such as integers), - artifacts like step-wise behaviour can occur. Please carefully consider the domain of the function - when interpreting this plot.", - placement = "auto" - ) - ), - - textInput('ERTPlot.Max', - label = F_MAX_LABEL, - value = ''), - - checkboxInput('ERTPlot.show.ERT', - label = 'Show/hide ERT', - value = T), - - checkboxInput( - 'ERTPlot.show.mean', - label = 'Show/hide PAR-1', - value = F - ) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Penalized Average Runtime", content = "PAR-1 score is ther average of running time values, - where non-successful runs are counted as evaluation budget B.", - placement = "auto" - ) - ), - - checkboxInput('ERTPlot.show.CI', - label = 'Show/hide mean +/- sd', - value = F), - - checkboxInput('ERTPlot.show.Quantiles', - label = 'Show/hide outer quantiles', - value = F) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are - 2% and 98% by default. This can be changed in the settings-tab.", - placement = "auto" - ) - ), - - checkboxInput('ERTPlot.show.median', - label = 'Show/hide median', - value = F), - - checkboxInput('ERTPlot.semilogx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('ERTPlot.semilogy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - box(title = HTML('

Additional Options

'), collapsible = T, collapsed = T, width = width, solidHeader = T, status = 'info', - checkboxInput('ERTPlot.inclueOpts', - label = "Include optimal points found by each algorithm", - value = F), - checkboxInput('ERTPlot.show.runs', - label = 'Show individual runs', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_popover( - title = "Individual runs", content = "This procedure can be slow when many - runs are present in the data. Please use with caution.", - placement = "auto" - ) - ), - numericInput("ERTPlot.Additional.Budget", "Choose Custom Budget for ERT calculation", value = NA) %>% - shinyInput_label_embed( - custom_icon("info") %>% - bs_embed_popover( - title = "Custom Budget", content = "This will use this budget value instead of the ones in the - data for the ERT calculation. Any hitting times larger than this value are treated as unfinished runs. - Please use with caution.", - placement = "auto" - ) - ) - ), - hr(), - - selectInput('ERTPlot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Download', label = 'Download the figure') - - ), - - mainPanel( - width = 9, - column( - width = 12, - align = "center", - HTML_P('The mean, median, standard deviation and ERT of the runtime samples - are depicted against the best objective values. - The displayed elements (mean, median, standard deviations and ERT) - can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('ERT_PER_FUN') - ) - ) - ) - ) -} +ERT_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Runtime (ERT): single function

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + + selectInput('ERTPlot.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + HTML('

Range of the displayed target values

'), + + textInput('ERTPlot.Min', + label = F_MIN_LABEL, + value = '') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Interpolation", content = "The points on this plot are all interpollated where needed + to create solid lines. This means that for functions with limited domains (such as integers), + artifacts like step-wise behaviour can occur. Please carefully consider the domain of the function + when interpreting this plot.", + placement = "auto" + ) + ), + + textInput('ERTPlot.Max', + label = F_MAX_LABEL, + value = ''), + + checkboxInput('ERTPlot.show.ERT', + label = 'Show/hide ERT', + value = T), + + checkboxInput( + 'ERTPlot.show.mean', + label = 'Show/hide PAR-1', + value = F + ) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Penalized Average Runtime", content = "PAR-1 score is ther average of running time values, + where non-successful runs are counted as evaluation budget B.", + placement = "auto" + ) + ), + + checkboxInput('ERTPlot.show.CI', + label = 'Show/hide mean +/- sd', + value = F), + + checkboxInput('ERTPlot.show.Quantiles', + label = 'Show/hide outer quantiles', + value = F) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are + 2% and 98% by default. This can be changed in the settings-tab.", + placement = "auto" + ) + ), + + checkboxInput('ERTPlot.show.median', + label = 'Show/hide median', + value = F), + + checkboxInput('ERTPlot.semilogx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('ERTPlot.semilogy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + box(title = HTML('

Additional Options

'), collapsible = T, collapsed = T, width = width, solidHeader = T, status = 'info', + checkboxInput('ERTPlot.inclueOpts', + label = "Include optimal points found by each algorithm", + value = F), + checkboxInput('ERTPlot.show.runs', + label = 'Show individual runs', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_popover( + title = "Individual runs", content = "This procedure can be slow when many + runs are present in the data. Please use with caution.", + placement = "auto" + ) + ), + numericInput("ERTPlot.Additional.Budget", "Choose Custom Budget for ERT calculation", value = NA) %>% + shinyInput_label_embed( + custom_icon("info") %>% + bs_embed_popover( + title = "Custom Budget", content = "This will use this budget value instead of the ones in the + data for the ERT calculation. Any hitting times larger than this value are treated as unfinished runs. + Please use with caution.", + placement = "auto" + ) + ) + ), + hr(), + + selectInput('ERTPlot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Download', label = 'Download the figure') + + ), + + mainPanel( + width = 9, + column( + width = 12, + align = "center", + HTML_P('The mean, median, standard deviation and ERT of the runtime samples + are depicted against the best objective values. + The displayed elements (mean, median, standard deviations and ERT) + can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('ERT_PER_FUN') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/ERT_comparison_box.R b/inst/shiny-server/ui/ERT_comparison_box.R index fa4c5974..9da19179 100644 --- a/inst/shiny-server/ui/ERT_comparison_box.R +++ b/inst/shiny-server/ui/ERT_comparison_box.R @@ -1,128 +1,128 @@ -ERT_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Runtime Comparisons (across functions on one dimension)

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('ERTPlot.Aggr.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - - selectInput('ERTPlot.Aggr.Funcs', label = 'Select which functions to aggregate over:', - multiple = T, selected = NULL, choices = NULL), - - selectInput('ERTPlot.Aggr.Mode', label = 'Select the plotting mode', - choices = c('radar', 'line'), selected = 'radar'), - - checkboxInput('ERTPlot.Aggr.Ranking', - label = 'Use ranking instead of ERT-values', - value = T), - - checkboxInput('ERTPlot.Aggr.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F), - actionButton("ERTPlot.Aggr.Refresh", "Refresh the figure and table"), - hr(), - - selectInput('ERTPlot.Aggr.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Aggr.Download', label = 'Download the figure'), - hr(), - selectInput('ERTPlot.Aggr.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('ERTPlot.Aggr.DownloadTable', label = 'Download the table') - - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('The ERT of the runtime samples across all functions. - ERT is decided based on the target values in the table below, - with the default being the best reached f(x) by any of the - selected algorithms. Infinite ERTS are shown as - seperate dots on the graph.'), - plotlyOutput.IOHanalyzer('ERTPlot.Aggr.Plot'), - hr(), - HTML_P("The chosen target values per function are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("ERTPlot.Aggr.Targets"), - hr(), - HTML_P("The raw ERT-values are:"), - DT::dataTableOutput("ERTPlot.Aggr.ERTTable") - ) - ) - ) - ) -} - -#TODO: combine with other function using proper namespacing and modularity -ERT_comparison_box_dim <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Runtime Comparisons (across dimensions)

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('ERTPlot.Aggr_Dim.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('ERTPlot.Aggr_Dim.Mode', label = 'Select the plotting mode', - choices = c('radar', 'line'), selected = 'line'), - - checkboxInput('ERTPlot.Aggr_Dim.Ranking', - label = 'Use ranking instead of ERT-values', - value = F), - - checkboxInput('ERTPlot.Aggr_Dim.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - actionButton("ERTPlot.Aggr_Dim.Refresh", "Refresh the figure and table"), - hr(), - - selectInput('ERTPlot.Aggr_Dim.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ERTPlot.Aggr_Dim.Download', label = 'Download the figure'), - hr(), - selectInput('ERTPlot.Aggr_Dim.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('ERTPlot.Aggr_Dim.DownloadTable', label = 'Download the table') - - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('The ERT of the runtime samples across all functions. - ERT is decided based on the target values in the table below, - with the default being the best reached f(x) by any of the - selected algorithms. Infinite ERTS are shown as - seperate dots on the graph.'), - plotlyOutput.IOHanalyzer('ERTPlot.Aggr_Dim.Plot'), - hr(), - HTML_P("The chosen target values per dimension are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("ERTPlot.Aggr_Dim.Targets"), - hr(), - HTML_P("The raw ERT-values are:"), - DT::dataTableOutput("ERTPlot.Aggr_Dim.ERTTable") - ) - ) - ) - ) +ERT_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Runtime Comparisons (across functions on one dimension)

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('ERTPlot.Aggr.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + selectInput('ERTPlot.Aggr.Funcs', label = 'Select which functions to aggregate over:', + multiple = T, selected = NULL, choices = NULL), + + selectInput('ERTPlot.Aggr.Mode', label = 'Select the plotting mode', + choices = c('radar', 'line'), selected = 'radar'), + + checkboxInput('ERTPlot.Aggr.Ranking', + label = 'Use ranking instead of ERT-values', + value = T), + + checkboxInput('ERTPlot.Aggr.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F), + actionButton("ERTPlot.Aggr.Refresh", "Refresh the figure and table"), + hr(), + + selectInput('ERTPlot.Aggr.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Aggr.Download', label = 'Download the figure'), + hr(), + selectInput('ERTPlot.Aggr.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('ERTPlot.Aggr.DownloadTable', label = 'Download the table') + + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('The ERT of the runtime samples across all functions. + ERT is decided based on the target values in the table below, + with the default being the best reached f(x) by any of the + selected algorithms. Infinite ERTS are shown as + seperate dots on the graph.'), + plotlyOutput.IOHanalyzer('ERTPlot.Aggr.Plot'), + hr(), + HTML_P("The chosen target values per function are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("ERTPlot.Aggr.Targets"), + hr(), + HTML_P("The raw ERT-values are:"), + DT::dataTableOutput("ERTPlot.Aggr.ERTTable") + ) + ) + ) + ) +} + +#TODO: combine with other function using proper namespacing and modularity +ERT_comparison_box_dim <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Runtime Comparisons (across dimensions)

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('ERTPlot.Aggr_Dim.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('ERTPlot.Aggr_Dim.Mode', label = 'Select the plotting mode', + choices = c('radar', 'line'), selected = 'line'), + + checkboxInput('ERTPlot.Aggr_Dim.Ranking', + label = 'Use ranking instead of ERT-values', + value = F), + + checkboxInput('ERTPlot.Aggr_Dim.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + actionButton("ERTPlot.Aggr_Dim.Refresh", "Refresh the figure and table"), + hr(), + + selectInput('ERTPlot.Aggr_Dim.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ERTPlot.Aggr_Dim.Download', label = 'Download the figure'), + hr(), + selectInput('ERTPlot.Aggr_Dim.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('ERTPlot.Aggr_Dim.DownloadTable', label = 'Download the table') + + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('The ERT of the runtime samples across all functions. + ERT is decided based on the target values in the table below, + with the default being the best reached f(x) by any of the + selected algorithms. Infinite ERTS are shown as + seperate dots on the graph.'), + plotlyOutput.IOHanalyzer('ERTPlot.Aggr_Dim.Plot'), + hr(), + HTML_P("The chosen target values per dimension are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("ERTPlot.Aggr_Dim.Targets"), + hr(), + HTML_P("The raw ERT-values are:"), + DT::dataTableOutput("ERTPlot.Aggr_Dim.ERTTable") + ) + ) + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/Parrallel_coord_box.R b/inst/shiny-server/ui/Parrallel_coord_box.R index c6bcffcf..44b993f0 100644 --- a/inst/shiny-server/ui/Parrallel_coord_box.R +++ b/inst/shiny-server/ui/Parrallel_coord_box.R @@ -1,33 +1,33 @@ -Par_coord_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Parallel coordinate plot of final position

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - - selectInput('ParCoordPlot.Algs', label = 'Select which algorithm to plot:', - multiple = F, selected = NULL, choices = NULL), - - hr(), - - selectInput('ParCoordPlot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('ParCoordPlot.Download', label = 'Download the figure') - - ), - - mainPanel( - width = 9, - column( - width = 12, - align = "center", - HTML_P('The location of the best found solution by the algorithm, for each of the runs. - Each x-value corresponds to one coordinate of the solution.'), - plotlyOutput.IOHanalyzer('Parallel_Coord_Plot') - ) - ) - ) - ) -} +Par_coord_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Parallel coordinate plot of final position

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + + selectInput('ParCoordPlot.Algs', label = 'Select which algorithm to plot:', + multiple = F, selected = NULL, choices = NULL), + + hr(), + + selectInput('ParCoordPlot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('ParCoordPlot.Download', label = 'Download the figure') + + ), + + mainPanel( + width = 9, + column( + width = 12, + align = "center", + HTML_P('The location of the best found solution by the algorithm, for each of the runs. + Each x-value corresponds to one coordinate of the solution.'), + plotlyOutput.IOHanalyzer('Parallel_Coord_Plot') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_box.R b/inst/shiny-server/ui/fv_box.R index 4252fc72..f4d593bc 100644 --- a/inst/shiny-server/ui/fv_box.R +++ b/inst/shiny-server/ui/fv_box.R @@ -1,175 +1,175 @@ -fv_per_fct_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Expected Target Value (per function)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarPanel( - width = 3, - HTML('

Range of the displayed budget values

'), - - textInput('FCEPlot.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCEPlot.Max', label = RT_MAX_LABEL, value = ''), - selectInput('FCEPlot.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - checkboxInput('FCEPlot.show.mean', - label = 'Show/hide mean', - value = T), - - checkboxInput('FCEPlot.show.median', - label = 'Show/hide median', - value = F), - - checkboxInput('FCEPlot.show.CI', - label = 'Show/hide mean +/- sd', - value = F), - - checkboxInput('FCEPlot.show.IQR', - label = 'Show/hide interquartile range', - value = F), - - checkboxInput('FCEPlot.semilogx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('FCEPlot.semilogy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - checkboxInput('FCEPlot.show.runs', - label = 'Show individual runs', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_popover( - title = "Individual runs", content = "This procedure can be slow when many - runs are present in the data. Please use with caution.", - placement = "auto" - ) - ), - - hr(), - selectInput('FCEPlot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPlot.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The mean, median and standard deviation of the best function values - found with a fixed-budget of evaluations are depicted against the budget. - The displayed elements can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('FCE_PER_FUN') - ) - ) - ) -} - -fv_agg_box <- function(width = 12, height = '600px', collapsible = T, collapsed = T) { - box(title = HTML('

Expected Function values: all functions

'), - width = width, collapsible = collapsible, solidHeader = T, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - selectInput('FCEPlot.Multi.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput('FCEPlot.Multi.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('FCEPlot.Multi.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = T), - - actionButton('FCEPlot.Multi.PlotButton', label = 'Refresh the figure'), - hr(), - selectInput('FCEPlot.Multi.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPlot.Multi.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - plotlyOutput.IOHanalyzer('FCEPlot.Multi.Plot', aspect_ratio = 1) - ) - ) - ) - ) -} - -fv_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Expected Function Value comparisons

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - selectInput('FCEPlot.Aggr.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('FCEPlot.Aggr.Mode', label = 'Select the plotting mode', - choices = c('radar','line'), selected = 'radar'), - - selectInput('FCEPlot.Aggr.Aggregator', label = 'Create plot for all?', - choices = c('Functions','Dimensions'), selected = 'Functions'), - - checkboxInput('FCEPlot.Aggr.Ranking', - label = 'Use ranking instead of ERT-values', - value = T), - - checkboxInput('FCEPlot.Aggr.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F), - actionButton("FCEPlot.Aggr.Refresh", "Refresh the figure and table"), - # textInput('FCEPlot.Aggr.Targets', label = 'Choose the runtimes (comma-separated)'), - hr(), - selectInput('FCEPlot.Aggr.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPlot.Aggr.Download', label = 'Download the figure'), - hr(), - selectInput('FCEPlot.Aggr.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FCEPlot.Aggr.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - plotlyOutput.IOHanalyzer('FCEPlot.Aggr.Plot'), - hr(), - HTML_P("The chosen budget values per function are as follows (double click an entry to edit it):"), - DT::dataTableOutput("FCEPlot.Aggr.Targets"), - hr(), - HTML_P("The raw function-values are:"), - DT::dataTableOutput("FCEPlot.Aggr.FCETable") - ) - ) - ) - ) -} +fv_per_fct_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Expected Target Value (per function)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarPanel( + width = 3, + HTML('

Range of the displayed budget values

'), + + textInput('FCEPlot.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCEPlot.Max', label = RT_MAX_LABEL, value = ''), + selectInput('FCEPlot.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + checkboxInput('FCEPlot.show.mean', + label = 'Show/hide mean', + value = T), + + checkboxInput('FCEPlot.show.median', + label = 'Show/hide median', + value = F), + + checkboxInput('FCEPlot.show.CI', + label = 'Show/hide mean +/- sd', + value = F), + + checkboxInput('FCEPlot.show.IQR', + label = 'Show/hide interquartile range', + value = F), + + checkboxInput('FCEPlot.semilogx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('FCEPlot.semilogy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + checkboxInput('FCEPlot.show.runs', + label = 'Show individual runs', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_popover( + title = "Individual runs", content = "This procedure can be slow when many + runs are present in the data. Please use with caution.", + placement = "auto" + ) + ), + + hr(), + selectInput('FCEPlot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPlot.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The mean, median and standard deviation of the best function values + found with a fixed-budget of evaluations are depicted against the budget. + The displayed elements can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('FCE_PER_FUN') + ) + ) + ) +} + +fv_agg_box <- function(width = 12, height = '600px', collapsible = T, collapsed = T) { + box(title = HTML('

Expected Function values: all functions

'), + width = width, collapsible = collapsible, solidHeader = T, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + selectInput('FCEPlot.Multi.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput('FCEPlot.Multi.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('FCEPlot.Multi.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = T), + + actionButton('FCEPlot.Multi.PlotButton', label = 'Refresh the figure'), + hr(), + selectInput('FCEPlot.Multi.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPlot.Multi.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + plotlyOutput.IOHanalyzer('FCEPlot.Multi.Plot', aspect_ratio = 1) + ) + ) + ) + ) +} + +fv_comparison_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Expected Function Value comparisons

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + selectInput('FCEPlot.Aggr.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('FCEPlot.Aggr.Mode', label = 'Select the plotting mode', + choices = c('radar','line'), selected = 'radar'), + + selectInput('FCEPlot.Aggr.Aggregator', label = 'Create plot for all?', + choices = c('Functions','Dimensions'), selected = 'Functions'), + + checkboxInput('FCEPlot.Aggr.Ranking', + label = 'Use ranking instead of ERT-values', + value = T), + + checkboxInput('FCEPlot.Aggr.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F), + actionButton("FCEPlot.Aggr.Refresh", "Refresh the figure and table"), + # textInput('FCEPlot.Aggr.Targets', label = 'Choose the runtimes (comma-separated)'), + hr(), + selectInput('FCEPlot.Aggr.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPlot.Aggr.Download', label = 'Download the figure'), + hr(), + selectInput('FCEPlot.Aggr.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FCEPlot.Aggr.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + plotlyOutput.IOHanalyzer('FCEPlot.Aggr.Plot'), + hr(), + HTML_P("The chosen budget values per function are as follows (double click an entry to edit it):"), + DT::dataTableOutput("FCEPlot.Aggr.Targets"), + hr(), + HTML_P("The raw function-values are:"), + DT::dataTableOutput("FCEPlot.Aggr.FCETable") + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_dist_box.R b/inst/shiny-server/ui/fv_dist_box.R index 5c635126..105e6fe9 100644 --- a/inst/shiny-server/ui/fv_dist_box.R +++ b/inst/shiny-server/ui/fv_dist_box.R @@ -1,96 +1,96 @@ -fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = 'Histogram of Fixed-Budget Targets', - width = width, collapsible = collapsible, - solidHeader = TRUE, collapsed = collapsed, - status = "primary", - sidebarPanel( - width = 2, - textInput('FCEPDF.Hist.Runtime', label = HTML('Select the budget value'), - value = ''), - selectInput('FCEPDF.Hist.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('Choose whether the histograms are overlaid in one plot - or separated in several subplots:'), - selectInput('FCEPDF.Hist.Mode', '', - choices = c("overlay", "subplot"), - selected = 'subplot'), - checkboxInput("FCEPDF.Hist.Equal", "Use equal bins for all algorithms", F), - - hr(), - selectInput('FCEPDF.Hist.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPDF.Hist.Download', label = 'Download the figure') - - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML('

- This histogram counts the number of runs whose best-so-far function - values within the first \\(B\\) evaluations is between \\(v_i\\) and - \\(v_{i+1}\\). The buckets \\([v_i,v_{i+1})\\) are chosen automatically - according to the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= - 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) - and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. - The displayed algorithms can be selected by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.

'), - plotlyOutput.IOHanalyzer('FCE_HIST') - ) - ) - ) -} - -fv_pdf_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = 'Empirical Probability Density Function of Fixed-Budget Function Values', - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - HTML('Select the budget for which the distribution of best-so-far function values is shown'), - - textInput('FCEPDF.Bar.Runtime', label = '', value = ''), - selectInput('FCEPDF.Bar.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - checkboxInput('FCEPDF.Bar.Samples', label = 'Show runtime samples', value = T), - checkboxInput('FCEPDF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = T), - - hr(), - selectInput('FCEPDF.Bar.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEPDF.Bar.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML('

- The plot shows, for the budget selected on the left, the distribution - of the best-so-far function values of the individual runs (dots), and an estimated distribution of the probability mass function. - The displayed algorithms can be selected by clicking on the legend on the right. A tooltip and toolbar - appear when hovering over the figure. A csv file with the runtime data can be downloaded from the - Data Summary tab.'), - plotlyOutput.IOHanalyzer('FCE_PDF') - ) - ) - ) - ) -} +fv_histgram_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = 'Histogram of Fixed-Budget Targets', + width = width, collapsible = collapsible, + solidHeader = TRUE, collapsed = collapsed, + status = "primary", + sidebarPanel( + width = 2, + textInput('FCEPDF.Hist.Runtime', label = HTML('Select the budget value'), + value = ''), + selectInput('FCEPDF.Hist.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('Choose whether the histograms are overlaid in one plot + or separated in several subplots:'), + selectInput('FCEPDF.Hist.Mode', '', + choices = c("overlay", "subplot"), + selected = 'subplot'), + checkboxInput("FCEPDF.Hist.Equal", "Use equal bins for all IDs", F), + + hr(), + selectInput('FCEPDF.Hist.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPDF.Hist.Download', label = 'Download the figure') + + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML('

+ This histogram counts the number of runs whose best-so-far function + values within the first \\(B\\) evaluations is between \\(v_i\\) and + \\(v_{i+1}\\). The buckets \\([v_i,v_{i+1})\\) are chosen automatically + according to the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= + 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) + and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. + The displayed IDs can be selected by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.

'), + plotlyOutput.IOHanalyzer('FCE_HIST') + ) + ) + ) +} + +fv_pdf_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = 'Empirical Probability Density Function of Fixed-Budget Function Values', + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + HTML('Select the budget for which the distribution of best-so-far function values is shown'), + + textInput('FCEPDF.Bar.Runtime', label = '', value = ''), + selectInput('FCEPDF.Bar.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + checkboxInput('FCEPDF.Bar.Samples', label = 'Show runtime samples', value = T), + checkboxInput('FCEPDF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = T), + + hr(), + selectInput('FCEPDF.Bar.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEPDF.Bar.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML('

+ The plot shows, for the budget selected on the left, the distribution + of the best-so-far function values of the individual runs (dots), and an estimated distribution of the probability mass function. + The displayed IDs can be selected by clicking on the legend on the right. A tooltip and toolbar + appear when hovering over the figure. A csv file with the runtime data can be downloaded from the + Data Summary tab.'), + plotlyOutput.IOHanalyzer('FCE_PDF') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_ecdf_box.R b/inst/shiny-server/ui/fv_ecdf_box.R index e9fc1214..d6570787 100644 --- a/inst/shiny-server/ui/fv_ecdf_box.R +++ b/inst/shiny-server/ui/fv_ecdf_box.R @@ -1,136 +1,136 @@ -fv_ecdf_single_budget_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Empirical Cumulative Distribution of - the Fixed-Budget Values: Single Budgets

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarLayout( - sidebarPanel( - width = 3, - selectInput('FCEECDF.Single.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('Select the budgets for which EDCF curves are displayed '), - textInput('FCEECDF.Single.Target', label = HTML('

\\(B_1\\)

'), value = ''), - checkboxInput('FCEECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('Each EDCF curve shows the proportion of the runs that have found - within the given budget B a solution of at least the required target - value given by the x-axis. The displayed curves can be selected - by clicking on the legend on the right. A tooltip and toolbar - appears when hovering over the figure.

'), - plotlyOutput.IOHanalyzer("FCE_ECDF_PER_TARGET") - ) - ) - ) - ) -} - -fv_ecdf_agg_budgets_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Empirical Cumulative Distribution - of the Fixed-Budget Values: Aggregation

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('FCEECDF.Mult.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('

Set the range and the granularity of the budgets - taken into account in the ECDF curve. The plot will show the ECDF curves - for evenly spaced budgets.

'), - textInput('FCEECDF.Mult.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCEECDF.Mult.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCEECDF.Mult.Step', label = RT_STEP_LABEL, value = ''), -# -# checkboxInput('FCEECDF.Mult.Targets', -# label = 'Show ECDF for each budget', -# value = F), - - checkboxInput('FCEECDF.Mult.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = F), - hr(), - selectInput('FCEECDF.Mult.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEECDF.Mult.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The evenly spaced budget values are:'), - verbatimTextOutput('FCE_RT_GRID'), - - HTML_P('The fraction of (run,budget) pairs \\((i,B)\\) satisfying that the best - solution that the algorithm has found in the \\(i\\)-th run within the - first \\(B\\) evaluations has quality at most \\(v\\) is plotted - against the target value \\(v\\). The displayed elements can be switched - on and off by clicking on the legend on the right. A tooltip and - toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('FCE_ECDF_AGGR') - ) - ) - ) -} - -fv_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Area Under the ECDF

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('FCEECDF.AUC.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('

Set the range and the granularity of the evenly spaced budgets.

'), - textInput('FCEECDF.AUC.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCEECDF.AUC.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCEECDF.AUC.Step', label = RT_STEP_LABEL, value = ''), - - hr(), - selectInput('FCEECDF.AUC.Format', label = 'select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FCEECDF.AUC.Download', label = 'download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The area under the ECDF is - caculated for the sequence of budget values specified on the left. The displayed - values are normalized against the maximal target value recorded for - each algorithm. Intuitively, the smaller the area, the better the algorithm. - The displayed algorithms can be selected by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer("FCE_AUC") - ) - ) - ) -} +fv_ecdf_single_budget_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Empirical Cumulative Distribution of + the Fixed-Budget Values: Single Budgets

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarLayout( + sidebarPanel( + width = 3, + selectInput('FCEECDF.Single.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('Select the budgets for which EDCF curves are displayed '), + textInput('FCEECDF.Single.Target', label = HTML('

\\(B_1\\)

'), value = ''), + checkboxInput('FCEECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('Each EDCF curve shows the proportion of the runs that have found + within the given budget B a solution of at least the required target + value given by the x-axis. The displayed curves can be selected + by clicking on the legend on the right. A tooltip and toolbar + appears when hovering over the figure.

'), + plotlyOutput.IOHanalyzer("FCE_ECDF_PER_TARGET") + ) + ) + ) + ) +} + +fv_ecdf_agg_budgets_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Empirical Cumulative Distribution + of the Fixed-Budget Values: Aggregation

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('FCEECDF.Mult.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('

Set the range and the granularity of the budgets + taken into account in the ECDF curve. The plot will show the ECDF curves + for evenly spaced budgets.

'), + textInput('FCEECDF.Mult.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCEECDF.Mult.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCEECDF.Mult.Step', label = RT_STEP_LABEL, value = ''), +# +# checkboxInput('FCEECDF.Mult.Targets', +# label = 'Show ECDF for each budget', +# value = F), + + checkboxInput('FCEECDF.Mult.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = F), + hr(), + selectInput('FCEECDF.Mult.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEECDF.Mult.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The evenly spaced budget values are:'), + verbatimTextOutput('FCE_RT_GRID'), + + HTML_P('The fraction of (run,budget) pairs \\((i,B)\\) satisfying that the best + solution that the algorithm has found in the \\(i\\)-th run within the + first \\(B\\) evaluations has quality at most \\(v\\) is plotted + against the target value \\(v\\). The displayed elements can be switched + on and off by clicking on the legend on the right. A tooltip and + toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('FCE_ECDF_AGGR') + ) + ) + ) +} + +fv_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Area Under the ECDF

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('FCEECDF.AUC.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('

Set the range and the granularity of the evenly spaced budgets.

'), + textInput('FCEECDF.AUC.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCEECDF.AUC.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCEECDF.AUC.Step', label = RT_STEP_LABEL, value = ''), + + hr(), + selectInput('FCEECDF.AUC.Format', label = 'select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FCEECDF.AUC.Download', label = 'download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The area under the ECDF is + caculated for the sequence of budget values specified on the left. The displayed + values are normalized against the maximal target value recorded for + each algorithm. Intuitively, the smaller the area, the better the algorithm. + The displayed IDs can be selected by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer("FCE_AUC") + ) + ) + ) +} diff --git a/inst/shiny-server/ui/fv_par_box.R b/inst/shiny-server/ui/fv_par_box.R index 4c816a13..763b5e8d 100644 --- a/inst/shiny-server/ui/fv_par_box.R +++ b/inst/shiny-server/ui/fv_par_box.R @@ -1,134 +1,134 @@ -fv_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Expected Parameter Value (per function)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarPanel( - width = 3, - HTML('

Range of the budget values (\\(x\\) axis)

'), - - textInput('FV_PAR.Plot.Min', label = RT_MIN_LABEL, value = ''), - textInput('FV_PAR.Plot.Max', label = RT_MAX_LABEL, value = ''), - selectInput('FV_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), - - selectInput('FV_PAR.Plot.Algs', 'Select which algorithms to plot:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('FV_PAR.Plot.show.mean', label = 'Mean/median', - choices = c('mean', 'median'), - selected = 'mean'), - selectInput('FV_PAR.Plot.CI', "Show standard deviations or quantiles", - choices = c('Standard Deviation', 'Outer Quantiles', 'None'), - selected = 'None') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are - 2% and 98% by default. This can be changed in the settings-tab.", - placement = "auto" - ) - ), - checkboxInput('FV_PAR.Plot.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('FV_PAR.Plot.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_tooltip( - title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" - ) - ), - - hr(), - selectInput('FV_PAR.Plot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('FV_PAR.Plot.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The mean or median of internal parameters of the algorithm - found with a fixed-budget of evaluations are depicted against the budget. - The displayed elements can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('FV_PAR.Plot.Figure') - ) - ) - ) -} - -fv_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Statistics at Chosen Runtime Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-budget parameter values for evenly spaced budget values.'), - - textInput('FV_PAR.Summary.Min', label = RT_MIN_LABEL, value = ''), - textInput('FV_PAR.Summary.Max', label = RT_MAX_LABEL, value = ''), - textInput('FV_PAR.Summary.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FV_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Summary.Algid', 'Algorithms', choices = NULL, selected = NULL), - selectInput('FV_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('FV_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV_PAR.Summary.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', - includeMarkdown('markdown/FV_PAR_SUMMARY_TABLE.Rmd'), '
')), - DT::dataTableOutput('table_FV_PAR_summary') - ) - ) -} - -fv_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Sample at Chosen Runtime Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-budget parameter values for evenly spaced budget values.'), - - textInput('FV_PAR.Sample.Min', label = RT_MIN_LABEL, value = ''), - textInput('FV_PAR.Sample.Max', label = RT_MAX_LABEL, value = ''), - textInput('FV_PAR.Sample.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FV_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FV_PAR.Sample.Algid', 'Algorithms', choices = NULL, selected = NULL), - selectInput('FV_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('FV_PAR.Sample.Format', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - selectInput('FV_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV_PAR.Sample.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML('

This table shows for each selected algorithm \\(A\\), - each selected runtime value \\(b\\), and each run \\(r\\) the parameter value - observed when the after a total budget of \\(b\\) function evaluations has been used.

'), - DT::dataTableOutput('table_FV_PAR_SAMPLE') - ) - ) -} +fv_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Expected Parameter Value (per function)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarPanel( + width = 3, + HTML('

Range of the budget values (\\(x\\) axis)

'), + + textInput('FV_PAR.Plot.Min', label = RT_MIN_LABEL, value = ''), + textInput('FV_PAR.Plot.Max', label = RT_MAX_LABEL, value = ''), + selectInput('FV_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), + + selectInput('FV_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('FV_PAR.Plot.show.mean', label = 'Mean/median', + choices = c('mean', 'median'), + selected = 'mean'), + selectInput('FV_PAR.Plot.CI', "Show standard deviations or quantiles", + choices = c('Standard Deviation', 'Outer Quantiles', 'None'), + selected = 'None') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are + 2% and 98% by default. This can be changed in the settings-tab.", + placement = "auto" + ) + ), + checkboxInput('FV_PAR.Plot.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('FV_PAR.Plot.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_tooltip( + title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" + ) + ), + + hr(), + selectInput('FV_PAR.Plot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('FV_PAR.Plot.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The mean or median of internal parameters of the algorithm + found with a fixed-budget of evaluations are depicted against the budget. + The displayed elements can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('FV_PAR.Plot.Figure') + ) + ) + ) +} + +fv_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Statistics at Chosen Runtime Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-budget parameter values for evenly spaced budget values.'), + + textInput('FV_PAR.Summary.Min', label = RT_MIN_LABEL, value = ''), + textInput('FV_PAR.Summary.Max', label = RT_MAX_LABEL, value = ''), + textInput('FV_PAR.Summary.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FV_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FV_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FV_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('FV_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV_PAR.Summary.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', + includeMarkdown('markdown/FV_PAR_SUMMARY_TABLE.Rmd'), '
')), + DT::dataTableOutput('table_FV_PAR_summary') + ) + ) +} + +fv_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Sample at Chosen Runtime Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-budget parameter values for evenly spaced budget values.'), + + textInput('FV_PAR.Sample.Min', label = RT_MIN_LABEL, value = ''), + textInput('FV_PAR.Sample.Max', label = RT_MAX_LABEL, value = ''), + textInput('FV_PAR.Sample.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FV_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FV_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('FV_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('FV_PAR.Sample.Format', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + selectInput('FV_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV_PAR.Sample.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML('

This table shows for each selected algorithm \\(A\\), + each selected runtime value \\(b\\), and each run \\(r\\) the parameter value + observed when the after a total budget of \\(b\\) function evaluations has been used.

'), + DT::dataTableOutput('table_FV_PAR_SAMPLE') + ) + ) +} diff --git a/inst/shiny-server/ui/fv_stats_box.R b/inst/shiny-server/ui/fv_stats_box.R index fc7a9d26..892a502b 100644 --- a/inst/shiny-server/ui/fv_stats_box.R +++ b/inst/shiny-server/ui/fv_stats_box.R @@ -1,118 +1,118 @@ -fv_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Hypothesis Testing

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.Overview.Algid', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - textInput('FV_Stats.Overview.Target', label = RT_TAR_LABEL), - textInput('FV_Stats.Overview.Alpha', - label = HTML('

Significe level \\(\\alpha\\)

'), - value = 0.01), - # numericInput('FV_Stats.Overview.Samples', - # label = 'size of the bootstrap sample', - # min = 1, max = 1000, step = 1, value = 0), - hr(), - selectInput('FV_Stats.Overview.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.Overview.DownloadTable', label = 'Download the table'), - hr(), - selectInput('FV_Stats.Overview.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('FV_Stats.Overview.DownloadHeatmap', - label = 'Download the heatmap') - # downloadButton('FV_Stats.Overview.DownloadNetwork', - # label = 'Download the network-graph', status = F) - - ), - - mainPanel( - width = 9, - HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of - algorithms, in order to determine which algorithm gives a significantly - smaller running time distribution. The resulting p-values are arranged in a matrix, where - each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: - the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), - DT::dataTableOutput('FV_Stats.Overview.Pmatrix') - ), - - fluidRow( - column( - width = 12, - HTML('
'), - HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, - are visualized in a heatmap (left) and a network (right). - In each cell (row, column) of the heatmap, the alternative hypothesis is again: - the row algorithm has smaller (better) running time than the column. - The color indicates: -
    -
  • Red: A row is better than the column
    • -
  • Blue: A row is worse than the column
    • -
  • Gray: no significant distinction between the row and column
    • -
- On the right subplot, the partial order resulted from the test is visualized as a network, - where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than - \\(B\\) with the specified \\(\\alpha\\) value.') - ) - ), - - fluidRow( - column( - width = 6, align = 'center', - HTML('
'), - plotlyOutput.IOHanalyzer('FV_Stats.Overview.Heatmap', aspect_ratio = 1) - ), - column( - width = 6, align = 'center', - HTML('
'), - plotOutput("FV_Stats.Overview.Graph", height = '70vh') - ) - ) - ) -} - -fv_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Glicko2-based ranking

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('FV_Stats.Glicko.Algid', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('FV_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - textInput('FV_Stats.Glicko.Nrgames', - label = "Number of games per (function,dimension) pair", - value = 25), - actionButton('FV_Stats.Glicko.Create', 'Create / Update Ranking'), - hr(), - selectInput('FV_Stats.Glicko.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('FV_Stats.Glicko.Download', label = 'Download the figure'), - hr(), - selectInput('FV_Stats.Glicko.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('FV_Stats.Glicko.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. - Every round, for every function and dimension of the datasetlist, - each pair of algorithms competes. This competition samples a random function value for the - provided runtime (maximum used among all algorithms). Whichever algorithm has the better - function value wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), - HTML_P("The chosen budget values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("FV_Stats.Glicko.Targets"), - hr(), - HTML_P("The results of the ranking are:"), - plotlyOutput.IOHanalyzer("FV_Stats.Glicko.Candlestick"), - DT::dataTableOutput('FV_Stats.Glicko.Dataframe') - ) - ) +fv_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Hypothesis Testing

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + textInput('FV_Stats.Overview.Target', label = RT_TAR_LABEL), + textInput('FV_Stats.Overview.Alpha', + label = HTML('

Significe level \\(\\alpha\\)

'), + value = 0.01), + # numericInput('FV_Stats.Overview.Samples', + # label = 'size of the bootstrap sample', + # min = 1, max = 1000, step = 1, value = 0), + hr(), + selectInput('FV_Stats.Overview.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.Overview.DownloadTable', label = 'Download the table'), + hr(), + selectInput('FV_Stats.Overview.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('FV_Stats.Overview.DownloadHeatmap', + label = 'Download the heatmap') + # downloadButton('FV_Stats.Overview.DownloadNetwork', + # label = 'Download the network-graph', status = F) + + ), + + mainPanel( + width = 9, + HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of + algorithms, in order to determine which algorithm gives a significantly + smaller running time distribution. The resulting p-values are arranged in a matrix, where + each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: + the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), + DT::dataTableOutput('FV_Stats.Overview.Pmatrix') + ), + + fluidRow( + column( + width = 12, + HTML('
'), + HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, + are visualized in a heatmap (left) and a network (right). + In each cell (row, column) of the heatmap, the alternative hypothesis is again: + the row algorithm has smaller (better) running time than the column. + The color indicates: +
    +
  • Red: A row is better than the column
    • +
  • Blue: A row is worse than the column
    • +
  • Gray: no significant distinction between the row and column
    • +
+ On the right subplot, the partial order resulted from the test is visualized as a network, + where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than + \\(B\\) with the specified \\(\\alpha\\) value.') + ) + ), + + fluidRow( + column( + width = 6, align = 'center', + HTML('
'), + plotlyOutput.IOHanalyzer('FV_Stats.Overview.Heatmap', aspect_ratio = 1) + ), + column( + width = 6, align = 'center', + HTML('
'), + plotOutput("FV_Stats.Overview.Graph", height = '70vh') + ) + ) + ) +} + +fv_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Glicko2-based ranking

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FV_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('FV_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + textInput('FV_Stats.Glicko.Nrgames', + label = "Number of games per (function,dimension) pair", + value = 25), + actionButton('FV_Stats.Glicko.Create', 'Create / Update Ranking'), + hr(), + selectInput('FV_Stats.Glicko.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('FV_Stats.Glicko.Download', label = 'Download the figure'), + hr(), + selectInput('FV_Stats.Glicko.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('FV_Stats.Glicko.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. + Every round, for every function and dimension of the datasetlist, + each pair of algorithms competes. This competition samples a random function value for the + provided runtime (maximum used among all algorithms). Whichever algorithm has the better + function value wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), + HTML_P("The chosen budget values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("FV_Stats.Glicko.Targets"), + hr(), + HTML_P("The results of the ranking are:"), + plotlyOutput.IOHanalyzer("FV_Stats.Glicko.Candlestick"), + DT::dataTableOutput('FV_Stats.Glicko.Dataframe') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/fv_summary_box.R b/inst/shiny-server/ui/fv_summary_box.R index bd523377..f71f37ad 100644 --- a/inst/shiny-server/ui/fv_summary_box.R +++ b/inst/shiny-server/ui/fv_summary_box.R @@ -1,90 +1,88 @@ -fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Data Overview

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('FCESummary.Overview.Algid', 'Algorithms', choices = NULL, selected = NULL), - #TODO: implement this button - hr(), - selectInput('FCESummary.Overview.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FCESummary.Overview.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('table_FV_overview') - ) - ) -} - -fv_stats_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Target Statistics at Chosen Budget Values

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML(FCE_GRID_INPUT_TEXT), - - textInput('FCESummary.Statistics.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCESummary.Statistics.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCESummary.Statistics.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FCESummary.Statistics.Single', label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? - Once toggled, only \\(B_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Statistics.Algid', 'Algorithms', choices = NULL, selected = NULL), - hr(), - selectInput('FCESummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FCESummary.Statistics.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', - includeMarkdown('markdown/TAR_SUMMARY_TABLE.Rmd'), - '
')), - DT::dataTableOutput('FCE_SUMMARY') - ) - ) -} - -fv_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Original Target Samples

'), - width = width, solidHeader = TRUE, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML(FCE_GRID_INPUT_TEXT), - - textInput('FCESummary.Sample.Min', label = RT_MIN_LABEL, value = ''), - textInput('FCESummary.Sample.Max', label = RT_MAX_LABEL, value = ''), - textInput('FCESummary.Sample.Step', label = RT_STEP_LABEL, value = ''), - checkboxInput('FCESummary.Sample.Single', - label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? - Once toggled, only \\(B_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('FCESummary.Sample.Algid', 'Algorithms', - choices = NULL, selected = NULL), - - selectInput('FCESummary.Sample.Format', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - hr(), - selectInput('FCESummary.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FCESummary.Sample.Download", "Save the aligned runtime samples") - ), - - mainPanel( - width = 9, - HTML("
This table shows for each selected algorithm \\(A\\), - each selected target value \\(f(x)\\), and each run \\(r\\) - the number \\(T(A,f(x),r)\\) of evaluations performed by the - algorithm until it evaluated for the first time a solution of - quality at least \\(f(x)\\).
"), - br(), - DT::dataTableOutput('FCE_SAMPLE') - ) - ) -} +fv_overview_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Data Overview

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('FCESummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + #TODO: implement this button + hr(), + selectInput('FCESummary.Overview.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FCESummary.Overview.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('table_FV_overview') + ) + ) +} + +fv_stats_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Target Statistics at Chosen Budget Values

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML(FCE_GRID_INPUT_TEXT), + + textInput('FCESummary.Statistics.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCESummary.Statistics.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCESummary.Statistics.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FCESummary.Statistics.Single', label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? + Once toggled, only \\(B_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FCESummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('FCESummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FCESummary.Statistics.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', + includeMarkdown('markdown/TAR_SUMMARY_TABLE.Rmd'), + '
')), + DT::dataTableOutput('FCE_SUMMARY') + ) + ) +} + +fv_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Original Target Samples

'), + width = width, solidHeader = TRUE, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML(FCE_GRID_INPUT_TEXT), + + textInput('FCESummary.Sample.Min', label = RT_MIN_LABEL, value = ''), + textInput('FCESummary.Sample.Max', label = RT_MAX_LABEL, value = ''), + textInput('FCESummary.Sample.Step', label = RT_STEP_LABEL, value = ''), + checkboxInput('FCESummary.Sample.Single', + label = HTML('

\\(B_{\\text{min}} = B_{\\text{max}}\\)? + Once toggled, only \\(B_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('FCESummary.Sample.ID', 'Select which IDs to include:', + choices = NULL, selected = NULL, multiple = T), + + selectInput('FCESummary.Sample.Format', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + hr(), + selectInput('FCESummary.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FCESummary.Sample.Download", "Save the aligned runtime samples") + ), + + mainPanel( + width = 9, + HTML("
This table shows for each selected algorithm \\(A\\), + each selected target value \\(f(x)\\), and each run \\(r\\) + the number \\(T(A,f(x),r)\\) of evaluations performed by the + algorithm until it evaluated for the first time a solution of + quality at least \\(f(x)\\).
"), + br(), + DT::dataTableOutput('FCE_SAMPLE') + ) + ) +} diff --git a/inst/shiny-server/ui/overview_box.R b/inst/shiny-server/ui/overview_box.R index 2779b482..5896306d 100644 --- a/inst/shiny-server/ui/overview_box.R +++ b/inst/shiny-server/ui/overview_box.R @@ -1,37 +1,37 @@ -general_overview_box_single <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Overview of Selected Function

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('Overview.Single.Algid', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - hr(), - selectInput('Overview.Single.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("Overview.Single.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('Overview.Single.Table') - ) - ) -} - -general_overview_box_all <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Overview of All Available Functions

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('Overview.All.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("Overview.All.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('Overview.All.Table') - ) - ) +general_overview_box_single <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Overview of Selected Function

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('Overview.Single.ID', 'IDs:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('Overview.Single.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("Overview.Single.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('Overview.Single.Table') + ) + ) +} + +general_overview_box_all <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Overview of All Available Functions

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('Overview.All.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("Overview.All.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('Overview.All.Table') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/report_generation_box.R b/inst/shiny-server/ui/report_generation_box.R index 0150e150..5b71705a 100644 --- a/inst/shiny-server/ui/report_generation_box.R +++ b/inst/shiny-server/ui/report_generation_box.R @@ -1,119 +1,119 @@ -main_report_box <- function(width = 6, collapsible = T, collapsed = F) { - box(title = HTML('

Report Content

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - HTML("Select which components should be in the report. \n All relevant settings (bounds, targets, scaling...) - will be taken from the relevant tab if available, or set to a default value."), - checkboxInput("Report.Introduction", "Introduction", T), - fixed_target_report_box(), - fixed_budget_report_box(), - parameter_box(), - hr(), - downloadButton("Report.Generate", "Generate report pdf"), - downloadButton("Report.Generate.Tar", "Generate report tar", style="float:right") - ) -} - -fixed_target_report_box <- function(){ - box(title = "Fixed-Target", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "blue", - box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.Overview", "Data Overview", F), - conditionalSelection("Report.RT.Overview",'input["Report.RT.Overview"]'), - checkboxInput("Report.RT.Statistics", "Runtime Statistics", F), - conditionalSelection("Report.RT.Statistics",'input["Report.RT.Statistics"]'), - checkboxInput("Report.RT.Samples", "Runtime Samples", F), - conditionalSelection("Report.RT.Samples",'input["Report.RT.Samples"]') - ), - box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.Single_ERT", "Runtime plot (per function)", F), - conditionalSelection("Report.RT.Single_ERT",'input["Report.RT.Single_ERT"]'), - checkboxInput("Report.RT.Multi_ERT", "Runtime plot (all functions)", F), - conditionalSelection("Report.RT.Multi_ERT",'input["Report.RT.Multi_ERT"]', fid=F), - checkboxInput("Report.RT.Rank", "Runtime ranking", F), - conditionalSelection("Report.RT.Rank",'input["Report.RT.Rank"]', fid=F) #, plottype=c("radar", "line")) - ), - box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.Histogram", "Histogram of hitting times", F), - conditionalSelection("Report.RT.Histogram",'input["Report.RT.Histogram"]'), # plottype = c("subplot","overlay")), - checkboxInput("Report.RT.PMF", "Probability Mass Function", F), - conditionalSelection("Report.RT.PMF",'input["Report.RT.PMF"]') - ), - box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.RT.ECDF_Single_Target", "Single-target ECDF", F), - conditionalSelection("Report.RT.ECDF_Single_Target",'input["Report.RT.ECDF_Single_Target"]'), - checkboxInput("Report.RT.ECDF_Single_Function", "Single-function ECDF", F), - conditionalSelection("Report.RT.ECDF_Single_Function",'input["Report.RT.ECDF_Single_Function"]'), - checkboxInput("Report.RT.ECDF_Aggregated", "Aggregated ECDF", F), - conditionalSelection("Report.RT.ECDF_Aggregated",'input["Report.RT.ECDF_Aggregated"]', F, F), - checkboxInput("Report.RT.ECDF_AUC", "AUC of ECDF", F), - conditionalSelection("Report.RT.ECDF_AUC",'input["Report.RT.ECDF_AUC"]') - ) - ) -} - -fixed_budget_report_box <- function(){ - box(title = "Fixed-Budget", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "blue", - box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.Overview", "Data Overview", F), - conditionalSelection("Report.FV.Overview",'input["Report.FV.Overview"]'), - checkboxInput("Report.FV.Statistics", "Runtime Statistics", F), - conditionalSelection("Report.FV.Statistics",'input["Report.FV.Statistics"]'), - checkboxInput("Report.FV.Samples", "Runtime Samples", F), - conditionalSelection("Report.FV.Samples",'input["Report.FV.Samples"]') - ), - box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.Single_FCE", "Runtime plot (per function)", F), - conditionalSelection("Report.FV.Single_FCE",'input["Report.FV.Single_FCE"]'), - checkboxInput("Report.FV.Multi_FCE", "Runtime plot (all functions)", F), - conditionalSelection("Report.FV.Multi_FCE",'input["Report.FV.Multi_FCE"]', fid=F), - checkboxInput("Report.FV.Rank", "Runtime ranking", F), - conditionalSelection("Report.FV.Rank",'input["Report.FV.Rank"]', fid=F) #, plottype=c("radar", "line")) - ), - box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.Histogram", "Histogram of hitting times", F), - conditionalSelection("Report.FV.Histogram",'input["Report.FV.Histogram"]'), # plottype = c("subplot","overlay")), - checkboxInput("Report.FV.PMF", "Probability Mass Function", F), - conditionalSelection("Report.FV.PMF",'input["Report.FV.PMF"]') - ), - box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "light-blue", - checkboxInput("Report.FV.ECDF_Single_Target", "Single-target ECDF", F), - conditionalSelection("Report.FV.ECDF_Single_Target",'input["Report.FV.ECDF_Single_Target"]'), - checkboxInput("Report.FV.ECDF_Single_Function", "Single-function ECDF", F), - conditionalSelection("Report.FV.ECDF_Single_Function",'input["Report.FV.ECDF_Single_Function"]'), - checkboxInput("Report.FV.ECDF_AUC", "AUC of ECDF", F), - conditionalSelection("Report.FV.ECDF_AUC",'input["Report.FV.ECDF_AUC"]') - ) - ) -} - -parameter_box <- function(){ - box(title = "Parameters", collapsible = T, collapsed = T, width = 11, status = "primary", - background = "blue", - checkboxInput("Report.Param.Plot", "Parameter plot", F), - conditionalSelection("Report.Param.Plot",'input["Report.Param.Plot"]'), - checkboxInput("Report.Param.Statistics", "Parameter statistics", F), - conditionalSelection("Report.Param.Statistics",'input["Report.Param.Statistics"]') - ) -} - -conditionalSelection <- function(id, condition, fid=T, dim=T, alg=T, scalex=F, scaley=F, plottype=NULL){ - ns <- NS(id) - conditionalPanel(condition = condition, - if (fid) selectInput(ns("FuncId"), label = "Function ids", choices = NULL, selected = NULL, multiple = T), - if (dim) selectInput(ns("DIM"), label = "Dimensions", choices = NULL, selected = NULL, multiple = T), - if (alg) selectInput(ns("Alg"), label = "Algorithms", choices = NULL, selected = NULL, multiple = T), - if (scalex) checkboxInput(ns("Scalex"), label = "Scale x-axis logarithmically", value = F), - if (scaley) checkboxInput(ns("Scaley"), label = "Scale y-axis logarithmically", value = F), - if (!is.null(plottype)) selectInput(ns("Type"), label = "Plot type", choices = plottype, selected = plottype[[1]]) - ) +main_report_box <- function(width = 6, collapsible = T, collapsed = F) { + box(title = HTML('

Report Content

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + HTML("Select which components should be in the report. \n All relevant settings (bounds, targets, scaling...) + will be taken from the relevant tab if available, or set to a default value."), + checkboxInput("Report.Introduction", "Introduction", T), + fixed_target_report_box(), + fixed_budget_report_box(), + parameter_box(), + hr(), + downloadButton("Report.Generate", "Generate report pdf"), + downloadButton("Report.Generate.Tar", "Generate report tar", style="float:right") + ) +} + +fixed_target_report_box <- function(){ + box(title = "Fixed-Target", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "blue", + box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.Overview", "Data Overview", F), + conditionalSelection("Report.RT.Overview",'input["Report.RT.Overview"]'), + checkboxInput("Report.RT.Statistics", "Runtime Statistics", F), + conditionalSelection("Report.RT.Statistics",'input["Report.RT.Statistics"]'), + checkboxInput("Report.RT.Samples", "Runtime Samples", F), + conditionalSelection("Report.RT.Samples",'input["Report.RT.Samples"]') + ), + box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.Single_ERT", "Runtime plot (per function)", F), + conditionalSelection("Report.RT.Single_ERT",'input["Report.RT.Single_ERT"]'), + checkboxInput("Report.RT.Multi_ERT", "Runtime plot (all functions)", F), + conditionalSelection("Report.RT.Multi_ERT",'input["Report.RT.Multi_ERT"]', fid=F), + checkboxInput("Report.RT.Rank", "Runtime ranking", F), + conditionalSelection("Report.RT.Rank",'input["Report.RT.Rank"]', fid=F) #, plottype=c("radar", "line")) + ), + box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.Histogram", "Histogram of hitting times", F), + conditionalSelection("Report.RT.Histogram",'input["Report.RT.Histogram"]'), # plottype = c("subplot","overlay")), + checkboxInput("Report.RT.PMF", "Probability Mass Function", F), + conditionalSelection("Report.RT.PMF",'input["Report.RT.PMF"]') + ), + box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.RT.ECDF_Single_Target", "Single-target ECDF", F), + conditionalSelection("Report.RT.ECDF_Single_Target",'input["Report.RT.ECDF_Single_Target"]'), + checkboxInput("Report.RT.ECDF_Single_Function", "Single-function ECDF", F), + conditionalSelection("Report.RT.ECDF_Single_Function",'input["Report.RT.ECDF_Single_Function"]'), + checkboxInput("Report.RT.ECDF_Aggregated", "Aggregated ECDF", F), + conditionalSelection("Report.RT.ECDF_Aggregated",'input["Report.RT.ECDF_Aggregated"]', F, F), + checkboxInput("Report.RT.ECDF_AUC", "AUC of ECDF", F), + conditionalSelection("Report.RT.ECDF_AUC",'input["Report.RT.ECDF_AUC"]') + ) + ) +} + +fixed_budget_report_box <- function(){ + box(title = "Fixed-Budget", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "blue", + box(title = "Data Summary", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.Overview", "Data Overview", F), + conditionalSelection("Report.FV.Overview",'input["Report.FV.Overview"]'), + checkboxInput("Report.FV.Statistics", "Runtime Statistics", F), + conditionalSelection("Report.FV.Statistics",'input["Report.FV.Statistics"]'), + checkboxInput("Report.FV.Samples", "Runtime Samples", F), + conditionalSelection("Report.FV.Samples",'input["Report.FV.Samples"]') + ), + box(title = "Expected Runtime", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.Single_FCE", "Runtime plot (per function)", F), + conditionalSelection("Report.FV.Single_FCE",'input["Report.FV.Single_FCE"]'), + checkboxInput("Report.FV.Multi_FCE", "Runtime plot (all functions)", F), + conditionalSelection("Report.FV.Multi_FCE",'input["Report.FV.Multi_FCE"]', fid=F), + checkboxInput("Report.FV.Rank", "Runtime ranking", F), + conditionalSelection("Report.FV.Rank",'input["Report.FV.Rank"]', fid=F) #, plottype=c("radar", "line")) + ), + box(title = "Probability Mass Function", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.Histogram", "Histogram of hitting times", F), + conditionalSelection("Report.FV.Histogram",'input["Report.FV.Histogram"]'), # plottype = c("subplot","overlay")), + checkboxInput("Report.FV.PMF", "Probability Mass Function", F), + conditionalSelection("Report.FV.PMF",'input["Report.FV.PMF"]') + ), + box(title = "Cumulative Distirbution", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "light-blue", + checkboxInput("Report.FV.ECDF_Single_Target", "Single-target ECDF", F), + conditionalSelection("Report.FV.ECDF_Single_Target",'input["Report.FV.ECDF_Single_Target"]'), + checkboxInput("Report.FV.ECDF_Single_Function", "Single-function ECDF", F), + conditionalSelection("Report.FV.ECDF_Single_Function",'input["Report.FV.ECDF_Single_Function"]'), + checkboxInput("Report.FV.ECDF_AUC", "AUC of ECDF", F), + conditionalSelection("Report.FV.ECDF_AUC",'input["Report.FV.ECDF_AUC"]') + ) + ) +} + +parameter_box <- function(){ + box(title = "Parameters", collapsible = T, collapsed = T, width = 11, status = "primary", + background = "blue", + checkboxInput("Report.Param.Plot", "Parameter plot", F), + conditionalSelection("Report.Param.Plot",'input["Report.Param.Plot"]'), + checkboxInput("Report.Param.Statistics", "Parameter statistics", F), + conditionalSelection("Report.Param.Statistics",'input["Report.Param.Statistics"]') + ) +} + +conditionalSelection <- function(id, condition, fid=T, dim=T, alg=T, scalex=F, scaley=F, plottype=NULL){ + ns <- NS(id) + conditionalPanel(condition = condition, + if (fid) selectInput(ns("FuncId"), label = "Function ids", choices = NULL, selected = NULL, multiple = T), + if (dim) selectInput(ns("DIM"), label = "Dimensions", choices = NULL, selected = NULL, multiple = T), + if (alg) selectInput(ns("Alg"), label = "Algorithms", choices = NULL, selected = NULL, multiple = T), + if (scalex) checkboxInput(ns("Scalex"), label = "Scale x-axis logarithmically", value = F), + if (scaley) checkboxInput(ns("Scaley"), label = "Scale y-axis logarithmically", value = F), + if (!is.null(plottype)) selectInput(ns("Type"), label = "Plot type", choices = plottype, selected = plottype[[1]]) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/rt_dist_box.R b/inst/shiny-server/ui/rt_dist_box.R index d40aca2f..1631b4ef 100644 --- a/inst/shiny-server/ui/rt_dist_box.R +++ b/inst/shiny-server/ui/rt_dist_box.R @@ -1,114 +1,114 @@ -rt_histogram_box <- function(width = 12, collapsed = T, collapsible = T) { - box(title = 'Histogram of Fixed-Target Runtimes', - width = width, collapsible = collapsible, solidHeader = TRUE, collapsed = collapsed, - status = "primary", - sidebarPanel( - width = 2, - textInput('RTPMF.Hist.Target', label = HTML('Select the target value'), - value = ''), - selectInput( - 'RTPMF.Hist.Algs', - label = 'Select which algorithms to plot', - multiple = T, - selected = NULL, - choices = NULL - ) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", - content = alg_select_info, - placement = "auto" - ) - ), - - HTML_P('Choose whether the histograms are overlaid in one plot - or separated in several subplots:'), - selectInput('RTPMF.Hist.Mode', '', - choices = c("overlay", "subplot"), - selected = 'subplot'), - checkboxInput("RTPMF.Hist.Equal", "Use equal bins for all algorithms", F), - hr(), - selectInput('RTPMF.Hist.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTPMF.Hist.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML_P('This histogram counts how many runs needed between - \\(t\\) and \\(t+1\\) function evaluations. The bins - \\([t,t+1)\\) are chosen automatically. The bin size is determined - by the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= - 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) - and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. - The displayed algorithms can be selected by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('RT_HIST', aspect_ratio = 16/14) - ) - ) - ) -} - -rt_pmf_box <- function(width = 12, collapsed = T, collapsible = T) { - box(title = 'Empirical Probability Mass Function of the Runtime', - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 2, - textInput('RTPMF.Bar.Target', label = 'Select the target value', value = ''), - - selectInput( - 'RTPMF.Bar.Algs', - label = 'Select which algorithms to plot:', - multiple = T, - selected = NULL, - choices = NULL - ) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", - content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput('RTPMF.Bar.Sample', label = 'Show runtime for each run', value = T), - checkboxInput('RTPMF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = F), - - hr(), - selectInput('RTPMF.Bar.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTPMF.Bar.Download', label = 'Download the figure') - ), - - mainPanel( - width = 10, - column( - width = 12, align = "center", - HTML('

Warning! The - probability mass function of the runtime is approximated by the - treating the runtime as a continuous random variable and - applying the kernel estimation (KDE):

'), - HTML('

- The plot shows the distribution of the first hitting - times of the individual runs (dots), and an estimated - distribution of the probability mass function. - The displayed algorithms can be selected by clicking on - the legend on the right. A tooltip and toolbar - appear when hovering over the figure. This also includes the - option to download the plot as png file. A csv file with the runtime - data can be downlowaded from the - Data Summary tab.'), - plotlyOutput.IOHanalyzer('RT_PMF') - ) - ) - ) - ) -} +rt_histogram_box <- function(width = 12, collapsed = T, collapsible = T) { + box(title = 'Histogram of Fixed-Target Runtimes', + width = width, collapsible = collapsible, solidHeader = TRUE, collapsed = collapsed, + status = "primary", + sidebarPanel( + width = 2, + textInput('RTPMF.Hist.Target', label = HTML('Select the target value'), + value = ''), + selectInput( + 'RTPMF.Hist.Algs', + label = 'Select which IDs to include:', + multiple = T, + selected = NULL, + choices = NULL + ) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", + content = alg_select_info, + placement = "auto" + ) + ), + + HTML_P('Choose whether the histograms are overlaid in one plot + or separated in several subplots:'), + selectInput('RTPMF.Hist.Mode', '', + choices = c("overlay", "subplot"), + selected = 'subplot'), + checkboxInput("RTPMF.Hist.Equal", "Use equal bins for all algorithms", F), + hr(), + selectInput('RTPMF.Hist.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTPMF.Hist.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML_P('This histogram counts how many runs needed between + \\(t\\) and \\(t+1\\) function evaluations. The bins + \\([t,t+1)\\) are chosen automatically. The bin size is determined + by the so-called Freedman–Diaconis rule: \\(\\text{Bin size}= + 2\\frac{Q_3 - Q_1}{\\sqrt[3]{n}}\\), where \\(Q_1, Q_3\\) are the \\(25\\%\\) + and \\(75\\%\\) percentile of the runtime and \\(n\\) is the sample size. + The displayed algorithms can be selected by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('RT_HIST', aspect_ratio = 16/14) + ) + ) + ) +} + +rt_pmf_box <- function(width = 12, collapsed = T, collapsible = T) { + box(title = 'Empirical Probability Mass Function of the Runtime', + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 2, + textInput('RTPMF.Bar.Target', label = 'Select the target value', value = ''), + + selectInput( + 'RTPMF.Bar.Algs', + label = 'Select which IDs to include:', + multiple = T, + selected = NULL, + choices = NULL + ) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", + content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput('RTPMF.Bar.Sample', label = 'Show runtime for each run', value = T), + checkboxInput('RTPMF.Bar.Logy', label = 'Scale y axis \\(\\log_{10}\\)', value = F), + + hr(), + selectInput('RTPMF.Bar.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTPMF.Bar.Download', label = 'Download the figure') + ), + + mainPanel( + width = 10, + column( + width = 12, align = "center", + HTML('

Warning! The + probability mass function of the runtime is approximated by the + treating the runtime as a continuous random variable and + applying the kernel estimation (KDE):

'), + HTML('

+ The plot shows the distribution of the first hitting + times of the individual runs (dots), and an estimated + distribution of the probability mass function. + The displayed algorithms can be selected by clicking on + the legend on the right. A tooltip and toolbar + appear when hovering over the figure. This also includes the + option to download the plot as png file. A csv file with the runtime + data can be downlowaded from the + Data Summary tab.'), + plotlyOutput.IOHanalyzer('RT_PMF') + ) + ) + ) + ) +} diff --git a/inst/shiny-server/ui/rt_ecdf_box.R b/inst/shiny-server/ui/rt_ecdf_box.R index cd33a5d6..5ccffbd6 100644 --- a/inst/shiny-server/ui/rt_ecdf_box.R +++ b/inst/shiny-server/ui/rt_ecdf_box.R @@ -1,256 +1,256 @@ -rt_ecdf_single_target_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Empirical Cumulative Distribution: Single target

'), - width = width, collapsible = collapsible, solidHeader = TRUE, - status = "primary", collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - selectInput('RTECDF.Single.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('Select the target values for which EDCF curves are displayed'), - textInput('RTECDF.Single.Target', label = HTML('

\\(f_{target}\\)

'), - value = ''), - - checkboxInput('RTECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) - ), - - mainPanel( - width = 9, - column( - width = 12, - align = "center", - HTML_P('Each EDCF curve shows the proportion of the runs - that have found a solution of at least the required - target value within the budget given by the \\(x\\)-axis. - The displayed curves can be selected by clicking on the legend on the right. A tooltip - and toolbar appears when hovering over the figure. - This also includes the option to download the plot as png file.'), - plotlyOutput.IOHanalyzer("RT_ECDF"), - HTML_P('The approximated Area Under the Curve of this displayed single-target ECDF is:'), - DT::dataTableOutput('AUC_GRID_GENERATED_SINGLE') - ) - ) - ) - ) -} - -rt_ecdf_agg_targets_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Aggregated Empirical Cumulative - Distribution: Single function

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('RTECDF.Multi.Algs', label = 'Select which algorithms to plot:', - multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - HTML('

Set the range and the granularity - of the quality targets taken into account in the ECDF curve. - The plot will show the ECDF curves for evenly spaced target values.

'), - textInput('RTECDF.Multi.Min', label = F_MIN_LABEL, value = ''), - textInput('RTECDF.Multi.Max', label = F_MAX_LABEL, value = ''), - textInput('RTECDF.Multi.Step', label = F_STEP_LABEL, value = ''), - # checkboxInput('RTECDF.Multi.Targets', - # label = 'Show ECDFs for each target', - # value = F), - checkboxInput('RTECDF.Multi.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - hr(), - selectInput('RTECDF.Multi.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTECDF.Multi.Download', label = 'download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The evenly spaced target values are:'), - verbatimTextOutput('RT_GRID'), - HTML_P('The fraction of (run,target value) - pairs \\((i,v)\\) satisfying that the best solution that the algorithm has - found in the \\(i\\)-th run within the given time budget \\(t\\) has quality at least - \\(v\\) is plotted against the available budget \\(t\\). The displayed elements can be switched - on and off by clicking on the legend on the right. A tooltip - and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('RT_ECDF_AGGR'), - HTML_P('The approximated Area Under the Curve of this displayed single-function ECDF is:'), - DT::dataTableOutput('AUC_GRID_GENERATED_FUNC') - ) - ) - ) -} - -rt_ecdf_agg_fct_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Aggregated Empirical Cumulative - Distribution: All functions

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput( - 'RTECDF.Aggr.Algs', - label = 'Select which algorithms to plot:', - multiple = T, - selected = NULL, - choices = NULL - ) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - - checkboxInput("RTECDF.Aggr.Func", "Aggregate functions", value = T), - checkboxInput("RTECDF.Aggr.Dim", "Aggregate dimensions", value = F), - checkboxInput("RTECDF.Aggr.Logx", "Scale x axis \\(\\log_{10}\\)", value = T), - checkboxInput("RTECDF.Aggr.Logy", "Scale y axis \\(\\log_{10}\\)", value = F) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Scaling", content = "The logorithmic scaling might cause some visual issues - when the smallest y-values are (very close to) 0. - Please be mindful of this fact when using this option.", - placement = "auto" - ) - ), - - br(), - actionButton( - "RTECDF.Aggr.Refresh", - label = HTML('

Refresh the figure

') - ), - - hr(), - selectInput("RTECDF.Aggr.Target_type", label = "Select the spacing for the - automatically generated ECDF-targets:", - choices = c('linear', 'log-linear', 'bbob'), - selected = 'linear') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Default targets", content = "The log-linear spacing only works correctly - when no negative target values are present in the data. The BBOB-spacing is pre-defined - to 51 log-linear targets between 10^2 and 10^-8.", - placement = "auto" - ) - ), - numericInput("RTECDF.Aggr.Target_number", label = "Select the number of ECDF-targets to - generate for each function/dimension", value = 10, min = 1, max = 100), - - HTML_P('Alternatively, you can download the table containing the target values for each - (function, dimension)-pair and edit the table as you want. Please keep - the file format when modifying it.'), - downloadButton('RTECDF.Aggr.Table.Download', label = 'Download the table of targets'), - br(), - br(), - br(), - - HTML_P('Upload the table you just downloaded and edited'), - fileInput( - "RTECDF.Aggr.Table.Upload", - label = NULL, - multiple = FALSE, - accept = c( - "text/csv", - "text/comma-separated-values,text/plain", - ".csv" - ) - ), - - hr(), - selectInput('RTECDF.Aggr.Format', label = 'figure format to download', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTECDF.Aggr.Download', label = 'Download the figure'), - hr(), - selectInput('RTECDF.AUC.Table.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('RTECDF.AUC.Table.Download', label = 'Download the AUC table') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - hr(), - - HTML_P('The fraction of (run,target value, ...) - pairs \\((i,v, ...)\\) satisfying that the best solution that the algorithm has - found in the \\(i\\)-th (run of function \\(f\\) in dimension \\(d\\)) within - the given time budget \\(t\\) has quality at least \\(v\\) is plotted against - the available budget \\(t\\). The displayed elements can be switched - on and off by clicking on the legend on the right. A tooltip - and toolbar appears when hovering over the figure. Aggregation over - functions and dimension can be switched on or off using the checkboxes on - the left; when aggregation is off the selected function / dimension - is chosen according the the value in the bottom-left selection-box.'), - plotlyOutput.IOHanalyzer('RT_ECDF_MULT'), - HTML_P('The approximated Area Under the Curve of this displayed ECDF is:'), - DT::dataTableOutput('AUC_GRID_GENERATED'), - HTML_P('The selected targets are:'), - DT::dataTableOutput('RT_GRID_GENERATED') - ) - ) - ) -} - -# rt_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { -# box( -# title = HTML('

Area Under the ECDF

'), -# width = width, collapsible = collapsible, collapsed = collapsed, -# solidHeader = T, status = "primary", -# sidebarPanel( -# width = 3, -# selectInput('RTECDF.AUC.Algs', label = 'Select which algorithms to plot:', -# multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( -# custom_icon() %>% -# bs_embed_popover( -# title = "Algorithm selection", content = alg_select_info, -# placement = "auto" -# ) -# ), -# HTML('

Set the range and the granularity of -# the evenly spaced quality targets taken into account in the plot.

'), -# textInput('RTECDF.AUC.Min', label = F_MIN_LABEL, value = ''), -# textInput('RTECDF.AUC.Max', label = F_MAX_LABEL, value = ''), -# textInput('RTECDF.AUC.Step', label = F_STEP_LABEL, value = ''), -# -# hr(), -# selectInput('RTECDF.AUC.Format', label = 'Select the figure format', -# choices = supported_fig_format, selected = 'pdf'), -# downloadButton('RTECDF.AUC.Download', label = 'Download the figure') -# ), -# -# mainPanel( -# width = 9, -# column( -# width = 12, align = "center", -# HTML_P('The area under the ECDF is -# caculated for the sequence of target values specified on the left. The displayed -# values are normalized against the maximal number of function evaluations for -# each algorithm. Intuitively, the larger the area, the better the algorithm. -# The displayed algorithms can be selected by clicking on the legend on the right. -# A tooltip and toolbar appears when hovering over the figure. -# This also includes the option to download the plot as png file.'), -# plotlyOutput.IOHanalyzer("RT_AUC") -# ) -# ) -# ) -# } +rt_ecdf_single_target_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Empirical Cumulative Distribution: Single target

'), + width = width, collapsible = collapsible, solidHeader = TRUE, + status = "primary", collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + selectInput('RTECDF.Single.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('Select the target values for which EDCF curves are displayed'), + textInput('RTECDF.Single.Target', label = HTML('

\\(f_{target}\\)

'), + value = ''), + + checkboxInput('RTECDF.Single.Logx', label = 'Scale x axis \\(\\log_{10}\\)', value = F) + ), + + mainPanel( + width = 9, + column( + width = 12, + align = "center", + HTML_P('Each EDCF curve shows the proportion of the runs + that have found a solution of at least the required + target value within the budget given by the \\(x\\)-axis. + The displayed curves can be selected by clicking on the legend on the right. A tooltip + and toolbar appears when hovering over the figure. + This also includes the option to download the plot as png file.'), + plotlyOutput.IOHanalyzer("RT_ECDF"), + HTML_P('The approximated Area Under the Curve of this displayed single-target ECDF is:'), + DT::dataTableOutput('AUC_GRID_GENERATED_SINGLE') + ) + ) + ) + ) +} + +rt_ecdf_agg_targets_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Aggregated Empirical Cumulative + Distribution: Single function

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('RTECDF.Multi.Algs', label = 'Select which IDs to include:', + multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + HTML('

Set the range and the granularity + of the quality targets taken into account in the ECDF curve. + The plot will show the ECDF curves for evenly spaced target values.

'), + textInput('RTECDF.Multi.Min', label = F_MIN_LABEL, value = ''), + textInput('RTECDF.Multi.Max', label = F_MAX_LABEL, value = ''), + textInput('RTECDF.Multi.Step', label = F_STEP_LABEL, value = ''), + # checkboxInput('RTECDF.Multi.Targets', + # label = 'Show ECDFs for each target', + # value = F), + checkboxInput('RTECDF.Multi.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + hr(), + selectInput('RTECDF.Multi.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTECDF.Multi.Download', label = 'download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The evenly spaced target values are:'), + verbatimTextOutput('RT_GRID'), + HTML_P('The fraction of (run,target value) + pairs \\((i,v)\\) satisfying that the best solution that the algorithm has + found in the \\(i\\)-th run within the given time budget \\(t\\) has quality at least + \\(v\\) is plotted against the available budget \\(t\\). The displayed elements can be switched + on and off by clicking on the legend on the right. A tooltip + and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('RT_ECDF_AGGR'), + HTML_P('The approximated Area Under the Curve of this displayed single-function ECDF is:'), + DT::dataTableOutput('AUC_GRID_GENERATED_FUNC') + ) + ) + ) +} + +rt_ecdf_agg_fct_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Aggregated Empirical Cumulative + Distribution: All functions

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput( + 'RTECDF.Aggr.Algs', + label = 'Select which IDs to include:', + multiple = T, + selected = NULL, + choices = NULL + ) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + + checkboxInput("RTECDF.Aggr.Func", "Aggregate functions", value = T), + checkboxInput("RTECDF.Aggr.Dim", "Aggregate dimensions", value = F), + checkboxInput("RTECDF.Aggr.Logx", "Scale x axis \\(\\log_{10}\\)", value = T), + checkboxInput("RTECDF.Aggr.Logy", "Scale y axis \\(\\log_{10}\\)", value = F) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Scaling", content = "The logorithmic scaling might cause some visual issues + when the smallest y-values are (very close to) 0. + Please be mindful of this fact when using this option.", + placement = "auto" + ) + ), + + br(), + actionButton( + "RTECDF.Aggr.Refresh", + label = HTML('

Refresh the figure

') + ), + + hr(), + selectInput("RTECDF.Aggr.Target_type", label = "Select the spacing for the + automatically generated ECDF-targets:", + choices = c('linear', 'log-linear', 'bbob'), + selected = 'linear') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Default targets", content = "The log-linear spacing only works correctly + when no negative target values are present in the data. The BBOB-spacing is pre-defined + to 51 log-linear targets between 10^2 and 10^-8.", + placement = "auto" + ) + ), + numericInput("RTECDF.Aggr.Target_number", label = "Select the number of ECDF-targets to + generate for each function/dimension", value = 10, min = 1, max = 100), + + HTML_P('Alternatively, you can download the table containing the target values for each + (function, dimension)-pair and edit the table as you want. Please keep + the file format when modifying it.'), + downloadButton('RTECDF.Aggr.Table.Download', label = 'Download the table of targets'), + br(), + br(), + br(), + + HTML_P('Upload the table you just downloaded and edited'), + fileInput( + "RTECDF.Aggr.Table.Upload", + label = NULL, + multiple = FALSE, + accept = c( + "text/csv", + "text/comma-separated-values,text/plain", + ".csv" + ) + ), + + hr(), + selectInput('RTECDF.Aggr.Format', label = 'figure format to download', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTECDF.Aggr.Download', label = 'Download the figure'), + hr(), + selectInput('RTECDF.AUC.Table.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('RTECDF.AUC.Table.Download', label = 'Download the AUC table') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + hr(), + + HTML_P('The fraction of (run,target value, ...) + pairs \\((i,v, ...)\\) satisfying that the best solution that the algorithm has + found in the \\(i\\)-th (run of function \\(f\\) in dimension \\(d\\)) within + the given time budget \\(t\\) has quality at least \\(v\\) is plotted against + the available budget \\(t\\). The displayed elements can be switched + on and off by clicking on the legend on the right. A tooltip + and toolbar appears when hovering over the figure. Aggregation over + functions and dimension can be switched on or off using the checkboxes on + the left; when aggregation is off the selected function / dimension + is chosen according the the value in the bottom-left selection-box.'), + plotlyOutput.IOHanalyzer('RT_ECDF_MULT'), + HTML_P('The approximated Area Under the Curve of this displayed ECDF is:'), + DT::dataTableOutput('AUC_GRID_GENERATED'), + HTML_P('The selected targets are:'), + DT::dataTableOutput('RT_GRID_GENERATED') + ) + ) + ) +} + +# rt_ecdf_auc_box <- function(width = 12, collapsible = T, collapsed = T) { +# box( +# title = HTML('

Area Under the ECDF

'), +# width = width, collapsible = collapsible, collapsed = collapsed, +# solidHeader = T, status = "primary", +# sidebarPanel( +# width = 3, +# selectInput('RTECDF.AUC.Algs', label = 'Select which IDs to include:', +# multiple = T, selected = NULL, choices = NULL) %>% shinyInput_label_embed( +# custom_icon() %>% +# bs_embed_popover( +# title = "ID selection", content = alg_select_info, +# placement = "auto" +# ) +# ), +# HTML('

Set the range and the granularity of +# the evenly spaced quality targets taken into account in the plot.

'), +# textInput('RTECDF.AUC.Min', label = F_MIN_LABEL, value = ''), +# textInput('RTECDF.AUC.Max', label = F_MAX_LABEL, value = ''), +# textInput('RTECDF.AUC.Step', label = F_STEP_LABEL, value = ''), +# +# hr(), +# selectInput('RTECDF.AUC.Format', label = 'Select the figure format', +# choices = supported_fig_format, selected = 'pdf'), +# downloadButton('RTECDF.AUC.Download', label = 'Download the figure') +# ), +# +# mainPanel( +# width = 9, +# column( +# width = 12, align = "center", +# HTML_P('The area under the ECDF is +# caculated for the sequence of target values specified on the left. The displayed +# values are normalized against the maximal number of function evaluations for +# each algorithm. Intuitively, the larger the area, the better the algorithm. +# The displayed algorithms can be selected by clicking on the legend on the right. +# A tooltip and toolbar appears when hovering over the figure. +# This also includes the option to download the plot as png file.'), +# plotlyOutput.IOHanalyzer("RT_AUC") +# ) +# ) +# ) +# } diff --git a/inst/shiny-server/ui/rt_par_box.R b/inst/shiny-server/ui/rt_par_box.R index 0345e581..2feef3ad 100644 --- a/inst/shiny-server/ui/rt_par_box.R +++ b/inst/shiny-server/ui/rt_par_box.R @@ -1,134 +1,134 @@ -rt_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Expected Parameter Value (per function)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = TRUE, status = "primary", - sidebarPanel( - width = 3, - HTML('

Range of the function values (\\(x\\) axis)

'), - - textInput('RT_PAR.Plot.Min', label = F_MIN_LABEL, value = ''), - textInput('RT_PAR.Plot.Max', label = F_MAX_LABEL, value = ''), - selectInput('RT_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), - - selectInput('RT_PAR.Plot.Algs', 'Select which algorithms to plot:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Algorithm selection", content = alg_select_info, - placement = "auto" - ) - ), - selectInput('RT_PAR.Plot.show.mean', label = 'Mean/median', - choices = c('mean', 'median'), - selected = 'mean'), - selectInput('RT_PAR.Plot.CI', "Show standard deviations or quantiles", - choices = c('Standard Deviation', 'Outer Quantiles', 'None'), - selected = 'None') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are - 2% and 98% by default. This can be changed in the settings-tab.", - placement = "auto" - ) - ), - checkboxInput('RT_PAR.Plot.Logx', - label = 'Scale x axis \\(\\log_{10}\\)', - value = T), - - checkboxInput('RT_PAR.Plot.Logy', - label = 'Scale y axis \\(\\log_{10}\\)', - value = F) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_tooltip( - title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" - ) - ), - - hr(), - selectInput('RT_PAR.Plot.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('RT_PAR.Plot.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - HTML_P('The mean or median of internal parameters of the algorithm - found with a fixed-budget of evaluations are depicted against the budget. - The displayed elements can be switched on and off by clicking on the legend on the right. - A tooltip and toolbar appears when hovering over the figure.'), - plotlyOutput.IOHanalyzer('RT_PAR.Plot.Figure') - ) - ) - ) -} - -rt_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Statistics at Chosen Target Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-target parameter values for evenly spaced target values.'), - - textInput('RT_PAR.Summary.Min', label = F_MIN_LABEL, value = ''), - textInput('RT_PAR.Summary.Max', label = F_MAX_LABEL, value = ''), - textInput('RT_PAR.Summary.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RT_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Summary.Algid', 'Algorithms', choices = NULL, selected = NULL), - selectInput('RT_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('RT_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT_PAR.Summary.Download", "Save this table as csv") - ), - - mainPanel( - width = 9, - HTML(paste0('
', - includeMarkdown('markdown/PAR_SUMMARY_TABLE.Rmd'), '
')), - DT::dataTableOutput('table_RT_PAR_summary') - ) - ) -} - -rt_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Parameter Sample at Chosen Target Values

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - HTML_P('Set the range and the granularity of the results. - The table will show fixed-target parameter values for evenly spaced target values.'), - - textInput('RT_PAR.Sample.Min', label = F_MIN_LABEL, value = ''), - textInput('RT_PAR.Sample.Max', label = F_MAX_LABEL, value = ''), - textInput('RT_PAR.Sample.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RT_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('RT_PAR.Sample.Algid', 'Algorithms', choices = NULL, selected = NULL), - selectInput('RT_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), - hr(), - selectInput('RT_PAR.Sample.Format', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - selectInput('RT_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT_PAR.Sample.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML('

This table shows for each selected algorithm \\(A\\), - each selected target value \\(f(x)\\), and each run \\(r\\) the parameter value - observed when the target value \\(f(x)\\) is reached for the first time.

'), - DT::dataTableOutput('table_RT_PAR_SAMPLE') - ) - ) -} +rt_par_expected_value_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Expected Parameter Value (per function)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = TRUE, status = "primary", + sidebarPanel( + width = 3, + HTML('

Range of the function values (\\(x\\) axis)

'), + + textInput('RT_PAR.Plot.Min', label = F_MIN_LABEL, value = ''), + textInput('RT_PAR.Plot.Max', label = F_MAX_LABEL, value = ''), + selectInput('RT_PAR.Plot.Params', 'Parameters', choices = NULL, selected = NULL, multiple = T), + + selectInput('RT_PAR.Plot.Algs', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T) %>% shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "ID selection", content = alg_select_info, + placement = "auto" + ) + ), + selectInput('RT_PAR.Plot.show.mean', label = 'Mean/median', + choices = c('mean', 'median'), + selected = 'mean'), + selectInput('RT_PAR.Plot.CI', "Show standard deviations or quantiles", + choices = c('Standard Deviation', 'Outer Quantiles', 'None'), + selected = 'None') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Outer quantiles", content = "This method uses the highest and lowest quantiles, which are + 2% and 98% by default. This can be changed in the settings-tab.", + placement = "auto" + ) + ), + checkboxInput('RT_PAR.Plot.Logx', + label = 'Scale x axis \\(\\log_{10}\\)', + value = T), + + checkboxInput('RT_PAR.Plot.Logy', + label = 'Scale y axis \\(\\log_{10}\\)', + value = F) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_tooltip( + title = "Be mindful of using logarithmic scaling when the parameter-values can be <= 0" + ) + ), + + hr(), + selectInput('RT_PAR.Plot.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('RT_PAR.Plot.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + HTML_P('The mean or median of internal parameters of the algorithm + found with a fixed-budget of evaluations are depicted against the budget. + The displayed elements can be switched on and off by clicking on the legend on the right. + A tooltip and toolbar appears when hovering over the figure.'), + plotlyOutput.IOHanalyzer('RT_PAR.Plot.Figure') + ) + ) + ) +} + +rt_par_summary_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Statistics at Chosen Target Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-target parameter values for evenly spaced target values.'), + + textInput('RT_PAR.Summary.Min', label = F_MIN_LABEL, value = ''), + textInput('RT_PAR.Summary.Max', label = F_MAX_LABEL, value = ''), + textInput('RT_PAR.Summary.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RT_PAR.Summary.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('RT_PAR.Summary.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RT_PAR.Summary.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('RT_PAR.Summary.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT_PAR.Summary.Download", "Save this table as csv") + ), + + mainPanel( + width = 9, + HTML(paste0('
', + includeMarkdown('markdown/PAR_SUMMARY_TABLE.Rmd'), '
')), + DT::dataTableOutput('table_RT_PAR_summary') + ) + ) +} + +rt_par_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Parameter Sample at Chosen Target Values

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + HTML_P('Set the range and the granularity of the results. + The table will show fixed-target parameter values for evenly spaced target values.'), + + textInput('RT_PAR.Sample.Min', label = F_MIN_LABEL, value = ''), + textInput('RT_PAR.Sample.Max', label = F_MAX_LABEL, value = ''), + textInput('RT_PAR.Sample.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RT_PAR.Sample.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('RT_PAR.Sample.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + selectInput('RT_PAR.Sample.Param', 'Parameters', choices = NULL, selected = NULL), + hr(), + selectInput('RT_PAR.Sample.Format', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + selectInput('RT_PAR.Sample.FileFormat', 'File-Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT_PAR.Sample.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML('

This table shows for each selected algorithm \\(A\\), + each selected target value \\(f(x)\\), and each run \\(r\\) the parameter value + observed when the target value \\(f(x)\\) is reached for the first time.

'), + DT::dataTableOutput('table_RT_PAR_SAMPLE') + ) + ) +} diff --git a/inst/shiny-server/ui/rt_portfolio_box.R b/inst/shiny-server/ui/rt_portfolio_box.R index 0807394c..da955139 100644 --- a/inst/shiny-server/ui/rt_portfolio_box.R +++ b/inst/shiny-server/ui/rt_portfolio_box.R @@ -1,97 +1,97 @@ -rt_shapleys_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

Contribution to portfolio (Shapley-values)

'), - width = width, collapsible = collapsible, collapsed = collapsed, - solidHeader = T, status = "primary", - sidebarPanel( - width = 3, - selectInput('RTportfolio.Shapley.Algs', 'Algorithms to include:', multiple = T, selected = NULL, choices = NULL), - selectInput("RTportfolio.Shapley.Funcs", "Functions to include:", multiple = T, selected = NULL, choices = NULL), - selectInput("RTportfolio.Shapley.Dims", "Dimensions to include:", multiple = T, selected = NULL, choices = NULL), - hr(), - checkboxInput("RTportfolio.Shapley.Logx", "Scale runtimes to sample at \\(\\log_{10}\\)", value = T), - numericInput("RTportfolio.Shapley.Groupsize", "Maximum permutation size:", value = 5, min = 1, max = 100) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Groupsize", content = "This parameter controls how many groups of permutations are used to - calculate the Shapley-values. Larger values give more accurate estimates, but take longer to compute.", - placement = "auto" - ) - ), - numericInput("RTportfolio.Shapley.Permsize", "Maximum permutation size", value = 0) %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Scaling", content = "This parameter controls the maximum size of permutations to include - in the calculation of the marginal contribution. So, if this parameter is low, only small portfolios - will be considered. Higher values lead to more accurate Shapley-values (relative to eachother) but take - longer to compute.", - placement = "auto" - ) - ), - - br(), - actionButton( - "RTportfolio.Shapley.Refresh", - label = HTML('

Refresh the figure

') - ), - - hr(), - selectInput("RTportfolio.Shapley.Target_type", label = "Select the spacing for the - automatically generated ECDF-targets:", - choices = c('linear', 'log-linear', 'bbob'), - selected = 'linear') %>% - shinyInput_label_embed( - custom_icon() %>% - bs_embed_popover( - title = "Default targets", content = "The log-linear spacing only works correctly - when no negative target values are present in the data. The BBOB-spacing is pre-defined - to 51 log-linear targets between 10^2 and 10^-8.", - placement = "auto" - ) - ), - numericInput("RTportfolio.Shapley.Target_number", label = "Select the number of ECDF-targets to - generate for each function/dimension", value = 10, min = 1, max = 100), - - HTML_P('Alternatively, you can download the table containing the target values for each - (function, dimension)-pair and edit the table as you want. Please keep - the file format when modifying it.'), - downloadButton('RTportfolio.Shapley.Table.Download', label = 'Download the table of targets'), - br(), - br(), - br(), - - HTML_P('Upload the table you just downloaded and edited'), - fileInput( - "RTportfolio.Shapley.Table.Upload", - label = NULL, - multiple = FALSE, - accept = c( - "text/csv", - "text/comma-separated-values,text/plain", - ".csv" - ) - ), - - hr(), - selectInput('RTportfolio.Shapley.Format', label = 'figure format to download', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - - downloadButton('RTportfolio.Shapley.Download', label = 'Download the figure') - ), - - mainPanel( - width = 9, - column( - width = 12, align = "center", - hr(), - - HTML_P('DESCRIPTION OF SHAPLEY VALUES HERE'), - plotlyOutput.IOHanalyzer('RT_SHAPLEY'), - HTML_P('The selected targets are:'), - DT::dataTableOutput('RT_SHAPLEY_TARGETS_GENERATED') - ) - ) - ) +rt_shapleys_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

Contribution to portfolio (Shapley-values)

'), + width = width, collapsible = collapsible, collapsed = collapsed, + solidHeader = T, status = "primary", + sidebarPanel( + width = 3, + selectInput('RTportfolio.Shapley.Algs', 'Algorithms to include:', multiple = T, selected = NULL, choices = NULL), + selectInput("RTportfolio.Shapley.Funcs", "Functions to include:", multiple = T, selected = NULL, choices = NULL), + selectInput("RTportfolio.Shapley.Dims", "Dimensions to include:", multiple = T, selected = NULL, choices = NULL), + hr(), + checkboxInput("RTportfolio.Shapley.Logx", "Scale runtimes to sample at \\(\\log_{10}\\)", value = T), + numericInput("RTportfolio.Shapley.Groupsize", "Maximum permutation size:", value = 5, min = 1, max = 100) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Groupsize", content = "This parameter controls how many groups of permutations are used to + calculate the Shapley-values. Larger values give more accurate estimates, but take longer to compute.", + placement = "auto" + ) + ), + numericInput("RTportfolio.Shapley.Permsize", "Maximum permutation size", value = 0) %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Scaling", content = "This parameter controls the maximum size of permutations to include + in the calculation of the marginal contribution. So, if this parameter is low, only small portfolios + will be considered. Higher values lead to more accurate Shapley-values (relative to eachother) but take + longer to compute.", + placement = "auto" + ) + ), + + br(), + actionButton( + "RTportfolio.Shapley.Refresh", + label = HTML('

Refresh the figure

') + ), + + hr(), + selectInput("RTportfolio.Shapley.Target_type", label = "Select the spacing for the + automatically generated ECDF-targets:", + choices = c('linear', 'log-linear', 'bbob'), + selected = 'linear') %>% + shinyInput_label_embed( + custom_icon() %>% + bs_embed_popover( + title = "Default targets", content = "The log-linear spacing only works correctly + when no negative target values are present in the data. The BBOB-spacing is pre-defined + to 51 log-linear targets between 10^2 and 10^-8.", + placement = "auto" + ) + ), + numericInput("RTportfolio.Shapley.Target_number", label = "Select the number of ECDF-targets to + generate for each function/dimension", value = 10, min = 1, max = 100), + + HTML_P('Alternatively, you can download the table containing the target values for each + (function, dimension)-pair and edit the table as you want. Please keep + the file format when modifying it.'), + downloadButton('RTportfolio.Shapley.Table.Download', label = 'Download the table of targets'), + br(), + br(), + br(), + + HTML_P('Upload the table you just downloaded and edited'), + fileInput( + "RTportfolio.Shapley.Table.Upload", + label = NULL, + multiple = FALSE, + accept = c( + "text/csv", + "text/comma-separated-values,text/plain", + ".csv" + ) + ), + + hr(), + selectInput('RTportfolio.Shapley.Format', label = 'figure format to download', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + + downloadButton('RTportfolio.Shapley.Download', label = 'Download the figure') + ), + + mainPanel( + width = 9, + column( + width = 12, align = "center", + hr(), + + HTML_P('DESCRIPTION OF SHAPLEY VALUES HERE'), + plotlyOutput.IOHanalyzer('RT_SHAPLEY'), + HTML_P('The selected targets are:'), + DT::dataTableOutput('RT_SHAPLEY_TARGETS_GENERATED') + ) + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/rt_summary_box.R b/inst/shiny-server/ui/rt_summary_box.R index 8ed1ddb9..9a3d35f2 100644 --- a/inst/shiny-server/ui/rt_summary_box.R +++ b/inst/shiny-server/ui/rt_summary_box.R @@ -1,94 +1,92 @@ -rt_stats_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Runtime Statistics at Chosen Target Values

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Set the range and the granularity of the results. - The table will show fixed-target runtimes for evenly spaced target values.

'), - - textInput('RTSummary.Statistics.Min', label = F_MIN_LABEL, value = ''), - textInput('RTSummary.Statistics.Max', label = F_MAX_LABEL, value = ''), - textInput('RTSummary.Statistics.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RTSummary.Statistics.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - selectInput('RTSummary.Statistics.Algid', 'Algorithms', choices = NULL, selected = NULL), - hr(), - selectInput('RTSummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RTSummary.Statistics.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', includeMarkdown('markdown/RT_SUMMARY_TABLE.Rmd'),'
')), - DT::dataTableOutput('table_RT_summary') - ) - ) -} - -rt_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Original Runtime Samples

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Set the range and the granularity of the results. - The table will show fixed-target runtimes for evenly spaced target values.

'), - - textInput('RTSummary.Sample.Min', label = F_MIN_LABEL, value = ''), - textInput('RTSummary.Sample.Max', label = F_MAX_LABEL, value = ''), - textInput('RTSummary.Sample.Step', label = F_STEP_LABEL, value = ''), - checkboxInput('RTSummary.Sample.Single', - label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? - Once toggled, only \\(f_{\\text{min}}\\) is - used to generate the table on the right.

'), value = FALSE), - - # TODO: do we need this log scaling? - # checkboxInput('F_LOGSPACE_DATA_SUMMARY', - # label = HTML('Evenly space target values in \\(log_{10}\\) space')), - selectInput('RTSummary.Sample.Algid', 'Algorithms', - choices = NULL, selected = NULL), - - hr(), - selectInput('RTSummary.Sample.DownloadFormat', 'Format of the table', - choices = c('long', 'wide'), selected = 'wide'), - selectInput('RTSummary.Sample.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), - - downloadButton("RTSummary.Sample.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML('

This table shows for each selected algorithm \\(A\\), - each selected target value \\(f(x)\\), and each run \\(r\\) - the number \\(T(A,f(x),r)\\) of evaluations performed by the - algorithm until it evaluated for the first time a solution of - quality at least \\(f(x)\\).

'), - DT::dataTableOutput('table_RT_sample') - ) - ) -} - -rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Data Overview

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('RTSummary.Overview.Algid', 'Algorithms', choices = NULL, selected = NULL), - hr(), - selectInput('RTSummary.Overview.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RTSummary.Overview.Download", "Save this table") - ), - - mainPanel( - width = 9, - HTML(paste0('
', includeMarkdown('markdown/RT_OVERVIEW_TABLE.Rmd'), '
')), - DT::dataTableOutput('table_RT_overview') - ) - ) -} - +rt_stats_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Runtime Statistics at Chosen Target Values

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Set the range and the granularity of the results. + The table will show fixed-target runtimes for evenly spaced target values.

'), + + textInput('RTSummary.Statistics.Min', label = F_MIN_LABEL, value = ''), + textInput('RTSummary.Statistics.Max', label = F_MAX_LABEL, value = ''), + textInput('RTSummary.Statistics.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RTSummary.Statistics.Single', label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + selectInput('RTSummary.Statistics.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('RTSummary.Statistics.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RTSummary.Statistics.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', includeMarkdown('markdown/RT_SUMMARY_TABLE.Rmd'),'
')), + DT::dataTableOutput('table_RT_summary') + ) + ) +} + +rt_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Original Runtime Samples

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Set the range and the granularity of the results. + The table will show fixed-target runtimes for evenly spaced target values.

'), + + textInput('RTSummary.Sample.Min', label = F_MIN_LABEL, value = ''), + textInput('RTSummary.Sample.Max', label = F_MAX_LABEL, value = ''), + textInput('RTSummary.Sample.Step', label = F_STEP_LABEL, value = ''), + checkboxInput('RTSummary.Sample.Single', + label = HTML('

\\(f_{\\text{min}} = f_{\\text{max}}\\)? + Once toggled, only \\(f_{\\text{min}}\\) is + used to generate the table on the right.

'), value = FALSE), + + # TODO: do we need this log scaling? + # checkboxInput('F_LOGSPACE_DATA_SUMMARY', + # label = HTML('Evenly space target values in \\(log_{10}\\) space')), + selectInput('RTSummary.Sample.ID', 'Select which IDs to include:', + choices = NULL, selected = NULL, multiple = T), + + hr(), + selectInput('RTSummary.Sample.DownloadFormat', 'Format of the table', + choices = c('long', 'wide'), selected = 'wide'), + selectInput('RTSummary.Sample.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), + + downloadButton("RTSummary.Sample.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML('

This table shows for each selected algorithm \\(A\\), + each selected target value \\(f(x)\\), and each run \\(r\\) + the number \\(T(A,f(x),r)\\) of evaluations performed by the + algorithm until it evaluated for the first time a solution of + quality at least \\(f(x)\\).

'), + DT::dataTableOutput('table_RT_sample') + ) + ) +} + +rt_overview_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Data Overview

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RTSummary.Overview.ID', 'Select which IDs to include:', choices = NULL, selected = NULL, multiple = T), + hr(), + selectInput('RTSummary.Overview.Format', 'File-format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RTSummary.Overview.Download", "Save this table") + ), + + mainPanel( + width = 9, + HTML(paste0('
', includeMarkdown('markdown/RT_OVERVIEW_TABLE.Rmd'), '
')), + DT::dataTableOutput('table_RT_overview') + ) + ) +} + diff --git a/inst/shiny-server/ui/settings.R b/inst/shiny-server/ui/settings.R index 33ade29a..28da0f74 100644 --- a/inst/shiny-server/ui/settings.R +++ b/inst/shiny-server/ui/settings.R @@ -1,87 +1,92 @@ -color_settings_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Color Settings

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarLayout( - sidebarPanel( - width = 3, - selectInput(inputId = "Settings.Color.Scheme", label = "Color schemes", - choices = c("Default", "Variant 1", "Variant 2", "Variant 3", "Custom")), - conditionalPanel(condition = 'input["Settings.Color.Scheme"] == "Custom"', - downloadButton("Settings.Color.Example","Download an example color settings file"), - fileInput("Settings.Color.Upload","Upload a color settings file") - ), - colourInput("Settings.Color.Bg", "Plot background colour", value = "#E6E6E6"), - colourInput("Settings.Color.Grid", "Plot gridline colour", value = "#FFFFFF"), - colourInput("Settings.Color.Tick", "Plot ticks colour", value = "#333333"), - numericInput("Settings.Color.Linewidth", "Line Width", value = 2), - numericInput("Settings.Color.Markersize", "Marker Size", value = 4), - selectInput("Settings.Legend.Location", "Legend location", - c("Outside, right", "Inside, right", "Inside, left", "Below", "Custom"), "Below"), - conditionalPanel(condition = 'input["Settings.Legend.Location"] == "Custom"', - numericInput("Settings.Legend.LocationX", "X-position of Legend", value = 0.5, step = 0.05), - numericInput("Settings.Legend.LocationY", "Y-position of Legend", value = -0.2, step = 0.05) - ), - downloadButton("Settings.Plot.Download", label = "Download sample plot") - ), - - mainPanel( - width = 9, - column( - width = 12, - align = 'center', - HTML_P('Example of the current colorscheme.'), - plotlyOutput.IOHanalyzer('Settings.Color.Plot') - ) - ) - ) - ) -} - -general_settings_box <- function(width=12, collapsible = T, collapsed = F) { - box(title = HTML('

General settings

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - mainPanel( - width = 12, - column( - width = 3, - align = "Left", - HTML_P('Set the general properties'), - #TODO: get probabilities from get_probability and put them as default - textInput("Settings.General.Probs", label = "Probability-quantiles", - value = "0.02,0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.98"), - numericInput("Settings.General.Max_samples", - label = "Maximum samples shown per algorithm", - value = 100), - selectInput("Settings.General.Backend", "Plotting backend", - c('plotly', 'ggplot2'), 'plotly'), - numericInput("Settings.General.Precision", label = "Function value precision (digits)", - value = 2), - hr(), - downloadButton("Settings.Download", "Download current general settings file"), - fileInput("Settings.Upload", "Upload a settings file", accept = "rds") - ), - column( - width = 3, - align = "Left", - HTML_P('Set the figure download properties'), - selectInput("Settings.Download.Preset", label = "Choose preset for download and font sizes", - choices = c("Default", "Paper-1col", "Paper-2col"), selected = "Default"), - hr(), - numericInput("Settings.Download.Width", label = "Image width (px)", value = 1000, min = 100, max = 4096), - numericInput("Settings.Download.Height", label = "Image height (px)", value = 1000, min = 100, max = 4096) - ), - column( - width = 3, - align = "Left", - HTML_P('Set the figure fontsizes'), - numericInput("Settings.Font.Title", label = "Title", value = 16, min = 8, max = 100), - numericInput("Settings.Font.Label", label = "Axis labels", value = 16, min = 8, max = 100), - numericInput("Settings.Font.Legend", label = "Legend", value = 13, min = 8, max = 100), - numericInput("Settings.Font.Tick", label = "Ticks", value = 12, min = 8, max = 100) - ) - ) - - ) +color_settings_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Color Settings

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarLayout( + sidebarPanel( + width = 3, + selectInput(inputId = "Settings.Color.Scheme", label = "Color schemes", + choices = c("Default", "Variant 1", "Variant 2", "Variant 3", "Custom")), + conditionalPanel(condition = 'input["Settings.Color.Scheme"] == "Custom"', + downloadButton("Settings.Color.Example","Download an example color settings file"), + fileInput("Settings.Color.Upload","Upload a color settings file") + ), + colourInput("Settings.Color.Bg", "Plot background colour", value = "#E6E6E6"), + colourInput("Settings.Color.Grid", "Plot gridline colour", value = "#FFFFFF"), + colourInput("Settings.Color.Tick", "Plot ticks colour", value = "#333333"), + numericInput("Settings.Color.Linewidth", "Line Width", value = 2), + numericInput("Settings.Color.Markersize", "Marker Size", value = 4), + selectInput("Settings.Legend.Location", "Legend location", + c("Outside, right", "Inside, right", "Inside, left", "Below", "Custom"), "Below"), + conditionalPanel(condition = 'input["Settings.Legend.Location"] == "Custom"', + numericInput("Settings.Legend.LocationX", "X-position of Legend", value = 0.5, step = 0.05), + numericInput("Settings.Legend.LocationY", "Y-position of Legend", value = -0.2, step = 0.05) + ), + downloadButton("Settings.Plot.Download", label = "Download sample plot") + ), + + mainPanel( + width = 9, + column( + width = 12, + align = 'center', + HTML_P('Example of the current colorscheme.'), + plotlyOutput.IOHanalyzer('Settings.Color.Plot') + ) + ) + ) + ) +} + +general_settings_box <- function(width=12, collapsible = T, collapsed = F) { + box(title = HTML('

General settings

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + mainPanel( + width = 12, + column( + width = 3, + align = "Left", + HTML_P('Set the general properties'), + #TODO: get probabilities from get_probability and put them as default + textInput("Settings.General.Probs", label = "Probability-quantiles", + value = "0.02,0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.98"), + numericInput("Settings.General.Max_samples", + label = "Maximum samples shown per algorithm", + value = 100), + selectInput("Settings.General.Backend", "Plotting backend", + c('plotly', 'ggplot2'), 'plotly'), + numericInput("Settings.General.Precision", label = "Function value precision (digits)", + value = 2), + hr(), + HTML_P("ID variables"), + selectInput("Settings.ID.Variables", label = "Attributes used to create IDs", + multiple = T, selected = NULL, choices = NULL), + checkboxInput("Settings.Use_Funcname", "Use function names instead of IDs"), + hr(), + downloadButton("Settings.Download", "Download current general settings file"), + fileInput("Settings.Upload", "Upload a settings file", accept = "rds") + ), + column( + width = 3, + align = "Left", + HTML_P('Set the figure download properties'), + selectInput("Settings.Download.Preset", label = "Choose preset for download and font sizes", + choices = c("Default", "Paper-1col", "Paper-2col"), selected = "Default"), + hr(), + numericInput("Settings.Download.Width", label = "Image width (px)", value = 1000, min = 100, max = 4096), + numericInput("Settings.Download.Height", label = "Image height (px)", value = 1000, min = 100, max = 4096) + ), + column( + width = 3, + align = "Left", + HTML_P('Set the figure fontsizes'), + numericInput("Settings.Font.Title", label = "Title", value = 16, min = 8, max = 100), + numericInput("Settings.Font.Label", label = "Axis labels", value = 16, min = 8, max = 100), + numericInput("Settings.Font.Legend", label = "Legend", value = 13, min = 8, max = 100), + numericInput("Settings.Font.Tick", label = "Ticks", value = 12, min = 8, max = 100) + ) + ) + + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/sidebar_menu.R b/inst/shiny-server/ui/sidebar_menu.R index 761e8e7c..3294a140 100644 --- a/inst/shiny-server/ui/sidebar_menu.R +++ b/inst/shiny-server/ui/sidebar_menu.R @@ -1,49 +1,49 @@ -sidebar_menu <- function() { - sidebarMenu(id = "tabs", - menuItem("Upload Data", tabName = "upload", icon = icon('upload', lib = 'glyphicon'), - selected = T), - menuItem("General Overview", tabName = "overview", icon = icon("table")), - - menuItem("Fixed-Target Results", tabName = "ERT", icon = icon("file-text-o"), - menuItem("Single Function", tabName = "RT_single", icon = icon("line-chart"), selected = F, - menuSubItem("Data Summary", tabName = "ERT_data", icon = icon("table")), - menuSubItem("Expected Runtime", tabName = "ERT_convergence_single", icon = icon("line-chart"), selected = F), - menuSubItem("Probability Mass Function", tabName = "RT_PMF", icon = icon("bar-chart"), selected = F), - menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_single", icon = icon("line-chart"), selected = F), - menuSubItem("Algorithm Parameters", tabName = "RT_PARAMETER", icon = icon('file-text-o'), selected = F), - menuSubItem("Statistics", tabName = "RT_Statistics_single", icon = icon("file-text-o"), selected = F) - ), - menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, - menuSubItem("Data Summary", tabName = "RT_table_multi", icon = icon("table"), selected = F), - menuSubItem("Expected Runtime", tabName = "ERT_convergence_aggr", icon = icon("line-chart"), selected = F), - menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_aggr", icon = icon("line-chart"), selected = F), - menuSubItem("Deep Statistics", tabName = "RT_DSC", icon = icon("not-equal"), selected = F), - menuSubItem("Ranking", tabName = "RT_Statistics_aggr", icon = icon("file-text-o"), selected = F), - menuSubItem("Portfolio", tabName = "RT_portfolio", icon = icon("bar-chart"), selected = F) - ) - ), - - menuItem("Fixed-Budget Results", tabName = "FCE", icon = icon("file-text-o"), - menuItem("Single Function", tabName = "FCE_single", icon = icon("line-chart"), selected = F, - menuSubItem("Data Summary", tabName = "FCE_DATA", icon = icon("table")), - menuSubItem("Expected Target Value", tabName = "FCE_convergence_single", icon = icon("bar-chart")), - menuSubItem("Probability Density Function", tabName = "FCE_PDF", icon = icon("bar-chart"), selected = F), - menuSubItem("Cumulative Distribution", tabName = "FCE_ECDF", icon = icon("line-chart")), - menuSubItem("Algorithm Parameters", tabName = "FCE_PARAMETER", icon = icon('file-text-o'), selected = F), - menuSubItem('Statistics', tabName = "FCE_Statistics_single", icon = icon('file-text-o')) - ), - menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, - menuSubItem("Data Summary", tabName = "FV_table_multi", icon = icon("table"), selected = F), - menuSubItem("Expected Target Value", tabName = "FCE_convergence_aggr", icon = icon("bar-chart")), - menuSubItem('Deep Statistics', tabName = "FCE_DSC", icon = icon('not-equal')), - menuSubItem('Ranking', tabName = "FCE_Statistics_aggr", icon = icon('file-text-o')) - ) - ), - menuItem("Position Information", tabName = "Positions", icon = icon("arrows-alt")), - # menuItem("Data Format", tabName = "dataformat", icon = icon("fas fa-database")), - menuItem("About", tabName = "about", icon = icon("question")), - menuItem("Settings", tabName = "Settings", icon = icon("cog")) - # , - # menuItem("Report", tabName = "Report", icon = icon("file-pdf")) - ) -} +sidebar_menu <- function() { + sidebarMenu(id = "tabs", + menuItem("Upload Data", tabName = "upload", icon = icon('upload', lib = 'glyphicon'), + selected = T), + menuItem("General Overview", tabName = "overview", icon = icon("table")), + + menuItem("Fixed-Target Results", tabName = "ERT", icon = icon("file-text-o"), + menuItem("Single Function", tabName = "RT_single", icon = icon("line-chart"), selected = F, + menuSubItem("Data Summary", tabName = "ERT_data", icon = icon("table")), + menuSubItem("Expected Runtime", tabName = "ERT_convergence_single", icon = icon("line-chart"), selected = F), + menuSubItem("Probability Mass Function", tabName = "RT_PMF", icon = icon("bar-chart"), selected = F), + menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_single", icon = icon("line-chart"), selected = F), + menuSubItem("Algorithm Parameters", tabName = "RT_PARAMETER", icon = icon('file-text-o'), selected = F), + menuSubItem("Statistics", tabName = "RT_Statistics_single", icon = icon("file-text-o"), selected = F) + ), + menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, + menuSubItem("Data Summary", tabName = "RT_table_multi", icon = icon("table"), selected = F), + menuSubItem("Expected Runtime", tabName = "ERT_convergence_aggr", icon = icon("line-chart"), selected = F), + menuSubItem("Cumulative Distribution", tabName = "RT_ECDF_aggr", icon = icon("line-chart"), selected = F), + menuSubItem("Deep Statistics", tabName = "RT_DSC", icon = icon("not-equal"), selected = F), + menuSubItem("Ranking", tabName = "RT_Statistics_aggr", icon = icon("file-text-o"), selected = F), + menuSubItem("Portfolio", tabName = "RT_portfolio", icon = icon("bar-chart"), selected = F) + ) + ), + + menuItem("Fixed-Budget Results", tabName = "FCE", icon = icon("file-text-o"), + menuItem("Single Function", tabName = "FCE_single", icon = icon("line-chart"), selected = F, + menuSubItem("Data Summary", tabName = "FCE_DATA", icon = icon("table")), + menuSubItem("Expected Target Value", tabName = "FCE_convergence_single", icon = icon("bar-chart")), + menuSubItem("Probability Density Function", tabName = "FCE_PDF", icon = icon("bar-chart"), selected = F), + menuSubItem("Cumulative Distribution", tabName = "FCE_ECDF", icon = icon("line-chart")), + menuSubItem("Algorithm Parameters", tabName = "FCE_PARAMETER", icon = icon('file-text-o'), selected = F), + menuSubItem('Statistics', tabName = "FCE_Statistics_single", icon = icon('file-text-o')) + ), + menuItem("Multiple Functions", tabName = "RT_aggr", icon = icon("bar-chart"), selected = F, + menuSubItem("Data Summary", tabName = "FV_table_multi", icon = icon("table"), selected = F), + menuSubItem("Expected Target Value", tabName = "FCE_convergence_aggr", icon = icon("bar-chart")), + menuSubItem('Deep Statistics', tabName = "FCE_DSC", icon = icon('not-equal')), + menuSubItem('Ranking', tabName = "FCE_Statistics_aggr", icon = icon('file-text-o')) + ) + ), + menuItem("Position Information", tabName = "Positions", icon = icon("arrows-alt")), + # menuItem("Data Format", tabName = "dataformat", icon = icon("fas fa-database")), + menuItem("About", tabName = "about", icon = icon("question")), + menuItem("Settings", tabName = "Settings", icon = icon("cog")) + # , + # menuItem("Report", tabName = "Report", icon = icon("file-pdf")) + ) +} diff --git a/inst/shiny-server/ui/stats_box.R b/inst/shiny-server/ui/stats_box.R index 5ae0e15f..b93782a1 100644 --- a/inst/shiny-server/ui/stats_box.R +++ b/inst/shiny-server/ui/stats_box.R @@ -1,125 +1,125 @@ -rt_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { - box(title = HTML('

Hypothesis Testing

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.Overview.Algid', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - textInput('RT_Stats.Overview.Target', label = F_TAR_LABEL), - textInput('RT_Stats.Overview.Alpha', - label = HTML('

Significe level \\(\\alpha\\)

'), - value = 0.01), - numericInput('RT_Stats.Overview.Samples', - label = 'Size of the bootstrap sample', - min = 1, max = 1000, step = 1, value = 0) %>% - shinyInput_label_embed( - custom_icon("exclamation-triangle") %>% - bs_embed_tooltip( - title = "Using bootstrapped running times might result in misleading significance - levels. Use only when absolutely sure bootstrapping is applicable." - ) - ), - hr(), - selectInput('RT_Stats.Overview.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton('RT_Stats.Overview.DownloadTable', label = 'Download the table'), - hr(), - selectInput('RT_Stats.Overview.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format[[1]]), - downloadButton('RT_Stats.Overview.DownloadHeatmap', - label = 'Download the heatmap') - # downloadButton('RT_Stats.Overview.DownloadNetwork', - # label = 'Download the network-graph', status = F) - - ), - - mainPanel( - width = 9, - HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of - algorithms, in order to determine which algorithm gives a significantly - smaller running time distribution. The resulting p-values are arranged in a matrix, where - each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: - the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), - DT::dataTableOutput('RT_Stats.Overview.Pmatrix') - ), - - fluidRow( - column( - width = 12, - HTML('
'), - HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, - are visualized in a heatmap (left) and a network (right). - In each cell (row, column) of the heatmap, the alternative hypothesis is again: - the row algorithm has smaller (better) running time than the column. - The color indicates: -
    -
  • Red: A row is better than the column
    • -
  • Blue: A row is worse than the column
    • -
  • Gray: no significant distinction between the row and column
    • -
- On the right subplot, the partial order resulted from the test is visualized as a network, - where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than - \\(B\\) with the specified \\(\\alpha\\) value.') - ) - ), - - fluidRow( - column( - width = 6, align = 'center', - HTML('
'), - plotlyOutput.IOHanalyzer('RT_Stats.Overview.Heatmap', aspect_ratio = 1) - ), - column( - width = 6, align = 'center', - HTML('
'), - plotOutput("RT_Stats.Overview.Graph", height = '70vh') - ) - ) - ) -} - -rt_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Glicko2-based ranking

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - selectInput('RT_Stats.Glicko.Algid', 'Algorithms to compare', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, - selected = NULL, multiple = T), - selectInput('RT_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, - selected = NULL, multiple = T), - textInput('RT_Stats.Glicko.Nrgames', - label = "Number of games per (function,dimension) pair", - value = 25), - actionButton('RT_Stats.Glicko.Create', 'Create Ranking'), - hr(), - selectInput('RT_Stats.Glicko.Format', label = 'Select the figure format', - choices = supported_fig_format, selected = supported_fig_format), - - downloadButton('RT_Stats.Glicko.Download', label = 'Download the figure'), - hr(), - selectInput('RT_Stats.Glicko.TableFormat', label = 'Select the table format', - choices = supported_table_format, selected = supported_table_format), - downloadButton('RT_Stats.Glicko.DownloadTable', label = 'Download the table') - ), - - mainPanel( - width = 9, - HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. - Every round, for every function and dimension of the datasetlist, - each pair of algorithms competes. This competition samples a random runtime for the - provided target (best achieved target among all algorithms). Whichever algorithm has the lower - runtime wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), - HTML_P("The chosen target function values per (function, dimension)-pair are as follows - (double click an entry to edit it):"), - DT::dataTableOutput("RT_Stats.Glicko.Targets"), - hr(), - HTML_P("The results of the ranking are:"), - plotlyOutput.IOHanalyzer("RT_Stats.Glicko.Candlestick"), - DT::dataTableOutput('RT_Stats.Glicko.Dataframe') - ) - ) -} +rt_heatmap_box <- function(width = 12, collapsible = T, collapsed = F) { + box(title = HTML('

Hypothesis Testing

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.Overview.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + textInput('RT_Stats.Overview.Target', label = F_TAR_LABEL), + textInput('RT_Stats.Overview.Alpha', + label = HTML('

Significe level \\(\\alpha\\)

'), + value = 0.01), + numericInput('RT_Stats.Overview.Samples', + label = 'Size of the bootstrap sample', + min = 1, max = 1000, step = 1, value = 0) %>% + shinyInput_label_embed( + custom_icon("exclamation-triangle") %>% + bs_embed_tooltip( + title = "Using bootstrapped running times might result in misleading significance + levels. Use only when absolutely sure bootstrapping is applicable." + ) + ), + hr(), + selectInput('RT_Stats.Overview.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton('RT_Stats.Overview.DownloadTable', label = 'Download the table'), + hr(), + selectInput('RT_Stats.Overview.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format[[1]]), + downloadButton('RT_Stats.Overview.DownloadHeatmap', + label = 'Download the heatmap') + # downloadButton('RT_Stats.Overview.DownloadNetwork', + # label = 'Download the network-graph', status = F) + + ), + + mainPanel( + width = 9, + HTML_P('The Kolmogorov-Smirnov test is performed on empirical CDFs of running times for each pair of + algorithms, in order to determine which algorithm gives a significantly + smaller running time distribution. The resulting p-values are arranged in a matrix, where + each cell \\((i, j)\\) contains a p-value from the test with the alternative hypothesis: + the running time of algorithm \\(i\\) is smaller (thus better) than that of \\(j\\).'), + DT::dataTableOutput('RT_Stats.Overview.Pmatrix') + ), + + fluidRow( + column( + width = 12, + HTML('
'), + HTML_P('Decisions of the test, based on the \\(p-\\) value matrix and the \\(\\alpha\\) value, + are visualized in a heatmap (left) and a network (right). + In each cell (row, column) of the heatmap, the alternative hypothesis is again: + the row algorithm has smaller (better) running time than the column. + The color indicates: +
    +
  • Red: A row is better than the column
    • +
  • Blue: A row is worse than the column
    • +
  • Gray: no significant distinction between the row and column
    • +
+ On the right subplot, the partial order resulted from the test is visualized as a network, + where an arrow from algorithm \\(A\\) to \\(B\\) indicates \\(A\\) is siginicantly better than + \\(B\\) with the specified \\(\\alpha\\) value.') + ) + ), + + fluidRow( + column( + width = 6, align = 'center', + HTML('
'), + plotlyOutput.IOHanalyzer('RT_Stats.Overview.Heatmap', aspect_ratio = 1) + ), + column( + width = 6, align = 'center', + HTML('
'), + plotOutput("RT_Stats.Overview.Graph", height = '70vh') + ) + ) + ) +} + +rt_glicko2_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Glicko2-based ranking

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + selectInput('RT_Stats.Glicko.ID', 'Algorithms to compare', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.Glicko.Funcid', 'Functions to use', choices = NULL, + selected = NULL, multiple = T), + selectInput('RT_Stats.Glicko.Dim', 'Dimensions to use', choices = NULL, + selected = NULL, multiple = T), + textInput('RT_Stats.Glicko.Nrgames', + label = "Number of games per (function,dimension) pair", + value = 25), + actionButton('RT_Stats.Glicko.Create', 'Create Ranking'), + hr(), + selectInput('RT_Stats.Glicko.Format', label = 'Select the figure format', + choices = supported_fig_format, selected = supported_fig_format), + + downloadButton('RT_Stats.Glicko.Download', label = 'Download the figure'), + hr(), + selectInput('RT_Stats.Glicko.TableFormat', label = 'Select the table format', + choices = supported_table_format, selected = supported_table_format), + downloadButton('RT_Stats.Glicko.DownloadTable', label = 'Download the table') + ), + + mainPanel( + width = 9, + HTML_P('The Glicko2 This procedure ranks algorithms based on a glico2-procedure. + Every round, for every function and dimension of the datasetlist, + each pair of algorithms competes. This competition samples a random runtime for the + provided target (best achieved target among all algorithms). Whichever algorithm has the lower + runtime wins the game. Then, from these games, the glico2-rating is used to determine the ranking.'), + HTML_P("The chosen target function values per (function, dimension)-pair are as follows + (double click an entry to edit it):"), + DT::dataTableOutput("RT_Stats.Glicko.Targets"), + hr(), + HTML_P("The results of the ranking are:"), + plotlyOutput.IOHanalyzer("RT_Stats.Glicko.Candlestick"), + DT::dataTableOutput('RT_Stats.Glicko.Dataframe') + ) + ) +} diff --git a/inst/shiny-server/ui/tables_multi_func_box.R b/inst/shiny-server/ui/tables_multi_func_box.R index 7402a11f..dd986317 100644 --- a/inst/shiny-server/ui/tables_multi_func_box.R +++ b/inst/shiny-server/ui/tables_multi_func_box.R @@ -1,104 +1,104 @@ -multi_function_ert_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Statistics

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('RT.MultiERT.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.MultiERT.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.MultiERT.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('RT.MultiERT.Target', 'Target Value', value = ''), - hr(), - selectInput('RT.MultiERT.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT.MultiERT.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('RT.MultiERT.Table') - ) - ) -} - - -multi_function_sample_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Hitting Times

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('RT.Multisample.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('RT.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('RT.Multisample.Target', 'Target Value', value = ''), - hr(), - selectInput('RT.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), - selectInput('RT.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("RT.Multisample.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('RT.Multisample.Table') - ) - ) -} - - -multi_function_fv_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Statistics

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('FV.MultiFV.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.MultiFV.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.MultiFV.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('FV.MultiFV.Target', 'Budget Value', value = ''), - hr(), - selectInput('FV.MultiFV.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV.MultiFV.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('FV.MultiFV.Table') - ) - ) -} - - -multi_function_sample_box_fv <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Multi-Function Hitting Times

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - sidebarPanel( - width = 3, - HTML('

Select which algorithms to show.

'), - - selectInput('FV.Multisample.AlgId', 'Algorithms', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), - selectInput('FV.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), - hr(), - textInput('FV.Multisample.Target', 'Budget Value', value = ''), - hr(), - selectInput('FV.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), - selectInput('FV.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), - downloadButton("FV.Multisample.Download", "Save this table") - ), - - mainPanel( - width = 9, - DT::dataTableOutput('FV.Multisample.Table') - ) - ) +multi_function_ert_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Statistics

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('RT.MultiERT.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.MultiERT.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.MultiERT.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('RT.MultiERT.Target', 'Target Value', value = ''), + hr(), + selectInput('RT.MultiERT.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT.MultiERT.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('RT.MultiERT.Table') + ) + ) +} + + +multi_function_sample_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Hitting Times

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('RT.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('RT.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('RT.Multisample.Target', 'Target Value', value = ''), + hr(), + selectInput('RT.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), + selectInput('RT.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("RT.Multisample.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('RT.Multisample.Table') + ) + ) +} + + +multi_function_fv_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Statistics

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('FV.MultiFV.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.MultiFV.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.MultiFV.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('FV.MultiFV.Target', 'Budget Value', value = ''), + hr(), + selectInput('FV.MultiFV.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV.MultiFV.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('FV.MultiFV.Table') + ) + ) +} + + +multi_function_sample_box_fv <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

Multi-Function Hitting Times

'), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + sidebarPanel( + width = 3, + HTML('

Select which IDs to include:

'), + + selectInput('FV.Multisample.ID', 'Algorithms', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.Multisample.FuncId', 'Functions', choices = NULL, selected = NULL, multiple = T), + selectInput('FV.Multisample.DIM', 'Dimensions', choices = NULL, selected = NULL, multiple = T), + hr(), + textInput('FV.Multisample.Target', 'Budget Value', value = ''), + hr(), + selectInput('FV.Multisample.mode', 'Table style', choices = c('long', 'wide'), selected = 'wide'), + selectInput('FV.Multisample.Format', 'Format', choices = supported_table_format, selected = supported_table_format[[1]]), + downloadButton("FV.Multisample.Download", "Save this table") + ), + + mainPanel( + width = 9, + DT::dataTableOutput('FV.Multisample.Table') + ) + ) } \ No newline at end of file diff --git a/inst/shiny-server/ui/upload_box.R b/inst/shiny-server/ui/upload_box.R index eb836cb1..0b30367e 100644 --- a/inst/shiny-server/ui/upload_box.R +++ b/inst/shiny-server/ui/upload_box.R @@ -1,149 +1,149 @@ -upload_box <- function(width = 12, collapsible = T, collapsed = T, # TODO: find a way to include all potential arguments - height = '75vh') { - box( - title = HTML('

Upload Data

'), - width = width, height = height, collapsed = collapsed, collapsible = collapsible, - solidHeader = T, status = "primary", - sidebarPanel( - width = 12, - - HTML_P('IOHexperimenter, Nevergrad, and BBOB/COCO data files are accepted. - For alternative data files, please convert them to the format described here.'), - - selectInput( - 'upload.data_format', - label = "Please choose the format of your datasets", - choices = c(AUTOMATIC, TWO_COL), - selected = AUTOMATIC, width = '60%' - ) %>% - shinyInput_label_embed( - custom_icon("info") %>% - bs_embed_tooltip( - title = "The IOHprofiler, COCO and Nevergrad formats can be automatically detected." - ) - ), - - selectInput('upload.maximization', - label = "Maximization or minimization?", - choices = c(AUTOMATIC,"MAXIMIZE", "MINIMIZE"), - selected = AUTOMATIC, width = '60%') , - - HTML('

When the dataset is huge, - the alignment can take a very long time. In this case, you could toggle - the efficient mode to subsample the dataset. However, - the precision of data will be compromised.

'), - - checkboxInput('upload.subsampling', - label = HTML('

- Efficient mode

'), - value = F), - - fileInput("upload.add_zip", - label = HTML('

- Please choose a zip file containing the - benchmark data

'), - multiple = TRUE, accept = c("Application/zip", ".zip", - ".csv", 'bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')), - - actionButton('upload.remove_data', - label = HTML('

- Remove all the data

')) - ) - ) -} - -upload_prompt_box <- function(width = 12, collapsible = T, collapsed = T) { - box(title = HTML('

Data Processing Prompt

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - - verbatimTextOutput('process_data_promt'), - # modifying CSS for 'process_data_promt' - tags$head(tags$style("#process_data_promt{ - color:black; font-size:12px; font-style:italic; - max-height: 500px; - overflow-y:visible; overflow-x: auto; - white-space: pre-wrap; - white-space: -moz-pre-wrap; - white-space: -pre-wrap; - white-space: -o-pre-wrap; - word-wrap: normal; - background: ghostwhite;}")) - ) -} - -data_list_box <- function(width = 12, collapsible = T, collapsed = T) { - box( - title = HTML('

List of Processed Data

'), - width = width, solidHeader = T, status = "primary", - collapsible = collapsible, collapsed = collapsed, - dataTableOutput('data_info') - ) -} - -repository_box <- function(width = 12, collapsible = F, collapsed = T, - height = '75vh') { - box( - title = HTML('

Load Data from Repository

'), - width = width, height = height, collapsed = collapsed, collapsible = collapsible, - solidHeader = T, status = "primary", - sidebarPanel( - width = 12, - - HTML_P("Load the data from the available repositories. There are currently three available sources: -
    -
  • - Data generated with the PBO-suite, implemented in the IOHexperimenter -
    • -
  • - All data generated by the nevergrad benchmarking framework -
    • -
  • - The majority of the publicly available benchmark data on the single-objective BBOB framework -
    • "), - - selectInput('repository.type', label = "Select the dataset source", - choices = NULL, selected = NULL, width = '50%'), - - selectInput('repository.dataset', - label = "Select the dataset", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - selectInput('repository.funcId', - label = "Please choose the function", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - selectInput('repository.dim', - label = "Please choose the dimension", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - selectInput('repository.algId', - label = "Please choose the algorithm", - choices = NULL, selected = NULL, width = '50%', multiple = T), - - shinyjs::disabled( - actionButton('repository.load_button', 'load data') - ) - ) - ) -} - -welcome_bar <- function(width = 12, collapsible = T, collapsed = F) { - box( - title = HTML('

    Welcome to IOHanalyzer!

    '), - width = width, collapsed = collapsed, collapsible = collapsible, - solidHeader = T, status = "primary", - mainPanel( - width = 12, - HTML(paste0( - '', - includeMarkdown('markdown/welcome.md'), - '') - )), - fluidRow(valueBoxOutput("VersionBox"), - valueBoxOutput("WikiBox"), - valueBoxOutput("ContactBox") - ) - ) -} - +upload_box <- function(width = 12, collapsible = T, collapsed = T, # TODO: find a way to include all potential arguments + height = '75vh') { + box( + title = HTML('

    Upload Data

    '), + width = width, height = height, collapsed = collapsed, collapsible = collapsible, + solidHeader = T, status = "primary", + sidebarPanel( + width = 12, + + HTML_P('IOHexperimenter, Nevergrad, and BBOB/COCO data files are accepted. + For alternative data files, please convert them to the format described here.'), + + selectInput( + 'upload.data_format', + label = "Please choose the format of your datasets", + choices = c(AUTOMATIC, TWO_COL), + selected = AUTOMATIC, width = '60%' + ) %>% + shinyInput_label_embed( + custom_icon("info") %>% + bs_embed_tooltip( + title = "The IOHprofiler, COCO and Nevergrad formats can be automatically detected." + ) + ), + + selectInput('upload.maximization', + label = "Maximization or minimization?", + choices = c(AUTOMATIC,"MAXIMIZE", "MINIMIZE"), + selected = AUTOMATIC, width = '60%') , + + HTML('

    When the dataset is huge, + the alignment can take a very long time. In this case, you could toggle + the efficient mode to subsample the dataset. However, + the precision of data will be compromised.

    '), + + checkboxInput('upload.subsampling', + label = HTML('

    + Efficient mode

    '), + value = F), + + fileInput("upload.add_zip", + label = HTML('

    + Please choose a zip file containing the + benchmark data

    '), + multiple = TRUE, accept = c("Application/zip", ".zip", + ".csv", 'bz2', 'bz', 'gz', 'tar', 'tgz', 'tar.gz', 'xz')), + + actionButton('upload.remove_data', + label = HTML('

    + Remove all the data

    ')) + ) + ) +} + +upload_prompt_box <- function(width = 12, collapsible = T, collapsed = T) { + box(title = HTML('

    Data Processing Prompt

    '), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + + verbatimTextOutput('process_data_promt'), + # modifying CSS for 'process_data_promt' + tags$head(tags$style("#process_data_promt{ + color:black; font-size:12px; font-style:italic; + max-height: 500px; + overflow-y:visible; overflow-x: auto; + white-space: pre-wrap; + white-space: -moz-pre-wrap; + white-space: -pre-wrap; + white-space: -o-pre-wrap; + word-wrap: normal; + background: ghostwhite;}")) + ) +} + +data_list_box <- function(width = 12, collapsible = T, collapsed = T) { + box( + title = HTML('

    List of Processed Data

    '), + width = width, solidHeader = T, status = "primary", + collapsible = collapsible, collapsed = collapsed, + dataTableOutput('data_info') + ) +} + +repository_box <- function(width = 12, collapsible = F, collapsed = T, + height = '75vh') { + box( + title = HTML('

    Load Data from Repository

    '), + width = width, height = height, collapsed = collapsed, collapsible = collapsible, + solidHeader = T, status = "primary", + sidebarPanel( + width = 12, + + HTML_P("Load the data from the available repositories. There are currently three available sources: +
      +
    • + Data generated with the PBO-suite, implemented in the IOHexperimenter +
      • +
    • + All data generated by the nevergrad benchmarking framework +
      • +
    • + The majority of the publicly available benchmark data on the single-objective BBOB framework +
      • "), + + selectInput('repository.type', label = "Select the dataset source", + choices = NULL, selected = NULL, width = '50%'), + + selectInput('repository.dataset', + label = "Select the dataset", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + selectInput('repository.funcId', + label = "Please choose the function", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + selectInput('repository.dim', + label = "Please choose the dimension", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + selectInput('repository.ID', + label = "Please choose the algorithm", + choices = NULL, selected = NULL, width = '50%', multiple = T), + + shinyjs::disabled( + actionButton('repository.load_button', 'load data') + ) + ) + ) +} + +welcome_bar <- function(width = 12, collapsible = T, collapsed = F) { + box( + title = HTML('

      Welcome to IOHanalyzer!

      '), + width = width, collapsed = collapsed, collapsible = collapsible, + solidHeader = T, status = "primary", + mainPanel( + width = 12, + HTML(paste0( + '', + includeMarkdown('markdown/welcome.md'), + '') + )), + fluidRow(valueBoxOutput("VersionBox"), + valueBoxOutput("WikiBox"), + valueBoxOutput("ContactBox") + ) + ) +} + diff --git a/inst/shiny-server/ui/variable.R b/inst/shiny-server/ui/variable.R index 9b303446..5a25071d 100644 --- a/inst/shiny-server/ui/variable.R +++ b/inst/shiny-server/ui/variable.R @@ -1,37 +1,38 @@ -FCE_GRID_INPUT_TEXT <- '

      Set the range and the granularity of the results. - The table will show function values that have been reached within evenly - spaced evaluation budgets.

      ' - -RT_TAR_LABEL <- HTML('

      \\(B:\\) Budget value

      ') -RT_MIN_LABEL <- HTML('

      \\(B_{\\text{min}}:\\) Smallest budget value

      ') -RT_MAX_LABEL <- HTML('

      \\(B_{\\text{max}}:\\) Largest budget value

      ') -RT_STEP_LABEL <- HTML('

      \\(\\Delta B:\\) Granularity (step size)

      ') - -F_TAR_LABEL <- HTML('Target function value

      \\(f_{\\text{target}}:\\)

      ') -F_MIN_LABEL <- HTML('

      \\(f_{\\text{min}}:\\) Smallest target value

      ') -F_MAX_LABEL <- HTML('

      \\(f_{\\text{max}}:\\) Largest target value

      ') -F_STEP_LABEL <- HTML('

      \\(\\Delta f:\\) Granularity (step size)

      ') - -Pdsc_info <- "Practical Deep Comparison uses this - as a threshold parameter to determine when two performance measure values are - equal or different from a practical point of view. - Since the practical significance - is handled with preprocessing using the predefined threshold in sequential order of - the obtained runs, to make more robust analysis a Monte-Carlo simulation is required." - -Pdsc_mc_info <- "Practical Deep Comparison uses this - to determine the number of monte-carlo simulations to perform. - The Monte-Carlo simulation involves preprocessing done with different orders of the - independent runs before comparing the distribution of the data. - This only has effect if threshold for practical significance > 0" - -HTML_P <- function(s) HTML(paste0('

      ', s, '

      ')) - -alg_select_info <- "Use this option to select which algorithms to plot. - This will hava an effect the dowloaded plot, - as opposed to using the legend-entries to show or hide algorithms " - -custom_icon <- function(name = NULL){ - if (is.null(name)) name <- "info-circle" - htmltools::tags$a(shiny::icon(name = name), href = "javascript:void(0);") +FCE_GRID_INPUT_TEXT <- '

      Set the range and the granularity of the results. + The table will show function values that have been reached within evenly + spaced evaluation budgets.

      ' + +RT_TAR_LABEL <- HTML('

      \\(B:\\) Budget value

      ') +RT_MIN_LABEL <- HTML('

      \\(B_{\\text{min}}:\\) Smallest budget value

      ') +RT_MAX_LABEL <- HTML('

      \\(B_{\\text{max}}:\\) Largest budget value

      ') +RT_STEP_LABEL <- HTML('

      \\(\\Delta B:\\) Granularity (step size)

      ') + +F_TAR_LABEL <- HTML('Target function value

      \\(f_{\\text{target}}:\\)

      ') +F_MIN_LABEL <- HTML('

      \\(f_{\\text{min}}:\\) Smallest target value

      ') +F_MAX_LABEL <- HTML('

      \\(f_{\\text{max}}:\\) Largest target value

      ') +F_STEP_LABEL <- HTML('

      \\(\\Delta f:\\) Granularity (step size)

      ') + +Pdsc_info <- "Practical Deep Comparison uses this + as a threshold parameter to determine when two performance measure values are + equal or different from a practical point of view. + Since the practical significance + is handled with preprocessing using the predefined threshold in sequential order of + the obtained runs, to make more robust analysis a Monte-Carlo simulation is required." + +Pdsc_mc_info <- "Practical Deep Comparison uses this + to determine the number of monte-carlo simulations to perform. + The Monte-Carlo simulation involves preprocessing done with different orders of the + independent runs before comparing the distribution of the data. + This only has effect if threshold for practical significance > 0" + +HTML_P <- function(s) HTML(paste0('

      ', s, '

      ')) + +alg_select_info <- "Use this option to select which IDs to plot. + This will have an effect the dowloaded plot, + as opposed to using the legend-entries to show or hide algorithms. + The attributes which define the ID can be changed in the settings." + +custom_icon <- function(name = NULL){ + if (is.null(name)) name <- "info-circle" + htmltools::tags$a(shiny::icon(name = name), href = "javascript:void(0);") } \ No newline at end of file diff --git a/man/AUC.Rd b/man/AUC.Rd index e0bbe3ce..30a2df1a 100644 --- a/man/AUC.Rd +++ b/man/AUC.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{AUC} -\alias{AUC} -\alias{AUC.ECDF} -\title{Area Under Curve (Empirical Cumulative Dsitribution Function)} -\usage{ -AUC(fun, from = NULL, to = NULL) - -\method{AUC}{ECDF}(fun, from = NULL, to = NULL) -} -\arguments{ -\item{fun}{A ECDF object.} - -\item{from}{double. Starting point of the area on x-axis} - -\item{to}{double. Ending point of the area on x-axis} -} -\value{ -a object of type 'ECDF' -} -\description{ -Area Under Curve (Empirical Cumulative Dsitribution Function) -} -\examples{ -ecdf <- ECDF(dsl,c(12,14)) -AUC(ecdf, 0, 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{AUC} +\alias{AUC} +\alias{AUC.ECDF} +\title{Area Under Curve (Empirical Cumulative Dsitribution Function)} +\usage{ +AUC(fun, from = NULL, to = NULL) + +\method{AUC}{ECDF}(fun, from = NULL, to = NULL) +} +\arguments{ +\item{fun}{A ECDF object.} + +\item{from}{double. Starting point of the area on x-axis} + +\item{to}{double. Ending point of the area on x-axis} +} +\value{ +a object of type 'ECDF' +} +\description{ +Area Under Curve (Empirical Cumulative Dsitribution Function) +} +\examples{ +ecdf <- ECDF(dsl,c(12,14)) +AUC(ecdf, 0, 100) +} diff --git a/man/DataSet.Rd b/man/DataSet.Rd index 6d377f4c..3306e293 100644 --- a/man/DataSet.Rd +++ b/man/DataSet.Rd @@ -1,41 +1,41 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{DataSet} -\alias{DataSet} -\title{Constructor of S3 class 'DataSet'} -\usage{ -DataSet(info, verbose = F, maximization = NULL, format = IOHprofiler, - subsampling = FALSE) -} -\arguments{ -\item{info}{A List. Contains a set of in a *.info file.} - -\item{verbose}{Logical.} - -\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization? -Set to NULL to determine automatically based on format} - -\item{format}{A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL"} - -\item{subsampling}{Logical. Whether *.cdat files are subsampled?} -} -\value{ -A S3 object 'DataSet' -} -\description{ -DataSet contains the following attributes - * funId - * DIM - * algId - * datafile - * instance - * maxEvals - * finalFunEvals - * comment - * Additional attributes based on the original format -} -\examples{ -path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') -info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) -DataSet(info[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{DataSet} +\alias{DataSet} +\title{Constructor of S3 class 'DataSet'} +\usage{ +DataSet(info, verbose = F, maximization = NULL, format = IOHprofiler, + subsampling = FALSE) +} +\arguments{ +\item{info}{A List. Contains a set of in a *.info file.} + +\item{verbose}{Logical.} + +\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization? +Set to NULL to determine automatically based on format} + +\item{format}{A character. The format of data source, either 'IOHProfiler', 'COCO' or 'TWO_COL"} + +\item{subsampling}{Logical. Whether *.cdat files are subsampled?} +} +\value{ +A S3 object 'DataSet' +} +\description{ +DataSet contains the following attributes + * funId + * DIM + * algId + * datafile + * instance + * maxEvals + * finalFunEvals + * comment + * Additional attributes based on the original format +} +\examples{ +path <- system.file('extdata', 'ONE_PLUS_LAMDA_EA', package = 'IOHanalyzer') +info <- read_index_file(file.path(path, 'IOHprofiler_f1_i1.info')) +DataSet(info[[1]]) +} diff --git a/man/DataSetList.Rd b/man/DataSetList.Rd index 624504a3..61230cc9 100644 --- a/man/DataSetList.Rd +++ b/man/DataSetList.Rd @@ -1,45 +1,50 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{DataSetList} -\alias{DataSetList} -\title{S3 constructor of the 'DataSetList'} -\usage{ -DataSetList(path = NULL, verbose = T, print_fun = NULL, - maximization = NULL, format = IOHprofiler, subsampling = FALSE) -} -\arguments{ -\item{path}{Path to the data files. Will look for all .info-files in this directory and use -the corresponding datafiles to create the DataSetList} - -\item{verbose}{Logical.} - -\item{print_fun}{Function used to print output when in verbose mode} - -\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization?} - -\item{format}{A character. The format of data source, options are: -\itemize{ -\item'IOHProfiler' -\item'COCO' -\item'TWO_COL' -\item'COCO_BIOBJ' -\item'NEVERGRAD' -\item'SOS' -} -These formats are specified in more detail in our github wiki.} - -\item{subsampling}{Logical. Whether *.cdat files are subsampled?} -} -\value{ -A DataSetList object -} -\description{ -Attributes - funId - DIM - algId -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -DataSetList(path) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{DataSetList} +\alias{DataSetList} +\title{S3 constructor of the 'DataSetList'} +\usage{ +DataSetList(path = NULL, verbose = T, print_fun = NULL, + maximization = NULL, format = IOHprofiler, subsampling = FALSE, + full_aggregation = TRUE) +} +\arguments{ +\item{path}{Path to the data files. Will look for all .info-files in this directory and use +the corresponding datafiles to create the DataSetList} + +\item{verbose}{Logical.} + +\item{print_fun}{Function used to print output when in verbose mode} + +\item{maximization}{Logical. Whether the underlying optimization algorithm performs a maximization?} + +\item{format}{A character. The format of data source, options are: +\itemize{ +\item'IOHProfiler' +\item'COCO' +\item'TWO_COL' +\item'COCO_BIOBJ' +\item'NEVERGRAD' +\item'SOS' +} +These formats are specified in more detail in our github wiki.} + +\item{subsampling}{Logical. Whether *.cdat files are subsampled?} + +\item{full_aggregation}{If True, individual DataSets are aggregated as much as possible: all DataSets +with the same algorithmname, function id and dimension are combined together. This leads to information loss +related to static variables, so only use if that information is not required.} +} +\value{ +A DataSetList object +} +\description{ +Attributes + funId + DIM + algId +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +DataSetList(path) +} diff --git a/man/ECDF.Rd b/man/ECDF.Rd index 674c6fdf..28077296 100644 --- a/man/ECDF.Rd +++ b/man/ECDF.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{ECDF} -\alias{ECDF} -\alias{ECDF.DataSet} -\alias{ECDF.DataSetList} -\title{Empirical Cumulative Dsitribution Function of Runtime of a single data set} -\usage{ -ECDF(ds, ftarget, ...) - -\method{ECDF}{DataSet}(ds, ftarget, ...) - -\method{ECDF}{DataSetList}(ds, ftarget, ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object.} - -\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} - -\item{...}{Arguments passed to other methods} -} -\value{ -a object of type 'ECDF' -} -\description{ -Empirical Cumulative Dsitribution Function of Runtime of a single data set -} -\examples{ -ECDF(dsl,c(12,14)) -ECDF(dsl[[1]],c(12,14)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{ECDF} +\alias{ECDF} +\alias{ECDF.DataSet} +\alias{ECDF.DataSetList} +\title{Empirical Cumulative Dsitribution Function of Runtime of a single data set} +\usage{ +ECDF(ds, ftarget, ...) + +\method{ECDF}{DataSet}(ds, ftarget, ...) + +\method{ECDF}{DataSetList}(ds, ftarget, ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object.} + +\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} + +\item{...}{Arguments passed to other methods} +} +\value{ +a object of type 'ECDF' +} +\description{ +Empirical Cumulative Dsitribution Function of Runtime of a single data set +} +\examples{ +ECDF(dsl,c(12,14)) +ECDF(dsl[[1]],c(12,14)) +} diff --git a/man/IOH_plot_ly_default.Rd b/man/IOH_plot_ly_default.Rd index 9b8c9553..d44af531 100644 --- a/man/IOH_plot_ly_default.Rd +++ b/man/IOH_plot_ly_default.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{IOH_plot_ly_default} -\alias{IOH_plot_ly_default} -\title{Template for creating plots in the IOHanalyzer-style} -\usage{ -IOH_plot_ly_default(title = NULL, x.title = NULL, y.title = NULL) -} -\arguments{ -\item{title}{Title for the plot} - -\item{x.title}{X-axis label} - -\item{y.title}{Y-axis label} -} -\description{ -Template for creating plots in the IOHanalyzer-style -} -\examples{ -IOH_plot_ly_default("Example plot","x-axis","y-axis") -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{IOH_plot_ly_default} +\alias{IOH_plot_ly_default} +\title{Template for creating plots in the IOHanalyzer-style} +\usage{ +IOH_plot_ly_default(title = NULL, x.title = NULL, y.title = NULL) +} +\arguments{ +\item{title}{Title for the plot} + +\item{x.title}{X-axis label} + +\item{y.title}{Y-axis label} +} +\description{ +Template for creating plots in the IOHanalyzer-style +} +\examples{ +IOH_plot_ly_default("Example plot","x-axis","y-axis") +} diff --git a/man/IOHanalyzer.Rd b/man/IOHanalyzer.Rd index 41ca129d..650ff685 100644 --- a/man/IOHanalyzer.Rd +++ b/man/IOHanalyzer.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer.R -\docType{package} -\name{IOHanalyzer} -\alias{IOHanalyzer} -\alias{IOHanalyzer-package} -\title{IOHanalyzer: Data Analysis Part of IOHprofiler} -\description{ -The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). -This module provides statistical analysis methods for the benchmark data generated by -optimization heuristics, which can be visualized through a -web-based interface. The benchmark data is usually generated by the -experimentation module, called IOHexperimenter. IOHanalyzer also supports -the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. -} -\section{Functions}{ - -The IOHanalyzer consists of 3 main functionalities: -\itemize{ -\item Reading and alligning data from different heuristics, such as IOHExperimenter. -This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions -\item Processing and summarizing this data -\item Creating various plots -} -} - -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -dsList <- DataSetList(path) -summary(dsList) -Plot.RT.Single_Func(dsList[1]) - -\dontrun{ -runServer() -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer.R +\docType{package} +\name{IOHanalyzer} +\alias{IOHanalyzer} +\alias{IOHanalyzer-package} +\title{IOHanalyzer: Data Analysis Part of IOHprofiler} +\description{ +The data analysis module for the Iterative Optimization Heuristics Profiler (IOHprofiler). +This module provides statistical analysis methods for the benchmark data generated by +optimization heuristics, which can be visualized through a +web-based interface. The benchmark data is usually generated by the +experimentation module, called IOHexperimenter. IOHanalyzer also supports +the widely used COCO (Comparing Continuous Optimisers) data format for benchmarking. +} +\section{Functions}{ + +The IOHanalyzer consists of 3 main functionalities: +\itemize{ +\item Reading and alligning data from different heuristics, such as IOHExperimenter. +This is done using the \code{\link{DataSet}} and \code{\link{DataSetList}} functions +\item Processing and summarizing this data +\item Creating various plots +} +} + +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +dsList <- DataSetList(path) +summary(dsList) +Plot.RT.Single_Func(dsList[1]) + +\dontrun{ +runServer() +} +} diff --git a/man/IOHanlyzer-deprecated.Rd b/man/IOHanlyzer-deprecated.Rd index 38b68118..b932c49b 100644 --- a/man/IOHanlyzer-deprecated.Rd +++ b/man/IOHanlyzer-deprecated.Rd @@ -1,11 +1,11 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{IOHanlyzer-deprecated} -\alias{IOHanlyzer-deprecated} -\title{Deprecated function in package \pkg{IOHanalyzer}} -\description{ -The functions listed below are deprecated and will be defunct in - the near future. When possible, alternative functions with similar - functionality are also mentioned. -} -\keyword{internal} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{IOHanlyzer-deprecated} +\alias{IOHanlyzer-deprecated} +\title{Deprecated function in package \pkg{IOHanalyzer}} +\description{ +The functions listed below are deprecated and will be defunct in + the near future. When possible, alternative functions with similar + functionality are also mentioned. +} +\keyword{internal} diff --git a/man/Plot.FV.Aggregated.Rd b/man/Plot.FV.Aggregated.Rd index ef91b3a2..a11be39d 100644 --- a/man/Plot.FV.Aggregated.Rd +++ b/man/Plot.FV.Aggregated.Rd @@ -1,42 +1,42 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Aggregated} -\alias{Plot.FV.Aggregated} -\alias{Plot.FV.Aggregated.DataSetList} -\title{Plot expected function value-based comparison over multiple functions or dimensions} -\usage{ -Plot.FV.Aggregated(dsList, aggr_on = "funcId", runtimes = NULL, - plot_mode = "radar", use_rank = F, scale.ylog = T, fvs = NULL) - -\method{Plot.FV.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", - runtimes = NULL, plot_mode = "radar", use_rank = F, - scale.ylog = T, fvs = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} - -\item{runtimes}{Custom list of function-value targets, one for each function or dimension.} - -\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} - -\item{use_rank}{Wheter to use a ranking system. If False, the actual expected function- -values will be used.} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{fvs}{Pre-calculated expected function-values for the provided runtimes Created by the -max_ERTs function of DataSetList. Can be provided to prevent needless computation -in recalculating ERTs when recreating this plot.} -} -\value{ -A plot of expected function value-based comparison on the provided functions - or dimensions of the DataSetList -} -\description{ -Plot expected function value-based comparison over multiple functions or dimensions -} -\examples{ -Plot.FV.Aggregated(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Aggregated} +\alias{Plot.FV.Aggregated} +\alias{Plot.FV.Aggregated.DataSetList} +\title{Plot expected function value-based comparison over multiple functions or dimensions} +\usage{ +Plot.FV.Aggregated(dsList, aggr_on = "funcId", runtimes = NULL, + plot_mode = "radar", use_rank = F, scale.ylog = T, fvs = NULL) + +\method{Plot.FV.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", + runtimes = NULL, plot_mode = "radar", use_rank = F, + scale.ylog = T, fvs = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} + +\item{runtimes}{Custom list of function-value targets, one for each function or dimension.} + +\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} + +\item{use_rank}{Wheter to use a ranking system. If False, the actual expected function- +values will be used.} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{fvs}{Pre-calculated expected function-values for the provided runtimes Created by the +max_ERTs function of DataSetList. Can be provided to prevent needless computation +in recalculating ERTs when recreating this plot.} +} +\value{ +A plot of expected function value-based comparison on the provided functions + or dimensions of the DataSetList +} +\description{ +Plot expected function value-based comparison over multiple functions or dimensions +} +\examples{ +Plot.FV.Aggregated(dsl) +} diff --git a/man/Plot.FV.ECDF_AUC.Rd b/man/Plot.FV.ECDF_AUC.Rd index d2c3c427..8e430fef 100644 --- a/man/Plot.FV.ECDF_AUC.Rd +++ b/man/Plot.FV.ECDF_AUC.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.ECDF_AUC} -\alias{Plot.FV.ECDF_AUC} -\alias{Plot.FV.ECDF_AUC.DataSetList} -\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} -\usage{ -Plot.FV.ECDF_AUC(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL) - -\method{Plot.FV.ECDF_AUC}{DataSetList}(dsList, rt_min = NULL, - rt_max = NULL, rt_step = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{rt_min}{The starting runtime} - -\item{rt_max}{The final runtime} - -\item{rt_step}{The spacing between starting and final runtimes} -} -\value{ -A radarplot of the area under the aggregated ECDF-curve of the DataSetList -} -\description{ -Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -} -\examples{ -Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.ECDF_AUC} +\alias{Plot.FV.ECDF_AUC} +\alias{Plot.FV.ECDF_AUC.DataSetList} +\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} +\usage{ +Plot.FV.ECDF_AUC(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL) + +\method{Plot.FV.ECDF_AUC}{DataSetList}(dsList, rt_min = NULL, + rt_max = NULL, rt_step = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{rt_min}{The starting runtime} + +\item{rt_max}{The final runtime} + +\item{rt_step}{The spacing between starting and final runtimes} +} +\value{ +A radarplot of the area under the aggregated ECDF-curve of the DataSetList +} +\description{ +Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +} +\examples{ +Plot.FV.ECDF_AUC(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.FV.ECDF_Per_Target.Rd b/man/Plot.FV.ECDF_Per_Target.Rd index a1c4c81c..9b15c256 100644 --- a/man/Plot.FV.ECDF_Per_Target.Rd +++ b/man/Plot.FV.ECDF_Per_Target.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.ECDF_Per_Target} -\alias{Plot.FV.ECDF_Per_Target} -\alias{Plot.FV.ECDF_Per_Target.DataSetList} -\title{Plot the empirical cumulative distriburtion as a function of the target values of -a DataSetList at certain target runtimes} -\usage{ -Plot.FV.ECDF_Per_Target(dsList, runtimes, scale.xlog = F, - scale.reverse = F) - -\method{Plot.FV.ECDF_Per_Target}{DataSetList}(dsList, runtimes, - scale.xlog = F, scale.reverse = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{runtimes}{The target runtimes} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the fucntion values of the DataSetList at the target runtimes -} -\description{ -Plot the empirical cumulative distriburtion as a function of the target values of -a DataSetList at certain target runtimes -} -\examples{ -Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.ECDF_Per_Target} +\alias{Plot.FV.ECDF_Per_Target} +\alias{Plot.FV.ECDF_Per_Target.DataSetList} +\title{Plot the empirical cumulative distriburtion as a function of the target values of +a DataSetList at certain target runtimes} +\usage{ +Plot.FV.ECDF_Per_Target(dsList, runtimes, scale.xlog = F, + scale.reverse = F) + +\method{Plot.FV.ECDF_Per_Target}{DataSetList}(dsList, runtimes, + scale.xlog = F, scale.reverse = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{runtimes}{The target runtimes} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the fucntion values of the DataSetList at the target runtimes +} +\description{ +Plot the empirical cumulative distriburtion as a function of the target values of +a DataSetList at certain target runtimes +} +\examples{ +Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10) +} diff --git a/man/Plot.FV.ECDF_Single_Func.Rd b/man/Plot.FV.ECDF_Single_Func.Rd index 39ec35a5..03a0c7ae 100644 --- a/man/Plot.FV.ECDF_Single_Func.Rd +++ b/man/Plot.FV.ECDF_Single_Func.Rd @@ -1,42 +1,42 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.ECDF_Single_Func} -\alias{Plot.FV.ECDF_Single_Func} -\alias{Plot.FV.ECDF_Single_Func.DataSetList} -\title{Plot the aggregated empirical cumulative distriburtion as a function of the function values of -a DataSetList.} -\usage{ -Plot.FV.ECDF_Single_Func(dsList, rt_min = NULL, rt_max = NULL, - rt_step = NULL, scale.xlog = F, show.per_target = F, - scale.reverse = F) - -\method{Plot.FV.ECDF_Single_Func}{DataSetList}(dsList, rt_min = NULL, - rt_max = NULL, rt_step = NULL, scale.xlog = F, - show.per_target = F, scale.reverse = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{rt_min}{The starting runtime} - -\item{rt_max}{The final runtime} - -\item{rt_step}{The spacing between starting and final runtimes} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{show.per_target}{Whether or not to show the individual ECDF-curves for each runtime} - -\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the function values of the DataSetList -} -\description{ -Plot the aggregated empirical cumulative distriburtion as a function of the function values of -a DataSetList. -} -\examples{ -Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.ECDF_Single_Func} +\alias{Plot.FV.ECDF_Single_Func} +\alias{Plot.FV.ECDF_Single_Func.DataSetList} +\title{Plot the aggregated empirical cumulative distriburtion as a function of the function values of +a DataSetList.} +\usage{ +Plot.FV.ECDF_Single_Func(dsList, rt_min = NULL, rt_max = NULL, + rt_step = NULL, scale.xlog = F, show.per_target = F, + scale.reverse = F) + +\method{Plot.FV.ECDF_Single_Func}{DataSetList}(dsList, rt_min = NULL, + rt_max = NULL, rt_step = NULL, scale.xlog = F, + show.per_target = F, scale.reverse = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{rt_min}{The starting runtime} + +\item{rt_max}{The final runtime} + +\item{rt_step}{The spacing between starting and final runtimes} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{show.per_target}{Whether or not to show the individual ECDF-curves for each runtime} + +\item{scale.reverse}{Whether or not to reverse the x-axis (when using minimization)} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the function values of the DataSetList +} +\description{ +Plot the aggregated empirical cumulative distriburtion as a function of the function values of +a DataSetList. +} +\examples{ +Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.FV.Histogram.Rd b/man/Plot.FV.Histogram.Rd index 15e9d762..dacf141f 100644 --- a/man/Plot.FV.Histogram.Rd +++ b/man/Plot.FV.Histogram.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Histogram} -\alias{Plot.FV.Histogram} -\alias{Plot.FV.Histogram.DataSetList} -\title{Plot histograms of the function values of a DataSetList at a certain target runtime} -\usage{ -Plot.FV.Histogram(dsList, runtime, plot_mode = "overlay", - use.equal.bins = F) - -\method{Plot.FV.Histogram}{DataSetList}(dsList, runtime, - plot_mode = "overlay", use.equal.bins = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{runtime}{The target runtime} - -\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either -'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} - -\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual -bin sizes for each algorithm} -} -\value{ -A plot of the histograms of the function values at a the - target runtime of the DataSetList -} -\description{ -Plot histograms of the function values of a DataSetList at a certain target runtime -} -\examples{ -Plot.FV.Histogram(subset(dsl, funcId == 1), 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Histogram} +\alias{Plot.FV.Histogram} +\alias{Plot.FV.Histogram.DataSetList} +\title{Plot histograms of the function values of a DataSetList at a certain target runtime} +\usage{ +Plot.FV.Histogram(dsList, runtime, plot_mode = "overlay", + use.equal.bins = F) + +\method{Plot.FV.Histogram}{DataSetList}(dsList, runtime, + plot_mode = "overlay", use.equal.bins = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{runtime}{The target runtime} + +\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either +'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} + +\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual +bin sizes for each algorithm} +} +\value{ +A plot of the histograms of the function values at a the + target runtime of the DataSetList +} +\description{ +Plot histograms of the function values of a DataSetList at a certain target runtime +} +\examples{ +Plot.FV.Histogram(subset(dsl, funcId == 1), 100) +} diff --git a/man/Plot.FV.Multi_Func.Rd b/man/Plot.FV.Multi_Func.Rd index e3ac971e..464f7157 100644 --- a/man/Plot.FV.Multi_Func.Rd +++ b/man/Plot.FV.Multi_Func.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Multi_Func} -\alias{Plot.FV.Multi_Func} -\alias{Plot.FV.Multi_Func.DataSetList} -\title{Plot FV-plots for multiple functions or dimensions} -\usage{ -Plot.FV.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, - backend = NULL) - -\method{Plot.FV.Multi_Func}{DataSetList}(dsList, scale.xlog = F, - scale.ylog = F, backend = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} -} -\value{ -A plot of Function-values of the DataSetList -} -\description{ -Plot FV-plots for multiple functions or dimensions -} -\examples{ -Plot.FV.Multi_Func(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Multi_Func} +\alias{Plot.FV.Multi_Func} +\alias{Plot.FV.Multi_Func.DataSetList} +\title{Plot FV-plots for multiple functions or dimensions} +\usage{ +Plot.FV.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, + backend = NULL) + +\method{Plot.FV.Multi_Func}{DataSetList}(dsList, scale.xlog = F, + scale.ylog = F, backend = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} +} +\value{ +A plot of Function-values of the DataSetList +} +\description{ +Plot FV-plots for multiple functions or dimensions +} +\examples{ +Plot.FV.Multi_Func(dsl) +} diff --git a/man/Plot.FV.PDF.Rd b/man/Plot.FV.PDF.Rd index 2d879670..02ed6ad9 100644 --- a/man/Plot.FV.PDF.Rd +++ b/man/Plot.FV.PDF.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.PDF} -\alias{Plot.FV.PDF} -\alias{Plot.FV.PDF.DataSetList} -\title{Plot probablity density function of the function values of a DataSetList at -a certain target runtime} -\usage{ -Plot.FV.PDF(dsList, runtime, show.sample = F, scale.ylog = F) - -\method{Plot.FV.PDF}{DataSetList}(dsList, runtime, show.sample = F, - scale.ylog = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{runtime}{The target runtime} - -\item{show.sample}{Whether or not to show the individual function value samples} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} -} -\value{ -A plot of the probablity density function of the runtimes at a the - target function value of the DataSetList -} -\description{ -Plot probablity density function of the function values of a DataSetList at -a certain target runtime -} -\examples{ -Plot.FV.PDF(subset(dsl, funcId == 1), 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.PDF} +\alias{Plot.FV.PDF} +\alias{Plot.FV.PDF.DataSetList} +\title{Plot probablity density function of the function values of a DataSetList at +a certain target runtime} +\usage{ +Plot.FV.PDF(dsList, runtime, show.sample = F, scale.ylog = F) + +\method{Plot.FV.PDF}{DataSetList}(dsList, runtime, show.sample = F, + scale.ylog = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{runtime}{The target runtime} + +\item{show.sample}{Whether or not to show the individual function value samples} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} +} +\value{ +A plot of the probablity density function of the runtimes at a the + target function value of the DataSetList +} +\description{ +Plot probablity density function of the function values of a DataSetList at +a certain target runtime +} +\examples{ +Plot.FV.PDF(subset(dsl, funcId == 1), 100) +} diff --git a/man/Plot.FV.Parameters.Rd b/man/Plot.FV.Parameters.Rd index 65ebb0c5..893cd455 100644 --- a/man/Plot.FV.Parameters.Rd +++ b/man/Plot.FV.Parameters.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Parameters} -\alias{Plot.FV.Parameters} -\alias{Plot.FV.Parameters.DataSetList} -\title{Plot the parameter values recorded in a DataSetList (aligned by budget)} -\usage{ -Plot.FV.Parameters(dsList, rt_min = NULL, rt_max = NULL, - algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, show.CI = F) - -\method{Plot.FV.Parameters}{DataSetList}(dsList, rt_min = NULL, - rt_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, - scale.ylog = F, show.mean = T, show.median = F, show.CI = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{rt_min}{The starting budget value.} - -\item{rt_max}{The final budget value.} - -\item{algids}{Which algorithms from dsList to use} - -\item{par_name}{Which parameters to create plots for; set to NULL to use all -parameters found in dsList.} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{show.mean}{Whether or not to show the mean parameter values} - -\item{show.median}{Whether or not to show the median parameter values} - -\item{show.CI}{Whether or not to show the standard deviation} -} -\value{ -A plot of for every recorded parameter in the DataSetList -} -\description{ -Plot the parameter values recorded in a DataSetList (aligned by budget) -} -\examples{ -Plot.FV.Parameters(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Parameters} +\alias{Plot.FV.Parameters} +\alias{Plot.FV.Parameters.DataSetList} +\title{Plot the parameter values recorded in a DataSetList (aligned by budget)} +\usage{ +Plot.FV.Parameters(dsList, rt_min = NULL, rt_max = NULL, + algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, show.CI = F) + +\method{Plot.FV.Parameters}{DataSetList}(dsList, rt_min = NULL, + rt_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, + scale.ylog = F, show.mean = T, show.median = F, show.CI = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{rt_min}{The starting budget value.} + +\item{rt_max}{The final budget value.} + +\item{algids}{Which algorithms from dsList to use} + +\item{par_name}{Which parameters to create plots for; set to NULL to use all +parameters found in dsList.} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{show.mean}{Whether or not to show the mean parameter values} + +\item{show.median}{Whether or not to show the median parameter values} + +\item{show.CI}{Whether or not to show the standard deviation} +} +\value{ +A plot of for every recorded parameter in the DataSetList +} +\description{ +Plot the parameter values recorded in a DataSetList (aligned by budget) +} +\examples{ +Plot.FV.Parameters(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.FV.Single_Func.Rd b/man/Plot.FV.Single_Func.Rd index ed014fa7..74fc741b 100644 --- a/man/Plot.FV.Single_Func.Rd +++ b/man/Plot.FV.Single_Func.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.FV.Single_Func} -\alias{Plot.FV.Single_Func} -\alias{Plot.FV.Single_Func.DataSetList} -\title{Plot lineplot of the expected function values of a DataSetList} -\usage{ -Plot.FV.Single_Func(dsList, RTstart = NULL, RTstop = NULL, - show.CI = F, show.mean = T, show.median = F, backend = NULL, - scale.xlog = F, scale.ylog = F, scale.reverse = F) - -\method{Plot.FV.Single_Func}{DataSetList}(dsList, RTstart = NULL, - RTstop = NULL, show.CI = F, show.mean = T, show.median = F, - backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{RTstart}{The starting runtime value.} - -\item{RTstop}{The final runtime value.} - -\item{show.CI}{Whether or not to show the standard deviations} - -\item{show.mean}{Whether or not to show the mean runtimes} - -\item{show.median}{Whether or not to show the median runtimes} - -\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} -} -\value{ -A plot of ERT-values of the DataSetList -} -\description{ -Plot lineplot of the expected function values of a DataSetList -} -\examples{ -Plot.FV.Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.FV.Single_Func} +\alias{Plot.FV.Single_Func} +\alias{Plot.FV.Single_Func.DataSetList} +\title{Plot lineplot of the expected function values of a DataSetList} +\usage{ +Plot.FV.Single_Func(dsList, RTstart = NULL, RTstop = NULL, + show.CI = F, show.mean = T, show.median = F, backend = NULL, + scale.xlog = F, scale.ylog = F, scale.reverse = F) + +\method{Plot.FV.Single_Func}{DataSetList}(dsList, RTstart = NULL, + RTstop = NULL, show.CI = F, show.mean = T, show.median = F, + backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{RTstart}{The starting runtime value.} + +\item{RTstop}{The final runtime value.} + +\item{show.CI}{Whether or not to show the standard deviations} + +\item{show.mean}{Whether or not to show the mean runtimes} + +\item{show.median}{Whether or not to show the median runtimes} + +\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} +} +\value{ +A plot of ERT-values of the DataSetList +} +\description{ +Plot lineplot of the expected function values of a DataSetList +} +\examples{ +Plot.FV.Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.Performviz.Rd b/man/Plot.Performviz.Rd index 16ef829d..bfe78c58 100644 --- a/man/Plot.Performviz.Rd +++ b/man/Plot.Performviz.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Performviz} -\alias{Plot.Performviz} -\title{Create the PerformViz plot} -\usage{ -Plot.Performviz(DSC_rank_result) -} -\arguments{ -\item{DSC_rank_result}{The result from a call to DSCtool rank service (`get_dsc_rank`)} -} -\value{ -A performviz plot -} -\description{ -From the paper: -} -\examples{ -\dontrun{ -Plot.Performviz(get_dsc_rank(dsl)) -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Performviz} +\alias{Plot.Performviz} +\title{Create the PerformViz plot} +\usage{ +Plot.Performviz(DSC_rank_result) +} +\arguments{ +\item{DSC_rank_result}{The result from a call to DSCtool rank service (`get_dsc_rank`)} +} +\value{ +A performviz plot +} +\description{ +From the paper: +} +\examples{ +\dontrun{ +Plot.Performviz(get_dsc_rank(dsl)) +} +} diff --git a/man/Plot.RT.Aggregated.Rd b/man/Plot.RT.Aggregated.Rd index 0a2e62c5..98aa73f3 100644 --- a/man/Plot.RT.Aggregated.Rd +++ b/man/Plot.RT.Aggregated.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Aggregated} -\alias{Plot.RT.Aggregated} -\alias{Plot.RT.Aggregated.DataSetList} -\title{Plot ERT-based comparison over multiple functions or dimensions} -\usage{ -Plot.RT.Aggregated(dsList, aggr_on = "funcId", targets = NULL, - plot_mode = "radar", use_rank = F, scale.ylog = T, maximize = T, - erts = NULL, inf.action = "overlap") - -\method{Plot.RT.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", - targets = NULL, plot_mode = "radar", use_rank = F, - scale.ylog = T, maximize = T, erts = NULL, - inf.action = "overlap") -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} - -\item{targets}{Custom list of function-value targets, one for each function or dimension.} - -\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} - -\item{use_rank}{Wheter to use a ranking system. If False, the actual ERT-values will be used.} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{maximize}{Wheter or not to the data is of a maximization problem} - -\item{erts}{Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function -of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating -this plot.} - -\item{inf.action}{How to handle infinite ERTs ('overlap' or 'jitter')} -} -\value{ -A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList -} -\description{ -Plot ERT-based comparison over multiple functions or dimensions -} -\examples{ -Plot.RT.Aggregated(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Aggregated} +\alias{Plot.RT.Aggregated} +\alias{Plot.RT.Aggregated.DataSetList} +\title{Plot ERT-based comparison over multiple functions or dimensions} +\usage{ +Plot.RT.Aggregated(dsList, aggr_on = "funcId", targets = NULL, + plot_mode = "radar", use_rank = F, scale.ylog = T, maximize = T, + erts = NULL, inf.action = "overlap") + +\method{Plot.RT.Aggregated}{DataSetList}(dsList, aggr_on = "funcId", + targets = NULL, plot_mode = "radar", use_rank = F, + scale.ylog = T, maximize = T, erts = NULL, + inf.action = "overlap") +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{aggr_on}{Whether to compare on functions ('funcId') or dimensions ('DIM')} + +\item{targets}{Custom list of function-value targets, one for each function or dimension.} + +\item{plot_mode}{How the plots should be created. Can be 'line' or 'radar'} + +\item{use_rank}{Wheter to use a ranking system. If False, the actual ERT-values will be used.} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{maximize}{Wheter or not to the data is of a maximization problem} + +\item{erts}{Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function +of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating +this plot.} + +\item{inf.action}{How to handle infinite ERTs ('overlap' or 'jitter')} +} +\value{ +A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList +} +\description{ +Plot ERT-based comparison over multiple functions or dimensions +} +\examples{ +Plot.RT.Aggregated(dsl) +} diff --git a/man/Plot.RT.ECDF_AUC.Rd b/man/Plot.RT.ECDF_AUC.Rd index d047fd5d..767f1cc6 100644 --- a/man/Plot.RT.ECDF_AUC.Rd +++ b/man/Plot.RT.ECDF_AUC.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_AUC} -\alias{Plot.RT.ECDF_AUC} -\alias{Plot.RT.ECDF_AUC.DataSetList} -\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} -\usage{ -Plot.RT.ECDF_AUC(dsList, fstart = NULL, fstop = NULL, fstep = NULL, - fval_formatter = as.integer) - -\method{Plot.RT.ECDF_AUC}{DataSetList}(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, fval_formatter = as.integer) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{fstart}{The starting function value} - -\item{fstop}{The final function value} - -\item{fstep}{The spacing between starting and final function values} - -\item{fval_formatter}{Function to format the function-value labels} -} -\value{ -A radarplot of the area under the aggregated ECDF-curve of the DataSetList -} -\description{ -Radarplot of the area under the aggregated ECDF-curve of a DataSetList. -} -\examples{ -Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_AUC} +\alias{Plot.RT.ECDF_AUC} +\alias{Plot.RT.ECDF_AUC.DataSetList} +\title{Radarplot of the area under the aggregated ECDF-curve of a DataSetList.} +\usage{ +Plot.RT.ECDF_AUC(dsList, fstart = NULL, fstop = NULL, fstep = NULL, + fval_formatter = as.integer) + +\method{Plot.RT.ECDF_AUC}{DataSetList}(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, fval_formatter = as.integer) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{fstart}{The starting function value} + +\item{fstop}{The final function value} + +\item{fstep}{The spacing between starting and final function values} + +\item{fval_formatter}{Function to format the function-value labels} +} +\value{ +A radarplot of the area under the aggregated ECDF-curve of the DataSetList +} +\description{ +Radarplot of the area under the aggregated ECDF-curve of a DataSetList. +} +\examples{ +Plot.RT.ECDF_AUC(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.RT.ECDF_Multi_Func.Rd b/man/Plot.RT.ECDF_Multi_Func.Rd index 27c19799..86442f78 100644 --- a/man/Plot.RT.ECDF_Multi_Func.Rd +++ b/man/Plot.RT.ECDF_Multi_Func.Rd @@ -1,32 +1,32 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_Multi_Func} -\alias{Plot.RT.ECDF_Multi_Func} -\alias{Plot.RT.ECDF_Multi_Func.DataSetList} -\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList. Aggregated over multiple functions or dimensions.} -\usage{ -Plot.RT.ECDF_Multi_Func(dsList, targets = NULL, scale.xlog = F) - -\method{Plot.RT.ECDF_Multi_Func}{DataSetList}(dsList, targets = NULL, - scale.xlog = F) -} -\arguments{ -\item{dsList}{A DataSetList.} - -\item{targets}{The target function values. Specified in a data.frame, as can be generated} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically -by the function 'get_ECDF_targets'} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the running times of the DataSetList -} -\description{ -Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList. Aggregated over multiple functions or dimensions. -} -\examples{ -Plot.RT.ECDF_Multi_Func(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_Multi_Func} +\alias{Plot.RT.ECDF_Multi_Func} +\alias{Plot.RT.ECDF_Multi_Func.DataSetList} +\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList. Aggregated over multiple functions or dimensions.} +\usage{ +Plot.RT.ECDF_Multi_Func(dsList, targets = NULL, scale.xlog = F) + +\method{Plot.RT.ECDF_Multi_Func}{DataSetList}(dsList, targets = NULL, + scale.xlog = F) +} +\arguments{ +\item{dsList}{A DataSetList.} + +\item{targets}{The target function values. Specified in a data.frame, as can be generated} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically +by the function 'get_ECDF_targets'} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the running times of the DataSetList +} +\description{ +Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList. Aggregated over multiple functions or dimensions. +} +\examples{ +Plot.RT.ECDF_Multi_Func(dsl) +} diff --git a/man/Plot.RT.ECDF_Per_Target.Rd b/man/Plot.RT.ECDF_Per_Target.Rd index e2668271..0d652fac 100644 --- a/man/Plot.RT.ECDF_Per_Target.Rd +++ b/man/Plot.RT.ECDF_Per_Target.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_Per_Target} -\alias{Plot.RT.ECDF_Per_Target} -\alias{Plot.RT.ECDF_Per_Target.DataSetList} -\title{Plot the empirical cumulative distriburtion as a function of the running times of -a DataSetList at certain target function values} -\usage{ -Plot.RT.ECDF_Per_Target(dsList, ftargets, scale.xlog = F) - -\method{Plot.RT.ECDF_Per_Target}{DataSetList}(dsList, ftargets, - scale.xlog = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftargets}{The target function values} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the running times of the DataSetList at the target function values -} -\description{ -Plot the empirical cumulative distriburtion as a function of the running times of -a DataSetList at certain target function values -} -\examples{ -Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_Per_Target} +\alias{Plot.RT.ECDF_Per_Target} +\alias{Plot.RT.ECDF_Per_Target.DataSetList} +\title{Plot the empirical cumulative distriburtion as a function of the running times of +a DataSetList at certain target function values} +\usage{ +Plot.RT.ECDF_Per_Target(dsList, ftargets, scale.xlog = F) + +\method{Plot.RT.ECDF_Per_Target}{DataSetList}(dsList, ftargets, + scale.xlog = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftargets}{The target function values} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the running times of the DataSetList at the target function values +} +\description{ +Plot the empirical cumulative distriburtion as a function of the running times of +a DataSetList at certain target function values +} +\examples{ +Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14) +} diff --git a/man/Plot.RT.ECDF_Single_Func.Rd b/man/Plot.RT.ECDF_Single_Func.Rd index 4c9bac6c..5174e043 100644 --- a/man/Plot.RT.ECDF_Single_Func.Rd +++ b/man/Plot.RT.ECDF_Single_Func.Rd @@ -1,38 +1,38 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.ECDF_Single_Func} -\alias{Plot.RT.ECDF_Single_Func} -\alias{Plot.RT.ECDF_Single_Func.DataSetList} -\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList.} -\usage{ -Plot.RT.ECDF_Single_Func(dsList, fstart = NULL, fstop = NULL, - fstep = NULL, show.per_target = F, scale.xlog = F) - -\method{Plot.RT.ECDF_Single_Func}{DataSetList}(dsList, fstart = NULL, - fstop = NULL, fstep = NULL, show.per_target = F, scale.xlog = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{fstart}{The starting function value} - -\item{fstop}{The final function value} - -\item{fstep}{The spacing between starting and final function values} - -\item{show.per_target}{Whether or not to show the individual ECDF-curves for each target} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} -} -\value{ -A plot of the empirical cumulative distriburtion as a function of -the running times of the DataSetList -} -\description{ -Plot the aggregated empirical cumulative distriburtion as a function of the running times of -a DataSetList. -} -\examples{ -Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.ECDF_Single_Func} +\alias{Plot.RT.ECDF_Single_Func} +\alias{Plot.RT.ECDF_Single_Func.DataSetList} +\title{Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList.} +\usage{ +Plot.RT.ECDF_Single_Func(dsList, fstart = NULL, fstop = NULL, + fstep = NULL, show.per_target = F, scale.xlog = F) + +\method{Plot.RT.ECDF_Single_Func}{DataSetList}(dsList, fstart = NULL, + fstop = NULL, fstep = NULL, show.per_target = F, scale.xlog = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{fstart}{The starting function value} + +\item{fstop}{The final function value} + +\item{fstep}{The spacing between starting and final function values} + +\item{show.per_target}{Whether or not to show the individual ECDF-curves for each target} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} +} +\value{ +A plot of the empirical cumulative distriburtion as a function of +the running times of the DataSetList +} +\description{ +Plot the aggregated empirical cumulative distriburtion as a function of the running times of +a DataSetList. +} +\examples{ +Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.RT.Histogram.Rd b/man/Plot.RT.Histogram.Rd index 5a839b84..6d733180 100644 --- a/man/Plot.RT.Histogram.Rd +++ b/man/Plot.RT.Histogram.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Histogram} -\alias{Plot.RT.Histogram} -\alias{Plot.RT.Histogram.DataSetList} -\title{Plot histograms of the runtimes of a DataSetList at a certain target function value} -\usage{ -Plot.RT.Histogram(dsList, ftarget, plot_mode = "overlay", - use.equal.bins = F) - -\method{Plot.RT.Histogram}{DataSetList}(dsList, ftarget, - plot_mode = "overlay", use.equal.bins = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value.} - -\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either -'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} - -\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual -bin sizes for each algorithm} -} -\value{ -A plot of the histograms of the runtimes at a the - target function value of the DataSetList -} -\description{ -Plot histograms of the runtimes of a DataSetList at a certain target function value -} -\examples{ -Plot.RT.Histogram(subset(dsl, funcId == 1), 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Histogram} +\alias{Plot.RT.Histogram} +\alias{Plot.RT.Histogram.DataSetList} +\title{Plot histograms of the runtimes of a DataSetList at a certain target function value} +\usage{ +Plot.RT.Histogram(dsList, ftarget, plot_mode = "overlay", + use.equal.bins = F) + +\method{Plot.RT.Histogram}{DataSetList}(dsList, ftarget, + plot_mode = "overlay", use.equal.bins = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value.} + +\item{plot_mode}{How to plot the different hisograms for each algorithm. Can be either +'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.} + +\item{use.equal.bins}{Whether to determine one bin size for all plots or have individual +bin sizes for each algorithm} +} +\value{ +A plot of the histograms of the runtimes at a the + target function value of the DataSetList +} +\description{ +Plot histograms of the runtimes of a DataSetList at a certain target function value +} +\examples{ +Plot.RT.Histogram(subset(dsl, funcId == 1), 14) +} diff --git a/man/Plot.RT.Multi_Func.Rd b/man/Plot.RT.Multi_Func.Rd index 1121a3ae..9dd528fb 100644 --- a/man/Plot.RT.Multi_Func.Rd +++ b/man/Plot.RT.Multi_Func.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Multi_Func} -\alias{Plot.RT.Multi_Func} -\alias{Plot.RT.Multi_Func.DataSetList} -\title{Plot ERT-plots for multiple functions or dimensions} -\usage{ -Plot.RT.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, - scale.reverse = F, backend = NULL) - -\method{Plot.RT.Multi_Func}{DataSetList}(dsList, scale.xlog = F, - scale.ylog = F, scale.reverse = F, backend = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function OR dimension).} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} - -\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} -} -\value{ -A plot of ERT-values of the DataSetList -} -\description{ -Plot ERT-plots for multiple functions or dimensions -} -\examples{ -Plot.RT.Multi_Func(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Multi_Func} +\alias{Plot.RT.Multi_Func} +\alias{Plot.RT.Multi_Func.DataSetList} +\title{Plot ERT-plots for multiple functions or dimensions} +\usage{ +Plot.RT.Multi_Func(dsList, scale.xlog = F, scale.ylog = F, + scale.reverse = F, backend = NULL) + +\method{Plot.RT.Multi_Func}{DataSetList}(dsList, scale.xlog = F, + scale.ylog = F, scale.reverse = F, backend = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function OR dimension).} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} + +\item{backend}{Which plotting library to use. Either 'plotly' or 'ggplot2'.} +} +\value{ +A plot of ERT-values of the DataSetList +} +\description{ +Plot ERT-plots for multiple functions or dimensions +} +\examples{ +Plot.RT.Multi_Func(dsl) +} diff --git a/man/Plot.RT.PMF.Rd b/man/Plot.RT.PMF.Rd index 8d270f5d..aa6c9db6 100644 --- a/man/Plot.RT.PMF.Rd +++ b/man/Plot.RT.PMF.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.PMF} -\alias{Plot.RT.PMF} -\alias{Plot.RT.PMF.DataSetList} -\title{Plot probablity mass function of the runtimes of a DataSetList at a certain target function value} -\usage{ -Plot.RT.PMF(dsList, ftarget, show.sample = F, scale.ylog = F, - backend = NULL) - -\method{Plot.RT.PMF}{DataSetList}(dsList, ftarget, show.sample = F, - scale.ylog = F, backend = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value.} - -\item{show.sample}{Whether or not to show the individual runtime samples} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} -} -\value{ -A plot of the probablity mass function of the runtimes at a the - target function value of the DataSetList -} -\description{ -Plot probablity mass function of the runtimes of a DataSetList at a certain target function value -} -\examples{ -Plot.RT.PMF(subset(dsl, funcId == 1), 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.PMF} +\alias{Plot.RT.PMF} +\alias{Plot.RT.PMF.DataSetList} +\title{Plot probablity mass function of the runtimes of a DataSetList at a certain target function value} +\usage{ +Plot.RT.PMF(dsList, ftarget, show.sample = F, scale.ylog = F, + backend = NULL) + +\method{Plot.RT.PMF}{DataSetList}(dsList, ftarget, show.sample = F, + scale.ylog = F, backend = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value.} + +\item{show.sample}{Whether or not to show the individual runtime samples} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} +} +\value{ +A plot of the probablity mass function of the runtimes at a the + target function value of the DataSetList +} +\description{ +Plot probablity mass function of the runtimes of a DataSetList at a certain target function value +} +\examples{ +Plot.RT.PMF(subset(dsl, funcId == 1), 14) +} diff --git a/man/Plot.RT.Parameters.Rd b/man/Plot.RT.Parameters.Rd index 1ed81362..9fc96bba 100644 --- a/man/Plot.RT.Parameters.Rd +++ b/man/Plot.RT.Parameters.Rd @@ -1,46 +1,46 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Parameters} -\alias{Plot.RT.Parameters} -\alias{Plot.RT.Parameters.DataSetList} -\title{Plot the parameter values recorded in a DataSetList (aligned by funcion value)} -\usage{ -Plot.RT.Parameters(dsList, f_min = NULL, f_max = NULL, - algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, - show.mean = T, show.median = F, show.CI = F) - -\method{Plot.RT.Parameters}{DataSetList}(dsList, f_min = NULL, - f_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, - scale.ylog = F, show.mean = T, show.median = F, show.CI = F) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{f_min}{The starting function value.} - -\item{f_max}{The final function value.} - -\item{algids}{Which algorithms from dsList to use} - -\item{par_name}{Which parameters to create plots for; set to NULL to use all -parameters found in dsList.} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{show.mean}{Whether or not to show the mean parameter values} - -\item{show.median}{Whether or not to show the median parameter values} - -\item{show.CI}{Whether or not to show the standard deviation} -} -\value{ -A plot of for every recorded parameter in the DataSetList -} -\description{ -Plot the parameter values recorded in a DataSetList (aligned by funcion value) -} -\examples{ -Plot.RT.Parameters(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Parameters} +\alias{Plot.RT.Parameters} +\alias{Plot.RT.Parameters.DataSetList} +\title{Plot the parameter values recorded in a DataSetList (aligned by funcion value)} +\usage{ +Plot.RT.Parameters(dsList, f_min = NULL, f_max = NULL, + algids = "all", par_name = NULL, scale.xlog = F, scale.ylog = F, + show.mean = T, show.median = F, show.CI = F) + +\method{Plot.RT.Parameters}{DataSetList}(dsList, f_min = NULL, + f_max = NULL, algids = "all", par_name = NULL, scale.xlog = F, + scale.ylog = F, show.mean = T, show.median = F, show.CI = F) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{f_min}{The starting function value.} + +\item{f_max}{The final function value.} + +\item{algids}{Which algorithms from dsList to use} + +\item{par_name}{Which parameters to create plots for; set to NULL to use all +parameters found in dsList.} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{show.mean}{Whether or not to show the mean parameter values} + +\item{show.median}{Whether or not to show the median parameter values} + +\item{show.CI}{Whether or not to show the standard deviation} +} +\value{ +A plot of for every recorded parameter in the DataSetList +} +\description{ +Plot the parameter values recorded in a DataSetList (aligned by funcion value) +} +\examples{ +Plot.RT.Parameters(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.RT.Single_Func.Rd b/man/Plot.RT.Single_Func.Rd index 2d5bc0b2..9577a353 100644 --- a/man/Plot.RT.Single_Func.Rd +++ b/man/Plot.RT.Single_Func.Rd @@ -1,53 +1,53 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.RT.Single_Func} -\alias{Plot.RT.Single_Func} -\alias{Plot.RT.Single_Func.DataSetList} -\title{Plot lineplot of the ERTs of a DataSetList} -\usage{ -Plot.RT.Single_Func(dsList, Fstart = NULL, Fstop = NULL, - show.ERT = T, show.CI = F, show.mean = F, show.median = F, - backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F, includeOpts = F, p = NULL) - -\method{Plot.RT.Single_Func}{DataSetList}(dsList, Fstart = NULL, - Fstop = NULL, show.ERT = T, show.CI = T, show.mean = F, - show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, - scale.reverse = F, includeOpts = F, p = NULL) -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{Fstart}{The starting function value.} - -\item{Fstop}{The final function value.} - -\item{show.ERT}{Whether or not to show the ERT-values} - -\item{show.CI}{Whether or not to show the standard deviations} - -\item{show.mean}{Whether or not to show the mean hitting times} - -\item{show.median}{Whether or not to show the median hitting times} - -\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} - -\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} - -\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} - -\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} - -\item{includeOpts}{Whether or not to include all best points reached by each algorithm} - -\item{p}{Existing plot to which to add the current data} -} -\value{ -A plot of ERT-values of the DataSetList -} -\description{ -Plot lineplot of the ERTs of a DataSetList -} -\examples{ -Plot.RT.Single_Func(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.RT.Single_Func} +\alias{Plot.RT.Single_Func} +\alias{Plot.RT.Single_Func.DataSetList} +\title{Plot lineplot of the ERTs of a DataSetList} +\usage{ +Plot.RT.Single_Func(dsList, Fstart = NULL, Fstop = NULL, + show.ERT = T, show.CI = F, show.mean = F, show.median = F, + backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F, includeOpts = F, p = NULL) + +\method{Plot.RT.Single_Func}{DataSetList}(dsList, Fstart = NULL, + Fstop = NULL, show.ERT = T, show.CI = T, show.mean = F, + show.median = F, backend = NULL, scale.xlog = F, scale.ylog = F, + scale.reverse = F, includeOpts = F, p = NULL) +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{Fstart}{The starting function value.} + +\item{Fstop}{The final function value.} + +\item{show.ERT}{Whether or not to show the ERT-values} + +\item{show.CI}{Whether or not to show the standard deviations} + +\item{show.mean}{Whether or not to show the mean hitting times} + +\item{show.median}{Whether or not to show the median hitting times} + +\item{backend}{Which plotting library to use. Can be 'plotly' or 'ggplot2'} + +\item{scale.xlog}{Whether or not to scale the x-axis logaritmically} + +\item{scale.ylog}{Whether or not to scale the y-axis logaritmically} + +\item{scale.reverse}{Wheter or not to reverse the x-axis (when using minimization)} + +\item{includeOpts}{Whether or not to include all best points reached by each algorithm} + +\item{p}{Existing plot to which to add the current data} +} +\value{ +A plot of ERT-values of the DataSetList +} +\description{ +Plot lineplot of the ERTs of a DataSetList +} +\examples{ +Plot.RT.Single_Func(subset(dsl, funcId == 1)) +} diff --git a/man/Plot.Stats.Glicko2_Candlestick.Rd b/man/Plot.Stats.Glicko2_Candlestick.Rd index 25f6d212..cf54c63e 100644 --- a/man/Plot.Stats.Glicko2_Candlestick.Rd +++ b/man/Plot.Stats.Glicko2_Candlestick.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Stats.Glicko2_Candlestick} -\alias{Plot.Stats.Glicko2_Candlestick} -\alias{Plot.Stats.Glicko2_Candlestick.DataSetList} -\title{Create a candlestick plot of Glicko2-rankings} -\usage{ -Plot.Stats.Glicko2_Candlestick(dsList, nr_rounds = 100, - glicko2_rank_df = NULL, which = "by_FV", target_dt = NULL) - -\method{Plot.Stats.Glicko2_Candlestick}{DataSetList}(dsList, - nr_rounds = 100, glicko2_rank_df = NULL, which = "by_FV", - target_dt = NULL) -} -\arguments{ -\item{dsList}{A DataSetList} - -\item{nr_rounds}{The number of rounds in the tournament} - -\item{glicko2_rank_df}{Optional. Dataframe containing the glicko2 rating to avoid needless recalculation.} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} - -\item{target_dt}{Optional: data table containing the targets for each function and dimension} -} -\description{ -Create a candlestick plot of Glicko2-rankings -} -\examples{ -Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Stats.Glicko2_Candlestick} +\alias{Plot.Stats.Glicko2_Candlestick} +\alias{Plot.Stats.Glicko2_Candlestick.DataSetList} +\title{Create a candlestick plot of Glicko2-rankings} +\usage{ +Plot.Stats.Glicko2_Candlestick(dsList, nr_rounds = 100, + glicko2_rank_df = NULL, which = "by_FV", target_dt = NULL) + +\method{Plot.Stats.Glicko2_Candlestick}{DataSetList}(dsList, + nr_rounds = 100, glicko2_rank_df = NULL, which = "by_FV", + target_dt = NULL) +} +\arguments{ +\item{dsList}{A DataSetList} + +\item{nr_rounds}{The number of rounds in the tournament} + +\item{glicko2_rank_df}{Optional. Dataframe containing the glicko2 rating to avoid needless recalculation.} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} + +\item{target_dt}{Optional: data table containing the targets for each function and dimension} +} +\description{ +Create a candlestick plot of Glicko2-rankings +} +\examples{ +Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2) +} diff --git a/man/Plot.Stats.Significance_Graph.Rd b/man/Plot.Stats.Significance_Graph.Rd index 6894fce3..5c77551c 100644 --- a/man/Plot.Stats.Significance_Graph.Rd +++ b/man/Plot.Stats.Significance_Graph.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Stats.Significance_Graph} -\alias{Plot.Stats.Significance_Graph} -\alias{Plot.Stats.Significance_Graph.DataSetList} -\title{Plot a network graph showing the statistically different algorithms} -\usage{ -Plot.Stats.Significance_Graph(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = "by_FV") - -\method{Plot.Stats.Significance_Graph}{DataSetList}(dsList, ftarget, - alpha = 0.01, bootstrap.size = 30, which = "by_FV") -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value to use} - -\item{alpha}{The cutoff for statistical significance} - -\item{bootstrap.size}{The amound of bootstrapped samples used} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} -} -\value{ -A graph showing the statistical significance between algorithms -} -\description{ -Plot a network graph showing the statistically different algorithms -} -\examples{ -Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Stats.Significance_Graph} +\alias{Plot.Stats.Significance_Graph} +\alias{Plot.Stats.Significance_Graph.DataSetList} +\title{Plot a network graph showing the statistically different algorithms} +\usage{ +Plot.Stats.Significance_Graph(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = "by_FV") + +\method{Plot.Stats.Significance_Graph}{DataSetList}(dsList, ftarget, + alpha = 0.01, bootstrap.size = 30, which = "by_FV") +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value to use} + +\item{alpha}{The cutoff for statistical significance} + +\item{bootstrap.size}{The amound of bootstrapped samples used} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} +} +\value{ +A graph showing the statistical significance between algorithms +} +\description{ +Plot a network graph showing the statistically different algorithms +} +\examples{ +Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16) +} diff --git a/man/Plot.Stats.Significance_Heatmap.Rd b/man/Plot.Stats.Significance_Heatmap.Rd index 9c3693e0..e50c1880 100644 --- a/man/Plot.Stats.Significance_Heatmap.Rd +++ b/man/Plot.Stats.Significance_Heatmap.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{Plot.Stats.Significance_Heatmap} -\alias{Plot.Stats.Significance_Heatmap} -\alias{Plot.Stats.Significance_Heatmap.DataSetList} -\title{Plot a heatmap showing the statistically different algorithms} -\usage{ -Plot.Stats.Significance_Heatmap(dsList, ftarget, alpha = 0.01, - bootstrap.size = 30, which = "by_FV") - -\method{Plot.Stats.Significance_Heatmap}{DataSetList}(dsList, ftarget, - alpha = 0.01, bootstrap.size = 30, which = "by_FV") -} -\arguments{ -\item{dsList}{A DataSetList (should consist of only one function and dimension).} - -\item{ftarget}{The target function value to use} - -\item{alpha}{The cutoff for statistical significance} - -\item{bootstrap.size}{The amound of bootstrapped samples used} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} -} -\value{ -A heatmap showing the statistical significance between algorithms -} -\description{ -Plot a heatmap showing the statistically different algorithms -} -\examples{ -Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{Plot.Stats.Significance_Heatmap} +\alias{Plot.Stats.Significance_Heatmap} +\alias{Plot.Stats.Significance_Heatmap.DataSetList} +\title{Plot a heatmap showing the statistically different algorithms} +\usage{ +Plot.Stats.Significance_Heatmap(dsList, ftarget, alpha = 0.01, + bootstrap.size = 30, which = "by_FV") + +\method{Plot.Stats.Significance_Heatmap}{DataSetList}(dsList, ftarget, + alpha = 0.01, bootstrap.size = 30, which = "by_FV") +} +\arguments{ +\item{dsList}{A DataSetList (should consist of only one function and dimension).} + +\item{ftarget}{The target function value to use} + +\item{alpha}{The cutoff for statistical significance} + +\item{bootstrap.size}{The amound of bootstrapped samples used} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} +} +\value{ +A heatmap showing the statistical significance between algorithms +} +\description{ +Plot a heatmap showing the statistically different algorithms +} +\examples{ +Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16) +} diff --git a/man/SP.Rd b/man/SP.Rd index c202d82e..cb6552d0 100644 --- a/man/SP.Rd +++ b/man/SP.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{SP} -\alias{SP} -\title{Estimator 'SP' for the Expected Running Time (ERT)} -\usage{ -SP(data, max_runtime) -} -\arguments{ -\item{data}{A dataframe or matrix. Each row stores the runtime sample points from -several runs} - -\item{max_runtime}{The budget to use for calculating ERT. If this is a vector, the largest value is taken. -Using this as a vector is being deprecated, and will be removed in a future update} -} -\value{ -A list containing ERTs, number of succesfull runs and the succes rate -} -\description{ -Estimator 'SP' for the Expected Running Time (ERT) -} -\examples{ -SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{SP} +\alias{SP} +\title{Estimator 'SP' for the Expected Running Time (ERT)} +\usage{ +SP(data, max_runtime) +} +\arguments{ +\item{data}{A dataframe or matrix. Each row stores the runtime sample points from +several runs} + +\item{max_runtime}{The budget to use for calculating ERT. If this is a vector, the largest value is taken. +Using this as a vector is being deprecated, and will be removed in a future update} +} +\value{ +A list containing ERTs, number of succesfull runs and the succes rate +} +\description{ +Estimator 'SP' for the Expected Running Time (ERT) +} +\examples{ +SP(dsl[[1]]$RT, max(dsl[[1]]$RT)) +} diff --git a/man/arrange.Rd b/man/arrange.Rd index 2f4ae67c..c36ce771 100644 --- a/man/arrange.Rd +++ b/man/arrange.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{arrange} -\alias{arrange} -\alias{arrange.DataSetList} -\title{S3 sort function for DataSetList} -\usage{ -arrange(dsl, ...) - -\method{arrange}{DataSetList}(dsl, ...) -} -\arguments{ -\item{dsl}{The DataSetList to sort} - -\item{...}{attribute by which `dsl` is sorted. Multiple attributes can be specified.} -} -\description{ -Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). -Default is as ascending, can be made descending by adding a - in front of the attribute. -Sorting accross multiple attributes is supported, in the order they are specified. -} -\examples{ -arrange(dsl, DIM, -funcId, algId) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{arrange} +\alias{arrange} +\alias{arrange.DataSetList} +\title{S3 sort function for DataSetList} +\usage{ +arrange(dsl, ...) + +\method{arrange}{DataSetList}(dsl, ...) +} +\arguments{ +\item{dsl}{The DataSetList to sort} + +\item{...}{attribute by which `dsl` is sorted. Multiple attributes can be specified.} +} +\description{ +Sorts a DataSetList based on the custom specified attributes ('algId', 'DIM' or 'funcId'). +Default is as ascending, can be made descending by adding a - in front of the attribute. +Sorting accross multiple attributes is supported, in the order they are specified. +} +\examples{ +arrange(dsl, DIM, -funcId, algId) +} diff --git a/man/as.character.DataSet.Rd b/man/as.character.DataSet.Rd index 9329de82..84a75c24 100644 --- a/man/as.character.DataSet.Rd +++ b/man/as.character.DataSet.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{as.character.DataSet} -\alias{as.character.DataSet} -\title{S3 generic as.character operator for DataSet} -\usage{ -\method{as.character}{DataSet}(x, verbose = F, ...) -} -\arguments{ -\item{x}{A DataSet object} - -\item{verbose}{Verbose mode, currently not implemented} - -\item{...}{Arguments passed to other methods} -} -\value{ -A short description of the DataSet -} -\description{ -S3 generic as.character operator for DataSet -} -\examples{ -as.character(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{as.character.DataSet} +\alias{as.character.DataSet} +\title{S3 generic as.character operator for DataSet} +\usage{ +\method{as.character}{DataSet}(x, verbose = F, ...) +} +\arguments{ +\item{x}{A DataSet object} + +\item{verbose}{Verbose mode, currently not implemented} + +\item{...}{Arguments passed to other methods} +} +\value{ +A short description of the DataSet +} +\description{ +S3 generic as.character operator for DataSet +} +\examples{ +as.character(dsl[[1]]) +} diff --git a/man/bootstrap_RT.Rd b/man/bootstrap_RT.Rd index 416a7da5..4ff38ccf 100644 --- a/man/bootstrap_RT.Rd +++ b/man/bootstrap_RT.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{bootstrap_RT} -\alias{bootstrap_RT} -\title{Bootstrapping for running time samples} -\usage{ -bootstrap_RT(x, max_eval, bootstrap.size) -} -\arguments{ -\item{x}{A numeric vector. A sample of the running time.} - -\item{max_eval}{A numeric vector, containing the maximal running time in -each run. It should have the same size as x} - -\item{bootstrap.size}{integer, the size of the bootstrapped sample} -} -\value{ -A numeric vector of the bootstrapped running time sample -} -\description{ -Bootstrapping for running time samples -} -\examples{ -ds <- dsl[[1]] -x <- get_RT_sample(ds, ftarget = 16, output = 'long') -max_eval <- get_maxRT(dsl, output = 'long') -bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{bootstrap_RT} +\alias{bootstrap_RT} +\title{Bootstrapping for running time samples} +\usage{ +bootstrap_RT(x, max_eval, bootstrap.size) +} +\arguments{ +\item{x}{A numeric vector. A sample of the running time.} + +\item{max_eval}{A numeric vector, containing the maximal running time in +each run. It should have the same size as x} + +\item{bootstrap.size}{integer, the size of the bootstrapped sample} +} +\value{ +A numeric vector of the bootstrapped running time sample +} +\description{ +Bootstrapping for running time samples +} +\examples{ +ds <- dsl[[1]] +x <- get_RT_sample(ds, ftarget = 16, output = 'long') +max_eval <- get_maxRT(dsl, output = 'long') +bootstrap_RT(x$RT, max_eval$maxRT, bootstrap.size = 30) +} diff --git a/man/c.DataSet.Rd b/man/c.DataSet.Rd index 2c395bc7..af05b5a7 100644 --- a/man/c.DataSet.Rd +++ b/man/c.DataSet.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{c.DataSet} -\alias{c.DataSet} -\title{S3 concatenation function for DataSet} -\usage{ -\method{c}{DataSet}(...) -} -\arguments{ -\item{...}{The DataSets to concatenate} -} -\value{ -A new DataSet -} -\description{ -Concatenation for DataSets. Combines multiple runs from separate DataSets -into a single DataSet object if all provided arguments have the same dimension, function ID and -algorithm ID, and each contains only a single run. Currently does not support parameter tracking -} -\examples{ -c(dsl[[1]], dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{c.DataSet} +\alias{c.DataSet} +\title{S3 concatenation function for DataSet} +\usage{ +\method{c}{DataSet}(...) +} +\arguments{ +\item{...}{The DataSets to concatenate} +} +\value{ +A new DataSet +} +\description{ +Concatenation for DataSets. Combines multiple runs from separate DataSets +into a single DataSet object if all provided arguments have the same dimension, function ID and +algorithm ID, and each contains only a single run. Currently does not support parameter tracking +} +\examples{ +c(dsl[[1]], dsl[[1]]) +} diff --git a/man/c.DataSetList.Rd b/man/c.DataSetList.Rd index 246d46b4..14571fa2 100644 --- a/man/c.DataSetList.Rd +++ b/man/c.DataSetList.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{c.DataSetList} -\alias{c.DataSetList} -\title{S3 concatenation function for DataSetList} -\usage{ -\method{c}{DataSetList}(...) -} -\arguments{ -\item{...}{The DataSetLists to concatenate} -} -\value{ -A new DataSetList -} -\description{ -S3 concatenation function for DataSetList -} -\examples{ -c(dsl[1], dsl[3]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{c.DataSetList} +\alias{c.DataSetList} +\title{S3 concatenation function for DataSetList} +\usage{ +\method{c}{DataSetList}(...) +} +\arguments{ +\item{...}{The DataSetLists to concatenate} +} +\value{ +A new DataSetList +} +\description{ +S3 concatenation function for DataSetList +} +\examples{ +c(dsl[1], dsl[3]) +} diff --git a/man/cat.DataSet.Rd b/man/cat.DataSet.Rd index d4b4ce7c..6fa1905e 100644 --- a/man/cat.DataSet.Rd +++ b/man/cat.DataSet.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{cat.DataSet} -\alias{cat.DataSet} -\title{S3 generic cat operator for DataSet} -\usage{ -cat.DataSet(x) -} -\arguments{ -\item{x}{A DataSet object} -} -\value{ -A short description of the DataSet -} -\description{ -S3 generic cat operator for DataSet -} -\examples{ -cat.DataSet(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{cat.DataSet} +\alias{cat.DataSet} +\title{S3 generic cat operator for DataSet} +\usage{ +cat.DataSet(x) +} +\arguments{ +\item{x}{A DataSet object} +} +\value{ +A short description of the DataSet +} +\description{ +S3 generic cat operator for DataSet +} +\examples{ +cat.DataSet(dsl[[1]]) +} diff --git a/man/change_id.Rd b/man/change_id.Rd new file mode 100644 index 00000000..6b4033a2 --- /dev/null +++ b/man/change_id.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{change_id} +\alias{change_id} +\title{Add unique identifiers to each DataSet in the provided DataSetList based on static attributes} +\usage{ +change_id(dsl, attrs) +} +\arguments{ +\item{dsl}{The DataSetList} + +\item{attrs}{The list of attributes to combine into a unique identifier} +} +\value{ +A new DataSetList object where the split has been done based on the provided attributes, and the unique +identifier has been added. +} +\description{ +Note that this function returns a new DataSetList object, since a split into new datasetlist has to be done to +ensure each dataset has exactly one unique identifier. +Note that only static attributes (see `get_static_attributes`) can be used to create unique identifiers. +} +\examples{ +change_id(dsl, c('instance')) +} diff --git a/man/check_dsc_configured.Rd b/man/check_dsc_configured.Rd index e685f58f..1405e520 100644 --- a/man/check_dsc_configured.Rd +++ b/man/check_dsc_configured.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{check_dsc_configured} -\alias{check_dsc_configured} -\title{Verify that the credentials for DSCtool have been set} -\usage{ -check_dsc_configured() -} -\description{ -This uses the keyring package to store and load credentials. -If the keyring package does not exists, it will default to look for -a config-file in the 'repository'-folder, under your home directory. -This can be changed by setting the option IOHprofiler.config_dir -If you already have an account, please call `set_DSC_credentials` -with the corresponding username and password. -If you don't have an account, you can register for one using `register_DSC` -} -\examples{ -check_dsc_configured() -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{check_dsc_configured} +\alias{check_dsc_configured} +\title{Verify that the credentials for DSCtool have been set} +\usage{ +check_dsc_configured() +} +\description{ +This uses the keyring package to store and load credentials. +If the keyring package does not exists, it will default to look for +a config-file in the 'repository'-folder, under your home directory. +This can be changed by setting the option IOHprofiler.config_dir +If you already have an account, please call `set_DSC_credentials` +with the corresponding username and password. +If you don't have an account, you can register for one using `register_DSC` +} +\examples{ +check_dsc_configured() +} diff --git a/man/check_format.Rd b/man/check_format.Rd index 8e11463f..dccedf7e 100644 --- a/man/check_format.Rd +++ b/man/check_format.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{check_format} -\alias{check_format} -\title{Check the format of data} -\usage{ -check_format(path) -} -\arguments{ -\item{path}{The path to the folder to check} -} -\value{ -The format of the data in the given folder. Either 'COCO', 'IOHprofiler', -'NEVERGRAD' or 'SOS'. -} -\description{ -Throws a warning when multiple formats are found in the same folder. -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") -check_format(path) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{check_format} +\alias{check_format} +\title{Check the format of data} +\usage{ +check_format(path) +} +\arguments{ +\item{path}{The path to the folder to check} +} +\value{ +The format of the data in the given folder. Either 'COCO', 'IOHprofiler', +'NEVERGRAD' or 'SOS'. +} +\description{ +Throws a warning when multiple formats are found in the same folder. +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") +check_format(path) +} diff --git a/man/clean_DataSetList.Rd b/man/clean_DataSetList.Rd index b2518a86..8a8caf1b 100644 --- a/man/clean_DataSetList.Rd +++ b/man/clean_DataSetList.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{clean_DataSetList} -\alias{clean_DataSetList} -\title{Clean DataSetList object by concatenating DataSets} -\usage{ -clean_DataSetList(dsList) -} -\arguments{ -\item{dsList}{The DataSetList object to clean} -} -\description{ -Concatenates all DataSets with the same algorith name, function id and dimension -} -\examples{ -clean_DataSetList(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{clean_DataSetList} +\alias{clean_DataSetList} +\title{Clean DataSetList object by concatenating DataSets} +\usage{ +clean_DataSetList(dsList) +} +\arguments{ +\item{dsList}{The DataSetList object to clean} +} +\description{ +Concatenates all DataSets with the same ID, function id and dimension +} +\examples{ +clean_DataSetList(dsl) +} diff --git a/man/dsl.Rd b/man/dsl.Rd index 2358e21a..d2729899 100644 --- a/man/dsl.Rd +++ b/man/dsl.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/data.R -\docType{data} -\name{dsl} -\alias{dsl} -\title{Example DataSetList used in tests / examples} -\format{DataSetList} -\usage{ -dsl -} -\description{ -A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D -} -\examples{ -summary(dsl) -} -\keyword{datasets} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data.R +\docType{data} +\name{dsl} +\alias{dsl} +\title{Example DataSetList used in tests / examples} +\format{DataSetList} +\usage{ +dsl +} +\description{ +A DataSetList containing DataSets on 2 IOHProfiler functions from 2 algorithms in 16D +} +\examples{ +summary(dsl) +} +\keyword{datasets} diff --git a/man/dsl_large.Rd b/man/dsl_large.Rd index 88df6756..36637559 100644 --- a/man/dsl_large.Rd +++ b/man/dsl_large.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/data.R -\docType{data} -\name{dsl_large} -\alias{dsl_large} -\title{Larger example DataSetList used in tests / examples} -\format{DataSetList} -\usage{ -dsl_large -} -\description{ -A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D -} -\examples{ -summary(dsl_large) -} -\keyword{datasets} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data.R +\docType{data} +\name{dsl_large} +\alias{dsl_large} +\title{Larger example DataSetList used in tests / examples} +\format{DataSetList} +\usage{ +dsl_large +} +\description{ +A DataSetList containing DataSets on all IOHProfiler functions from 11 algorithms in 100D +} +\examples{ +summary(dsl_large) +} +\keyword{datasets} diff --git a/man/equals-.DataSet.Rd b/man/equals-.DataSet.Rd index 417eb711..a3b9513a 100644 --- a/man/equals-.DataSet.Rd +++ b/man/equals-.DataSet.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{==.DataSet} -\alias{==.DataSet} -\title{S3 generic == operator for DataSets} -\usage{ -\method{==}{DataSet}(dsL, dsR) -} -\arguments{ -\item{dsL}{A `DataSet` object} - -\item{dsR}{A `DataSet` object} -} -\value{ -True if the DataSets contain the same function, dimension and algorithm, -and have the exact same attributes -} -\description{ -S3 generic == operator for DataSets -} -\examples{ -dsl[[1]] == dsl[[2]] -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{==.DataSet} +\alias{==.DataSet} +\title{S3 generic == operator for DataSets} +\usage{ +\method{==}{DataSet}(dsL, dsR) +} +\arguments{ +\item{dsL}{A `DataSet` object} + +\item{dsR}{A `DataSet` object} +} +\value{ +True if the DataSets contain the same function, dimension and algorithm, +and have the exact same attributes +} +\description{ +S3 generic == operator for DataSets +} +\examples{ +dsl[[1]] == dsl[[2]] +} diff --git a/man/fast_RT_samples.Rd b/man/fast_RT_samples.Rd index 45c2cd4c..fa3c8a14 100644 --- a/man/fast_RT_samples.Rd +++ b/man/fast_RT_samples.Rd @@ -1,18 +1,18 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{fast_RT_samples} -\alias{fast_RT_samples} -\title{Function to get just the RT samples needed, without any formatting to improve speed} -\usage{ -fast_RT_samples(RT_mat, target, maximization = F) -} -\arguments{ -\item{RT_mat}{A matrix containing the RT-values of a dataset} - -\item{target}{Which target-value to use} - -\item{maximization}{Whether maximization is needed or not} -} -\description{ -Function to get just the RT samples needed, without any formatting to improve speed -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{fast_RT_samples} +\alias{fast_RT_samples} +\title{Function to get just the RT samples needed, without any formatting to improve speed} +\usage{ +fast_RT_samples(RT_mat, target, maximization = F) +} +\arguments{ +\item{RT_mat}{A matrix containing the RT-values of a dataset} + +\item{target}{Which target-value to use} + +\item{maximization}{Whether maximization is needed or not} +} +\description{ +Function to get just the RT samples needed, without any formatting to improve speed +} diff --git a/man/generate_data.AUC.Rd b/man/generate_data.AUC.Rd index 4dd79451..31956512 100644 --- a/man/generate_data.AUC.Rd +++ b/man/generate_data.AUC.Rd @@ -1,31 +1,31 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.AUC} -\alias{generate_data.AUC} -\title{Generate dataframe containing the AUC for any ECDF-curves} -\usage{ -generate_data.AUC(dsList, targets, scale_log = F, which = "by_RT", - dt_ecdf = NULL, multiple_x = FALSE) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale_log}{Whether to use logarithmic scaling or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{dt_ecdf}{A data table of the ECDF to avoid needless recomputations. Will take preference if it -is provided together with dsList and targets} - -\item{multiple_x}{Boolean, whether to get only the total AUC or get stepwise AUC values} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.AUC(dsl, get_ECDF_targets(dsl)) -generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.AUC} +\alias{generate_data.AUC} +\title{Generate dataframe containing the AUC for any ECDF-curves} +\usage{ +generate_data.AUC(dsList, targets, scale_log = F, which = "by_RT", + dt_ecdf = NULL, multiple_x = FALSE) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale_log}{Whether to use logarithmic scaling or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{dt_ecdf}{A data table of the ECDF to avoid needless recomputations. Will take preference if it +is provided together with dsList and targets} + +\item{multiple_x}{Boolean, whether to get only the total AUC or get stepwise AUC values} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.AUC(dsl, get_ECDF_targets(dsl)) +generate_data.AUC(NULL, NULL, dt_ecdf = generate_data.ECDF(dsl, get_ECDF_targets(dsl))) +} diff --git a/man/generate_data.Aggr.Rd b/man/generate_data.Aggr.Rd index afb0f11e..c7d23e05 100644 --- a/man/generate_data.Aggr.Rd +++ b/man/generate_data.Aggr.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.Aggr} -\alias{generate_data.Aggr} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.Aggr(dsList, aggr_on = "funcId", targets = NULL, - which = "by_RT") -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{aggr_on}{Which attribute to use for aggregation. Either 'funcId' or 'DIM'} - -\item{targets}{Optional list of target values (Runtime or target value)} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.Aggr(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.Aggr} +\alias{generate_data.Aggr} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.Aggr(dsList, aggr_on = "funcId", targets = NULL, + which = "by_RT") +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{aggr_on}{Which attribute to use for aggregation. Either 'funcId' or 'DIM'} + +\item{targets}{Optional list of target values (Runtime or target value)} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.Aggr(dsl) +} diff --git a/man/generate_data.ECDF.Rd b/man/generate_data.ECDF.Rd index dcf2df3a..fc1e5365 100644 --- a/man/generate_data.ECDF.Rd +++ b/man/generate_data.ECDF.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.ECDF} -\alias{generate_data.ECDF} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.ECDF(dsList, targets, scale_log = F, which = "by_RT", - use_full_range = TRUE) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{use_full_range}{Whether or not to use the full range of the x-axis or cut it off as soon as -all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.ECDF} +\alias{generate_data.ECDF} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.ECDF(dsList, targets, scale_log = F, which = "by_RT", + use_full_range = TRUE) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{use_full_range}{Whether or not to use the full range of the x-axis or cut it off as soon as +all algorithms reach 98\% success (+10\% buffer). Only supported in the case of one function and dimension} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) +} diff --git a/man/generate_data.ECDF_raw.Rd b/man/generate_data.ECDF_raw.Rd index 8ddbbc49..0c3ea4de 100644 --- a/man/generate_data.ECDF_raw.Rd +++ b/man/generate_data.ECDF_raw.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.ECDF_raw} -\alias{generate_data.ECDF_raw} -\title{Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of} -\usage{ -generate_data.ECDF_raw(dsList, targets, scale_log = F) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} -} -\description{ -This provides an unaggregated version of the function `generate_data.ECDF`. -} -\examples{ -generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.ECDF_raw} +\alias{generate_data.ECDF_raw} +\title{Generate dataframe of a the unaggregated values of individual algorithms. Stripped-down version of} +\usage{ +generate_data.ECDF_raw(dsList, targets, scale_log = F) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} +} +\description{ +This provides an unaggregated version of the function `generate_data.ECDF`. +} +\examples{ +generate_data.ECDF_raw(subset(dsl, funcId == 1), c(10, 15, 16)) +} diff --git a/man/generate_data.PMF.Rd b/man/generate_data.PMF.Rd index 5922249a..4d0d9127 100644 --- a/man/generate_data.PMF.Rd +++ b/man/generate_data.PMF.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.PMF} -\alias{generate_data.PMF} -\title{Generate dataframe of a single function/dimension pair for creating PDF or PMF plots} -\usage{ -generate_data.PMF(dsList, target, which = "by_RT") -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{target}{The target value (Runtime or target value)} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.PMF} +\alias{generate_data.PMF} +\title{Generate dataframe of a single function/dimension pair for creating PDF or PMF plots} +\usage{ +generate_data.PMF(dsList, target, which = "by_RT") +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{target}{The target value (Runtime or target value)} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +} diff --git a/man/generate_data.Parameters.Rd b/man/generate_data.Parameters.Rd index 85edb3a6..492b4c55 100644 --- a/man/generate_data.Parameters.Rd +++ b/man/generate_data.Parameters.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.Parameters} -\alias{generate_data.Parameters} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.Parameters(dsList, which = "by_RT", scale_log = F) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.Parameters(subset(dsl, funcId == 1)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.Parameters} +\alias{generate_data.Parameters} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.Parameters(dsList, which = "by_RT", scale_log = F) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.Parameters(subset(dsl, funcId == 1)) +} diff --git a/man/generate_data.Single_Function.Rd b/man/generate_data.Single_Function.Rd index 6c10b5bd..c71f6478 100644 --- a/man/generate_data.Single_Function.Rd +++ b/man/generate_data.Single_Function.Rd @@ -1,32 +1,32 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.Single_Function} -\alias{generate_data.Single_Function} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.Single_Function(dsList, start = NULL, stop = NULL, - scale_log = F, which = "by_RT", include_opts = F, budget = NULL) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{start}{Optional start value (Runtime or target value)} - -\item{stop}{Optional end value (Runtime or target value)} - -\item{scale_log}{Wheterh to use logarithmic scaling or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{include_opts}{Whether or not to also include the best value hit by each algorithm to -the generated datapoints} - -\item{budget}{Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works -in fixed_target mode.} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.Single_Function} +\alias{generate_data.Single_Function} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.Single_Function(dsList, start = NULL, stop = NULL, + scale_log = F, which = "by_RT", include_opts = F, budget = NULL) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{start}{Optional start value (Runtime or target value)} + +\item{stop}{Optional end value (Runtime or target value)} + +\item{scale_log}{Wheterh to use logarithmic scaling or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{include_opts}{Whether or not to also include the best value hit by each algorithm to +the generated datapoints} + +\item{budget}{Optional; overwrites the budget of each individual algorithm when doing ERT calculations. Only works +in fixed_target mode.} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +} diff --git a/man/generate_data.hist.Rd b/man/generate_data.hist.Rd index 93f215eb..7cd46dd8 100644 --- a/man/generate_data.hist.Rd +++ b/man/generate_data.hist.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{generate_data.hist} -\alias{generate_data.hist} -\title{Generate dataframe of a single function/dimension pair} -\usage{ -generate_data.hist(dsList, target, use.equal.bins = F, which = "by_RT") -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{target}{The target value (Runtime or target value)} - -\item{use.equal.bins}{Whether all bins should be equal size for each algorithm or not} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} -} -\description{ -This function generates a dataframe which can be easily plotted using the `plot_general_data`-function -} -\examples{ -generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{generate_data.hist} +\alias{generate_data.hist} +\title{Generate dataframe of a single function/dimension pair} +\usage{ +generate_data.hist(dsList, target, use.equal.bins = F, which = "by_RT") +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{target}{The target value (Runtime or target value)} + +\item{use.equal.bins}{Whether all bins should be equal size for each algorithm or not} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} +} +\description{ +This function generates a dataframe which can be easily plotted using the `plot_general_data`-function +} +\examples{ +generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') +} diff --git a/man/get_ECDF_targets.Rd b/man/get_ECDF_targets.Rd index 56fe681d..5d82a5d0 100644 --- a/man/get_ECDF_targets.Rd +++ b/man/get_ECDF_targets.Rd @@ -1,25 +1,25 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_ECDF_targets} -\alias{get_ECDF_targets} -\title{Generation of default ECDF-targets} -\usage{ -get_ECDF_targets(dsList, type = "log-linear", number_targets = 10) -} -\arguments{ -\item{dsList}{The DataSetList object for which to generate the targets} - -\item{type}{The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, -equal for all functions / dimensions)} - -\item{number_targets}{The amount of targets to generate} -} -\value{ -A data.table with 3 columns: funcId, DIM and target -} -\description{ -Generation of default ECDF-targets -} -\examples{ -get_ECDF_targets(dsl, 'linear', 10) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_ECDF_targets} +\alias{get_ECDF_targets} +\title{Generation of default ECDF-targets} +\usage{ +get_ECDF_targets(dsList, type = "log-linear", number_targets = 10) +} +\arguments{ +\item{dsList}{The DataSetList object for which to generate the targets} + +\item{type}{The way to generate the targets. Either 'log-linear', 'linear' or 'bbob' (51 fixed targets, +equal for all functions / dimensions)} + +\item{number_targets}{The amount of targets to generate} +} +\value{ +A data.table with 3 columns: funcId, DIM and target +} +\description{ +Generation of default ECDF-targets +} +\examples{ +get_ECDF_targets(dsl, 'linear', 10) +} diff --git a/man/get_ERT.Rd b/man/get_ERT.Rd index 1b2b333f..9e3ced27 100644 --- a/man/get_ERT.Rd +++ b/man/get_ERT.Rd @@ -1,37 +1,37 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_ERT} -\alias{get_ERT} -\alias{get_ERT.DataSet} -\alias{get_ERT.DataSetList} -\title{Get Expected RunTime} -\usage{ -get_ERT(ds, ftarget, budget, ...) - -\method{get_ERT}{DataSet}(ds, ftarget, budget = NULL, ...) - -\method{get_ERT}{DataSetList}(ds, ftarget, budget = NULL, - algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{ftarget}{The function target(s) for which to get the ERT} - -\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the runtime samples for each provided target -function value -} -\description{ -Get Expected RunTime -} -\examples{ -get_ERT(dsl, 14) -get_ERT(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_ERT} +\alias{get_ERT} +\alias{get_ERT.DataSet} +\alias{get_ERT.DataSetList} +\title{Get Expected RunTime} +\usage{ +get_ERT(ds, ftarget, budget, ...) + +\method{get_ERT}{DataSet}(ds, ftarget, budget = NULL, ...) + +\method{get_ERT}{DataSetList}(ds, ftarget, budget = NULL, + algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{ftarget}{The function target(s) for which to get the ERT} + +\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} + +\item{...}{Arguments passed to other methods} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the runtime samples for each provided target +function value +} +\description{ +Get Expected RunTime +} +\examples{ +get_ERT(dsl, 14) +get_ERT(dsl[[1]], 14) +} diff --git a/man/get_FV_overview.Rd b/man/get_FV_overview.Rd index a031cbf4..6ec007fe 100644 --- a/man/get_FV_overview.Rd +++ b/man/get_FV_overview.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_FV_overview} -\alias{get_FV_overview} -\alias{get_FV_overview.DataSet} -\alias{get_FV_overview.DataSetList} -\title{Get Function Value condensed overview} -\usage{ -get_FV_overview(ds, ...) - -\method{get_FV_overview}{DataSet}(ds, ...) - -\method{get_FV_overview}{DataSetList}(ds, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A `DataSet` or `DataSetList` object} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the algorithm ID, best, worst and mean reached function -values, the number of runs and available budget for the DataSet -} -\description{ -Get Function Value condensed overview -} -\examples{ -get_FV_overview(dsl) -get_FV_overview(dsl[[1]]) -get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_FV_overview} +\alias{get_FV_overview} +\alias{get_FV_overview.DataSet} +\alias{get_FV_overview.DataSetList} +\title{Get Function Value condensed overview} +\usage{ +get_FV_overview(ds, ...) + +\method{get_FV_overview}{DataSet}(ds, ...) + +\method{get_FV_overview}{DataSetList}(ds, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A `DataSet` or `DataSetList` object} + +\item{...}{Arguments passed to other methods} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the algorithm ID, best, worst and mean reached function +values, the number of runs and available budget for the DataSet +} +\description{ +Get Function Value condensed overview +} +\examples{ +get_FV_overview(dsl) +get_FV_overview(dsl[[1]]) +get_FV_overview(dsl, algorithm = '(1+1)_greedy_hill_climber_1') +} diff --git a/man/get_FV_sample.Rd b/man/get_FV_sample.Rd index 209eaa53..c75c0870 100644 --- a/man/get_FV_sample.Rd +++ b/man/get_FV_sample.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_FV_sample} -\alias{get_FV_sample} -\alias{get_FV_sample.DataSet} -\alias{get_FV_sample.DataSetList} -\title{Get Funtion Value Samples} -\usage{ -get_FV_sample(ds, ...) - -\method{get_FV_sample}{DataSet}(ds, runtime, output = "wide", ...) - -\method{get_FV_sample}{DataSetList}(ds, runtime, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{runtime}{A Numerical vector. Runtimes at which function values are reached} - -\item{output}{A String. The format of the output data: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the function value samples for each provided -target runtime -} -\description{ -Get Funtion Value Samples -} -\examples{ -get_FV_sample(dsl, 100) -get_FV_sample(dsl[[1]], 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_FV_sample} +\alias{get_FV_sample} +\alias{get_FV_sample.DataSet} +\alias{get_FV_sample.DataSetList} +\title{Get Funtion Value Samples} +\usage{ +get_FV_sample(ds, ...) + +\method{get_FV_sample}{DataSet}(ds, runtime, output = "wide", ...) + +\method{get_FV_sample}{DataSetList}(ds, runtime, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{runtime}{A Numerical vector. Runtimes at which function values are reached} + +\item{output}{A String. The format of the output data: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the function value samples for each provided +target runtime +} +\description{ +Get Funtion Value Samples +} +\examples{ +get_FV_sample(dsl, 100) +get_FV_sample(dsl[[1]], 100) +} diff --git a/man/get_FV_summary.Rd b/man/get_FV_summary.Rd index ec94fcdd..941b6408 100644 --- a/man/get_FV_summary.Rd +++ b/man/get_FV_summary.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_FV_summary} -\alias{get_FV_summary} -\alias{get_FV_summary.DataSet} -\alias{get_FV_summary.DataSetList} -\title{Get Function Value Summary} -\usage{ -get_FV_summary(ds, ...) - -\method{get_FV_summary}{DataSet}(ds, runtime, ...) - -\method{get_FV_summary}{DataSetList}(ds, runtime, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{runtime}{A Numerical vector. Runtimes at which function values are reached} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the function value statistics for each provided -target runtime value -} -\description{ -Get Function Value Summary -} -\examples{ -get_FV_summary(dsl, 100) -get_FV_summary(dsl[[1]], 100) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_FV_summary} +\alias{get_FV_summary} +\alias{get_FV_summary.DataSet} +\alias{get_FV_summary.DataSetList} +\title{Get Function Value Summary} +\usage{ +get_FV_summary(ds, ...) + +\method{get_FV_summary}{DataSet}(ds, runtime, ...) + +\method{get_FV_summary}{DataSetList}(ds, runtime, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{runtime}{A Numerical vector. Runtimes at which function values are reached} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the function value statistics for each provided +target runtime value +} +\description{ +Get Function Value Summary +} +\examples{ +get_FV_summary(dsl, 100) +get_FV_summary(dsl[[1]], 100) +} diff --git a/man/get_PAR_name.Rd b/man/get_PAR_name.Rd index 937c7f95..e618aa2e 100644 --- a/man/get_PAR_name.Rd +++ b/man/get_PAR_name.Rd @@ -1,27 +1,27 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{get_PAR_name} -\alias{get_PAR_name} -\alias{get_PAR_name.DataSet} -\title{Get the parameter names of the algorithm} -\usage{ -get_PAR_name(ds, which) - -\method{get_PAR_name}{DataSet}(ds, which = "by_FV") -} -\arguments{ -\item{ds}{A DataSet object} - -\item{which}{a string takes it value in `c('by_FV', 'by_RT')`, indicating the -parameters aligned against the running time (RT) or function value (FV). `'by_FV'` -is the default value.} -} -\value{ -a character list of paramter names, if recorded in the data set -} -\description{ -Get the parameter names of the algorithm -} -\examples{ -get_PAR_name(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{get_PAR_name} +\alias{get_PAR_name} +\alias{get_PAR_name.DataSet} +\title{Get the parameter names of the algorithm} +\usage{ +get_PAR_name(ds, which) + +\method{get_PAR_name}{DataSet}(ds, which = "by_FV") +} +\arguments{ +\item{ds}{A DataSet object} + +\item{which}{a string takes it value in `c('by_FV', 'by_RT')`, indicating the +parameters aligned against the running time (RT) or function value (FV). `'by_FV'` +is the default value.} +} +\value{ +a character list of paramter names, if recorded in the data set +} +\description{ +Get the parameter names of the algorithm +} +\examples{ +get_PAR_name(dsl[[1]]) +} diff --git a/man/get_PAR_sample.Rd b/man/get_PAR_sample.Rd index 204344d5..cdd586c2 100644 --- a/man/get_PAR_sample.Rd +++ b/man/get_PAR_sample.Rd @@ -1,45 +1,45 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_PAR_sample} -\alias{get_PAR_sample} -\alias{get_PAR_sample.DataSet} -\alias{get_PAR_sample.DataSetList} -\title{Get Parameter Value Samples} -\usage{ -get_PAR_sample(ds, idxValue, ...) - -\method{get_PAR_sample}{DataSet}(ds, idxValue, parId = "all", - which = "by_FV", output = "wide", ...) - -\method{get_PAR_sample}{DataSetList}(ds, idxValue, algorithm = "all", - ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. -The index value can either take its value in the range of running times, or function values. -Such a value type is signified by `which` parameter.} - -\item{...}{Arguments passed to other methods} - -\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} - -\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -is the default value.} - -\item{output}{A character. The format of the output data: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table object containing parameter values aligned at each given target value -} -\description{ -Get Parameter Value Samples -} -\examples{ -get_PAR_sample(dsl, 14) -get_PAR_sample(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_PAR_sample} +\alias{get_PAR_sample} +\alias{get_PAR_sample.DataSet} +\alias{get_PAR_sample.DataSetList} +\title{Get Parameter Value Samples} +\usage{ +get_PAR_sample(ds, idxValue, ...) + +\method{get_PAR_sample}{DataSet}(ds, idxValue, parId = "all", + which = "by_FV", output = "wide", ...) + +\method{get_PAR_sample}{DataSetList}(ds, idxValue, algorithm = "all", + ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. +The index value can either take its value in the range of running times, or function values. +Such a value type is signified by `which` parameter.} + +\item{...}{Arguments passed to other methods} + +\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} + +\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +is the default value.} + +\item{output}{A character. The format of the output data: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table object containing parameter values aligned at each given target value +} +\description{ +Get Parameter Value Samples +} +\examples{ +get_PAR_sample(dsl, 14) +get_PAR_sample(dsl[[1]], 14) +} diff --git a/man/get_PAR_summary.Rd b/man/get_PAR_summary.Rd index a96470b4..a234173c 100644 --- a/man/get_PAR_summary.Rd +++ b/man/get_PAR_summary.Rd @@ -1,43 +1,43 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_PAR_summary} -\alias{get_PAR_summary} -\alias{get_PAR_summary.DataSet} -\alias{get_PAR_summary.DataSetList} -\title{Get Parameter Value Summary} -\usage{ -get_PAR_summary(ds, idxValue, ...) - -\method{get_PAR_summary}{DataSet}(ds, idxValue, parId = "all", - which = "by_FV", ...) - -\method{get_PAR_summary}{DataSetList}(ds, idxValue, algorithm = "all", - ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. -The index value can either take its value in the range of running times, or function values. -Such a value type is signified by `which` parameter.} - -\item{...}{Arguments passed to other methods} - -\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} - -\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be -retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` -is the default value.} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table object containing basic statistics of parameter values aligned at each given target value -} -\description{ -Get Parameter Value Summary -} -\examples{ -get_PAR_summary(dsl, 14) -get_PAR_summary(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_PAR_summary} +\alias{get_PAR_summary} +\alias{get_PAR_summary.DataSet} +\alias{get_PAR_summary.DataSetList} +\title{Get Parameter Value Summary} +\usage{ +get_PAR_summary(ds, idxValue, ...) + +\method{get_PAR_summary}{DataSet}(ds, idxValue, parId = "all", + which = "by_FV", ...) + +\method{get_PAR_summary}{DataSetList}(ds, idxValue, algorithm = "all", + ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{idxValue}{A Numerical vector. Index values at which parameter values are observed. +The index value can either take its value in the range of running times, or function values. +Such a value type is signified by `which` parameter.} + +\item{...}{Arguments passed to other methods} + +\item{parId}{A character vector. Either 'all' or the name of parameters to be retrieved} + +\item{which}{A string takes values in `c('by_FV', 'by_RT')`, indicating the parameters to be +retrieved are aligned against the running time (RT) or function value (FV). `'by_FV'` +is the default value.} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table object containing basic statistics of parameter values aligned at each given target value +} +\description{ +Get Parameter Value Summary +} +\examples{ +get_PAR_summary(dsl, 14) +get_PAR_summary(dsl[[1]], 14) +} diff --git a/man/get_RT_overview.Rd b/man/get_RT_overview.Rd index 7f02a0aa..b454c27e 100644 --- a/man/get_RT_overview.Rd +++ b/man/get_RT_overview.Rd @@ -1,32 +1,32 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_RT_overview} -\alias{get_RT_overview} -\alias{get_RT_overview.DataSet} -\alias{get_RT_overview.DataSetList} -\title{Get Runtime Value condensed overview} -\usage{ -get_RT_overview(ds, ...) - -\method{get_RT_overview}{DataSet}(ds, ...) - -\method{get_RT_overview}{DataSetList}(ds, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the algorithm ID, minimum and maximum used evaluations, -number of runs and available budget for the DataSet -} -\description{ -Get Runtime Value condensed overview -} -\examples{ -get_RT_overview(dsl) -get_RT_overview(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_RT_overview} +\alias{get_RT_overview} +\alias{get_RT_overview.DataSet} +\alias{get_RT_overview.DataSetList} +\title{Get Runtime Value condensed overview} +\usage{ +get_RT_overview(ds, ...) + +\method{get_RT_overview}{DataSet}(ds, ...) + +\method{get_RT_overview}{DataSetList}(ds, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the algorithm ID, minimum and maximum used evaluations, +number of runs and available budget for the DataSet +} +\description{ +Get Runtime Value condensed overview +} +\examples{ +get_RT_overview(dsl) +get_RT_overview(dsl[[1]]) +} diff --git a/man/get_RT_sample.Rd b/man/get_RT_sample.Rd index 4aa676ed..d996bb97 100644 --- a/man/get_RT_sample.Rd +++ b/man/get_RT_sample.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_RT_sample} -\alias{get_RT_sample} -\alias{get_RT_sample.DataSet} -\alias{get_RT_sample.DataSetList} -\title{Get RunTime Sample} -\usage{ -get_RT_sample(ds, ftarget, ...) - -\method{get_RT_sample}{DataSet}(ds, ftarget, output = "wide", ...) - -\method{get_RT_sample}{DataSetList}(ds, ftarget, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} - -\item{...}{Arguments passed to other methods} - -\item{output}{A character determining the format of output data.table: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the runtime samples for each provided target -function value -} -\description{ -Get RunTime Sample -} -\examples{ -get_RT_sample(dsl, 14) -get_RT_sample(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_RT_sample} +\alias{get_RT_sample} +\alias{get_RT_sample.DataSet} +\alias{get_RT_sample.DataSetList} +\title{Get RunTime Sample} +\usage{ +get_RT_sample(ds, ftarget, ...) + +\method{get_RT_sample}{DataSet}(ds, ftarget, output = "wide", ...) + +\method{get_RT_sample}{DataSetList}(ds, ftarget, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{ftarget}{A Numerical vector. Function values at which runtime values are consumed} + +\item{...}{Arguments passed to other methods} + +\item{output}{A character determining the format of output data.table: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table containing the runtime samples for each provided target +function value +} +\description{ +Get RunTime Sample +} +\examples{ +get_RT_sample(dsl, 14) +get_RT_sample(dsl[[1]], 14) +} diff --git a/man/get_RT_summary.Rd b/man/get_RT_summary.Rd index 03bc7335..44c511d7 100644 --- a/man/get_RT_summary.Rd +++ b/man/get_RT_summary.Rd @@ -1,37 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_RT_summary} -\alias{get_RT_summary} -\alias{get_RT_summary.DataSet} -\alias{get_RT_summary.DataSetList} -\title{Get RunTime Summary} -\usage{ -get_RT_summary(ds, ftarget, budget, ...) - -\method{get_RT_summary}{DataSet}(ds, ftarget, budget = NULL, ...) - -\method{get_RT_summary}{DataSetList}(ds, ftarget, budget = NULL, - algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{ftarget}{The function target(s) for which to get the runtime summary} - -\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} - -\item{...}{Arguments passed to other methods} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table containing the runtime statistics for each provided target -function value -} -\description{ -Get RunTime Summary -} -\examples{ -get_RT_summary(dsl, 14) -get_RT_summary(dsl[[1]], 14) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_RT_summary} +\alias{get_RT_summary} +\alias{get_RT_summary.DataSet} +\alias{get_RT_summary.DataSetList} +\title{Get RunTime Summary} +\usage{ +get_RT_summary(ds, ftarget, budget, ...) + +\method{get_RT_summary}{DataSet}(ds, ftarget, budget = NULL, ...) + +\method{get_RT_summary}{DataSetList}(ds, ftarget, budget = NULL, ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{ftarget}{The function target(s) for which to get the runtime summary} + +\item{budget}{Optional; overwrites the budget found in ds for ERT-calculation} + +\item{...}{Arguments passed to other methods} +} +\value{ +A data.table containing the runtime statistics for each provided target +function value +} +\description{ +Get RunTime Summary +} +\examples{ +get_RT_summary(dsl, 14) +get_RT_summary(dsl[[1]], 14) +} diff --git a/man/get_algId.Rd b/man/get_algId.Rd index ca7ec9fc..d99a56cd 100644 --- a/man/get_algId.Rd +++ b/man/get_algId.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_algId} -\alias{get_algId} -\title{Get all algorithm ids present in a DataSetList} -\usage{ -get_algId(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A sorted list of all unique algorithm ids which occur in the DataSetList -} -\description{ -Get all algorithm ids present in a DataSetList -} -\examples{ -get_algId(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_algId} +\alias{get_algId} +\title{Get all algorithm ids present in a DataSetList} +\usage{ +get_algId(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A sorted list of all unique algorithm ids which occur in the DataSetList +} +\description{ +Get all algorithm ids present in a DataSetList +} +\examples{ +get_algId(dsl) +} diff --git a/man/get_color_scheme.Rd b/man/get_color_scheme.Rd index 45f829be..4e8c29e2 100644 --- a/man/get_color_scheme.Rd +++ b/man/get_color_scheme.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{get_color_scheme} -\alias{get_color_scheme} -\title{Get colors according to the current colorScheme of the IOHanalyzer} -\usage{ -get_color_scheme(algnames_in) -} -\arguments{ -\item{algnames_in}{List of algorithms for which to get colors} -} -\description{ -Get colors according to the current colorScheme of the IOHanalyzer -} -\examples{ -get_color_scheme(get_algId(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{get_color_scheme} +\alias{get_color_scheme} +\title{Get colors according to the current colorScheme of the IOHanalyzer} +\usage{ +get_color_scheme(ids_in) +} +\arguments{ +\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get colors} +} +\description{ +Get colors according to the current colorScheme of the IOHanalyzer +} +\examples{ +get_color_scheme(get_algId(dsl)) +} diff --git a/man/get_color_scheme_dt.Rd b/man/get_color_scheme_dt.Rd index e7297103..c0ac0e4b 100644 --- a/man/get_color_scheme_dt.Rd +++ b/man/get_color_scheme_dt.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{get_color_scheme_dt} -\alias{get_color_scheme_dt} -\title{Get datatable of current color (and linestyle) scheme to file} -\usage{ -get_color_scheme_dt() -} -\value{ -data.table object with 3 columns: algnames, colors, linestyles -} -\description{ -Get datatable of current color (and linestyle) scheme to file -} -\examples{ -get_color_scheme_dt() -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{get_color_scheme_dt} +\alias{get_color_scheme_dt} +\title{Get datatable of current color (and linestyle) scheme to file} +\usage{ +get_color_scheme_dt() +} +\value{ +data.table object with 3 columns: ids, colors, linestyles +} +\description{ +Get datatable of current color (and linestyle) scheme to file +} +\examples{ +get_color_scheme_dt() +} diff --git a/man/get_default_ECDF_targets.Rd b/man/get_default_ECDF_targets.Rd index e2823a7f..79a9e313 100644 --- a/man/get_default_ECDF_targets.Rd +++ b/man/get_default_ECDF_targets.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{get_default_ECDF_targets} -\alias{get_default_ECDF_targets} -\title{Generate ECDF targets for a DataSetList} -\usage{ -get_default_ECDF_targets(data, format_func = as.integer) -} -\arguments{ -\item{data}{A DataSetList} - -\item{format_func}{function to format the targets} -} -\value{ -a vector of targets -} -\description{ -Generate ECDF targets for a DataSetList -} -\examples{ -get_default_ECDF_targets(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{get_default_ECDF_targets} +\alias{get_default_ECDF_targets} +\title{Generate ECDF targets for a DataSetList} +\usage{ +get_default_ECDF_targets(data, format_func = as.integer) +} +\arguments{ +\item{data}{A DataSetList} + +\item{format_func}{function to format the targets} +} +\value{ +a vector of targets +} +\description{ +Generate ECDF targets for a DataSetList +} +\examples{ +get_default_ECDF_targets(dsl) +} diff --git a/man/get_dim.Rd b/man/get_dim.Rd index f31bb411..ce0fe941 100644 --- a/man/get_dim.Rd +++ b/man/get_dim.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_dim} -\alias{get_dim} -\title{Get all dimensions present in a DataSetList} -\usage{ -get_dim(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A sorted list of all unique dimensions which occur in the DataSetList -} -\description{ -Get all dimensions present in a DataSetList -} -\examples{ -get_dim(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_dim} +\alias{get_dim} +\title{Get all dimensions present in a DataSetList} +\usage{ +get_dim(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A sorted list of all unique dimensions which occur in the DataSetList +} +\description{ +Get all dimensions present in a DataSetList +} +\examples{ +get_dim(dsl) +} diff --git a/man/get_dsc_omnibus.Rd b/man/get_dsc_omnibus.Rd index 75cf4fa9..be9e0e5e 100644 --- a/man/get_dsc_omnibus.Rd +++ b/man/get_dsc_omnibus.Rd @@ -1,26 +1,26 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_dsc_omnibus} -\alias{get_dsc_omnibus} -\title{Perform omnibus statistical tests on the matrix of rankings from the DSCtool api} -\usage{ -get_dsc_omnibus(res, method = NULL, alpha = 0.05) -} -\arguments{ -\item{res}{The result of a call to the `get_dsc_rank`} - -\item{method}{Which method to use to do the tests. -Has be be one of the allowed ones in `res$valid_methods`. -When NULL, the first valid option is chosen by default} - -\item{alpha}{Threshold value for statistical significance} -} -\value{ -A named list containing the algorithm means -} -\description{ -Perform omnibus statistical tests on the matrix of rankings from the DSCtool api -} -\examples{ -get_dsc_omnibus(get_dsc_rank(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_dsc_omnibus} +\alias{get_dsc_omnibus} +\title{Perform omnibus statistical tests on the matrix of rankings from the DSCtool api} +\usage{ +get_dsc_omnibus(res, method = NULL, alpha = 0.05) +} +\arguments{ +\item{res}{The result of a call to the `get_dsc_rank`} + +\item{method}{Which method to use to do the tests. +Has be be one of the allowed ones in `res$valid_methods`. +When NULL, the first valid option is chosen by default} + +\item{alpha}{Threshold value for statistical significance} +} +\value{ +A named list containing the algorithm means +} +\description{ +Perform omnibus statistical tests on the matrix of rankings from the DSCtool api +} +\examples{ +get_dsc_omnibus(get_dsc_rank(dsl)) +} diff --git a/man/get_dsc_posthoc.Rd b/man/get_dsc_posthoc.Rd index acfc6e9d..44356ee7 100644 --- a/man/get_dsc_posthoc.Rd +++ b/man/get_dsc_posthoc.Rd @@ -1,36 +1,36 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_dsc_posthoc} -\alias{get_dsc_posthoc} -\title{Perform post-hoc processing on data from DSCtool} -\usage{ -get_dsc_posthoc(omni_res, nr_algs, nr_problems, base_algorithm = NULL, - method = "friedman", alpha = 0.05) -} -\arguments{ -\item{omni_res}{The result from a call to `get_dsc_omnibus`} - -\item{nr_algs}{The number of algorithms present in `omni_res`} - -\item{nr_problems}{The number of problems present in `omni_res`} - -\item{base_algorithm}{The base algorithm to which the other are compared. -This has to be present in `omni_res$algorithm_means` as an `algorithm` property} - -\item{method}{Either 'friedman' or 'friedman-aligned-rank'} - -\item{alpha}{Threshold value for statistical significance} -} -\value{ -A named list containing 4 types of analyses: -* Zvalue -* UnadjustedPValue -* Holm -* Hochberg -} -\description{ -Perform post-hoc processing on data from DSCtool -} -\examples{ -get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_dsc_posthoc} +\alias{get_dsc_posthoc} +\title{Perform post-hoc processing on data from DSCtool} +\usage{ +get_dsc_posthoc(omni_res, nr_algs, nr_problems, base_algorithm = NULL, + method = "friedman", alpha = 0.05) +} +\arguments{ +\item{omni_res}{The result from a call to `get_dsc_omnibus`} + +\item{nr_algs}{The number of algorithms present in `omni_res`} + +\item{nr_problems}{The number of problems present in `omni_res`} + +\item{base_algorithm}{The base algorithm to which the other are compared. +This has to be present in `omni_res$algorithm_means` as an `algorithm` property} + +\item{method}{Either 'friedman' or 'friedman-aligned-rank'} + +\item{alpha}{Threshold value for statistical significance} +} +\value{ +A named list containing 4 types of analyses: +* Zvalue +* UnadjustedPValue +* Holm +* Hochberg +} +\description{ +Perform post-hoc processing on data from DSCtool +} +\examples{ +get_dsc_posthoc(get_dsc_omnibus(get_dsc_rank(dsl)), 2, 2) +} diff --git a/man/get_dsc_rank.Rd b/man/get_dsc_rank.Rd index c6073f6f..b9aa00b3 100644 --- a/man/get_dsc_rank.Rd +++ b/man/get_dsc_rank.Rd @@ -1,47 +1,47 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_dsc_rank} -\alias{get_dsc_rank} -\title{Get the matrix of rankings using the DSCtool api for a DataSetList} -\usage{ -get_dsc_rank(dsList, targets = NULL, which = "by_RT", - test_type = "AD", alpha = 0.05, epsilon = 0, - monte_carlo_iterations = 0, na.correction = NULL) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{Optional list of target values (Runtime or target value)} - -\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} - -\item{test_type}{Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests} - -\item{alpha}{Threshold value for statistical significance} - -\item{epsilon}{Minimum threshold to have practical difference between algorithms (eDSC)} - -\item{monte_carlo_iterations}{How many monte-carlo-simulations to perform -(set to 0 to use regular DSC)} - -\item{na.correction}{How to deal with missing values. Only used in fixed-target perspective. -Options are: -- 'NULL': No correction is done. This will likely result in an error, as the DSCtool -does not allow for na values -- 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) -- 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) -- 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this -reverts to nr_runs * budget -- 'Remove-na' Removes all NA values} -} -\value{ -A named list containing a ranked-matrix which has the rankin of each algorithm -on each problem, as well as a list of which omnibus tests can be used to further process -this data. This can be further analyzed using `get_dsc_omnibus` -} -\description{ -Get the matrix of rankings using the DSCtool api for a DataSetList -} -\examples{ -get_dsc_rank(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_dsc_rank} +\alias{get_dsc_rank} +\title{Get the matrix of rankings using the DSCtool api for a DataSetList} +\usage{ +get_dsc_rank(dsList, targets = NULL, which = "by_RT", + test_type = "AD", alpha = 0.05, epsilon = 0, + monte_carlo_iterations = 0, na.correction = NULL) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{Optional list of target values (Runtime or target value)} + +\item{which}{Whether to use a fixed-target 'by_RT' perspective or fixed-budget 'by_FV'} + +\item{test_type}{Either 'AD' for Anderson-Darling or KS for Kolmogorov-Smirnov tests} + +\item{alpha}{Threshold value for statistical significance} + +\item{epsilon}{Minimum threshold to have practical difference between algorithms (eDSC)} + +\item{monte_carlo_iterations}{How many monte-carlo-simulations to perform +(set to 0 to use regular DSC)} + +\item{na.correction}{How to deal with missing values. Only used in fixed-target perspective. +Options are: +- 'NULL': No correction is done. This will likely result in an error, as the DSCtool +does not allow for na values +- 'PAR-1' Replace missing values with Budget (budget taken from relevant DataSet) +- 'PAR-10' Replace missing values with 10*Budget (budget taken from relevant DataSet) +- 'ERT' Replace NA values with the Expected Running Time. If all values are NA, this +reverts to nr_runs * budget +- 'Remove-na' Removes all NA values} +} +\value{ +A named list containing a ranked-matrix which has the rankin of each algorithm +on each problem, as well as a list of which omnibus tests can be used to further process +this data. This can be further analyzed using `get_dsc_omnibus` +} +\description{ +Get the matrix of rankings using the DSCtool api for a DataSetList +} +\examples{ +get_dsc_rank(dsl) +} diff --git a/man/get_funcId.Rd b/man/get_funcId.Rd index e570ea41..847b6f08 100644 --- a/man/get_funcId.Rd +++ b/man/get_funcId.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_funcId} -\alias{get_funcId} -\title{Get all function ids present in a DataSetList} -\usage{ -get_funcId(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A sorted list of all unique function ids which occur in the DataSetList -} -\description{ -Get all function ids present in a DataSetList -} -\examples{ -get_funcId(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funcId} +\alias{get_funcId} +\title{Get all function ids present in a DataSetList} +\usage{ +get_funcId(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A sorted list of all unique function ids which occur in the DataSetList +} +\description{ +Get all function ids present in a DataSetList +} +\examples{ +get_funcId(dsl) +} diff --git a/man/get_funcName.Rd b/man/get_funcName.Rd new file mode 100644 index 00000000..fc2e5519 --- /dev/null +++ b/man/get_funcName.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funcName} +\alias{get_funcName} +\title{Get all function names present in a DataSetList} +\usage{ +get_funcName(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list of all unique function names which occur in the DataSetList +} +\description{ +Get all function names present in a DataSetList +} +\examples{ +get_funcName(dsl) +} diff --git a/man/get_funvals.Rd b/man/get_funvals.Rd index 4152144a..9cbcca3f 100644 --- a/man/get_funvals.Rd +++ b/man/get_funvals.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_funvals} -\alias{get_funvals} -\title{Get all function values present in a DataSetList} -\usage{ -get_funvals(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A list matrices of all function values which occur in the DataSetList -} -\description{ -Get all function values present in a DataSetList -} -\examples{ -get_funvals(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_funvals} +\alias{get_funvals} +\title{Get all function values present in a DataSetList} +\usage{ +get_funvals(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list matrices of all function values which occur in the DataSetList +} +\description{ +Get all function values present in a DataSetList +} +\examples{ +get_funvals(dsl) +} diff --git a/man/get_id.Rd b/man/get_id.Rd new file mode 100644 index 00000000..74b5ad5a --- /dev/null +++ b/man/get_id.Rd @@ -0,0 +1,33 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_id} +\alias{get_id} +\alias{get_id.DataSet} +\alias{get_id.DataSetList} +\title{Get condensed overview of datasets} +\usage{ +get_id(ds, ...) + +\method{get_id}{DataSet}(ds, ...) + +\method{get_id}{DataSetList}(ds, ...) +} +\arguments{ +\item{ds}{The DataSetList} + +\item{...}{Arguments passed to other methods} +} +\value{ +The list of unique identiefiers present in dsl +} +\description{ +Get the unique identifiers for each DataSet in the provided DataSetList +} +\details{ +If no unique identifier is set (using `change_id` or done in DataSet construction from 1.6.0 onwards), +this function falls back on returning the algorith id (from `get_aldId`)to ensure backwards compatibility +} +\examples{ +get_id(dsl) +get_id(dsl[[1]]) +} diff --git a/man/get_line_style.Rd b/man/get_line_style.Rd index fd4f4b83..36d46b9d 100644 --- a/man/get_line_style.Rd +++ b/man/get_line_style.Rd @@ -1,17 +1,17 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{get_line_style} -\alias{get_line_style} -\title{Get line styles according to the current styleScheme of the IOHanalyzer} -\usage{ -get_line_style(algnames_in) -} -\arguments{ -\item{algnames_in}{List of algorithms for which to get linestyles} -} -\description{ -Get line styles according to the current styleScheme of the IOHanalyzer -} -\examples{ -get_line_style(get_algId(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{get_line_style} +\alias{get_line_style} +\title{Get line styles according to the current styleScheme of the IOHanalyzer} +\usage{ +get_line_style(ids_in) +} +\arguments{ +\item{ids_in}{List of algorithms (or custom ids, see `change_id`) for which to get linestyles} +} +\description{ +Get line styles according to the current styleScheme of the IOHanalyzer +} +\examples{ +get_line_style(get_algId(dsl)) +} diff --git a/man/get_marg_contrib_ecdf.Rd b/man/get_marg_contrib_ecdf.Rd index be2e725f..2200786a 100644 --- a/man/get_marg_contrib_ecdf.Rd +++ b/man/get_marg_contrib_ecdf.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_marg_contrib_ecdf} -\alias{get_marg_contrib_ecdf} -\title{Get the marginal contribution of an algorithm to a portfolio} -\usage{ -get_marg_contrib_ecdf(alg, perm, j, dt) -} -\arguments{ -\item{alg}{The result from a call to `get_dsc_omnibus`} - -\item{perm}{The permutation of algorithms to which is being contributed} - -\item{j}{At which point in the permutation the contribution should be measured} - -\item{dt}{The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`)} -} -\description{ -Based on the contribution to the ECDF-curve of the VBS of the portfolio -} -\examples{ -dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) -get_marg_contrib_ecdf(get_algId(dsl)[[1]], get_algId(dsl), 1, dt) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_marg_contrib_ecdf} +\alias{get_marg_contrib_ecdf} +\title{Get the marginal contribution of an algorithm to a portfolio} +\usage{ +get_marg_contrib_ecdf(id, perm, j, dt) +} +\arguments{ +\item{id}{The id for which to get the contribution} + +\item{perm}{The permutation of algorithms to which is being contributed} + +\item{j}{At which point in the permutation the contribution should be measured} + +\item{dt}{The datatable in which the raw ecdf-values are stored (see `generate_data.ECDF_raw`)} +} +\description{ +Based on the contribution to the ECDF-curve of the VBS of the portfolio +} +\examples{ +dt <- generate_data.ECDF_raw(dsl, get_ECDF_targets(dsl)) +get_marg_contrib_ecdf(get_id(dsl)[[1]], get_id(dsl), 1, dt) +} diff --git a/man/get_maxRT.Rd b/man/get_maxRT.Rd index 3d7191ae..f762b834 100644 --- a/man/get_maxRT.Rd +++ b/man/get_maxRT.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_maxRT} -\alias{get_maxRT} -\alias{get_maxRT.DataSet} -\alias{get_maxRT.DataSetList} -\title{Get the maximal running time} -\usage{ -get_maxRT(ds, ...) - -\method{get_maxRT}{DataSet}(ds, output = "wide", ...) - -\method{get_maxRT}{DataSetList}(ds, algorithm = "all", ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} - -\item{output}{The format of the outputted table: 'wide' or 'long'} - -\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} -} -\value{ -A data.table object containing the algorithm ID and the running time -when the algorithm terminates in each run -} -\description{ -Get the maximal running time -} -\examples{ -get_maxRT(dsl) -get_maxRT(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_maxRT} +\alias{get_maxRT} +\alias{get_maxRT.DataSet} +\alias{get_maxRT.DataSetList} +\title{Get the maximal running time} +\usage{ +get_maxRT(ds, ...) + +\method{get_maxRT}{DataSet}(ds, output = "wide", ...) + +\method{get_maxRT}{DataSetList}(ds, algorithm = "all", ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} + +\item{output}{The format of the outputted table: 'wide' or 'long'} + +\item{algorithm}{DEPRECATED, will be removed in next release. Which algorithms in the DataSetList to consider.} +} +\value{ +A data.table object containing the algorithm ID and the running time +when the algorithm terminates in each run +} +\description{ +Get the maximal running time +} +\examples{ +get_maxRT(dsl) +get_maxRT(dsl[[1]]) +} diff --git a/man/get_overview.Rd b/man/get_overview.Rd index d32eeefd..18697ec8 100644 --- a/man/get_overview.Rd +++ b/man/get_overview.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R, R/DataSetList.R -\name{get_overview} -\alias{get_overview} -\alias{get_overview.DataSet} -\alias{get_overview.DataSetList} -\title{Get condensed overview of datasets} -\usage{ -get_overview(ds, ...) - -\method{get_overview}{DataSet}(ds, ...) - -\method{get_overview}{DataSetList}(ds, ...) -} -\arguments{ -\item{ds}{A DataSet or DataSetList object} - -\item{...}{Arguments passed to other methods} -} -\value{ -A data.table containing some basic information about the provided DataSet(List) -} -\description{ -Get condensed overview of datasets -} -\examples{ -get_overview(dsl) -get_overview(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R, R/DataSetList.R +\name{get_overview} +\alias{get_overview} +\alias{get_overview.DataSet} +\alias{get_overview.DataSetList} +\title{Get condensed overview of datasets} +\usage{ +get_overview(ds, ...) + +\method{get_overview}{DataSet}(ds, ...) + +\method{get_overview}{DataSetList}(ds, ...) +} +\arguments{ +\item{ds}{A DataSet or DataSetList object} + +\item{...}{Arguments passed to other methods} +} +\value{ +A data.table containing some basic information about the provided DataSet(List) +} +\description{ +Get condensed overview of datasets +} +\examples{ +get_overview(dsl) +get_overview(dsl[[1]]) +} diff --git a/man/get_parId.Rd b/man/get_parId.Rd index 95ddad93..b64e87fd 100644 --- a/man/get_parId.Rd +++ b/man/get_parId.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_parId} -\alias{get_parId} -\title{Get all parameter ids present in a DataSetList} -\usage{ -get_parId(dsList, which = "by_FV") -} -\arguments{ -\item{dsList}{The DataSetList} - -\item{which}{A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned -by the running time (RT) or the function value (FV). Note that parameters in each case are -not necessary the same.} -} -\value{ -A sorted list of all unique parameter ids which occur in the DataSetList -} -\description{ -Get all parameter ids present in a DataSetList -} -\examples{ -get_parId(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_parId} +\alias{get_parId} +\title{Get all parameter ids present in a DataSetList} +\usage{ +get_parId(dsList, which = "by_FV") +} +\arguments{ +\item{dsList}{The DataSetList} + +\item{which}{A string takes values in `c('by_FV', 'by_RT')`. To choose the parameters aligned +by the running time (RT) or the function value (FV). Note that parameters in each case are +not necessary the same.} +} +\value{ +A sorted list of all unique parameter ids which occur in the DataSetList +} +\description{ +Get all parameter ids present in a DataSetList +} +\examples{ +get_parId(dsl) +} diff --git a/man/get_runtimes.Rd b/man/get_runtimes.Rd index dd910f07..7a75d6f7 100644 --- a/man/get_runtimes.Rd +++ b/man/get_runtimes.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{get_runtimes} -\alias{get_runtimes} -\title{Get all runtime values present in a DataSetList} -\usage{ -get_runtimes(dsList) -} -\arguments{ -\item{dsList}{The DataSetLsit} -} -\value{ -A list matrices of all runtime values which occur in the DataSetList -} -\description{ -Get all runtime values present in a DataSetList -} -\examples{ -get_runtimes(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_runtimes} +\alias{get_runtimes} +\title{Get all runtime values present in a DataSetList} +\usage{ +get_runtimes(dsList) +} +\arguments{ +\item{dsList}{The DataSetLsit} +} +\value{ +A list matrices of all runtime values which occur in the DataSetList +} +\description{ +Get all runtime values present in a DataSetList +} +\examples{ +get_runtimes(dsl) +} diff --git a/man/get_shapley_values.Rd b/man/get_shapley_values.Rd index 28ddc9e0..cbeb9855 100644 --- a/man/get_shapley_values.Rd +++ b/man/get_shapley_values.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_shapley_values} -\alias{get_shapley_values} -\title{Get the shapley-values of a portfolio of algorithms} -\usage{ -get_shapley_values(dsList, targets, scale.log = T, group_size = 5, - max_perm_size = 10, normalize = T) -} -\arguments{ -\item{dsList}{The DataSetList object} - -\item{targets}{A list or data.table containing the targets per function / dimension. If this is -a data.table, it needs columns 'target', 'DIM' and 'funcId'} - -\item{scale.log}{Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not} - -\item{group_size}{How many permutation groups will be considered} - -\item{max_perm_size}{The maximum limit for permutations to be considered} - -\item{normalize}{Whether or not to ensure the resulting values will be in [0,1]} -} -\description{ -Based on the contribution to the ECDF-curve of the VBS of the portfolio -} -\examples{ -get_shapley_values(dsl, get_ECDF_targets(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_shapley_values} +\alias{get_shapley_values} +\title{Get the shapley-values of a portfolio of algorithms} +\usage{ +get_shapley_values(dsList, targets, scale.log = T, group_size = 5, + max_perm_size = 10, normalize = T) +} +\arguments{ +\item{dsList}{The DataSetList object} + +\item{targets}{A list or data.table containing the targets per function / dimension. If this is +a data.table, it needs columns 'target', 'DIM' and 'funcId'} + +\item{scale.log}{Whether to use logarithmic scaling for the runtimes at which the ecdf will be sampled or not} + +\item{group_size}{How many permutation groups will be considered} + +\item{max_perm_size}{The maximum limit for permutations to be considered} + +\item{normalize}{Whether or not to ensure the resulting values will be in [0,1]} +} +\description{ +Based on the contribution to the ECDF-curve of the VBS of the portfolio +} +\examples{ +get_shapley_values(dsl, get_ECDF_targets(dsl)) +} diff --git a/man/get_static_attribute_values.Rd b/man/get_static_attribute_values.Rd new file mode 100644 index 00000000..33a34eea --- /dev/null +++ b/man/get_static_attribute_values.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_static_attribute_values} +\alias{get_static_attribute_values} +\title{Get all options for a specific attribute which can be used to subset a DataSetList} +\usage{ +get_static_attribute_values(dsl, attribute) +} +\arguments{ +\item{dsl}{The DataSetList} + +\item{attribute}{the name of the attribute for which to get the available options in dsl} +} +\value{ +The list of options for the specified attribute +} +\description{ +This is a more generic version of the existing `get_dim`, `get_funcId` and `get_algId` functions. +Note the only attributes returned by `get_static_attributes` are supported in this funcion +} +\examples{ +get_static_attribute_values(dsl, 'funcId') +} diff --git a/man/get_static_attributes.Rd b/man/get_static_attributes.Rd new file mode 100644 index 00000000..aad011bc --- /dev/null +++ b/man/get_static_attributes.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{get_static_attributes} +\alias{get_static_attributes} +\title{Get all attributes which can be used to subset a DataSetList} +\usage{ +get_static_attributes(dsl) +} +\arguments{ +\item{dsl}{The DataSetList} +} +\value{ +The list of available attributes +} +\description{ +Get all attributes which can be used to subset a DataSetList +} +\examples{ +get_static_attributes(dsl) +} diff --git a/man/get_target_dt.Rd b/man/get_target_dt.Rd index df418bec..4305a7cd 100644 --- a/man/get_target_dt.Rd +++ b/man/get_target_dt.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{get_target_dt} -\alias{get_target_dt} -\title{Generate datatables of runtime or function value targets for a DataSetList} -\usage{ -get_target_dt(dsList, which = "by_RT") -} -\arguments{ -\item{dsList}{A DataSetList} - -\item{which}{Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets} -} -\value{ -a data.table of targets -} -\description{ -Only one target is generated per (function, dimension)-pair, as opposed to the -function `get_default_ECDF_targets`, which generates multiple targets. -} -\examples{ -get_target_dt(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{get_target_dt} +\alias{get_target_dt} +\title{Generate datatables of runtime or function value targets for a DataSetList} +\usage{ +get_target_dt(dsList, which = "by_RT") +} +\arguments{ +\item{dsList}{A DataSetList} + +\item{which}{Whether to generate fixed-target ('by_FV') or fixed-budget ('by_RT') targets} +} +\value{ +a data.table of targets +} +\description{ +Only one target is generated per (function, dimension)-pair, as opposed to the +function `get_default_ECDF_targets`, which generates multiple targets. +} +\examples{ +get_target_dt(dsl) +} diff --git a/man/glicko2_ranking.Rd b/man/glicko2_ranking.Rd index 085167a0..013ceccb 100644 --- a/man/glicko2_ranking.Rd +++ b/man/glicko2_ranking.Rd @@ -1,33 +1,33 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{glicko2_ranking} -\alias{glicko2_ranking} -\title{Glicko2 raning of algorithms} -\usage{ -glicko2_ranking(dsl, nr_rounds = 100, which = "by_FV", - target_dt = NULL) -} -\arguments{ -\item{dsl}{The DataSetList, can contain multiple functions and dimensions, but should have the -same algorithms for all of them} - -\item{nr_rounds}{The number of rounds to run. More rounds leads to a more accurate ranking.} - -\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} - -\item{target_dt}{Custom data.table target value to use. When NULL, this is selected automatically.} -} -\value{ -A dataframe containing the glicko2-ratings and some additional info -} -\description{ -This procedure ranks algorithms based on a glicko2-procedure. -Every round (total nr_rounds), for every function and dimension of the datasetlist, -each pair of algorithms competes. This competition samples a random runtime for the -provided target (defaults to best achieved target). Whichever algorithm has the lower -runtime wins the game. Then, from these games, the glicko2-rating is determined. -} -\examples{ -glicko2_ranking(dsl, nr_round = 25) -glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{glicko2_ranking} +\alias{glicko2_ranking} +\title{Glicko2 raning of algorithms} +\usage{ +glicko2_ranking(dsl, nr_rounds = 100, which = "by_FV", + target_dt = NULL) +} +\arguments{ +\item{dsl}{The DataSetList, can contain multiple functions and dimensions, but should have the +same algorithms for all of them} + +\item{nr_rounds}{The number of rounds to run. More rounds leads to a more accurate ranking.} + +\item{which}{Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective} + +\item{target_dt}{Custom data.table target value to use. When NULL, this is selected automatically.} +} +\value{ +A dataframe containing the glicko2-ratings and some additional info +} +\description{ +This procedure ranks algorithms based on a glicko2-procedure. +Every round (total nr_rounds), for every function and dimension of the datasetlist, +each pair of algorithms competes. This competition samples a random runtime for the +provided target (defaults to best achieved target). Whichever algorithm has the lower +runtime wins the game. Then, from these games, the glicko2-rating is determined. +} +\examples{ +glicko2_ranking(dsl, nr_round = 25) +glicko2_ranking(dsl, nr_round = 25, which = 'by_RT') +} diff --git a/man/limit.data.Rd b/man/limit.data.Rd index acfd9ebb..cb2175cd 100644 --- a/man/limit.data.Rd +++ b/man/limit.data.Rd @@ -1,19 +1,19 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{limit.data} -\alias{limit.data} -\title{Reduce the size of the data set by evenly subsampling the records} -\usage{ -limit.data(df, n) -} -\arguments{ -\item{df}{The data to subsample} - -\item{n}{The amount of samples} -} -\value{ -A smaller data.frame -} -\description{ -Reduce the size of the data set by evenly subsampling the records -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{limit.data} +\alias{limit.data} +\title{Reduce the size of the data set by evenly subsampling the records} +\usage{ +limit.data(df, n) +} +\arguments{ +\item{df}{The data to subsample} + +\item{n}{The amount of samples} +} +\value{ +A smaller data.frame +} +\description{ +Reduce the size of the data set by evenly subsampling the records +} diff --git a/man/max_ERTs.Rd b/man/max_ERTs.Rd index b370ac57..e18371c4 100644 --- a/man/max_ERTs.Rd +++ b/man/max_ERTs.Rd @@ -1,30 +1,30 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{max_ERTs} -\alias{max_ERTs} -\alias{max_ERTs.DataSetList} -\title{Get the ERT-values for all DataSets in a DataSetList at certain targets} -\usage{ -max_ERTs(dsList, aggr_on = "funcId", targets = NULL, maximize = T) - -\method{max_ERTs}{DataSetList}(dsList, aggr_on = "funcId", - targets = NULL, maximize = T) -} -\arguments{ -\item{dsList}{The DataSetLsit} - -\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} - -\item{targets}{Predifined target function-values. Should be one for each function/dimension} - -\item{maximize}{Whether the DataSetList is from a maximization or minimization problem} -} -\value{ -A data.table containing ERT-values -} -\description{ -Get the ERT-values for all DataSets in a DataSetList at certain targets -} -\examples{ -max_ERTs(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{max_ERTs} +\alias{max_ERTs} +\alias{max_ERTs.DataSetList} +\title{Get the ERT-values for all DataSets in a DataSetList at certain targets} +\usage{ +max_ERTs(dsList, aggr_on = "funcId", targets = NULL, maximize = T) + +\method{max_ERTs}{DataSetList}(dsList, aggr_on = "funcId", + targets = NULL, maximize = T) +} +\arguments{ +\item{dsList}{The DataSetLsit} + +\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} + +\item{targets}{Predifined target function-values. Should be one for each function/dimension} + +\item{maximize}{Whether the DataSetList is from a maximization or minimization problem} +} +\value{ +A data.table containing ERT-values +} +\description{ +Get the ERT-values for all DataSets in a DataSetList at certain targets +} +\examples{ +max_ERTs(dsl) +} diff --git a/man/mean_FVs.Rd b/man/mean_FVs.Rd index 37a89e2f..3094936d 100644 --- a/man/mean_FVs.Rd +++ b/man/mean_FVs.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/IOHanalyzer-deprecated.R -\name{mean_FVs} -\alias{mean_FVs} -\alias{mean_FVs.DataSetList} -\title{Get the expected function-values for all DataSets in a DataSetList at certain runtimes} -\usage{ -mean_FVs(dsList, aggr_on = "funcId", runtimes = NULL) - -\method{mean_FVs}{DataSetList}(dsList, aggr_on = "funcId", - runtimes = NULL) -} -\arguments{ -\item{dsList}{The DataSetLsit} - -\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} - -\item{runtimes}{Predifined target runtimes-values. Should be one for each function/dimension} -} -\value{ -A data.table containing expected fucntion-values -} -\description{ -Get the expected function-values for all DataSets in a DataSetList at certain runtimes -} -\examples{ -mean_FVs(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/IOHanalyzer-deprecated.R +\name{mean_FVs} +\alias{mean_FVs} +\alias{mean_FVs.DataSetList} +\title{Get the expected function-values for all DataSets in a DataSetList at certain runtimes} +\usage{ +mean_FVs(dsList, aggr_on = "funcId", runtimes = NULL) + +\method{mean_FVs}{DataSetList}(dsList, aggr_on = "funcId", + runtimes = NULL) +} +\arguments{ +\item{dsList}{The DataSetLsit} + +\item{aggr_on}{Whether to aggregate on 'funcId' or 'DIM'.} + +\item{runtimes}{Predifined target runtimes-values. Should be one for each function/dimension} +} +\value{ +A data.table containing expected fucntion-values +} +\description{ +Get the expected function-values for all DataSets in a DataSetList at certain runtimes +} +\examples{ +mean_FVs(dsl) +} diff --git a/man/pairwise.test.Rd b/man/pairwise.test.Rd index 914c322f..963594b5 100644 --- a/man/pairwise.test.Rd +++ b/man/pairwise.test.Rd @@ -1,45 +1,45 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{pairwise.test} -\alias{pairwise.test} -\alias{pairwise.test.list} -\alias{pairwise.test.DataSetList} -\title{Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times -among a data set} -\usage{ -pairwise.test(x, ...) - -\method{pairwise.test}{list}(x, max_eval, bootstrap.size = 30, ...) - -\method{pairwise.test}{DataSetList}(x, ftarget, bootstrap.size = 0, - which = "by_FV", ...) -} -\arguments{ -\item{x}{either a list that contains running time sample for each algorithm as -sub-lists, or a DataSetList object} - -\item{...}{all other options} - -\item{max_eval}{list that contains the maximal running time for each algorithm -as sub-lists} - -\item{bootstrap.size}{integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping} - -\item{ftarget}{float, the target value used to determine the running / hitting} - -\item{which}{wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison -time} -} -\value{ -A matrix containing p-values of the test -} -\description{ -This function performs a Kolmogorov-Smirnov test on each pair of -algorithms in the input x to determine which algorithm gives a significantly -smaller running time. The resulting p-values are arranged in a matrix, where -each cell (i, j) contains a p-value from the test with alternative hypothesis: -the running time of algorithm i is smaller (thus better) than that of j. -} -\examples{ -pairwise.test(subset(dsl, funcId == 1), 16) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{pairwise.test} +\alias{pairwise.test} +\alias{pairwise.test.list} +\alias{pairwise.test.DataSetList} +\title{Performs a pairwise Kolmogorov-Smirnov test on the bootstrapped running times +among a data set} +\usage{ +pairwise.test(x, ...) + +\method{pairwise.test}{list}(x, max_eval, bootstrap.size = 30, ...) + +\method{pairwise.test}{DataSetList}(x, ftarget, bootstrap.size = 0, + which = "by_FV", ...) +} +\arguments{ +\item{x}{either a list that contains running time sample for each algorithm as +sub-lists, or a DataSetList object} + +\item{...}{all other options} + +\item{max_eval}{list that contains the maximal running time for each algorithm +as sub-lists} + +\item{bootstrap.size}{integer, the size of the bootstrapped sample. Set to 0 to disable bootstrapping} + +\item{ftarget}{float, the target value used to determine the running / hitting} + +\item{which}{wheter to do fixed-target ('by_FV') or fixed-budget ('by_RT') comparison +time} +} +\value{ +A matrix containing p-values of the test +} +\description{ +This function performs a Kolmogorov-Smirnov test on each pair of +algorithms in the input x to determine which algorithm gives a significantly +smaller running time. The resulting p-values are arranged in a matrix, where +each cell (i, j) contains a p-value from the test with alternative hypothesis: +the running time of algorithm i is smaller (thus better) than that of j. +} +\examples{ +pairwise.test(subset(dsl, funcId == 1), 16) +} diff --git a/man/plot_general_data.Rd b/man/plot_general_data.Rd index 965196d7..b9aeb184 100644 --- a/man/plot_general_data.Rd +++ b/man/plot_general_data.Rd @@ -1,57 +1,57 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plotDataSetList.R -\name{plot_general_data} -\alias{plot_general_data} -\title{General function for plotting within IOHanalyzer} -\usage{ -plot_general_data(df, x_attr = "algId", y_attr = "vals", - type = "violin", legend_attr = "algId", scale.xlog = F, - scale.ylog = F, scale.reverse = F, p = NULL, x_title = NULL, - y_title = NULL, plot_title = NULL, upper_attr = NULL, - lower_attr = NULL, subplot_attr = NULL, show.legend = F, - inf.action = "none", ...) -} -\arguments{ -\item{df}{The dataframe containing the data to plot. It should contain at least two columns: -'x_attr' and 'y_attr'} - -\item{x_attr}{The column to specify the x_axis. Default is 'algId'} - -\item{y_attr}{The column to specify the y_axis} - -\item{type}{The type of plot to use. Currently available: 'violin', 'line', 'radar', -'bar', hist' and 'ribbon'} - -\item{legend_attr}{Default is 'algId' This is also used for the selection of colorschemes} - -\item{scale.xlog}{Logarithmic scaling of x-axis} - -\item{scale.ylog}{Logarithmic scaling of y-axis} - -\item{scale.reverse}{Decreasing or increasing x-axis} - -\item{p}{A previously existing plot on which to add traces. If NULL, a new canvas is created} - -\item{x_title}{Title of x-axis. Defaults to x_attr} - -\item{y_title}{Title of x-axis. Defaults to x_attr} - -\item{plot_title}{Title of x-axis. Defaults to no title} - -\item{upper_attr}{When using ribbon-plot, this can be used to create a shaded area. -Only works in combination with`lower_attr` and `type` == 'ribbon'} - -\item{lower_attr}{When using ribbon-plot, this can be used to create a shaded area. -Only works in combination with`upper_attr` and `type` == 'ribbon'} - -\item{subplot_attr}{Which attribute of the dataframe to use for creating subplots} - -\item{show.legend}{Whether or not to include a legend} - -\item{inf.action}{How to deal with infinite values. Can be 'none', 'overlap' or 'jitter'} - -\item{...}{Additional parameters for the add_trace function} -} -\description{ -General function for plotting within IOHanalyzer -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plotDataSetList.R +\name{plot_general_data} +\alias{plot_general_data} +\title{General function for plotting within IOHanalyzer} +\usage{ +plot_general_data(df, x_attr = "ID", y_attr = "vals", + type = "violin", legend_attr = "ID", scale.xlog = F, + scale.ylog = F, scale.reverse = F, p = NULL, x_title = NULL, + y_title = NULL, plot_title = NULL, upper_attr = NULL, + lower_attr = NULL, subplot_attr = NULL, show.legend = F, + inf.action = "none", ...) +} +\arguments{ +\item{df}{The dataframe containing the data to plot. It should contain at least two columns: +'x_attr' and 'y_attr'} + +\item{x_attr}{The column to specify the x_axis. Default is 'algId'} + +\item{y_attr}{The column to specify the y_axis} + +\item{type}{The type of plot to use. Currently available: 'violin', 'line', 'radar', +'bar', hist' and 'ribbon'} + +\item{legend_attr}{Default is 'algId' This is also used for the selection of colorschemes} + +\item{scale.xlog}{Logarithmic scaling of x-axis} + +\item{scale.ylog}{Logarithmic scaling of y-axis} + +\item{scale.reverse}{Decreasing or increasing x-axis} + +\item{p}{A previously existing plot on which to add traces. If NULL, a new canvas is created} + +\item{x_title}{Title of x-axis. Defaults to x_attr} + +\item{y_title}{Title of x-axis. Defaults to x_attr} + +\item{plot_title}{Title of x-axis. Defaults to no title} + +\item{upper_attr}{When using ribbon-plot, this can be used to create a shaded area. +Only works in combination with`lower_attr` and `type` == 'ribbon'} + +\item{lower_attr}{When using ribbon-plot, this can be used to create a shaded area. +Only works in combination with`upper_attr` and `type` == 'ribbon'} + +\item{subplot_attr}{Which attribute of the dataframe to use for creating subplots} + +\item{show.legend}{Whether or not to include a legend} + +\item{inf.action}{How to deal with infinite values. Can be 'none', 'overlap' or 'jitter'} + +\item{...}{Additional parameters for the add_trace function} +} +\description{ +General function for plotting within IOHanalyzer +} diff --git a/man/print.DataSet.Rd b/man/print.DataSet.Rd index 2654e253..72d10ac5 100644 --- a/man/print.DataSet.Rd +++ b/man/print.DataSet.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{print.DataSet} -\alias{print.DataSet} -\title{S3 generic print operator for DataSet} -\usage{ -\method{print}{DataSet}(x, ...) -} -\arguments{ -\item{x}{A DataSet object} - -\item{...}{Arguments passed to other methods} -} -\value{ -A short description of the DataSet -} -\description{ -S3 generic print operator for DataSet -} -\examples{ -print(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{print.DataSet} +\alias{print.DataSet} +\title{S3 generic print operator for DataSet} +\usage{ +\method{print}{DataSet}(x, ...) +} +\arguments{ +\item{x}{A DataSet object} + +\item{...}{Arguments passed to other methods} +} +\value{ +A short description of the DataSet +} +\description{ +S3 generic print operator for DataSet +} +\examples{ +print(dsl[[1]]) +} diff --git a/man/print.DataSetList.Rd b/man/print.DataSetList.Rd index 24c714bc..8547c7f7 100644 --- a/man/print.DataSetList.Rd +++ b/man/print.DataSetList.Rd @@ -1,19 +1,19 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{print.DataSetList} -\alias{print.DataSetList} -\title{S3 print function for DataSetList} -\usage{ -\method{print}{DataSetList}(x, ...) -} -\arguments{ -\item{x}{The DataSetList to print} - -\item{...}{Arguments for underlying print function?} -} -\description{ -S3 print function for DataSetList -} -\examples{ -print(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{print.DataSetList} +\alias{print.DataSetList} +\title{S3 print function for DataSetList} +\usage{ +\method{print}{DataSetList}(x, ...) +} +\arguments{ +\item{x}{The DataSetList to print} + +\item{...}{Arguments for underlying print function?} +} +\description{ +S3 print function for DataSetList +} +\examples{ +print(dsl) +} diff --git a/man/read_index_file.Rd b/man/read_index_file.Rd index 0e5398d7..e5e9cb28 100644 --- a/man/read_index_file.Rd +++ b/man/read_index_file.Rd @@ -1,21 +1,21 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{read_index_file} -\alias{read_index_file} -\title{Read .info files and extract information} -\usage{ -read_index_file(fname) -} -\arguments{ -\item{fname}{The path to the .info file} -} -\value{ -The data contained in the .info file -} -\description{ -Read .info files and extract information -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{read_index_file} +\alias{read_index_file} +\title{Read .info files and extract information} +\usage{ +read_index_file(fname) +} +\arguments{ +\item{fname}{The path to the .info file} +} +\value{ +The data contained in the .info file +} +\description{ +Read .info files and extract information +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +info <- read_index_file(file.path(path,"IOHprofiler_f1_i1.info")) +} diff --git a/man/register_DSC.Rd b/man/register_DSC.Rd index 541c2207..fe3d3961 100644 --- a/man/register_DSC.Rd +++ b/man/register_DSC.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{register_DSC} -\alias{register_DSC} -\title{Register an account to the DSCtool API} -\usage{ -register_DSC(name, username, affiliation, email, password = NULL) -} -\arguments{ -\item{name}{Your name} - -\item{username}{A usename to be identified with. Will be stored on keyring under 'DSCtool_name'} - -\item{affiliation}{Your affiliation (university / company)} - -\item{email}{Your email adress} - -\item{password}{The password to use. If NULL, this will be generated at random. -Will be stored on keyring under 'DSCtool'} -} -\description{ -This uses the keyring package to store and load credentials. -If you already have an account, please call `set_DSC_credentials` instead -} -\examples{ -\dontrun{ -register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{register_DSC} +\alias{register_DSC} +\title{Register an account to the DSCtool API} +\usage{ +register_DSC(name, username, affiliation, email, password = NULL) +} +\arguments{ +\item{name}{Your name} + +\item{username}{A usename to be identified with. Will be stored on keyring under 'DSCtool_name'} + +\item{affiliation}{Your affiliation (university / company)} + +\item{email}{Your email adress} + +\item{password}{The password to use. If NULL, this will be generated at random. +Will be stored on keyring under 'DSCtool'} +} +\description{ +This uses the keyring package to store and load credentials. +If you already have an account, please call `set_DSC_credentials` instead +} +\examples{ +\dontrun{ +register_DSC('John Doe', 'jdoe', 'Sample University', "j.doe.sample.com") +} +} diff --git a/man/runServer.Rd b/man/runServer.Rd index 3c6a21c4..07b9212f 100644 --- a/man/runServer.Rd +++ b/man/runServer.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/runServer.R -\name{runServer} -\alias{runServer} -\title{Create a shiny-server GUI to interactively use the IOHanalyzer} -\usage{ -runServer(port = getOption("shiny.port"), open_browser = TRUE) -} -\arguments{ -\item{port}{Optional; which port the server should be opened at. Defaults -to the option set for 'shiny.port'} - -\item{open_browser}{Whether or not to open a browser tab with the -IOHanalyzer GUI. Defaults to TRUE.} -} -\description{ -Create a shiny-server GUI to interactively use the IOHanalyzer -} -\examples{ -\dontrun{ -runServer(6563, TRUE) -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/runServer.R +\name{runServer} +\alias{runServer} +\title{Create a shiny-server GUI to interactively use the IOHanalyzer} +\usage{ +runServer(port = getOption("shiny.port"), open_browser = TRUE) +} +\arguments{ +\item{port}{Optional; which port the server should be opened at. Defaults +to the option set for 'shiny.port'} + +\item{open_browser}{Whether or not to open a browser tab with the +IOHanalyzer GUI. Defaults to TRUE.} +} +\description{ +Create a shiny-server GUI to interactively use the IOHanalyzer +} +\examples{ +\dontrun{ +runServer(6563, TRUE) +} +} diff --git a/man/save_plotly.Rd b/man/save_plotly.Rd index a8d8b80f..dd574553 100644 --- a/man/save_plotly.Rd +++ b/man/save_plotly.Rd @@ -1,28 +1,28 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{save_plotly} -\alias{save_plotly} -\title{Save plotly figure in multiple format} -\usage{ -save_plotly(p, file, width = NULL, height = NULL, ...) -} -\arguments{ -\item{p}{plotly object. The plot to be saved} - -\item{file}{String. The name of the figure file, with the extension of the required file-format} - -\item{width}{Optional. Width of the figure} - -\item{height}{Optional. Height of the figure} - -\item{...}{Additional arguments for orca} -} -\description{ -NOTE: This function requires orca to be installed -} -\examples{ -\dontrun{ -p <- Plot.RT.Single_Func(dsl[1]) -save_plotly(p, 'example_file.png') -} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{save_plotly} +\alias{save_plotly} +\title{Save plotly figure in multiple format} +\usage{ +save_plotly(p, file, width = NULL, height = NULL, ...) +} +\arguments{ +\item{p}{plotly object. The plot to be saved} + +\item{file}{String. The name of the figure file, with the extension of the required file-format} + +\item{width}{Optional. Width of the figure} + +\item{height}{Optional. Height of the figure} + +\item{...}{Additional arguments for orca} +} +\description{ +NOTE: This function requires orca to be installed +} +\examples{ +\dontrun{ +p <- Plot.RT.Single_Func(dsl[1]) +save_plotly(p, 'example_file.png') +} +} diff --git a/man/save_table.Rd b/man/save_table.Rd index 531a6bdf..09e4e3a3 100644 --- a/man/save_table.Rd +++ b/man/save_table.Rd @@ -1,23 +1,23 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{save_table} -\alias{save_table} -\title{Save DataTable in multiple formats} -\usage{ -save_table(df, file, format = NULL) -} -\arguments{ -\item{df}{The DataTable to store} - -\item{file}{String. The name of the figure file, with the extension of the required file-format} - -\item{format}{Optional, string. Overwrites the extension of the `file` parameter. If not specified while -file does not have an extension, it defaults to csv} -} -\description{ -Save DataTable in multiple formats -} -\examples{ -df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') -save_table(df, tempfile(fileext = ".md")) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{save_table} +\alias{save_table} +\title{Save DataTable in multiple formats} +\usage{ +save_table(df, file, format = NULL) +} +\arguments{ +\item{df}{The DataTable to store} + +\item{file}{String. The name of the figure file, with the extension of the required file-format} + +\item{format}{Optional, string. Overwrites the extension of the `file` parameter. If not specified while +file does not have an extension, it defaults to csv} +} +\description{ +Save DataTable in multiple formats +} +\examples{ +df <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') +save_table(df, tempfile(fileext = ".md")) +} diff --git a/man/scan_index_file.Rd b/man/scan_index_file.Rd index 441ba077..448f2611 100644 --- a/man/scan_index_file.Rd +++ b/man/scan_index_file.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readFiles.R -\name{scan_index_file} -\alias{scan_index_file} -\title{Scan *.info files for IOHProfiler or COCO} -\usage{ -scan_index_file(folder) -} -\arguments{ -\item{folder}{The folder containing the .info files} -} -\value{ -The paths to all found .info-files -} -\description{ -Scan *.info files for IOHProfiler or COCO -} -\note{ -This automatically filetrs our files of size 0 -} -\examples{ -path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") -scan_index_file(path) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readFiles.R +\name{scan_index_file} +\alias{scan_index_file} +\title{Scan *.info files for IOHProfiler or COCO} +\usage{ +scan_index_file(folder) +} +\arguments{ +\item{folder}{The folder containing the .info and .json files} +} +\value{ +The paths to all found .info and .json-files +} +\description{ +Scan *.info files for IOHProfiler or COCO +} +\note{ +This automatically filetrs our files of size 0 +} +\examples{ +path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") +scan_index_file(path) +} diff --git a/man/seq_FV.Rd b/man/seq_FV.Rd index ea063685..811042cb 100644 --- a/man/seq_FV.Rd +++ b/man/seq_FV.Rd @@ -1,35 +1,35 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{seq_FV} -\alias{seq_FV} -\title{Function for generating sequences of function values} -\usage{ -seq_FV(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, - scale = NULL) -} -\arguments{ -\item{FV}{A list of function values} - -\item{from}{Starting function value. Will be replaced by min(FV) if it is NULL or too small} - -\item{to}{Stopping function value. Will be replaced by max(FV) if it is NULL or too large} - -\item{by}{Stepsize of the sequence. Will be replaced if it is too small} - -\item{length.out}{Number of values in the sequence. -'by' takes preference if both it and length.out are provided.} - -\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a -linear or log-linear spacing respectively. If NULL, the scale will be predicted -based on FV} -} -\value{ -A sequence of function values -} -\description{ -Function for generating sequences of function values -} -\examples{ -FVall <- get_runtimes(dsl) -seq_FV(FVall, 10, 16, 1, scale='linear') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{seq_FV} +\alias{seq_FV} +\title{Function for generating sequences of function values} +\usage{ +seq_FV(FV, from = NULL, to = NULL, by = NULL, length.out = NULL, + scale = NULL) +} +\arguments{ +\item{FV}{A list of function values} + +\item{from}{Starting function value. Will be replaced by min(FV) if it is NULL or too small} + +\item{to}{Stopping function value. Will be replaced by max(FV) if it is NULL or too large} + +\item{by}{Stepsize of the sequence. Will be replaced if it is too small} + +\item{length.out}{Number of values in the sequence. +'by' takes preference if both it and length.out are provided.} + +\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a +linear or log-linear spacing respectively. If NULL, the scale will be predicted +based on FV} +} +\value{ +A sequence of function values +} +\description{ +Function for generating sequences of function values +} +\examples{ +FVall <- get_runtimes(dsl) +seq_FV(FVall, 10, 16, 1, scale='linear') +} diff --git a/man/seq_RT.Rd b/man/seq_RT.Rd index a242c0d0..74ebe44c 100644 --- a/man/seq_RT.Rd +++ b/man/seq_RT.Rd @@ -1,34 +1,34 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{seq_RT} -\alias{seq_RT} -\title{Function for generating sequences of runtime values} -\usage{ -seq_RT(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, - scale = "linear") -} -\arguments{ -\item{RT}{A list of runtime values} - -\item{from}{Starting runtime value. Will be replaced by min(RT) if it is NULL or too small} - -\item{to}{Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large} - -\item{by}{Stepsize of the sequence. Will be replaced if it is too small} - -\item{length.out}{Number of values in the sequence. -'by' takes preference if both it and length.out are provided.} - -\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a -linear or log-linear spacing respectively.} -} -\value{ -A sequence of runtime values -} -\description{ -Function for generating sequences of runtime values -} -\examples{ -RTall <- get_runtimes(dsl) -seq_RT(RTall, 0, 500, length.out=10, scale='log') -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{seq_RT} +\alias{seq_RT} +\title{Function for generating sequences of runtime values} +\usage{ +seq_RT(RT, from = NULL, to = NULL, by = NULL, length.out = NULL, + scale = "linear") +} +\arguments{ +\item{RT}{A list of runtime values} + +\item{from}{Starting runtime value. Will be replaced by min(RT) if it is NULL or too small} + +\item{to}{Stopping runtime value. Will be replaced by max(RT) if it is NULL or too large} + +\item{by}{Stepsize of the sequence. Will be replaced if it is too small} + +\item{length.out}{Number of values in the sequence. +'by' takes preference if both it and length.out are provided.} + +\item{scale}{Scaling of the sequence. Can be either 'linear' or 'log', indicating a +linear or log-linear spacing respectively.} +} +\value{ +A sequence of runtime values +} +\description{ +Function for generating sequences of runtime values +} +\examples{ +RTall <- get_runtimes(dsl) +seq_RT(RTall, 0, 500, length.out=10, scale='log') +} diff --git a/man/set_DSC_credentials.Rd b/man/set_DSC_credentials.Rd index 5366817d..bd6b51ab 100644 --- a/man/set_DSC_credentials.Rd +++ b/man/set_DSC_credentials.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/stats.R -\name{set_DSC_credentials} -\alias{set_DSC_credentials} -\title{Register an account to the DSCtool API} -\usage{ -set_DSC_credentials(username, password) -} -\arguments{ -\item{username}{The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name'} - -\item{password}{The password you use on DSCtool. Will be stored on keyring under 'DSCtool'} -} -\description{ -This uses the keyring package to store and load credentials. -If you already have an account, please call `add_DSC_credentials` instead -} -\examples{ -\dontrun{set_DSC_credentials('jdoe', 'monkey123')} -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stats.R +\name{set_DSC_credentials} +\alias{set_DSC_credentials} +\title{Register an account to the DSCtool API} +\usage{ +set_DSC_credentials(username, password) +} +\arguments{ +\item{username}{The usename you use on DSCtool. Will be stored on keyring under 'DSCtool_name'} + +\item{password}{The password you use on DSCtool. Will be stored on keyring under 'DSCtool'} +} +\description{ +This uses the keyring package to store and load credentials. +If you already have an account, please call `add_DSC_credentials` instead +} +\examples{ +\dontrun{set_DSC_credentials('jdoe', 'monkey123')} +} diff --git a/man/set_color_scheme.Rd b/man/set_color_scheme.Rd index cd81b978..af55f1d2 100644 --- a/man/set_color_scheme.Rd +++ b/man/set_color_scheme.Rd @@ -1,29 +1,29 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{set_color_scheme} -\alias{set_color_scheme} -\title{Set the colorScheme of the IOHanalyzer plots} -\usage{ -set_color_scheme(schemename, algnames, path = NULL) -} -\arguments{ -\item{schemename}{Three default colorschemes are implemented: -\itemize{ -\item Default -\item Variant 1 -\item Variant 2 -\item Variant 3 -} -And it is also possible to select "Custom", which allows uploading of a custom set of colors} - -\item{algnames}{The names of the algorithms for which to set the colors} - -\item{path}{The path to the file containing the colors to use. Only used if -schemename is "Custom"} -} -\description{ -Set the colorScheme of the IOHanalyzer plots -} -\examples{ -set_color_scheme("Default", get_algId(dsl)) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{set_color_scheme} +\alias{set_color_scheme} +\title{Set the colorScheme of the IOHanalyzer plots} +\usage{ +set_color_scheme(schemename, ids, path = NULL) +} +\arguments{ +\item{schemename}{Three default colorschemes are implemented: +\itemize{ +\item Default +\item Variant 1 +\item Variant 2 +\item Variant 3 +} +And it is also possible to select "Custom", which allows uploading of a custom set of colors} + +\item{ids}{The names of the algorithms (or custom ids, see `change_id`) for which to set the colors} + +\item{path}{The path to the file containing the colors to use. Only used if +schemename is "Custom"} +} +\description{ +Set the colorScheme of the IOHanalyzer plots +} +\examples{ +set_color_scheme("Default", get_algId(dsl)) +} diff --git a/man/sub-.DataSetList.Rd b/man/sub-.DataSetList.Rd index 2c79a2a1..227f217f 100644 --- a/man/sub-.DataSetList.Rd +++ b/man/sub-.DataSetList.Rd @@ -1,24 +1,24 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{[.DataSetList} -\alias{[.DataSetList} -\title{S3 extraction function for DataSetList} -\usage{ -\method{[}{DataSetList}(x, i, drop = FALSE) -} -\arguments{ -\item{x}{The DataSetList to use} - -\item{i}{The indices to extract} - -\item{drop}{Currently unused parameter} -} -\value{ -The DataSetList of the DataSets at indices i of DataSetList x -} -\description{ -S3 extraction function for DataSetList -} -\examples{ -dsl[c(1, 3)] -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{[.DataSetList} +\alias{[.DataSetList} +\title{S3 extraction function for DataSetList} +\usage{ +\method{[}{DataSetList}(x, i, drop = FALSE) +} +\arguments{ +\item{x}{The DataSetList to use} + +\item{i}{The indices to extract} + +\item{drop}{Currently unused parameter} +} +\value{ +The DataSetList of the DataSets at indices i of DataSetList x +} +\description{ +S3 extraction function for DataSetList +} +\examples{ +dsl[c(1, 3)] +} diff --git a/man/subset.DataSet.Rd b/man/subset.DataSet.Rd new file mode 100644 index 00000000..0997c696 --- /dev/null +++ b/man/subset.DataSet.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{subset.DataSet} +\alias{subset.DataSet} +\title{S3 subset function for DataSet} +\usage{ +\method{subset}{DataSet}(x, mask, ...) +} +\arguments{ +\item{x}{The DataSet from which to get a subset} + +\item{mask}{The mask (as boolean list) to use when subsetting. The length should be equal to the number of runs +present in the provided dataset object x.} + +\item{...}{Arguments passed to underlying subset method (not yet supported)} +} +\value{ +A new DataSet +} +\description{ +Subset for DataSets. Based on the provided mask, the relevant data is taken from the given DataSet +and turned into a new DataSet object. +} +\examples{ +subset(dsl[[1]], c(0,1,1,1,0,0,0,0,0,0,0)) +} diff --git a/man/subset.DataSetList.Rd b/man/subset.DataSetList.Rd index 1928cb51..ab325b8a 100644 --- a/man/subset.DataSetList.Rd +++ b/man/subset.DataSetList.Rd @@ -1,23 +1,30 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{subset.DataSetList} -\alias{subset.DataSetList} -\title{Filter a DataSetList by some criteria} -\usage{ -\method{subset}{DataSetList}(x, ...) -} -\arguments{ -\item{x}{The DataSetLsit} - -\item{...}{The condition to filter on. Can be any expression which assigns True or False -to a DataSet object, such as DIM == 625 or funcId == 2} -} -\value{ -The filtered DataSetList -} -\description{ -Filter a DataSetList by some criteria -} -\examples{ -subset(dsl, funcId == 1) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{subset.DataSetList} +\alias{subset.DataSetList} +\title{Filter a DataSetList by some criteria} +\usage{ +\method{subset}{DataSetList}(x, ...) +} +\arguments{ +\item{x}{The DataSetList} + +\item{...}{The conditions to filter on. Can be any expression which assigns True or False +to a DataSet object, such as DIM == 625 or funcId == 2. Usage of && and || is only supported on default attributes +(funcId, algId, DIM), not on combinations of with other attributes (e.g. instance). In those cases, & and | should +be used respectively. Alternatively, this can be used as a keyword argument named 'text', with the condition as a +string to be parsed. This allows exectution of subset commands on arbitrary variables in code.} +} +\value{ +The filtered DataSetList +} +\description{ +Filter a DataSetList by some criteria +} +\examples{ +subset(dsl, funcId == 1) +subset(dsl, funcId == 1 && DIM == 16) # Can use && and || for default attributes +subset(dsl, instance == 1) +subset(dsl, instance == 1 & funcId == 1) # Can use & and | for all attributes +subset(dsl, instance == 1, funcId == 1) # Comma-seperated conditions are treated as AND +} diff --git a/man/summary.DataSet.Rd b/man/summary.DataSet.Rd index 65688097..f52f92e7 100644 --- a/man/summary.DataSet.Rd +++ b/man/summary.DataSet.Rd @@ -1,22 +1,22 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSet.R -\name{summary.DataSet} -\alias{summary.DataSet} -\title{S3 generic summary operator for DataSet} -\usage{ -\method{summary}{DataSet}(object, ...) -} -\arguments{ -\item{object}{A DataSet object} - -\item{...}{Arguments passed to other methods} -} -\value{ -A summary of the DataSet containing both function-value and runtime based statistics. -} -\description{ -S3 generic summary operator for DataSet -} -\examples{ -summary(dsl[[1]]) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSet.R +\name{summary.DataSet} +\alias{summary.DataSet} +\title{S3 generic summary operator for DataSet} +\usage{ +\method{summary}{DataSet}(object, ...) +} +\arguments{ +\item{object}{A DataSet object} + +\item{...}{Arguments passed to other methods} +} +\value{ +A summary of the DataSet containing both function-value and runtime based statistics. +} +\description{ +S3 generic summary operator for DataSet +} +\examples{ +summary(dsl[[1]]) +} diff --git a/man/summary.DataSetList.Rd b/man/summary.DataSetList.Rd index a6a26e5d..401bec81 100644 --- a/man/summary.DataSetList.Rd +++ b/man/summary.DataSetList.Rd @@ -1,20 +1,20 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DataSetList.R -\name{summary.DataSetList} -\alias{summary.DataSetList} -\title{S3 summary function for DataSetList} -\usage{ -\method{summary}{DataSetList}(object, ...) -} -\arguments{ -\item{object}{The DataSetList to print} - -\item{...}{Arguments for underlying summary function?} -} -\description{ -Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every -DataSet in the DataSetList -} -\examples{ -summary(dsl) -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataSetList.R +\name{summary.DataSetList} +\alias{summary.DataSetList} +\title{S3 summary function for DataSetList} +\usage{ +\method{summary}{DataSetList}(object, ...) +} +\arguments{ +\item{object}{The DataSetList to print} + +\item{...}{Arguments for underlying summary function?} +} +\description{ +Prints the Function ID, Dimension, Algorithm Id, datafile location and comment for every +DataSet in the DataSetList +} +\examples{ +summary(dsl) +} diff --git a/src/align.cc b/src/align.cc index 4531c32e..0f1be787 100644 --- a/src/align.cc +++ b/src/align.cc @@ -1,146 +1,146 @@ -#include -#include -using namespace Rcpp; - -// [[Rcpp::export]] -NumericVector align_by_target_inner_loop(double t, int idxEvals, int idxTarget, -List data, NumericVector index, NumericMatrix next_lines, NumericVector curr_eval, bool maximization) { - - bool condition; - int n_row, n_col, iter, N; - N = data.size(); - NumericVector out = clone(curr_eval); - - for (int k = 0; k < N; k++) { - NumericMatrix d = as(data[k]); - n_row = d.nrow(); - n_col = d.ncol(); - iter = index[k]; - - while (true) { - if (maximization) { - condition = next_lines(k, idxTarget) >= t; - } else { - condition = next_lines(k, idxTarget) <= t; - } - - if (condition) { - out[k] = next_lines(k, idxEvals); - break; - } - - if (iter < (n_row - 1)) { - iter++; - for (int j = 0; j < n_col; j++) { - next_lines(k, j) = d(iter, j); - } - } else { - break; - } - } - index[k] = iter; - } - return out; -} - -// [[Rcpp::export]] -NumericVector c_impute(NumericVector x, NumericVector y, NumericVector rowname) { - int N = rowname.size(); - int L = x.size(); - NumericVector res(N, NA_REAL); - - int i = 0; - int j = 0; - while (i < N) { - if (j < (L - 1) && rowname[i] >= y[j]) { - if (rowname[i] < y[j + 1]) { - res[i] = x[j]; - ++j; - } else { - ++j; - continue; - } - // if the query value is larger than the last runtime value `y[L - 1]` - } else if (j == L - 1 && rowname[i] >= y[j]) { - res[i] = x[j]; - // take the previous function value if the next runtime in y[j] is not reached - } else if (rowname[i] < y[j] && j > 0) { - res[i] = x[j - 1]; - } - ++i; - } - return res; -} - -// [[Rcpp::export]] -NumericMatrix c_impute_running_time(NumericVector index, // index value recorded - NumericMatrix value, // value to align: runtime + parameters - NumericVector FV, // function values to match - bool maximization // the data are collected from a maximization problem? - ) { - int N = FV.size(); - int L = index.size(); - int M = value.ncol(); - NumericMatrix res(N, M); - std::fill(res.begin(), res.end(), NA_REAL); - - bool condition; - int j = 0; - for (int i = 0; i < N; ++i) { - while (j < L) { - if (maximization) { - condition = FV[i] > index[j]; - } else { - condition = FV[i] < index[j]; - } - - if (condition) { - ++j; - } else { - res(i, _) = value(j, _); - break; - } - } - } - return res; -} - -// TODO: Better comments -//' Align a list of data set by function values -//' -//' @param data the data -//' @param FV Function values -//' @param idxValue index of the function values -//' @param maximization Boolean -//' @param idxTarget index of the target -//' @noRd -// [[Rcpp::export]] -List c_align_running_time(List data, NumericVector FV, NumericVector idxValue, bool maximization, int idxTarget) { - int NC = data.size(); - int NR = FV.size(); - int M = idxValue.size(); - - List res(M); - for (int i = 0; i < M; i++) { - NumericMatrix aux(NR, NC); - rownames(aux) = FV; - res[i] = clone(aux); - } - - for (int i = 0; i < NC; i++) { - NumericMatrix d = data[i]; - NumericMatrix value(d.nrow(), M); - - for (int j = 0; j < M; j++) { - value(_, j) = d(_, idxValue[j]); - } - - NumericMatrix tmp = c_impute_running_time(d(_, idxTarget), value, FV, maximization); - - for (int k = 0; k < M; k++) { - NumericMatrix aux = res[k]; - aux(_, i) = tmp(_, k); - } - } - return res; -} +#include +#include +using namespace Rcpp; + +// [[Rcpp::export]] +NumericVector align_by_target_inner_loop(double t, int idxEvals, int idxTarget, +List data, NumericVector index, NumericMatrix next_lines, NumericVector curr_eval, bool maximization) { + + bool condition; + int n_row, n_col, iter, N; + N = data.size(); + NumericVector out = clone(curr_eval); + + for (int k = 0; k < N; k++) { + NumericMatrix d = as(data[k]); + n_row = d.nrow(); + n_col = d.ncol(); + iter = index[k]; + + while (true) { + if (maximization) { + condition = next_lines(k, idxTarget) >= t; + } else { + condition = next_lines(k, idxTarget) <= t; + } + + if (condition) { + out[k] = next_lines(k, idxEvals); + break; + } + + if (iter < (n_row - 1)) { + iter++; + for (int j = 0; j < n_col; j++) { + next_lines(k, j) = d(iter, j); + } + } else { + break; + } + } + index[k] = iter; + } + return out; +} + +// [[Rcpp::export]] +NumericVector c_impute(NumericVector x, NumericVector y, NumericVector rowname) { + int N = rowname.size(); + int L = x.size(); + NumericVector res(N, NA_REAL); + + int i = 0; + int j = 0; + while (i < N) { + if (j < (L - 1) && rowname[i] >= y[j]) { + if (rowname[i] < y[j + 1]) { + res[i] = x[j]; + ++j; + } else { + ++j; + continue; + } + // if the query value is larger than the last runtime value `y[L - 1]` + } else if (j == L - 1 && rowname[i] >= y[j]) { + res[i] = x[j]; + // take the previous function value if the next runtime in y[j] is not reached + } else if (rowname[i] < y[j] && j > 0) { + res[i] = x[j - 1]; + } + ++i; + } + return res; +} + +// [[Rcpp::export]] +NumericMatrix c_impute_running_time(NumericVector index, // index value recorded + NumericMatrix value, // value to align: runtime + parameters + NumericVector FV, // function values to match + bool maximization // the data are collected from a maximization problem? + ) { + int N = FV.size(); + int L = index.size(); + int M = value.ncol(); + NumericMatrix res(N, M); + std::fill(res.begin(), res.end(), NA_REAL); + + bool condition; + int j = 0; + for (int i = 0; i < N; ++i) { + while (j < L) { + if (maximization) { + condition = FV[i] > index[j]; + } else { + condition = FV[i] < index[j]; + } + + if (condition) { + ++j; + } else { + res(i, _) = value(j, _); + break; + } + } + } + return res; +} + +// TODO: Better comments +//' Align a list of data set by function values +//' +//' @param data the data +//' @param FV Function values +//' @param idxValue index of the function values +//' @param maximization Boolean +//' @param idxTarget index of the target +//' @noRd +// [[Rcpp::export]] +List c_align_running_time(List data, NumericVector FV, NumericVector idxValue, bool maximization, int idxTarget) { + int NC = data.size(); + int NR = FV.size(); + int M = idxValue.size(); + + List res(M); + for (int i = 0; i < M; i++) { + NumericMatrix aux(NR, NC); + rownames(aux) = FV; + res[i] = clone(aux); + } + + for (int i = 0; i < NC; i++) { + NumericMatrix d = data[i]; + NumericMatrix value(d.nrow(), M); + + for (int j = 0; j < M; j++) { + value(_, j) = d(_, idxValue[j]); + } + + NumericMatrix tmp = c_impute_running_time(d(_, idxTarget), value, FV, maximization); + + for (int k = 0; k < M; k++) { + NumericMatrix aux = res[k]; + aux(_, i) = tmp(_, k); + } + } + return res; +} diff --git a/src/read.cc b/src/read.cc index 9ff229e3..7eb4483d 100644 --- a/src/read.cc +++ b/src/read.cc @@ -1,68 +1,68 @@ -// [[Rcpp::plugins(cpp11)]] -#include -#include -#include -#include -#include -#include - -using namespace Rcpp; -using namespace std; - -Rcpp::NumericMatrix make_mat(Rcpp::List input_list){ - - unsigned int n = input_list.length(); - - if(n == 0) { - Rcpp::stop("Must supply a list with more than 1 element."); - } - - Rcpp::NumericVector testvals = input_list[0]; - unsigned int elems = testvals.length(); - - Rcpp::NumericMatrix result_mat = Rcpp::no_init(n, elems); - - for(unsigned int i = 0; i < n; i++) { - Rcpp::NumericVector row_val = input_list[i]; - - if(elems != (unsigned int)row_val.length()) { - Rcpp::stop("Length of row does not match matrix requirements"); - } - - result_mat(i, Rcpp::_) = row_val; - } - - return result_mat; -} - -// [[Rcpp::export]] -List c_read_dat(std::string dat, int NC, std::string leading) { - int i; - std::string line, val, replacement (" "); - std::ifstream fp(dat.c_str()); - std::regex re("\\s+|\t+"); // in case of multiple whitespaces or tab - - List dataSets, one_run; - NumericVector row(NC); - - while (std::getline(fp, line)) { - if (line.find(leading) == 0) { - if (one_run.size() != 0) { - dataSets.push_back(make_mat(one_run)); - } - one_run = Rcpp::List::create(); - continue; - } - - std::stringstream ss(line); - - for (i = 0; i < NC; ++i) { - std::getline(ss, val, ' '); - row[i] = std::stod(val); - } - - one_run.push_back(Rcpp::clone(row)); - } - dataSets.push_back(make_mat(one_run)); - return dataSets; -} +// [[Rcpp::plugins(cpp11)]] +#include +#include +#include +#include +#include +#include + +using namespace Rcpp; +using namespace std; + +Rcpp::NumericMatrix make_mat(Rcpp::List input_list){ + + unsigned int n = input_list.length(); + + if(n == 0) { + Rcpp::stop("Must supply a list with more than 1 element."); + } + + Rcpp::NumericVector testvals = input_list[0]; + unsigned int elems = testvals.length(); + + Rcpp::NumericMatrix result_mat = Rcpp::no_init(n, elems); + + for(unsigned int i = 0; i < n; i++) { + Rcpp::NumericVector row_val = input_list[i]; + + if(elems != (unsigned int)row_val.length()) { + Rcpp::stop("Length of row does not match matrix requirements"); + } + + result_mat(i, Rcpp::_) = row_val; + } + + return result_mat; +} + +// [[Rcpp::export]] +List c_read_dat(std::string dat, int NC, std::string leading) { + int i; + std::string line, val, replacement (" "); + std::ifstream fp(dat.c_str()); + std::regex re("\\s+|\t+"); // in case of multiple whitespaces or tab + + List dataSets, one_run; + NumericVector row(NC); + + while (std::getline(fp, line)) { + if (line.find(leading) == 0) { + if (one_run.size() != 0) { + dataSets.push_back(make_mat(one_run)); + } + one_run = Rcpp::List::create(); + continue; + } + + std::stringstream ss(line); + + for (i = 0; i < NC; ++i) { + std::getline(ss, val, ' '); + row[i] = std::stod(val); + } + + one_run.push_back(Rcpp::clone(row)); + } + dataSets.push_back(make_mat(one_run)); + return dataSets; +} diff --git a/tests/testthat.R b/tests/testthat.R index 121c9107..cc4f02e3 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -1,4 +1,4 @@ -library(testthat) -library(IOHanalyzer) - -test_check("IOHanalyzer") +library(testthat) +library(IOHanalyzer) + +test_check("IOHanalyzer") diff --git a/tests/testthat/test_DataSetList.R b/tests/testthat/test_DataSetList.R index e8863565..c93d42be 100644 --- a/tests/testthat/test_DataSetList.R +++ b/tests/testthat/test_DataSetList.R @@ -1,47 +1,47 @@ -library(testthat) -library(IOHanalyzer) - -context("Basic DataSetList functionality") - -test_that("Can DataSetLists be loaded?",{ - expect_true(any(match(class(dsl), "DataSetList"))) - expect_true(any(match(class(dsl[[1]]), "DataSet"))) -}) - -test_that("Validate reading of files",{ - path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") - expect_equal(check_format(path), "IOHprofiler") - dsl1 <- DataSetList(path) - - expect_true(any(match(class(dsl1), "DataSetList"))) - expect_equal(get_dim(dsl1), 100) - expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") - expect_equal(get_funcId(dsl1), 1) - - ds1 <- dsl1[[1]] - expect_true(any(match(class(ds1), "DataSet"))) - expect_equal(attr(ds1,'DIM'), 100) - expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") - expect_equal(attr(ds1,'suite'), "PBO") - expect_equal(attr(ds1,'funcId'), 1) - expect_true(all(attr(ds1,'instance') == 1)) - expect_equal(attr(ds1,'format'), "IOHprofiler") - expect_equal(attr(ds1,'maximization'), TRUE) -}) - -test_that("Validate overview, summary and sample functions",{ - expect_equal(get_FV_summary(dsl[1], 12)$"98%", 14) - expect_equal(get_FV_summary(dsl[[1]], 12)$"98%", 14) - expect_equal(get_RT_summary(dsl[1], 12)$"98%", 21) - expect_equal(get_RT_summary(dsl[[1]], 12)$"98%", 21) - expect_equal(get_FV_overview(dsl[1])$"mean reached", 16) - expect_equal(get_FV_overview(dsl[[1]])$"mean reached", 16) - expect_equal(get_RT_overview(dsl[[1]])$"runs", 11) - expect_equal(get_RT_overview(dsl[1])$"runs", 11) - expect_equal(get_FV_sample(dsl[[1]],12)$"run.5",13) - expect_equal(get_FV_sample(dsl[1],12)$"run.5",13) - expect_equal(get_RT_sample(dsl[[1]],12)$"run.5",10) - expect_equal(get_RT_sample(dsl[1],12)$"run.5",10) - expect_equal(min(get_funvals(dsl[1])),5) - expect_equal(min(get_runtimes(dsl[1])),1) -}) +library(testthat) +library(IOHanalyzer) + +context("Basic DataSetList functionality") + +test_that("Can DataSetLists be loaded?",{ + expect_true(any(match(class(dsl), "DataSetList"))) + expect_true(any(match(class(dsl[[1]]), "DataSet"))) +}) + +test_that("Validate reading of files",{ + path <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package = "IOHanalyzer") + expect_equal(check_format(path), "IOHprofiler") + dsl1 <- DataSetList(path) + + expect_true(any(match(class(dsl1), "DataSetList"))) + expect_equal(get_dim(dsl1), 100) + expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") + expect_equal(get_funcId(dsl1), 1) + + ds1 <- dsl1[[1]] + expect_true(any(match(class(ds1), "DataSet"))) + expect_equal(attr(ds1,'DIM'), 100) + expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") + expect_equal(attr(ds1,'suite'), "PBO") + expect_equal(attr(ds1,'funcId'), 1) + expect_true(all(attr(ds1,'instance') == 1)) + expect_equal(attr(ds1,'format'), "IOHprofiler") + expect_equal(attr(ds1,'maximization'), TRUE) +}) + +test_that("Validate overview, summary and sample functions",{ + expect_equal(get_FV_summary(dsl[1], 12)$"98%", 14) + expect_equal(get_FV_summary(dsl[[1]], 12)$"98%", 14) + expect_equal(get_RT_summary(dsl[1], 12)$"98%", 21) + expect_equal(get_RT_summary(dsl[[1]], 12)$"98%", 21) + expect_equal(get_FV_overview(dsl[1])$"mean reached", 16) + expect_equal(get_FV_overview(dsl[[1]])$"mean reached", 16) + expect_equal(get_RT_overview(dsl[[1]])$"runs", 11) + expect_equal(get_RT_overview(dsl[1])$"runs", 11) + expect_equal(get_FV_sample(dsl[[1]],12)$"run.5",13) + expect_equal(get_FV_sample(dsl[1],12)$"run.5",13) + expect_equal(get_RT_sample(dsl[[1]],12)$"run.5",10) + expect_equal(get_RT_sample(dsl[1],12)$"run.5",10) + expect_equal(min(get_funvals(dsl[1])),5) + expect_equal(min(get_runtimes(dsl[1])),1) +}) diff --git a/tests/testthat/test_DataSetList_named_with_strings.R b/tests/testthat/test_DataSetList_named_with_strings.R index bca267fe..a6aab6a8 100644 --- a/tests/testthat/test_DataSetList_named_with_strings.R +++ b/tests/testthat/test_DataSetList_named_with_strings.R @@ -1,62 +1,62 @@ -library(testthat) -library(IOHanalyzer) - -context("Basic DataSetList functionality, but with String-Named Functions") - - - -test_that("Validate reading of files",{ - path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") - expect_equal(check_format(path1), "IOHprofiler") - dsl1 <- DataSetList(path1) - - expect_true(any(match(class(dsl1), "DataSetList"))) - expect_equal(get_dim(dsl1), 100) - expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") - expect_equal(get_funcId(dsl1), c('bla', 'blubb')) - - ds1 <- dsl1[[1]] - expect_true(any(match(class(ds1), "DataSet"))) - expect_equal(attr(ds1,'DIM'), 100) - expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") - #expect_equal(attr(ds1,'suite'), "PBO") - expect_equal(attr(ds1,'funcId'), 'bla') - expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) - expect_equal(attr(ds1,'format'), "IOHprofiler") - expect_equal(attr(ds1,'maximization'), TRUE) - - ds1 <- dsl1[[2]] - expect_true(any(match(class(ds1), "DataSet"))) - expect_equal(attr(ds1,'DIM'), 100) - expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") - #expect_equal(attr(ds1,'suite'), "PBO") - expect_equal(attr(ds1,'funcId'), 'blubb') - expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) - expect_equal(attr(ds1,'format'), "IOHprofiler") - expect_equal(attr(ds1,'maximization'), TRUE) -}) - -test_that("Validate overview, summary and sample functions",{ - path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") - expect_equal(check_format(path1), "IOHprofiler") - dsl1 <- DataSetList(path1) - - path2 <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") - expect_equal(check_format(path2), "IOHprofiler") - dsl2 <- DataSetList(path2) - - expect_equal(get_FV_summary(dsl1[1], 12)$"98%", get_FV_summary(dsl2[1], 12)$"98%") - expect_equal(get_FV_summary(dsl1[[1]], 12)$"98%", get_FV_summary(dsl2[[1]], 12)$"98%") - expect_equal(get_RT_summary(dsl1[1], 12)$"98%", get_RT_summary(dsl2[1], 12)$"98%") - expect_equal(get_RT_summary(dsl1[[1]], 12)$"98%", get_RT_summary(dsl2[[1]], 12)$"98%") - expect_equal(get_FV_overview(dsl1[1])$"mean reached", get_FV_overview(dsl2[1])$"mean reached") - expect_equal(get_FV_overview(dsl1[[1]])$"mean reached", get_FV_overview(dsl2[[1]])$"mean reached") - expect_equal(get_RT_overview(dsl1[[1]])$"runs", get_RT_overview(dsl2[[1]])$"runs") - expect_equal(get_RT_overview(dsl1[1])$"runs", get_RT_overview(dsl2[1])$"runs") - expect_equal(get_FV_sample(dsl1[[1]],12)$"run.5",get_FV_sample(dsl2[[1]],12)$"run.5") - expect_equal(get_FV_sample(dsl1[1],12)$"run.5",get_FV_sample(dsl2[1],12)$"run.5") - expect_equal(get_RT_sample(dsl1[[1]],12)$"run.5",get_RT_sample(dsl2[[1]],12)$"run.5") - expect_equal(get_RT_sample(dsl1[1],12)$"run.5",get_RT_sample(dsl2[1],12)$"run.5") - expect_equal(min(get_funvals(dsl1[1])),min(get_funvals(dsl2[1]))) - expect_equal(min(get_runtimes(dsl1[1])),min(get_runtimes(dsl2[1]))) -}) +library(testthat) +library(IOHanalyzer) + +context("Basic DataSetList functionality, but with String-Named Functions") + + + +test_that("Validate reading of files",{ + path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") + expect_equal(check_format(path1), "IOHprofiler") + dsl1 <- DataSetList(path1) + + expect_true(any(match(class(dsl1), "DataSetList"))) + expect_equal(get_dim(dsl1), 100) + expect_equal(get_algId(dsl1), "ONE_PLUS_LAMDA_EA") + expect_equal(get_funcId(dsl1), c('bla', 'blubb')) + + ds1 <- dsl1[[1]] + expect_true(any(match(class(ds1), "DataSet"))) + expect_equal(attr(ds1,'DIM'), 100) + expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") + #expect_equal(attr(ds1,'suite'), "PBO") + expect_equal(attr(ds1,'funcId'), 'bla') + expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) + expect_equal(attr(ds1,'format'), "IOHprofiler") + expect_equal(attr(ds1,'maximization'), TRUE) + + ds1 <- dsl1[[2]] + expect_true(any(match(class(ds1), "DataSet"))) + expect_equal(attr(ds1,'DIM'), 100) + expect_equal(attr(ds1,'algId'), "ONE_PLUS_LAMDA_EA") + #expect_equal(attr(ds1,'suite'), "PBO") + expect_equal(attr(ds1,'funcId'), 'blubb') + expect_equal(attr(ds1,'instance'), c(1,1,1,1,1,1,1,1,1,1)) + expect_equal(attr(ds1,'format'), "IOHprofiler") + expect_equal(attr(ds1,'maximization'), TRUE) +}) + +test_that("Validate overview, summary and sample functions",{ + path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") + expect_equal(check_format(path1), "IOHprofiler") + dsl1 <- DataSetList(path1) + + path2 <- system.file("extdata", "ONE_PLUS_LAMDA_EA", package="IOHanalyzer") + expect_equal(check_format(path2), "IOHprofiler") + dsl2 <- DataSetList(path2) + + expect_equal(get_FV_summary(dsl1[1], 12)$"98%", get_FV_summary(dsl2[1], 12)$"98%") + expect_equal(get_FV_summary(dsl1[[1]], 12)$"98%", get_FV_summary(dsl2[[1]], 12)$"98%") + expect_equal(get_RT_summary(dsl1[1], 12)$"98%", get_RT_summary(dsl2[1], 12)$"98%") + expect_equal(get_RT_summary(dsl1[[1]], 12)$"98%", get_RT_summary(dsl2[[1]], 12)$"98%") + expect_equal(get_FV_overview(dsl1[1])$"mean reached", get_FV_overview(dsl2[1])$"mean reached") + expect_equal(get_FV_overview(dsl1[[1]])$"mean reached", get_FV_overview(dsl2[[1]])$"mean reached") + expect_equal(get_RT_overview(dsl1[[1]])$"runs", get_RT_overview(dsl2[[1]])$"runs") + expect_equal(get_RT_overview(dsl1[1])$"runs", get_RT_overview(dsl2[1])$"runs") + expect_equal(get_FV_sample(dsl1[[1]],12)$"run.5",get_FV_sample(dsl2[[1]],12)$"run.5") + expect_equal(get_FV_sample(dsl1[1],12)$"run.5",get_FV_sample(dsl2[1],12)$"run.5") + expect_equal(get_RT_sample(dsl1[[1]],12)$"run.5",get_RT_sample(dsl2[[1]],12)$"run.5") + expect_equal(get_RT_sample(dsl1[1],12)$"run.5",get_RT_sample(dsl2[1],12)$"run.5") + expect_equal(min(get_funvals(dsl1[1])),min(get_funvals(dsl2[1]))) + expect_equal(min(get_runtimes(dsl1[1])),min(get_runtimes(dsl2[1]))) +}) diff --git a/tests/testthat/test_data_generation.R b/tests/testthat/test_data_generation.R index 1828aa23..221182f4 100644 --- a/tests/testthat/test_data_generation.R +++ b/tests/testthat/test_data_generation.R @@ -1,116 +1,116 @@ -context("Data generation for plotting") - -test_that("Single-function info",{ - dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "target", "ERT", - "lower", "upper", "mean", "median") %in% colnames(dt))) - - dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "runtime", - "lower", "upper", "mean", "median") %in% colnames(dt))) - expect_false("ERT" %in% colnames(dt)) - expect_error(generate_data.Single_Function(dsl)) -}) - -test_that("PMF-data", { - dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "target", "RT") %in% colnames(dt))) - expect_false("f(x)" %in% colnames(dt)) - - dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 100, which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("DIM", "funcId", "algId", "runtime", "f(x)") %in% colnames(dt))) - expect_false("RT" %in% colnames(dt)) - expect_error(generate_data.PMF(dsl)) -}) - -test_that("Histogram-data", { - dt <- generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "y", "width", "text", "algId") %in% colnames(dt))) - - dt <- generate_data.hist(subset(dsl, funcId == 1), target = 100, which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "y", "width", "text", "algId") %in% colnames(dt))) - expect_error(generate_data.hist(dsl)) -}) - -test_that("ECDF-data (single function)", { - dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) - - dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) - expect_error(generate_data.ECDF(dsl, c(10, 15, 16))) -}) - -test_that("ECDF-data (multiple functions, auto-generated targets)", { - targets <- get_ECDF_targets(dsl, 'linear', 3) - expect_true(is.data.table(targets)) - - dt <- generate_data.ECDF(dsl, targets) - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) - - dt <- generate_data.ECDF(dsl, c(1, 10, 100), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "mean", "algId") %in% colnames(dt))) - expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) -}) - -test_that("AUC-data", { - dt <- generate_data.AUC(subset(dsl, funcId == 1), c(10, 13, 16)) - expect_true(is.data.table(dt)) - expect_true(all(c("x", "auc", "algId") %in% colnames(dt))) - expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) - - dt <- generate_data.AUC(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("x", "auc", "algId") %in% colnames(dt))) - expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) - - dt <- generate_data.AUC(dsl, get_ECDF_targets(dsl)) - expect_true(is.data.table(dt)) - subset(dsl, funcId == 1) - expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) -}) - -test_that("Parameter-data", { - dt <- generate_data.Parameters(subset(dsl, funcId == 1)) - expect_true(is.data.table(dt)) - expect_true(all(c("algId", "runtime", "parId", - "lower", "upper", "mean", "median") %in% colnames(dt))) - - dt <- generate_data.Parameters(subset(dsl, funcId == 1), which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("algId", "target", "parId", - "lower", "upper", "mean", "median") %in% colnames(dt))) - - expect_error(generate_data.Parameters(dsl)) -}) - -test_that("Aggregated data for multiple functions / dimensions", { - targets <- get_target_dt(dsl) - expect_true(is.data.table(targets)) - expect_true(all(c("funcId", "DIM", "target") %in% colnames(targets))) - - dt <- generate_data.Aggr(dsl, targets = targets) - expect_true(is.data.table(dt)) - expect_true(all(c("algId", "target", "rank", "DIM", "funcId", "value", - "median") %in% colnames(dt))) - - dt <- generate_data.Aggr(dsl, which = 'by_FV') - expect_true(is.data.table(dt)) - expect_true(all(c("algId", "runtime", "rank", "DIM", "funcId", "value", - "median") %in% colnames(dt))) - - expect_error(generate_data.Aggr(dsl, targets = c(12, 16))) -}) +context("Data generation for plotting") + +test_that("Single-function info",{ + dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_RT') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "target", "ERT", + "lower", "upper", "mean", "median") %in% colnames(dt))) + + dt <- generate_data.Single_Function(subset(dsl, funcId == 1), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "runtime", + "lower", "upper", "mean", "median") %in% colnames(dt))) + expect_false("ERT" %in% colnames(dt)) + expect_error(generate_data.Single_Function(dsl)) +}) + +test_that("PMF-data", { + dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 15, which = 'by_RT') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "target", "RT") %in% colnames(dt))) + expect_false("f(x)" %in% colnames(dt)) + + dt <- generate_data.PMF(subset(dsl, funcId == 1), target = 100, which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("DIM", "funcId", "ID", "runtime", "f(x)") %in% colnames(dt))) + expect_false("RT" %in% colnames(dt)) + expect_error(generate_data.PMF(dsl)) +}) + +test_that("Histogram-data", { + dt <- generate_data.hist(subset(dsl, funcId == 1), target = 15, which = 'by_RT') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) + + dt <- generate_data.hist(subset(dsl, funcId == 1), target = 100, which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "y", "width", "text", "ID") %in% colnames(dt))) + expect_error(generate_data.hist(dsl)) +}) + +test_that("ECDF-data (single function)", { + dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(10, 15, 16)) + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) + + dt <- generate_data.ECDF(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) + expect_error(generate_data.ECDF(dsl, c(10, 15, 16))) +}) + +test_that("ECDF-data (multiple functions, auto-generated targets)", { + targets <- get_ECDF_targets(dsl, 'linear', 3) + expect_true(is.data.table(targets)) + + dt <- generate_data.ECDF(dsl, targets) + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) + + dt <- generate_data.ECDF(dsl, c(1, 10, 100), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "mean", "ID") %in% colnames(dt))) + expect_true(all(dt[['mean']] <= 1) && all(dt[['mean']] >= 0) ) +}) + +test_that("AUC-data", { + dt <- generate_data.AUC(subset(dsl, funcId == 1), c(10, 13, 16)) + expect_true(is.data.table(dt)) + expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) + expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) + + dt <- generate_data.AUC(subset(dsl, funcId == 1), c(1, 10, 100), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("x", "auc", "ID") %in% colnames(dt))) + expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) + + dt <- generate_data.AUC(dsl, get_ECDF_targets(dsl)) + expect_true(is.data.table(dt)) + subset(dsl, funcId == 1) + expect_true(all(dt[['auc']] <= 1) && all(dt[['auc']] >= 0) ) +}) + +test_that("Parameter-data", { + dt <- generate_data.Parameters(subset(dsl, funcId == 1)) + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "runtime", "parId", + "lower", "upper", "mean", "median") %in% colnames(dt))) + + dt <- generate_data.Parameters(subset(dsl, funcId == 1), which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "target", "parId", + "lower", "upper", "mean", "median") %in% colnames(dt))) + + expect_error(generate_data.Parameters(dsl)) +}) + +test_that("Aggregated data for multiple functions / dimensions", { + targets <- get_target_dt(dsl) + expect_true(is.data.table(targets)) + expect_true(all(c("funcId", "DIM", "target") %in% colnames(targets))) + + dt <- generate_data.Aggr(dsl, targets = targets) + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "target", "rank", "DIM", "funcId", "value", + "median") %in% colnames(dt))) + + dt <- generate_data.Aggr(dsl, which = 'by_FV') + expect_true(is.data.table(dt)) + expect_true(all(c("ID", "runtime", "rank", "DIM", "funcId", "value", + "median") %in% colnames(dt))) + + expect_error(generate_data.Aggr(dsl, targets = c(12, 16))) +}) diff --git a/tests/testthat/test_diagram_examples_with_string_names.R b/tests/testthat/test_diagram_examples_with_string_names.R index e554cfa7..c0a203e3 100644 --- a/tests/testthat/test_diagram_examples_with_string_names.R +++ b/tests/testthat/test_diagram_examples_with_string_names.R @@ -1,36 +1,36 @@ -library(testthat) -library(IOHanalyzer) - -context("Test the Diagram Examples with String Names") - - - -test_that("Test the diagram examples with string names",{ - path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") - expect_equal(check_format(path1), "IOHprofiler") - dsl1 <- DataSetList(path1) - - Plot.RT.Single_Func(subset(dsl1, funcId == 'bla')) - Plot.FV.Single_Func(subset(dsl1, funcId == 'bla')) - Plot.RT.PMF(subset(dsl1, funcId == 'bla'), 14) - Plot.RT.Histogram(subset(dsl1, funcId == 'bla'), 14) - Plot.RT.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 14) - Plot.RT.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) - suppressWarnings(Plot.RT.ECDF_AUC(subset(dsl1, funcId == 'bla'))) - Plot.FV.PDF(subset(dsl1, funcId == 'bla'), 100) - Plot.FV.Histogram(subset(dsl1, funcId == 'bla'), 100) - Plot.FV.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 10) - Plot.FV.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) - suppressWarnings(Plot.FV.ECDF_AUC(subset(dsl1, funcId == 'bla'))) - Plot.RT.Parameters(subset(dsl1, funcId == 'bla')) - Plot.FV.Parameters(subset(dsl1, funcId == 'bla')) - Plot.RT.ECDF_Multi_Func(dsl1) - Plot.RT.Multi_Func(dsl1) - Plot.RT.Aggregated(dsl1) - Plot.FV.Aggregated(dsl1) - Plot.FV.Multi_Func(dsl1) - Plot.Stats.Significance_Heatmap(subset(dsl1, funcId == 'bla'), 16) - Plot.Stats.Significance_Graph(subset(dsl1, funcId == 'bla'), 16) - #this will fail also with the original data, since only one alg present - #Plot.Stats.Glicko2_Candlestick(dsl1, nr_rounds=2) -}) +library(testthat) +library(IOHanalyzer) + +context("Test the Diagram Examples with String Names") + + + +test_that("Test the diagram examples with string names",{ + path1 <- system.file("extdata", "ONE_PLUS_LAMDA_EA_ws", package="IOHanalyzer") + expect_equal(check_format(path1), "IOHprofiler") + dsl1 <- DataSetList(path1) + + Plot.RT.Single_Func(subset(dsl1, funcId == 'bla')) + Plot.FV.Single_Func(subset(dsl1, funcId == 'bla')) + Plot.RT.PMF(subset(dsl1, funcId == 'bla'), 14) + Plot.RT.Histogram(subset(dsl1, funcId == 'bla'), 14) + Plot.RT.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 14) + Plot.RT.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) + suppressWarnings(Plot.RT.ECDF_AUC(subset(dsl1, funcId == 'bla'))) + Plot.FV.PDF(subset(dsl1, funcId == 'bla'), 100) + Plot.FV.Histogram(subset(dsl1, funcId == 'bla'), 100) + Plot.FV.ECDF_Per_Target(subset(dsl1, funcId == 'bla'), 10) + Plot.FV.ECDF_Single_Func(subset(dsl1, funcId == 'bla')) + suppressWarnings(Plot.FV.ECDF_AUC(subset(dsl1, funcId == 'bla'))) + Plot.RT.Parameters(subset(dsl1, funcId == 'bla')) + Plot.FV.Parameters(subset(dsl1, funcId == 'bla')) + Plot.RT.ECDF_Multi_Func(dsl1) + Plot.RT.Multi_Func(dsl1) + Plot.RT.Aggregated(dsl1) + Plot.FV.Aggregated(dsl1) + Plot.FV.Multi_Func(dsl1) + Plot.Stats.Significance_Heatmap(subset(dsl1, funcId == 'bla'), 16) + Plot.Stats.Significance_Graph(subset(dsl1, funcId == 'bla'), 16) + #this will fail also with the original data, since only one alg present + #Plot.Stats.Glicko2_Candlestick(dsl1, nr_rounds=2) +})