diff --git a/_extensions/openproblems-bio/custommdx/_extension.yml b/_extensions/openproblems-bio/custommdx/_extension.yml
new file mode 100644
index 0000000..2fd91f2
--- /dev/null
+++ b/_extensions/openproblems-bio/custommdx/_extension.yml
@@ -0,0 +1,25 @@
+title: Custom MDX
+author: OpenProblems
+contributes:
+ formats:
+ md:
+ output-ext: mdx
+ quarto-custom-format: custommdx-md
+ preserve-yaml: true
+ wrap: none
+ filters:
+ - at: post-render
+ path: custommdx_postrender.lua
+
+ variant: >-
+ gfm+pipe_tables+tex_math_dollars+header_attributes+raw_html
+ +all_symbols_escapable+backtick_code_blocks+fenced_code_blocks
+ +space_in_atx_header+intraword_underscores+lists_without_preceding_blankline
+ +shortcut_reference_links+footnotes+definition_lists+startnum
+
+ fig-width: 8
+ fig-height: 5
+ fig-format: svg
+
+ html-math-method: webtex
+
diff --git a/_extensions/openproblems-bio/custommdx/custommdx_postrender.lua b/_extensions/openproblems-bio/custommdx/custommdx_postrender.lua
new file mode 100644
index 0000000..71fcfa3
--- /dev/null
+++ b/_extensions/openproblems-bio/custommdx/custommdx_postrender.lua
@@ -0,0 +1,81 @@
+local function jsx(content)
+ return pandoc.RawBlock("markdown", content)
+end
+
+-- Check for the custom format (IMPORTANT: This is the correct way)
+local function isCustomMdxOutput()
+ return true
+ -- local name = param("quarto-custom-format", "")
+ -- print("Shortcode: ", name)
+ -- return name == "custommdx-md"
+end
+
+quarto._quarto.ast.add_renderer("Callout", isCustomMdxOutput, function(node)
+ return jsx(('\n%s\n'):format(
+ node.type:gsub("callout%-", ""), -- Extract "note", "tip", etc.
+ pandoc.write(pandoc.Pandoc(node.content), "markdown") -- Convert content to Markdown
+ ))
+end)
+
+quarto._quarto.ast.add_renderer("Tabset", isCustomMdxOutput, function(node)
+ local groupId = ""
+ local group = node.attr.attributes["group"]
+ if group then
+ groupId = ([[ group="%s"]]):format(group)
+ end
+
+ local tabValues = {}
+ for i = 1, #node.tabs do
+ tabValues[i] = pandoc.utils.stringify(node.tabs[i].title)
+ end
+
+ local tabs = pandoc.Div({})
+ tabs.content:insert(jsx("'))
+
+ for i = 1, #node.tabs do
+ local content = node.tabs[i].content
+ local title = node.tabs[i].title
+ tabs.content:insert(jsx(([[]]):format(pandoc.utils.stringify(title))))
+ if type(content) == "table" then
+ tabs.content:extend(content)
+ else
+ tabs.content:insert(content)
+ end
+ tabs.content:insert(jsx(""))
+ end
+
+ tabs.content:insert(jsx(""))
+ return tabs
+end)
+
+quarto._quarto.ast.add_renderer("FloatRefTarget", isCustomMdxOutput, function(float)
+ float = quarto.doc.crossref.decorate_caption_with_crossref(float)
+
+ -- note: is there a better way to do this?
+ local captionMd = pandoc.write(pandoc.Pandoc({ float.caption_long }), FORMAT, PANDOC_WRITER_OPTIONS)
+
+ return pandoc.Blocks({
+ jsx((''):format(float.identifier, captionMd)),
+ pandoc.Div(quarto.utils.as_blocks(float.content)),
+ jsx('')
+ })
+end)
+
+-- Remove simple HTML comments from text
+local function RawInline(el)
+ el.text = el.text:gsub("", "")
+ return el
+end
+
+-- Remove simple HTML comments from text
+local function RawBlock(el)
+ el.text = el.text:gsub("", "")
+ return el
+end
+
+return {
+ {
+ RawInline = RawInline,
+ RawBlock = RawBlock,
+ }
+}
diff --git a/_extensions/debruine/glossary/_extension.yml b/_extensions/rcannood/glossary/_extension.yml
similarity index 100%
rename from _extensions/debruine/glossary/_extension.yml
rename to _extensions/rcannood/glossary/_extension.yml
diff --git a/_extensions/debruine/glossary/glossary.lua b/_extensions/rcannood/glossary/glossary.lua
similarity index 84%
rename from _extensions/debruine/glossary/glossary.lua
rename to _extensions/rcannood/glossary/glossary.lua
index 3557149..d36fa8c 100644
--- a/_extensions/debruine/glossary/glossary.lua
+++ b/_extensions/rcannood/glossary/glossary.lua
@@ -55,10 +55,6 @@ end
-- Main Glossary Function Shortcode
return {
["glossary"] = function(args, kwargs2, meta)
- -- this will only run for HTML documents
- if not quarto.doc.isFormat("html:js") then
- return pandoc.Null()
- end
-- fix issue with kwargs
local kwargs = {}
@@ -102,12 +98,7 @@ return {
label = pandoc.utils.stringify(kwargs["label"])
end
- local url = "/reference/glossary/index.qmd#" .. term
- local infoIcon = pandoc.RawInline('html', '')
- local combinedLabel = pandoc.Span({pandoc.Str(label), infoIcon})
- local link = pandoc.Link(combinedLabel, url, "", pandoc.Attr("class", {"quarto-xref"}))
-
- return link
+ return pandoc.RawInline("markdown", "" .. label .. "")
end
end
}
diff --git a/_extensions/rcannood/openproblemsdocs/_extension.yml b/_extensions/rcannood/openproblemsdocs/_extension.yml
new file mode 100644
index 0000000..79c2e42
--- /dev/null
+++ b/_extensions/rcannood/openproblemsdocs/_extension.yml
@@ -0,0 +1,7 @@
+title: OpenProblemsDocs
+author: Robrecht Cannoodt
+version: 1.0.0
+quarto-required: ">=1.2.0"
+contributes:
+ filters:
+ - openproblemsdocs.lua
\ No newline at end of file
diff --git a/_extensions/rcannood/openproblemsdocs/openproblemsdocs.lua b/_extensions/rcannood/openproblemsdocs/openproblemsdocs.lua
new file mode 100644
index 0000000..d390580
--- /dev/null
+++ b/_extensions/rcannood/openproblemsdocs/openproblemsdocs.lua
@@ -0,0 +1,22 @@
+
+return {
+ {
+ -- wrap all divs with class="callout" in a Callout element
+ Callout = function(el)
+ local calloutContent = pandoc.utils.stringify(el.content)
+ return pandoc.RawBlock("markdown", "\n" .. calloutContent .. "\n")
+ end,
+ -- wrap all codeblocks with class="mermaid" in a Mermaid element
+ CodeBlock = function(el)
+ if el.attr.classes:includes("mermaid") then
+ local escapedTest = el.text:gsub("`", "\\`")
+ return pandoc.RawBlock("markdown", "")
+ end
+ end,
+ -- remove simple html comments from text
+ RawBlock = function(el)
+ el.text = el.text:gsub("", "")
+ return el
+ end
+ }
+}
diff --git a/_include/_clone_repo_openproblems.qmd b/_include/_clone_repo_openproblems.qmd
index a90d874..f3be3cb 100644
--- a/_include/_clone_repo_openproblems.qmd
+++ b/_include/_clone_repo_openproblems.qmd
@@ -1,5 +1,5 @@
-```{r setup_openporblems, include=FALSE}
+```{r setup_openproblems, include=FALSE}
# discard local changes from the /openproblems repo and
# sets the work directory to that directory.
repo_path <- system("git rev-parse --show-toplevel", intern = TRUE)
diff --git a/_include/_quarto_helpers.qmd b/_include/_quarto_helpers.qmd
new file mode 100644
index 0000000..05c5d88
--- /dev/null
+++ b/_include/_quarto_helpers.qmd
@@ -0,0 +1,50 @@
+```{r include=FALSE}
+# contents is a glob, e.g. "./*/index.qmd"
+# use it to list all files
+library(purrr, warn.conflicts = FALSE)
+library(dplyr, warn.conflicts = FALSE)
+library(stringr, warn.conflicts = FALSE)
+
+# search for qmd files in a directory
+listing <- function(
+ dir = ".",
+ pattern = "^([^/_][^/]*|[^/_][^/]*/index)\\.qmd$"
+) {
+ qmd_files <- list.files(
+ dir,
+ recursive = TRUE
+ ) |>
+ keep(~grepl(pattern, .)) |>
+ discard(~ . == "index.qmd")
+
+ df <- map_dfr(qmd_files, function(qmd_file) {
+ file <- file.path(dir, qmd_file)
+ tryCatch({
+ lines <- readr::read_lines(file)
+ yamlText <- lines[seq_len(which(lines == "---")[2] - 1)]
+ data <- yaml::yaml.load(yamlText)
+ for (col in names(data)) {
+ if (length(data[[col]]) != 1) {
+ data[[col]] <- list(data[[col]])
+ }
+ }
+ df_row <- as.data.frame(data)
+ df_row$path <- file |>
+ str_replace_all("/index\\.qmd", "") |>
+ str_replace_all("\\.qmd", "")
+ df_row
+ }, error = function(e) {
+ message("Error reading ", file, ": ", e$message)
+ return(NULL)
+ })
+ })
+
+ df$description <- df$description %||% ""
+ df$order <- df$order %||% 0
+
+ df |>
+ arrange(order, title, path) |>
+ mutate(str = glue::glue("* [**{title}**]({path}){ifelse(description != '', paste0(': ', description), '')}")) |>
+ pull(str)
+}
+```
\ No newline at end of file
diff --git a/_quarto.yml b/_quarto.yml
index 095bbea..bc09f30 100644
--- a/_quarto.yml
+++ b/_quarto.yml
@@ -2,8 +2,20 @@ project:
output-dir: _docs
format:
- docusaurus-md:
- toc: true
- fig-format: svg
+ custommdx-md:
+ bibliography: library.bib
+ glossary: _glossary.md
+
+ # gfm:
+ # toc: false
+ # fig-format: svg
+ # bibliography: library.bib
+ # output-ext: mdx
+ # glossary: _glossary.md
+ # variant: "+yaml_metadata_block"
+
execute:
freeze: auto
+
+# filters:
+# - custommdx
diff --git a/advanced_topics/create_a_dataset_loader.qmd b/advanced_topics/create_a_dataset_loader.qmd
index d9586ed..a078b31 100644
--- a/advanced_topics/create_a_dataset_loader.qmd
+++ b/advanced_topics/create_a_dataset_loader.qmd
@@ -7,7 +7,7 @@ engine: knitr
{{< include /_include/_clone_repo_openproblems.qmd >}}
{{< include /_include/_evaluate_code.qmd >}}
-A {{< glossary "dataset loader" >}} is used to generate one or more {{< glossary "raw dataset" label="raw datasets" >}}. A raw dataset is processed by the [Dataset preprocessing workflow](/reference/openproblems/src-datasets.qmd) to create a {{< glossary "common dataset" >}} that can be used by multiple benchmarking tasks.
+A {{< glossary "dataset loader" >}} is used to generate one or more {{< glossary "raw dataset" label="raw datasets" >}}. A raw dataset is processed by the [Dataset preprocessing workflow](/reference/openproblems/src-datasets) to create a {{< glossary "common dataset" >}} that can be used by multiple benchmarking tasks.
This guide will show you how to create a new {{< glossary Viash >}} component to fetch or generate datasets.
@@ -21,7 +21,7 @@ Common datasets are created by generating raw datasets with a data loader and ru
{{< include ../_figures/_project_structure_dataflow.qmd >}}
-See the [reference documentation](../reference/openproblems/src-datasets.qmd) for more information on how each of these steps works.
+See the [reference documentation](../reference/openproblems/src-datasets) for more information on how each of these steps works.
```{r setup-packages, include=FALSE}
library(tidyverse)
@@ -78,9 +78,9 @@ platforms:
HERE
```
-:::{.panel-tabset group="language"}
+
-## Python
+:::{#Python}
Contents of `src/datasets/loaders/myloader/config.vsh.yaml`:
@@ -88,7 +88,9 @@ Contents of `src/datasets/loaders/myloader/config.vsh.yaml`:
src/datasets/loaders/myloader/config.vsh.yaml
```
-## R
+:::
+
+:::{#R}
Contents of `src/datasets/loaders/myloader/config.vsh.yaml`:
@@ -114,8 +116,11 @@ platforms:
packages: [ libhdf5-dev, libgeos-dev, python3, python3-pip, python3-dev, python-is-python3 ]
- type: nextflow
```
+
:::
+
+
For more parameter options, refer to ["Parameters"](#parameters) section.
@@ -179,9 +184,9 @@ adata.write_h5ad(par["output"], compression="gzip")
HERE
```
-:::{.panel-tabset group="language"}
+
-## Python
+:::{#Python}
Contents of `src/datasets/loaders/myloader/script.py`:
@@ -189,7 +194,9 @@ Contents of `src/datasets/loaders/myloader/script.py`:
src/datasets/loaders/myloader/script.py
```
-## R
+:::
+
+:::{#R}
Contents of `src/datasets/loaders/myloader/script.R`:
@@ -243,9 +250,10 @@ adata <- AnnData(
# Write to file
adata$write_h5ad(par[["output"]], compression = "gzip")
```
-
:::
+
+
## Step 4: Run the component
Try running your component! You can start off by running your script inside your IDE.
@@ -284,9 +292,9 @@ arguments:
You can then use the `n_obs` and `n_vars` values in the `par` object to get access to the runtime parameters:
-:::{.panel-tabset group="language"}
+
-## Python
+:::{#Python}
```python
obs = pd.DataFrame({
@@ -298,8 +306,8 @@ var = pd.DataFrame(
index=[f"Gene_{i}" for i in range(100)]
)
```
-
-## R
+:::
+:::{#R}
```r
obs <- data.frame(
@@ -311,9 +319,10 @@ var <- data.frame(
row.names = paste0("Gene_", seq_len(par$n_vars))
)
```
-
:::
+
+
## Format of a raw dataset object
```{r, include=FALSE}
diff --git a/advanced_topics/technology_stack.qmd b/advanced_topics/technology_stack.qmd
index 44021cb..47c5f53 100644
--- a/advanced_topics/technology_stack.qmd
+++ b/advanced_topics/technology_stack.qmd
@@ -23,4 +23,4 @@ We use a variety of technologies to ensure the reliability, reproducibility, and
8. **Standardized Dataset Loaders and Processing Pipeline**: We offer a library of common datasets to be reused across tasks whenever possible. These datasets are derived from sources like CELLxGENE census and GEO and are processed by a standardized dataset processing pipeline. This uniformity in data processing ensures consistent data quality and structure across different tasks, facilitating comparability and consistency in benchmarking results.
-.](../images/repositories-thin.svg){#fig-project-structure}
\ No newline at end of file
+.](../images/repositories-thin.svg){#fig-project-structure}
diff --git a/create_component/add_a_control.qmd b/create_component/add_a_control.qmd
index 0276ee8..66421f6 100644
--- a/create_component/add_a_control.qmd
+++ b/create_component/add_a_control.qmd
@@ -16,7 +16,7 @@ In the following we will show examples for both Python and R.
Note that the Task template repo is used throughout the guide, so make sure to replace any occurrences of `"task_template"` with your task of interest.
:::{.callout-tip}
-Make sure you have followed the ["Getting started"](getting_started.qmd) guide.
+Make sure you have followed the ["Getting started"](getting_started) guide.
:::
## Step 1: Create a new component
@@ -92,9 +92,9 @@ The [Viash config](https://viash.io/reference/config/) contains metadata of your
This is what the `config.vsh.yaml` generated by the `create_component` component looks like:
-:::{.panel-tabset group="language"}
+
-### Python
+:::{#Python}
Contents of `config.vsh.yaml`
@@ -104,7 +104,9 @@ src/control_methods/my_python_method/config.vsh.yaml
-### R
+:::
+
+:::{#R}
Contents of `config.vsh.yaml`
@@ -116,6 +118,8 @@ src/control_methods/my_r_method/config.vsh.yaml
:::
+
+
### Required metadata fields
Please edit `info` section in the config file to fill in the necessary metadata.
@@ -166,9 +170,9 @@ e. Write output data to file
This is what the script generated by the `create_component` component looks like:
-:::{.panel-tabset group="language"}
+
-### Python
+:::{#Python}
Contents of `script.py`
@@ -178,7 +182,9 @@ src/control_methods/my_python_method/script.py
-### R
+:::
+
+:::{#R}
Contents of `script.R`
@@ -190,6 +196,8 @@ src/control_methods/my_r_method/script.R
:::
+
+
The required sections are explained here in more detail:
### a. Imports and libraries
@@ -238,7 +246,7 @@ viash test src/control_methods/true_labels/config.vsh.yaml
```
-Visit ["Run tests"](run_tests.qmd) for more information on running unit tests and how to interpret common error messages.
+Visit ["Run tests"](run_tests) for more information on running unit tests and how to interpret common error messages.
You can also run your component on local files using the `viash run` command. For example:
@@ -252,4 +260,4 @@ viash run src/tasks/label_projection/control_methods/my_python_method/config.vsh
## Next steps
-If your component works, please [create a pull request](../create_component/create_pull_request.qmd).
+If your component works, please [create a pull request](../create_component/create_pull_request).
diff --git a/create_component/add_a_method.qmd b/create_component/add_a_method.qmd
index 7a72571..eaadbab 100644
--- a/create_component/add_a_method.qmd
+++ b/create_component/add_a_method.qmd
@@ -16,7 +16,7 @@ In the following we will show examples for both Python and R.
Note that the Task template repo is used throughout the guide, so make sure to replace any occurrences of `"task_template"` with your task of interest.
:::{.callout-tip}
-Make sure you have followed the ["Getting started"](getting_started.qmd) guide.
+Make sure you have followed the ["Getting started"](getting_started) guide.
:::
## Step 1: Create a new component
@@ -91,9 +91,9 @@ The [Viash config](https://viash.io/reference/config/) contains metadata of your
This is what the `config.vsh.yaml` generated by the `create_component` component looks like:
-:::{.panel-tabset group="language"}
+
-### Python
+:::{#Python}
Contents of `config.vsh.yaml`
@@ -103,7 +103,9 @@ src/methods/my_python_method/config.vsh.yaml
-### R
+:::
+
+:::{#R}
Contents of `config.vsh.yaml`
@@ -115,6 +117,8 @@ src/methods/my_r_method/config.vsh.yaml
:::
+
+
### Required metadata fields
Please edit `info` section in the config file to fill in the necessary metadata.
@@ -164,9 +168,9 @@ e. Write output data to file
This is what the script generated by the `create_component` component looks like:
-:::{.panel-tabset group="language"}
+
-### Python
+:::{#Python}
Contents of `script.py`
@@ -176,7 +180,9 @@ src/methods/my_python_method/script.py
-### R
+:::
+
+:::{#R}
Contents of `script.R`
@@ -188,6 +194,8 @@ src/methods/my_r_method/script.R
:::
+
+
The required sections are explained here in more detail:
### a. Imports and libraries
@@ -237,7 +245,7 @@ viash test src/methods/logistic_regression/config.vsh.yaml
```
-Visit ["Run tests"](run_tests.qmd) for more information on running unit tests and how to interpret common error messages.
+Visit ["Run tests"](run_tests) for more information on running unit tests and how to interpret common error messages.
You can also run your component on local files using the `viash run` command. For example:
@@ -250,5 +258,5 @@ viash run src/methods/my_python_method/config.vsh.yaml -- \
## Next steps
-If your component works, please [create a pull request](../create_component/create_pull_request.qmd).
+If your component works, please [create a pull request](../create_component/create_pull_request).
diff --git a/create_component/add_a_metric.qmd b/create_component/add_a_metric.qmd
index 390c5b3..6be1318 100644
--- a/create_component/add_a_metric.qmd
+++ b/create_component/add_a_metric.qmd
@@ -16,7 +16,7 @@ In the following we will show examples for both Python and R.
Note that the Task template repo is used throughout the guide, so make sure to replace any occurrences of `"task_template"` with your task of interest.
:::{.callout-tip}
-Make sure you have followed the ["Getting started"](getting_started.qmd) guide.
+Make sure you have followed the ["Getting started"](getting_started) guide.
:::
@@ -86,9 +86,9 @@ The [Viash config](https://viash.io/reference/config/) contains metadata of your
This is what the `config.vsh.yaml` generated by the `create_component` component looks like:
-:::{.panel-tabset group="language"}
+
-### Python
+:::{#Python}
Contents of `config.vsh.yaml`
@@ -98,7 +98,9 @@ src/metrics/my_python_metric/config.vsh.yaml
-### R
+:::
+
+:::{#R}
Contents of `config.vsh.yaml`
@@ -110,6 +112,8 @@ src/metrics/my_r_metric/config.vsh.yaml
:::
+
+
### Required metadata fields
Please make sure that the following fields in the config file are filled in. The metrics component can contain several metric values these are listed in the `info.metrics`.
@@ -159,9 +163,9 @@ A component's script typically has five sections:
This is what the script generated by the `create_component` component looks like:
-:::{.panel-tabset group="language"}
+
-### Python
+:::{#Python}
Contents of `script.py`
@@ -176,7 +180,9 @@ cat src/metrics/my_python_metric/script.py
-### R
+:::
+
+:::{#R}
Contents of `script.R`
@@ -188,6 +194,8 @@ src/metrics/my_r_metric/script.R
:::
+
+
### Required sections
#### Imports and libraries
@@ -232,7 +240,7 @@ viash test src/metrics/accuracy/config.vsh.yaml
```
-Visit ["Run tests"](run_tests.qmd) for more information on running unit tests and how to interpret common error messages.
+Visit ["Run tests"](run_tests) for more information on running unit tests and how to interpret common error messages.
You can also run your component on local files using the `viash run` command. For example:
@@ -245,4 +253,4 @@ viash run src/metrics/my_python_metric/config.vsh.yaml -- \
## Next steps
-If your component works, please [create a pull request](../create_component/create_pull_request.qmd).
+If your component works, please [create a pull request](../create_component/create_pull_request).
diff --git a/create_component/first_contributions.qmd b/create_component/first_contributions.qmd
index 82c2960..fde8b1c 100644
--- a/create_component/first_contributions.qmd
+++ b/create_component/first_contributions.qmd
@@ -12,14 +12,14 @@ To contribute new functionality to OpenProblems, please follow the steps below:
- If someone is already working on it, post in that issue to express your interest or reach out to the contributors working on the issue directly.
- If there is no existing issue tracking the feature, create a new one.
-2. Review the [key concepts](../fundamentals/philosophy.qmd) of OpenProblems to get a good grasp of the framework as a whole.
+2. Review the [key concepts](../fundamentals/philosophy) of OpenProblems to get a good grasp of the framework as a whole.
-3. Install the [necessary requirements](../fundamentals/requirements.qmd) to work with OpenProblems.
+3. Install the [necessary requirements](../fundamentals/requirements) to work with OpenProblems.
-4. Follow the [getting started](getting_started.qmd) guide to fork and clone the repository and download the necessary test resources.
+4. Follow the [getting started](getting_started) guide to fork and clone the repository and download the necessary test resources.
-5. Add a [method](add_a_method.qmd), [metric](add_a_metric.qmd), or [control method](add_a_control.qmd) to the framework.
+5. Add a [method](add_a_method), [metric](add_a_metric), or [control method](add_a_control) to the framework.
-6. Run the provided [unit tests](run_tests.qmd) to ensure that the new functionality passes.
+6. Run the provided [unit tests](run_tests) to ensure that the new functionality passes.
-7. Create a [pull request](../create_component/create_pull_request.qmd) from your branch to the OpenProblems main branch. Don't forget to add an entry to the CHANGELOG.md file describing the proposed changes.
\ No newline at end of file
+7. Create a [pull request](../create_component/create_pull_request) from your branch to the OpenProblems main branch. Don't forget to add an entry to the CHANGELOG.md file describing the proposed changes.
\ No newline at end of file
diff --git a/create_component/getting_started.qmd b/create_component/getting_started.qmd
index 3c5bd02..0247e64 100644
--- a/create_component/getting_started.qmd
+++ b/create_component/getting_started.qmd
@@ -35,7 +35,7 @@ git clone --recursive
cd
```
-This will download a copy of the repository to your local machine. You can now make changes to the code, add new functionality, and commit your changes. Please commit your changes to a new branch following the [branch naming convention](/reference/naming/index.qmd).
+This will download a copy of the repository to your local machine. You can now make changes to the code, add new functionality, and commit your changes. Please commit your changes to a new branch following the [branch naming convention](/reference/naming/).
:::{.callout-note}
If somehow there are no files visible in the `common` submodule after cloning using the above command. Use the following command to update the submodule:
diff --git a/create_component/index.qmd b/create_component/index.qmd
index fa8e0b0..3dd14c8 100644
--- a/create_component/index.qmd
+++ b/create_component/index.qmd
@@ -2,9 +2,11 @@
title: Create component
description: Learn how to contribute a component to OpenProblems.
order: 20
-listing:
- template: ../../style/listing.ejs
- type: default
- contents: "./*.qmd"
page-navigation: true
---
+
+{{< include /_include/_quarto_helpers.qmd >}}
+
+```{r chapters, echo=FALSE, results='asis', warning=FALSE, message=FALSE, error=FALSE}
+listing()
+```
diff --git a/create_component/run_tests.qmd b/create_component/run_tests.qmd
index 3fc662e..501b078 100644
--- a/create_component/run_tests.qmd
+++ b/create_component/run_tests.qmd
@@ -64,7 +64,7 @@ viash ns test --parallel
## Common errors
-Below is a listing of common errors and how to solve them. If you come across any other problems, please take a look at our [troubleshooting](../more_information/troubleshooting.qmd) page, or reach out via [GitHub issues](https://github.com/openproblems-bio/openproblems/issues).
+Below is a listing of common errors and how to solve them. If you come across any other problems, please take a look at our [troubleshooting](../more_information/troubleshooting) page, or reach out via [GitHub issues](https://github.com/openproblems-bio/openproblems/issues).
### Assertion error
diff --git a/create_task/create_components.qmd b/create_task/create_components.qmd
index af13eba..4bfa645 100644
--- a/create_task/create_components.qmd
+++ b/create_task/create_components.qmd
@@ -16,22 +16,22 @@ Iterating over each of these steps will help you create the necessary components
## Step 1: Add a method
-Methods are the algorithms or approaches you want to benchmark in your task. To add a new method, follow the guide on [how to create a method](/create_component/add_a_method.qmd). Ensure that your method is compatible with the input/output format defined in your API and test it thoroughly.
+Methods are the algorithms or approaches you want to benchmark in your task. To add a new method, follow the guide on [how to create a method](/create_component/add_a_method). Ensure that your method is compatible with the input/output format defined in your API and test it thoroughly.
## Step 2: Add a control method
-Control methods serve as reference points for evaluating the performance of other methods in your task. They can be positive or negative controls, depending on their intended purpose. To add a control method method, follow the guide on [how to create a control method](/create_component/add_a_control.qmd). Make sure your control method method adheres to the input/output format defined in your API and test it for correctness.
+Control methods serve as reference points for evaluating the performance of other methods in your task. They can be positive or negative controls, depending on their intended purpose. To add a control method method, follow the guide on [how to create a control method](/create_component/add_a_control). Make sure your control method method adheres to the input/output format defined in your API and test it for correctness.
## Step 3: Add a metric
-Metrics are used to evaluate the performance of methods on your task. To add a new metric, follow the guide on [how to create a metric](/create_component/add_a_metric.qmd). Ensure that your metric is compatible with the input/output format defined in your API and test it to confirm its accuracy.
+Metrics are used to evaluate the performance of methods on your task. To add a new metric, follow the guide on [how to create a metric](/create_component/add_a_metric). Ensure that your metric is compatible with the input/output format defined in your API and test it to confirm its accuracy.
## Step 4: Run local tests
-Testing is a crucial part of the development process, ensuring that all components of your task function correctly and produce the expected results. To run tests on your components, follow the guide on [how to run tests](/create_component/run_tests.qmd). Regularly running tests as you develop your task will help you identify and fix any issues early on, leading to a more robust and reliable benchmarking task.
+Testing is a crucial part of the development process, ensuring that all components of your task function correctly and produce the expected results. To run tests on your components, follow the guide on [how to run tests](/create_component/run_tests). Regularly running tests as you develop your task will help you identify and fix any issues early on, leading to a more robust and reliable benchmarking task.
## Next steps
-Once you have iterated through these steps and have a working set of components for your task, you can proceed to integrate them into the OpenProblems framework and submit your task for review by [creating a pull request](/create_component/create_pull_request.qmd).
+Once you have iterated through these steps and have a working set of components for your task, you can proceed to integrate them into the OpenProblems framework and submit your task for review by [creating a pull request](/create_component/create_pull_request).
:::{.content-hidden}
TODO: add section on creating the remaining test resources to be able to run the unit tests.
diff --git a/create_task/dataset_processor.qmd b/create_task/dataset_processor.qmd
index 596619e..3a61418 100644
--- a/create_task/dataset_processor.qmd
+++ b/create_task/dataset_processor.qmd
@@ -33,12 +33,12 @@ The Dataset processor, like any component in OpenProblems, is a {{< glossary Via
# Step 1: Create the Viash config
-Start by creating the Viash config. Luckily, we already created its file format and argument list in the [previous section](design_api.qmd), so we can just import the interface using the `__merge__` field:
+Start by creating the Viash config. Luckily, we already created its file format and argument list in the [previous section](design_api), so we can just import the interface using the `__merge__` field:
-:::{.panel-tabset group="language"}
+
-# Python
+:::{#Python}
```{.yaml filename="src/data_processors/process_dataset/config.vsh.yaml"}
__merge__: ../../api/comp_data_processor.yaml #<1>
@@ -74,7 +74,9 @@ runners:
7. A [Docker engine](https://viash.io/guide/component/add-dependencies.html) to make the component reproducible.
8. A [Nextflow runner](https://viash.io/guide/nextflow_vdsl3/create-a-module.html) to allow turning this component into a Nextflow module (and also standalone pipeline).
-# R
+:::
+
+:::{#R}
```{.yaml filename="src/data_processors/process_dataset/config.vsh.yaml"}
@@ -113,12 +115,15 @@ runners:
:::
+
+
# Step 2: Create the script
Next, create the script which will split the common dataset into a train and solution dataset.
-:::{.panel-tabset group="language"}
-# Python
+
+
+:::{#Python}
```{.python filename="src/data_processors/process_dataset/script.py"}
import anndata as ad
@@ -152,7 +157,7 @@ output_dataset.write_h5ad(par["output_dataset"])
output_solution.write_h5ad(par["output_solution"])
```
-# R
+:::{#R}
```{.r filename="src/data_processors/process_dataset/script.R"}
library(anndata)
@@ -188,6 +193,8 @@ output_solution$write_h5ad(par[["output_solution"]])
:::
+
+
:::{.callout-tip}
You can use helper functions `subset_h5ad_by_format` defined in [`common/helper_functions/subset_h5ad_by_format.py`](https://github.com/openproblems-bio/common_resources/blob/main/helper_functions/subset_h5ad_by_format.py) to easily subset the AnnData to having only the slots specified by the corresponding API files. Examples of tasks where this is used: [Task template](https://github.com/openproblems-bio/task_template/blob/main/src/data_processors/process_dataset/script.py) [Dimensionality reduction](https://github.com/openproblems-bio/task_dimensionality_reduction/blob/main/src/data_processors/process_dataset/script.py).
:::
diff --git a/create_task/design_api.qmd b/create_task/design_api.qmd
index 354951a..ec43a6d 100644
--- a/create_task/design_api.qmd
+++ b/create_task/design_api.qmd
@@ -27,10 +27,10 @@ The [dimensionality reduction](https://github.com/openproblems-bio/task_dimensio
Having a formally defined API ensures consistency and interoperability across different components of your task. This makes it easier for others to contribute and build upon your work. Not only that, but creating API files (partially) automates the following steps:
-* Creating new [methods](../create_component/add_a_method.qmd) and [metrics](../create_component/add_a_metric.qmd) using the `create_component` scripts.
-* [Automated testing](../create_component/run_tests.qmd) of components.
+* Creating new [methods](../create_component/add_a_method) and [metrics](../create_component/add_a_metric) using the `create_component` scripts.
+* [Automated testing](../create_component/run_tests) of components.
* Generating [reference documentation](../reference/).
-* Implement a component for [processing the common datasets](dataset_processor.qmd).
+* Implement a component for [processing the common datasets](dataset_processor).
# How?
diff --git a/create_task/getting_started.qmd b/create_task/getting_started.qmd
index d6974cc..24650f9 100644
--- a/create_task/getting_started.qmd
+++ b/create_task/getting_started.qmd
@@ -17,7 +17,7 @@ Please check the OpenProblems [organisation](https://github.com/openproblems-bio
## Step 2: Reach out to the OpenProblems team
-If not, reach out to the OpenProblems team by creating a new task proposal. This can be done by creating a new issue in the OpenProblems repository using the [new task proposal template](https://github.com/openproblems-bio/openproblems/issues/new?assignees=&labels=task&projects=&template=new_task.yml). You can also reach out through [Discord](https://discord.gg/hDE5bYEcHF).
+If not, reach out to the OpenProblems team by creating a new task proposal. This can be done by creating a new issue in the OpenProblems repository using the [new task proposal template](https://github.com/openproblems-bio/openproblems/issues/new?assignees=&labels=task&projects=&template=new_task.yml). You can also reach out through [Discord](https://discord.gg/sDE7cM4PN7).
This collaborative process will help ensure that your task is well-defined, relevant, and compatible with the OpenProblems framework. Additionally, it informs others of your ongoing work in this area and establishes a foundation for potential collaboration. Check out [some examples](https://github.com/openproblems-bio/openproblems/labels/task) by filtering on the 'task' label on GitHub.
@@ -177,4 +177,4 @@ Now update the _viash.yaml file in the root of the task repository with the foll
## Ready, set, go!
-You are now well-equipped to begin [designing the API](design_api.qmd) for the new benchmarking task.
\ No newline at end of file
+You are now well-equipped to begin [designing the API](design_api) for the new benchmarking task.
\ No newline at end of file
diff --git a/create_task/index.qmd b/create_task/index.qmd
index 7a95c9e..7aafcdc 100644
--- a/create_task/index.qmd
+++ b/create_task/index.qmd
@@ -7,14 +7,14 @@ page-navigation: true
This chapter provides a step-by-step overview of the process of adding a new task. Here are the main sections:
-1. **[Task Structure](task_structure.qmd)**: The structure of a typical OpenProblems task, including its components and file formats.
+1. **[Task Structure](task_structure)**: The structure of a typical OpenProblems task, including its components and file formats.
-2. **[Getting Started](getting_started.qmd)**: How to get started with OpenProblems, including creating a new task proposal and repository.
+2. **[Getting Started](getting_started)**: How to get started with OpenProblems, including creating a new task proposal and repository.
-3. **[Design the API](design_api.qmd)**: Design a consistent API for your task components and input/output files, and learn how to create API files for each component.
+3. **[Design the API](design_api)**: Design a consistent API for your task components and input/output files, and learn how to create API files for each component.
-4. **[Dataset processor](dataset_processor.qmd)**: How to choose appropriate datasets for your task and create a dataset processing component to handle the data.
+4. **[Dataset processor](dataset_processor)**: How to choose appropriate datasets for your task and create a dataset processing component to handle the data.
-5. **[Create Components](create_components.qmd)**: Dive into the process of adding components to your task, such as dataset loaders, methods, control methods, and metrics.
+5. **[Create Components](create_components)**: Dive into the process of adding components to your task, such as dataset loaders, methods, control methods, and metrics.
-6. **[Create Workflow](create_workflow.qmd)**: Combine your components into a cohesive Nextflow workflow.
+6. **[Create Workflow](create_workflow)**: Combine your components into a cohesive Nextflow workflow.
diff --git a/fundamentals/common_commands.qmd b/fundamentals/common_commands.qmd
index 55dae38..e2f49fb 100644
--- a/fundamentals/common_commands.qmd
+++ b/fundamentals/common_commands.qmd
@@ -6,7 +6,7 @@ engine: knitr
---
Below is a list of common Viash commands to develop and test components.
-Check out this [Cheat sheet](/more_information/cheat_sheets.qmd) to get a printable version of the same information. Please visit the [Viash guide](https://viash.io/guide) or [reference documentation](https://viash.io/reference/) for more in-depth information.
+Check out this [Cheat sheet](/more_information/cheat_sheets) to get a printable version of the same information. Please visit the [Viash guide](https://viash.io/guide) or [reference documentation](https://viash.io/reference/) for more in-depth information.
:::{.callout-important}
Most of the common commands are added to a script in the `scripts` directory. You can run these scripts from the root directory of the project. Some scripts might need some additional arguments, which you can find in the script itself.
diff --git a/fundamentals/datasets.qmd b/fundamentals/datasets.qmd
index 759ea5e..5a1edc1 100644
--- a/fundamentals/datasets.qmd
+++ b/fundamentals/datasets.qmd
@@ -24,7 +24,7 @@ For more information on how these specifications are formatted, see ["Design the
# Common datasets
-OpenProblems offers a collection of common datasets that can be used to test components and run the benchmarking tasks. These datasets are generated by dataset loaders and processed by a common processing pipeline stored in [`src/datasets`](/reference/openproblems/src-datasets.qmd).
+OpenProblems offers a collection of common datasets that can be used to test components and run the benchmarking tasks. These datasets are generated by dataset loaders and processed by a common processing pipeline stored in [`src/datasets`](/reference/openproblems/src-datasets).
{{< include /_figures/_project_structure_dataset_workflow.qmd >}}
diff --git a/fundamentals/index.qmd b/fundamentals/index.qmd
index 2fda889..b3982fe 100644
--- a/fundamentals/index.qmd
+++ b/fundamentals/index.qmd
@@ -2,8 +2,10 @@
title: Fundamentals
description: The main philosophy and key concepts of the project.
order: 10
-listing:
- template: ../../style/listing.ejs
- type: default
- contents: "./*.qmd"
---
+
+{{< include /_include/_quarto_helpers.qmd >}}
+
+```{r chapters, echo=FALSE, results='asis', warning=FALSE, message=FALSE, error=FALSE}
+listing()
+```
diff --git a/fundamentals/philosophy.qmd b/fundamentals/philosophy.qmd
index 1fec2fd..3a7bd74 100644
--- a/fundamentals/philosophy.qmd
+++ b/fundamentals/philosophy.qmd
@@ -28,7 +28,7 @@ Viash is a meta-framework which allows generating modular {{< glossary Nextflow
Specifications for the format of AnnData files and the interface of the Viash component can be found in the `api` directory of each task, and are rendered in the readme of each task ([Example](https://github.com/openproblems-bio/openproblems/tree/main/src/tasks/label_projection)).
-Note that this task structure is an oversimplification -- both Dataset and Task workflows contain additional steps to [safeguard against common pitfalls](/advanced_topics/safeguards.qmd).
+Note that this task structure is an oversimplification -- both Dataset and Task workflows contain additional steps to [safeguard against common pitfalls](/advanced_topics/safeguards).
@@ -36,7 +36,7 @@ Note that this task structure is an oversimplification -- both Dataset and Task
## Reusability
We want contributors to be able to concentrate on the functionality of benchmarks, rather than the complexities of computational infrastructure.
-To this end, the [technologies used in OpenProblems](/advanced_topics/technology_stack.qmd) are chosen to maximize reproducibility, reusability and interoperability, while minimizing the amount of boilerplate code required to create new benchmarks.
+To this end, the [technologies used in OpenProblems](/advanced_topics/technology_stack) are chosen to maximize reproducibility, reusability and interoperability, while minimizing the amount of boilerplate code required to create new benchmarks.
Amongst others, these technologies include {{< glossary Viash >}} for defining computational components, {{< glossary AnnData >}} for storing data, {{< glossary Nextflow >}} for defining workflows, {{< glossary Docker >}} for containerization, and {{< glossary file format specification label="standardized file formats" >}} and {{< glossary component interface label="component interfaces" >}} for interoperability.
By leveraging these frameworks and tools, OpenProblems sets a high standard for reusability and efficiency in computational benchmarking, enabling developers to focus on innovation rather than busywork.
diff --git a/fundamentals/requirements.qmd b/fundamentals/requirements.qmd
index 95ab6bc..0297341 100644
--- a/fundamentals/requirements.qmd
+++ b/fundamentals/requirements.qmd
@@ -13,7 +13,7 @@ To start contributing to OpenProblems, you need to have the following software i
* Viash (>= 0.9.0)
* Nextflow (>= 21.04)
-The following sections provide instructions on how to install each of these tools. If you already have all of these tools installed, feel free to skip this boring page and go to the respective page on what you want to do either [Create component](../create_component/index.qmd) or [Create a new task](../create_task/index.qmd).
+The following sections provide instructions on how to install each of these tools. If you already have all of these tools installed, feel free to skip this boring page and go to the respective page on what you want to do either [Create component](../create_component/) or [Create a new task](../create_task/).
## Step 1: install Bash
diff --git a/index.qmd b/index.qmd
index 26c3436..41c1649 100644
--- a/index.qmd
+++ b/index.qmd
@@ -1,21 +1,19 @@
---
title: Documentation
order: 1
-listing:
- id: chapters
- template: ../style/listing.ejs
- type: default
- contents: "./*/index.qmd"
---
+{{< include /_include/_quarto_helpers.qmd >}}
+
Welcome to the OpenProblems documentation!
OpenProblems is a framework for machine learning scientists, computational biologists, and single-cell data analysts to benchmark and compare single-cell omics methods on a variety of tasks. We believe that improving reproducibility and transparency in research will accelerate progress in single-cell omics.
This documentation provides an overview of the framework and how to contribute to it. It is organized into the following chapters:
-:::{#chapters}
-:::
+```{r chapters, echo=FALSE, results='asis', warning=FALSE, message=FALSE, error=FALSE}
+listing()
+```
We encourage everyone who uses OpenProblems to contribute to the framework in any way they can. This can include submitting a pull request to improve the documentation, answering queries on the issue tracker, investigating bugs, reviewing other developers' pull requests, or simply reporting issues and giving a "thumbs up" to relevant issues.
@@ -23,5 +21,5 @@ One simple but important way to contribute is to spread the word about the libra
[{width=500px}](https://github.com/openproblems-bio/openproblems/stargazers)
-Finally, we want to emphasize that OpenProblems is an inclusive community and we expect all members to adhere to our [code of conduct](fundamentals/philosophy.qmd#inclusiveness). We hope that this documentation helps you get started with OpenProblems and we look forward to your contributions.
+Finally, we want to emphasize that OpenProblems is an inclusive community and we expect all members to adhere to our [code of conduct](fundamentals/philosophy#inclusiveness). We hope that this documentation helps you get started with OpenProblems and we look forward to your contributions.
diff --git a/more_information/index.qmd b/more_information/index.qmd
index ac4eaa8..a1b5fd3 100644
--- a/more_information/index.qmd
+++ b/more_information/index.qmd
@@ -2,8 +2,10 @@
title: More information
description: A collection of resources and reference materials designed to help you effectively navigate and contribute to the OpenProblems project.
order: 40
-listing:
- template: ../../style/listing.ejs
- type: default
- contents: "*.qmd"
----
\ No newline at end of file
+---
+
+{{< include /_include/_quarto_helpers.qmd >}}
+
+```{r chapters, echo=FALSE, results='asis', warning=FALSE, message=FALSE, error=FALSE}
+listing()
+```
diff --git a/reference/bibliography/biblio.css b/reference/bibliography/biblio.css
deleted file mode 100644
index 353f6be..0000000
--- a/reference/bibliography/biblio.css
+++ /dev/null
@@ -1,37 +0,0 @@
-.bibtexentry > .bibtex-citation {
- line-height: 1.05;
- margin-bottom: 10px;
-}
-
-.bibtexentry > .bibtex-citation:target {
- -webkit-animation: target-fade 3s;
- -moz-animation: target-fade 3s;
- -o-animation: target-fade 3s;
- animation: target-fade 3s;
- background-color: #ff414b44;
-}
-
-
-@-webkit-keyframes target-fade {
- from { background-color: #ff000d; }
- to { background-color: #ff414b44; }
-}
-
-@-moz-keyframes target-fade {
- from { background-color: #ff000d; }
- to { background-color: #ff414b44; }
-}
-@-o-keyframes target-fade {
- from { background-color: #ff000d; }
- to { background-color: #ff414b44; }
-}
-@keyframes target-fade {
- from { background-color: #ff000d; }
- to { background-color: #ff414b44; }
-}
-
-.bibtexentry > .bibtex-citation:target::before {
- font: 70% sans-serif;
- content: "►";
- margin-right: 0.25em;
-}
diff --git a/reference/bibliography/index.qmd b/reference/bibliography/index.qmd
deleted file mode 100644
index 1b0b28a..0000000
--- a/reference/bibliography/index.qmd
+++ /dev/null
@@ -1,89 +0,0 @@
----
-title: "Bibliography"
-description: "A list of all publications referred to in OpenProblems."
-title-block-banner: ../../../images/heros/bibliography.png
-title-block-banner-color: "#ffffff"
-page-layout: full
-css: [../../../style/style-hero.css, biblio.css]
-resources: ../../../library.bib
----
-
-```{=html}
-
-
-
-
-
-
-
-
-
-
-```
diff --git a/reference/index.qmd b/reference/index.qmd
index ba1cdba..f082853 100644
--- a/reference/index.qmd
+++ b/reference/index.qmd
@@ -2,8 +2,10 @@
title: Reference
description: Structured information about the OpenProblems project.
order: 90
-listing:
- template: ../../style/listing.ejs
- type: default
- contents: "*/index.qmd"
---
+
+{{< include /_include/_quarto_helpers.qmd >}}
+
+```{r chapters, echo=FALSE, results='asis', warning=FALSE, message=FALSE, error=FALSE}
+listing()
+```
diff --git a/reference/openproblems/index.qmd b/reference/openproblems/index.qmd
index 9aa5a54..37595a2 100644
--- a/reference/openproblems/index.qmd
+++ b/reference/openproblems/index.qmd
@@ -13,11 +13,11 @@ The main data flow of the pipeline is shown in @fig-dataflow. The common dataset
## Directory Structure
-* [`src/common`](src-common.qmd): This subdirectory contains helper components that helps with creating new components, unit testing other components, or managing task results.
-* [`src/datasets`](src-datasets.qmd): The dataset processing pipeline uses dataset loaders to create raw dataset files. The raw dataset files are then processed to generate common dataset files. Common dataset files are used in one or more tasks.
-* [`src/tasks/`](src-task_id.qmd): Each task should contain a data processor (to transform common datasets into task-specific datasets), methods, control methods (for quality control), and metrics.
-* [`resources_test`](test-resources.qmd): This directory contains the test resources required for running unit tests on the Viash components
-* [`target`](target.qmd): This directory contains the artifacts built from the Viash components in the `src` directory.
+* [`src/common`](src-common): This subdirectory contains helper components that helps with creating new components, unit testing other components, or managing task results.
+* [`src/datasets`](src-datasets): The dataset processing pipeline uses dataset loaders to create raw dataset files. The raw dataset files are then processed to generate common dataset files. Common dataset files are used in one or more tasks.
+* [`src/tasks/`](src-task_id): Each task should contain a data processor (to transform common datasets into task-specific datasets), methods, control methods (for quality control), and metrics.
+* [`resources_test`](test-resources): This directory contains the test resources required for running unit tests on the Viash components
+* [`target`](target): This directory contains the artifacts built from the Viash components in the `src` directory.
## Technology stack
diff --git a/renv.lock b/renv.lock
index 0b3975f..fd7c465 100644
--- a/renv.lock
+++ b/renv.lock
@@ -1372,7 +1372,7 @@
"Package": "openproblems",
"Version": "0.0.0.9000",
"Source": "GitHub",
- "Remotes": "openproblems.utils=local::../openproblems.utils",
+ "Remotes": "openproblems.utils=local::./_core",
"RemoteType": "github",
"RemoteHost": "api.github.com",
"RemoteRepo": "core",
@@ -1399,7 +1399,7 @@
"RemoteSubdir": "packages/r/openproblems.docs",
"RemoteRef": "main",
"RemoteSha": "d6a34ca0ebcfabdeac7e4f2e070143b5190631a4",
- "Remotes": "openproblems.utils=local::../openproblems.utils, openproblems=local::../openproblems",
+ "Remotes": "openproblems.utils=local::./_core, openproblems=local::../openproblems",
"Requirements": [
"R",
"cli",
diff --git a/script.sh b/script.sh
new file mode 100755
index 0000000..7a7bf21
--- /dev/null
+++ b/script.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+QUARTO_CHROMIUM_HEADLESS_MODE=new \
+ /home/rcannood/.local/share/quarto-1.6.40/bin/quarto render \
+ --profile evaluate_code
+
+rm -r ../website-new/src/routes/documentation
+cp -r _docs ../website-new/src/routes/documentation
diff --git a/style/listing.ejs b/style/listing.ejs
deleted file mode 100644
index 01bc608..0000000
--- a/style/listing.ejs
+++ /dev/null
@@ -1,9 +0,0 @@
-```{=html}
-