From 4f240746dbd8ac1e663cd3a2851f876c46bd05ed Mon Sep 17 00:00:00 2001 From: James Frost Date: Fri, 9 Jan 2026 16:49:41 +0000 Subject: [PATCH 1/8] Add search capability to CSET UI Additionally included is a conversion of the index to JSON Lines (In theory it should be more efficient, but in practice we still load the whole string first. At least it looks nicer in the index file.) Non-useful entries are removed from the index before it is written, so it only contains information the client will need. --- .../app/finish_website/bin/finish_website.py | 50 +- .../app/finish_website/file/html/index.html | 14 +- .../app/finish_website/file/html/script.js | 674 ++++++++++++++++-- .../app/finish_website/file/html/style.css | 227 ++++-- tests/workflow_utils/test_finish_website.py | 53 +- 5 files changed, 834 insertions(+), 184 deletions(-) diff --git a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py index f9a06218f..d970862fd 100755 --- a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py +++ b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py @@ -29,7 +29,7 @@ from importlib.metadata import version from pathlib import Path -from CSET._common import combine_dicts, sort_dict +from CSET._common import sort_dict logging.basicConfig( level=os.getenv("LOGLEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s" @@ -64,35 +64,25 @@ def install_website_skeleton(www_root_link: Path, www_content: Path): def construct_index(www_content: Path): """Construct the plot index.""" plots_dir = www_content / "plots" - index = {} - # Loop over all diagnostics and append to index. - for metadata_file in plots_dir.glob("**/*/meta.json"): - try: - with open(metadata_file, "rt", encoding="UTF-8") as fp: - plot_metadata = json.load(fp) - - category = plot_metadata["category"] - case_date = plot_metadata.get("case_date", "") - relative_url = str(metadata_file.parent.relative_to(plots_dir)) - - record = { - category: { - case_date if case_date else "Aggregation": { - relative_url: plot_metadata["title"].strip() - } - } - } - except (json.JSONDecodeError, KeyError, TypeError) as err: - logging.error("%s is invalid, skipping.\n%s", metadata_file, err) - continue - index = combine_dicts(index, record) - - # Sort index of diagnostics. - index = sort_dict(index) - - # Write out website index. - with open(plots_dir / "index.json", "wt", encoding="UTF-8") as fp: - json.dump(index, fp, indent=2) + with open(plots_dir / "index.jsonl", "wt", encoding="UTF-8") as index_fp: + # Loop over all diagnostics and append to index. The glob is sorted to + # ensure a consistent ordering. + for metadata_file in sorted(plots_dir.glob("**/*/meta.json")): + try: + with open(metadata_file, "rt", encoding="UTF-8") as plot_fp: + plot_metadata = json.load(plot_fp) + plot_metadata["path"] = str(metadata_file.parent.relative_to(plots_dir)) + # Remove keys that are not useful for the index. + plot_metadata.pop("description", None) + plot_metadata.pop("plots", None) + # Sort plot metadata. + plot_metadata = sort_dict(plot_metadata) + # Write metadata into website index. + json.dump(plot_metadata, index_fp, separators=(",", ":")) + index_fp.write("\n") + except (json.JSONDecodeError, KeyError, TypeError) as err: + logging.error("%s is invalid, skipping.\n%s", metadata_file, err) + continue def bust_cache(www_content: Path): diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/index.html b/src/CSET/cset_workflow/app/finish_website/file/html/index.html index b1b0a5de7..c7a53c6cc 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/index.html +++ b/src/CSET/cset_workflow/app/finish_website/file/html/index.html @@ -14,10 +14,20 @@

CSET

+ + + + +
+ Search facets +
+
-
- +
diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/script.js b/src/CSET/cset_workflow/app/finish_website/file/html/script.js index 265bb2289..0e05372e3 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/script.js +++ b/src/CSET/cset_workflow/app/finish_website/file/html/script.js @@ -1,6 +1,401 @@ // JavaScript code that is used by the pages. Plots should not rely on this // file, as it will not be stable. +/** Search query lexer and parser. + * + * EBNF to implement: + * + * query = expression ; + * + * expression = condition + * | expression , combiner ? , expression + * | "NOT" , expression + * | "(" , expression , ")" ; + * + * combiner = "AND" + * | "OR" ; + * + * condition = facet ? , operator ? , value ; + * + * facet = LITERAL , ":" ; + * + * value = LITERAL ; + * + * operator = NOT + * | GREATER_THAN + * | GREATER_THAN_OR_EQUALS + * | LESS_THAN + * | LESS_THAN_OR_EQUALS + * | NOT_EQUALS + * | EQUALS ; + */ + +class Literal { + constructor(value) { + this.value = value; + } +} + +class Facet { + constructor(value) { + this.value = value; + } +} + +const TOKEN_SPEC = new Map([ + ["Parenthesis_BEGIN", "\\("], + ["Parenthesis_END", "\\)"], + ["Operator_GREATER_THAN_OR_EQUALS", "<="], + ["Operator_GREATER_THAN", "<"], + ["Operator_LESS_THAN_OR_EQUALS", ">="], + ["Operator_LESS_THAN", ">"], + ["Operator_NOT_EQUALS", "!="], + ["Operator_EQUALS", "="], + ["Operator_NOT_IN", "!"], + ["Combiner_NOT", "\\bnot\\b"], + ["Combiner_AND", "\\band\\b"], + ["Combiner_OR", "\\bor\\b"], + ["LexOnly_WHITESPACE", "\\s+"], + ["LexOnly_FACET", "[a-z_\\-]+\\s*:"], + ["LexOnly_LITERAL", `'[^']*'|"[^"]*"|[^\\s\\(\\)]+`], +]); + +const TOKEN_REGEX = RegExp( + Array.from( + TOKEN_SPEC.entries().map((pair) => { + return `(?<${pair[0]}>${pair[1]})`; + }) + ).join("|"), + "ig" +); + +// Lex input string into tokens. +function lexer(query) { + const tokens = []; + for (const match of query.matchAll(TOKEN_REGEX)) { + // Split into tokens by matching the capture group names from TOKEN_SPEC. + if (!match.groups) { + throw new SyntaxError("Query did not consist of valid tokens."); + } + let [kind, value] = Object.entries(match.groups).filter( + (pair) => pair[1] !== undefined + )[0]; + + switch (kind) { + case "LexOnly_WHITESPACE": + continue; + case "LexOnly_FACET": + const facet_name = value.replace(/\s*:$/, ""); + tokens.push(new Facet(facet_name)); + break; + case "LexOnly_LITERAL": + if (/^".*"$|^'.+'$/.test(value)) { + value = value.slice(1, -1); + } + tokens.push(new Literal(value)); + break; + default: + tokens.push(kind); + break; + } + } + return tokens; +} + +class Condition { + constructor(value, facet = new Facet("title"), operator = "Operator_IN") { + if (typeof value == "function") { + this.func = value; + return; + } + + const v = value.value; + const f = facet.value; + let cond_func; + switch (operator) { + case "Operator_IN": + cond_func = function cond_in(d) { + return f in d && d[f].includes(v); + }; + break; + case "Operator_NOT_IN": + cond_func = function cond_nin(d) { + return f in d && !d[f].includes(v); + }; + break; + case "Operator_EQUALS": + cond_func = function cond_eq(d) { + return f in d && v == d[f]; + }; + break; + case "Operator_NOT_EQUALS": + cond_func = function cond_neq(d) { + return f in d && v != d[f]; + }; + break; + case "Operator_GREATER_THAN": + cond_func = function cond_gt(d) { + return f in d && v > d[f]; + }; + break; + case "Operator_GREATER_THAN_OR_EQUALS": + cond_func = function cond_gte(d) { + return f in d && v >= d[f]; + }; + break; + case "Operator_LESS_THAN": + cond_func = function cond_lt(d) { + return f in d && v < d[f]; + }; + break; + case "Operator_LESS_THAN_OR_EQUALS": + cond_func = function cond_lte(d) { + return f in d && v <= d[f]; + }; + break; + default: + throw new Error(`Invalid operator: ${operator}`); + } + this.func = cond_func; + } + + test(d) { + return this.func(d); + } + + // Implement self & other. + and(other) { + return new Condition((d) => this.test(d) && other.test(d)); + } + + // Implement self | other. + or(other) { + return new Condition((d) => this.test(d) || other.test(d)); + } + + // Implement ~self. + invert() { + return new Condition((d) => !this.test(d)); + } +} + +// Parse a grouped expression from a stream of tokens. +function parse_grouped_expression(tokens) { + if (tokens.length < 2 || tokens[0] !== "Parenthesis_BEGIN") { + return [0, null]; + } + let offset = 1; + let depth = 1; + while (depth > 0 && offset < tokens.length) { + switch (tokens[offset]) { + case "Parenthesis_BEGIN": + depth += 1; + break; + case "Parenthesis_END": + depth -= 1; + break; + } + offset += 1; + } + if (depth != 0) { + throw new Error("Unmatched parenthesis."); + } + // Recursively parse the grouped expression. + inner_expression = parse_expression(tokens.slice(1, offset - 1)); + return [offset, inner_expression]; +} + +// Parse a condition from a stream of tokens. +function parse_condition(tokens) { + if (tokens[0] instanceof Literal) { + // Just a value to search for. + const lt = tokens[0]; + return [1, new Condition(lt)]; + } else if ( + typeof tokens[0] === "string" && + tokens[0].startsWith("Operator.") && + tokens[1] instanceof Literal + ) { + // Value to search for with operator. + const op = tokens[0]; + const lt = tokens[1]; + return [2, new Condition(lt, op)]; + } else if (tokens[0] instanceof Facet && tokens[1] instanceof Literal) { + // Value to search for in facet. + const fc = tokens[0]; + const lt = tokens[1]; + return [2, new Condition(lt, (facet = fc))]; + } else if ( + tokens[0] instanceof Facet && + tokens[1].startsWith("Operator.") && + tokens[1] instanceof Literal + ) { + // Value to search for in facet with operator. + return [3, new Condition(lt, (facet = fc), (operator = op))]; + } else { + // Not matched as a condition. + return [0, null]; + } +} + +// Collapse all NOTs in a list of conditions. +function evaluate_not(conditions) { + const negated_conditions = []; + let index = 0; + while (index < conditions.length) { + if ( + conditions[index] == "Combiner_NOT" && + conditions[index + 1] == "Combiner_NOT" + ) { + // Skip double NOTs, as they negate each other. + index += 2; + } else if ( + conditions[index] == "Combiner_NOT" && + conditions[index + 1] instanceof Condition + ) { + const right = conditions[index + 1]; + negated_conditions.push(right.invert()); + index += 2; + } else if (conditions[index] != "Combiner_NOT") { + negated_conditions.push(conditions[index]); + index += 1; + } else { + throw new Error("Unprocessable NOT."); + } + } + return negated_conditions; +} + +// Collapse all explicit and implicit ANDs in a list of conditions. +function evaluate_and(conditions) { + const anded_conditions = []; + let index = 0; + while (index < conditions.length) { + let left; + if (anded_conditions.length) { + left = anded_conditions.pop(); + } else { + left = null; + } + + if ( + left instanceof Condition && + conditions[index] == "Combiner_AND" && + conditions[index + 1] instanceof Condition + ) { + const right = conditions[index + 1]; + anded_conditions.push(left.and(right)); + index += 2; + } else if (left instanceof Condition && conditions[index] instanceof Condition) { + const right = conditions[index]; + anded_conditions.push(left.and(right)); + index += 2; + } else if (conditions[index] != "Combiner_AND") { + if (left !== null) { + anded_conditions.push(left); + } + const right = conditions[index]; + anded_conditions.push(right); + index += 1; + } else { + throw new Error("Unprocessable AND."); + } + } + return anded_conditions; +} + +// Collapse all ORs in a list of conditions. +function evaluate_or(conditions) { + const ored_conditions = []; + let index = 0; + while (index < conditions.length) { + if ( + conditions[index] instanceof Condition && + conditions[index + 1] === "Combiner_OR" && + conditions[index + 2] + ) { + const left = conditions[index]; + const right = conditions[index + 2]; + ored_conditions.push(left.or(right)); + index += 3; + } else if (conditions[index] !== "Combiner_OR") { + ored_conditions.push(conditions[index]); + index += 1; + } else { + throw new Error("Unprocessable OR."); + } + } + return ored_conditions; +} + +// Parse an expression into a single Condition function. +function parse_expression(tokens) { + let conditions = []; + let index = 0; + while (index < tokens.length) { + console.log("Conditions:", conditions); + console.log("Token index:", index); + // Accounts for AND/OR/NOT. + if (typeof tokens[index] === "string" && tokens[index].startsWith("Combiner_")) { + conditions.push(tokens[index]); + index += 1; + continue; + } + + // Accounts for parentheses. + let [offset, condition] = parse_grouped_expression(tokens.slice(index)); + if (offset > 0 && condition !== null) { + conditions.push(condition); + index += offset; + continue; + } + + // Accounts for Facets, Operators, and Literals. + [offset, condition] = parse_condition(tokens.slice(index)); + if (offset > 0 && condition !== null) { + conditions.push(condition); + index += offset; + continue; + } + + console.error(tokens[index]); + throw new Error(`Unexpected token in expression: ${tokens[index]}`); + } + + // TODO: Investigate Pratt parsing for handling combiner precedence in a + // single pass. It should allow parsing them in the while loop above. + + // Evaluate NOTs first, left to right. + conditions = evaluate_not(conditions); + + // Evaluate ANDs second, left to right. + conditions = evaluate_and(conditions); + + // Evaluate ORs third, left to right. + conditions = evaluate_or(conditions); + + // Verify we have collapsed down to a single condition at this point. + if (conditions.length !== 1 || !conditions[0] instanceof Condition) { + throw new Error("Collapse should produce a single condition."); + } + + return conditions[0]; +} + +// Parse the query, returning a comparison function. +function query2condition(query) { + const tokens = lexer(query); + console.log("Tokens:", tokens); + if (tokens.length === 0) { + return new Condition((_) => true); + } + return parse_expression(tokens); +} + +/** + * End of query parser. + */ + // Toggle display of the extended description for plots. Global variable so it // can be referenced at plot insertion time. let description_shown = true; @@ -66,87 +461,137 @@ function ensure_dual_frame() { dual_frame.classList.remove("hidden"); } -function construct_sidebar_from_data(data) { - const sidebar = document.getElementById("plot-selector"); +function add_to_sidebar(record, facet_values) { + const diagnostics_list = document.getElementById("diagnostics"); + + // Add entry's display name. + const entry_title = document.createElement("h2"); + entry_title.textContent = record["title"]; + + // Create card for diagnostic. + const facets = document.createElement("dl"); + for (const facet in record) { + if (facet != "title" && facet != "path") { + const facet_node = document.createElement("div"); + const facet_name = document.createElement("dt"); + const facet_value = document.createElement("dd"); + facet_name.textContent = facet; + facet_value.textContent = record[facet]; + facet_node.append(facet_name, facet_value); + facets.append(facet_node); + // Record facet values. + if (!(facet in facet_values)) { + facet_values[facet] = new Set(); + } + const values = facet_values[facet]; + values.add(record[facet]); + } + } + + // Container element for plot position chooser buttons. + const position_chooser = document.createElement("div"); + position_chooser.classList.add("plot-position-chooser"); + + // Bind path to name in this scope to ensure it sticks around for callbacks. + const path = record["path"]; // Button icons. - const icons = { left: "◧", full: "▣", right: "◨" }; - - for (const category in data) { - // Details element for category. - const category_details = document.createElement("details"); - - // Title for category (summary element). - const category_summary = document.createElement("summary"); - category_summary.textContent = category; - category_details.append(category_summary); - - // Add each case date into category. - for (const case_date in data[category]) { - // Details element for case_date. - const case_details = document.createElement("details"); - - // Title for case_date. - const case_summary = document.createElement("summary"); - case_summary.textContent = case_date; - case_details.append(case_summary); - - // Menu of plots for this category and case_date. - const case_menu = document.createElement("menu"); - - // Add each plot. - for (const plot in data[category][case_date]) { - // Menu entry for plot. - const list_item = document.createElement("li"); - list_item.textContent = data[category][case_date][plot]; - - // Container element for plot position chooser buttons. - const position_chooser = document.createElement("div"); - position_chooser.classList.add("plot-position-chooser"); - - // Add buttons for each position. - for (const position of ["left", "full", "right"]) { - // Create button. - const button = document.createElement("button"); - button.classList.add(position); - button.textContent = icons[position]; - - // Add a callback updating the iframe when the link is clicked. - button.addEventListener("click", (event) => { - event.preventDefault(); - // Set the appropriate frame layout. - position == "full" ? ensure_single_frame() : ensure_dual_frame(); - document.getElementById(`plot-frame-${position}`).src = `${PLOTS_PATH}/${plot}`; - }); - - // Add button to chooser. - position_chooser.append(button); - } + const icons = { left: "◧", full: "▣", right: "◨", popup: "↗" }; - // Add position chooser to entry. - list_item.append(position_chooser); + // Add buttons for each position. + for (const position of ["left", "full", "right", "popup"]) { + // Create button. + const button = document.createElement("button"); + button.classList.add(position); + button.textContent = icons[position]; - // Add entry to the menu. - case_menu.append(list_item); + // Add a callback updating the iframe when the link is clicked. + button.addEventListener("click", (event) => { + event.preventDefault(); + // Open new window for popup. + if (position == "popup") { + window.open(`${PLOTS_PATH}/${path}`, "_blank", "popup,width=800,height=600"); + return; } + // Set the appropriate frame layout. + position == "full" ? ensure_single_frame() : ensure_dual_frame(); + document.getElementById(`plot-frame-${position}`).src = `${PLOTS_PATH}/${path}`; + }); + + // Add button to chooser. + position_chooser.append(button); + } + + // Create entry. + const entry = document.createElement("li"); - // Finish constructing this case and add to its category. - case_details.append(case_menu); - category_details.append(case_details); + // Add name, facets, and position chooser to entry. + entry.append(entry_title, facets, position_chooser); + + // Join entry to the DOM. + diagnostics_list.append(entry); +} + +function add_facet_dropdowns(facet_values) { + const fieldset = document.getElementById("filter-facets"); + + for (const facet in facet_values) { + const label = document.createElement("label"); + label.setAttribute("for", `facet-${facet}`); + label.textContent = facet; + const select = document.createElement("select"); + select.id = `facet-${facet}`; + select.name = facet; + const null_option = document.createElement("option"); + null_option.value = ""; + null_option.defaultSelected = true; + null_option.textContent = "--- Any ---"; + select.append(null_option); + // Sort facet values. + const values = Array.from(facet_values[facet]); + values.sort(); + for (const value of values) { + const option = document.createElement("option"); + option.textContent = value; + select.append(option); } + select.addEventListener("change", updateFacetQuery); - // Join category to the DOM. - sidebar.append(category_details); + // Add to DOM. + fieldset.append(label, select); + } +} + +// Update query based on facet dropdown value. +function updateFacetQuery(e) { + const facet = e.target.name; + const value = e.target.value; + const queryElem = document.getElementById("filter-query"); + const query = queryElem.value; + let new_query; + // Construct regular expression matching facet condition. + const pattern = RegExp(`${facet}:\\s*('[^']*'|"[^"]*"|[^ \\t\\(\\)]+)`, "i"); + if (value == "") { + // Facet unselected, remove from query. + new_query = query.replace(pattern, ""); + } else if (pattern.test(query)) { + // Facet value selected, update the query. + new_query = query.replace(pattern, `${facet}:"${value}"`); + } else { + // Facet value selected, add the query. + new_query = query + ` ${facet}:"${value}"`; } + queryElem.value = new_query.trim(); + doSearch(); } -// Plot selection sidebar +// Plot selection sidebar. function setup_plots_sidebar() { // Skip if there is no sidebar on page. if (!document.getElementById("plot-selector")) { return; } // Loading of plot index file, and adding them to the sidebar. - fetch(`${PLOTS_PATH}/index.json`) + fetch(`${PLOTS_PATH}/index.jsonl`) .then((response) => { // Display a message and stop if the fetch fails. if (!response.ok) { @@ -155,7 +600,27 @@ function setup_plots_sidebar() { window.alert(message); return; } - response.json().then(construct_sidebar_from_data); + response.text().then((data) => { + const facet_values = {}; + // Remove throbber now download has finished. + document.querySelector("#diagnostics > loading-throbber").remove(); + for (let line of data.split("\n")) { + line = line.trim(); + // Skip blank lines. + if (line.length) { + add_to_sidebar(JSON.parse(line), facet_values); + } + } + add_facet_dropdowns(facet_values); + // Do search if we already have a query specified in the URL. + const search = document.getElementById("filter-query"); + const params = new URLSearchParams(document.location.search); + const initial_query = params.get("q"); + if (initial_query) { + search.value = initial_query; + doSearch(); + } + }); }) .catch((err) => { // Catch non-HTTP fetch errors. @@ -176,7 +641,84 @@ function setup_clear_view_button() { clear_view_button.addEventListener("click", clear_frames); } +function setup_clear_search_button() { + const clear_search_button = document.getElementById("clear-query"); + clear_search_button.addEventListener("click", () => { + document.getElementById("filter-query").value = ""; + for (const select of document.querySelectorAll("#filter-facets select")) { + select.value = ""; + } + doSearch(); + }); +} + +// Filter the displayed diagnostics by the query. +function doSearch() { + const queryElem = document.getElementById("filter-query"); + const query = queryElem.value; + // Update URL in address bar to match current query, deleting if blank. + const url = new URL(document.location.href); + query ? url.searchParams.set("q", query) : url.searchParams.delete("q"); + // Updates the URL without reloading the page. + history.pushState(history.state, "", url.href); + + console.log("Search query:", query); + let condition; + try { + condition = query2condition(query); + // Set to an empty string to mark as valid. + queryElem.setCustomValidity(""); + } catch (error) { + console.error("Query failed to parse.", error); + // Add :invalid pseudoclass to input, so user gets feedback. + queryElem.setCustomValidity("Query failed to parse."); + return; + } + + // Filter all entries. + for (const entryElem of document.querySelectorAll("#diagnostics > li")) { + const entry = {}; + entry["title"] = entryElem.querySelector("h2").textContent; + for (const facet_node of entryElem.querySelector("dl").children) { + const facet = facet_node.firstChild.textContent; + const value = facet_node.lastChild.textContent; + entry[facet] = value; + } + + // Show entries matching filter and hide entries that don't. + if (condition.test(entry)) { + entryElem.classList.remove("hidden"); + } else { + entryElem.classList.add("hidden"); + } + } +} + +// For performance don't search on every keystroke immediately. Instead wait +// until quarter of a second of no typing has elapsed. To maximised perceived +// responsiveness immediately perform the search if a space is typed, as that +// indicates a completed search term. +let searchTimeoutID = undefined; +function debounce(e) { + clearTimeout(searchTimeoutID); + if (e.data == " ") { + doSearch(); + } else { + searchTimeoutID = setTimeout(doSearch, 250); + } +} + +// Diagnostic filtering searchbar. +function setup_search() { + const search = document.getElementById("filter-query"); + search.addEventListener("input", debounce); + // Immediately search if input is unfocused. + search.addEventListener("change", doSearch); +} + // Run everything. setup_description_toggle_button(); setup_clear_view_button(); +setup_clear_search_button(); setup_plots_sidebar(); +setup_search(); diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/style.css b/src/CSET/cset_workflow/app/finish_website/file/html/style.css index f8e929844..66035ca13 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/style.css +++ b/src/CSET/cset_workflow/app/finish_website/file/html/style.css @@ -20,83 +20,122 @@ a:hover { text-decoration-thickness: max(3px, 0.12em); } -nav > header { - margin: 8px; -} - -nav > header > h1 { - font-size: xx-large; - margin: 0; -} - -nav > header > button { - margin: 4px 0; -} - nav { display: inline; float: left; - width: 15em; + width: 35em; height: 100%; - overflow-x: hidden; overflow-y: scroll; background-color: whitesmoke; border-right: 1px solid black; -} -nav summary { - font-weight: bold; -} - -nav menu { - list-style: none; - padding: 0; - margin: 0; -} - -nav details { - margin: 8px; -} - -nav menu > li { - background-color: lightgrey; - margin: 8px 0; - padding: 0 4px; - border: 1px solid black; - font-size: small; - overflow-wrap: break-word; -} - -.plot-position-chooser { - display: flex; - margin-bottom: 0.25em; - font-size: medium; -} - -.plot-position-chooser button { - margin: 0 4px; - cursor: pointer; - width: 33%; -} - -.plot-position-chooser button.left { - background-color: #b8dcfd; - border-top-left-radius: 25% 50%; - border-bottom-left-radius: 25% 50%; -} -.plot-position-chooser button.full { - background-color: #c5e836; -} -.plot-position-chooser button.right { - background-color: #fdcabb; - border-top-right-radius: 25% 50%; - border-bottom-right-radius: 25% 50%; -} - -.plot-position-chooser button:focus, -.plot-position-chooser button:hover { - background-color: #1a1a1a; - color: white; + >header { + margin: 8px; + + >h1 { + font-size: xx-large; + margin: 0; + } + + >button { + margin: 4px 0; + } + + >search { + #filter-query { + width: 100%; + padding: 0.5em 1em; + margin: 8px 0; + border-radius: 3em; + + &:invalid { + box-shadow: 0px 0px 4px 4px red; + } + } + + fieldset select { + display: block; + margin-bottom: 4px; + } + } + } + + >ul { + list-style: none; + padding: 0; + margin-top: 8px; + + >li { + background-color: lightgrey; + margin: 8px 0; + padding: 4px; + border-top: 1px solid black; + border-bottom: 1px solid black; + overflow-wrap: break-word; + + >h2 { + margin: 0; + font-size: medium; + } + + >dl { + font-size: small; + margin: 4px 0; + + dt { + display: inline; + } + + dt:after { + content: ": "; + } + + dd { + display: inline; + margin-left: 0; + } + } + + .plot-position-chooser { + display: flex; + margin: 4px auto; + font-size: medium; + + button { + margin: 0 4px; + cursor: pointer; + width: 33%; + + &.left { + background-color: #b8dcfd; + border-top-left-radius: 3em; + border-bottom-left-radius: 3em; + } + + &.full { + background-color: #cae387; + } + + &.right { + background-color: #fdcabb; + border-top-right-radius: 3em; + border-bottom-right-radius: 3em; + } + + &.popup { + background-color: #e2dd92; + border-radius: 3em; + } + + &:focus, + &:hover { + background-color: #1a1a1a; + color: white; + } + } + } + } + } } .vsplit { @@ -130,7 +169,7 @@ main article { height: 100%; } -main article > iframe { +main article>iframe { border: none; height: 100%; width: 100%; @@ -139,3 +178,51 @@ main article > iframe { .hidden { display: none; } + +/* Loading throbber from https://cssloaders.github.io/ + +MIT License + +Copyright (c) 2020 Vineeth.TR + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ +loading-throbber { + display: block; + margin: 48px auto; + width: 48px; + height: 48px; + border: 5px solid black; + border-bottom-color: transparent; + border-radius: 50%; + box-sizing: border-box; + animation: rotation 1s linear infinite; +} + +@keyframes rotation { + 0% { + transform: rotate(0deg); + } + + 100% { + transform: rotate(360deg); + } +} + +/* End loading throbber. */ diff --git a/tests/workflow_utils/test_finish_website.py b/tests/workflow_utils/test_finish_website.py index eb2e29b5d..6187b24ed 100644 --- a/tests/workflow_utils/test_finish_website.py +++ b/tests/workflow_utils/test_finish_website.py @@ -79,9 +79,8 @@ def test_write_workflow_status(tmp_path): assert re.search(pattern, content) -def test_construct_index(monkeypatch, tmp_path): +def test_construct_index(tmp_path): """Test putting the index together.""" - monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", str(tmp_path)) plots_dir = tmp_path / "web/plots" plots_dir.mkdir(parents=True) @@ -102,17 +101,19 @@ def test_construct_index(monkeypatch, tmp_path): finish_website.construct_index(plots_dir.parent) # Check index. - index_file = plots_dir / "index.json" + index_file = plots_dir / "index.jsonl" assert index_file.is_file() with open(index_file, "rt", encoding="UTF-8") as fp: - index = json.load(fp) - expected = {"Category": {"20250101": {"p1": "P1", "p2": "P2"}}} + index = fp.read() + expected = ( + '{"case_date":"20250101","category":"Category","path":"p1","title":"P1"}\n' + '{"case_date":"20250101","category":"Category","path":"p2","title":"P2"}\n' + ) assert index == expected -def test_construct_index_aggregation_case(monkeypatch, tmp_path): +def test_construct_index_aggregation_case(tmp_path): """Construct the index from a diagnostics without a case date.""" - monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", str(tmp_path)) plots_dir = tmp_path / "web/plots" plots_dir.mkdir(parents=True) @@ -125,17 +126,40 @@ def test_construct_index_aggregation_case(monkeypatch, tmp_path): finish_website.construct_index(plots_dir.parent) # Check index. - index_file = plots_dir / "index.json" + index_file = plots_dir / "index.jsonl" assert index_file.is_file() with open(index_file, "rt", encoding="UTF-8") as fp: index = json.load(fp) - expected = {"Category": {"Aggregation": {"p1": "P1"}}} + expected = {"category": "Category", "path": "p1", "title": "P1"} assert index == expected -def test_construct_index_invalid(monkeypatch, tmp_path, caplog): +def test_construct_index_remove_keys(tmp_path): + """Unneeded keys are removed from the index.""" + plots_dir = tmp_path / "web/plots" + plots_dir.mkdir(parents=True) + + # Plot directories. + plot1 = plots_dir / "p1/meta.json" + plot1.parent.mkdir() + plot1.write_text( + '{"category": "Category", "title": "P1", "case_date": "20250101", "plots": ["a.png"], "description": "Foo"}' + ) + + # Construct index. + finish_website.construct_index(plots_dir.parent) + + # Check index. + index_file = plots_dir / "index.jsonl" + assert index_file.is_file() + with open(index_file, "rt", encoding="UTF-8") as fp: + index = json.loads(fp.readline()) + assert "plots" not in index + assert "description" not in index + + +def test_construct_index_invalid(tmp_path, caplog): """Test constructing index when metadata is invalid.""" - monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", str(tmp_path)) plots_dir = tmp_path / "web/plots" plots_dir.mkdir(parents=True) @@ -152,12 +176,9 @@ def test_construct_index_invalid(monkeypatch, tmp_path, caplog): assert level == logging.ERROR assert "p1/meta.json is invalid, skipping." in message - index_file = plots_dir / "index.json" + index_file = plots_dir / "index.jsonl" assert index_file.is_file() - with open(index_file, "rt", encoding="UTF-8") as fp: - index = json.load(fp) - expected = {} - assert index == expected + assert index_file.stat().st_size == 0 def test_entrypoint(monkeypatch): From 47b88e834178c7440ca006a09b3670524b7838f2 Mon Sep 17 00:00:00 2001 From: James Frost Date: Mon, 12 Jan 2026 16:10:53 +0000 Subject: [PATCH 2/8] Simplify filter parser language Removes redundant negative operators and allows facets to be any literal. Comparisons are now case insensitive. --- .../app/finish_website/file/html/script.js | 242 ++++++++---------- 1 file changed, 112 insertions(+), 130 deletions(-) diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/script.js b/src/CSET/cset_workflow/app/finish_website/file/html/script.js index 0e05372e3..243dbffc8 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/script.js +++ b/src/CSET/cset_workflow/app/finish_website/file/html/script.js @@ -3,7 +3,7 @@ /** Search query lexer and parser. * - * EBNF to implement: + * EBNF for filter. * * query = expression ; * @@ -17,17 +17,16 @@ * * condition = facet ? , operator ? , value ; * - * facet = LITERAL , ":" ; + * facet = LITERAL ; * * value = LITERAL ; * - * operator = NOT - * | GREATER_THAN - * | GREATER_THAN_OR_EQUALS + * operator = IN + * | EQUALS ; * | LESS_THAN + * | GREATER_THAN * | LESS_THAN_OR_EQUALS - * | NOT_EQUALS - * | EQUALS ; + * | GREATER_THAN_OR_EQUALS */ class Literal { @@ -36,37 +35,56 @@ class Literal { } } -class Facet { - constructor(value) { - this.value = value; - } -} - -const TOKEN_SPEC = new Map([ - ["Parenthesis_BEGIN", "\\("], - ["Parenthesis_END", "\\)"], - ["Operator_GREATER_THAN_OR_EQUALS", "<="], - ["Operator_GREATER_THAN", "<"], - ["Operator_LESS_THAN_OR_EQUALS", ">="], - ["Operator_LESS_THAN", ">"], - ["Operator_NOT_EQUALS", "!="], - ["Operator_EQUALS", "="], - ["Operator_NOT_IN", "!"], - ["Combiner_NOT", "\\bnot\\b"], - ["Combiner_AND", "\\band\\b"], - ["Combiner_OR", "\\bor\\b"], - ["LexOnly_WHITESPACE", "\\s+"], - ["LexOnly_FACET", "[a-z_\\-]+\\s*:"], - ["LexOnly_LITERAL", `'[^']*'|"[^"]*"|[^\\s\\(\\)]+`], -]); +const Parenthesis = { + BEGIN: "BEGIN", + END: "END", +}; + +const Operator = { + IN: "IN", + EQUALS: "EQUALS", + LESS_THAN: "LESS_THAN", + GREATER_THAN: "GREATER_THAN", + LESS_THAN_OR_EQUALS: "LESS_THAN_OR_EQUALS", + GREATER_THAN_OR_EQUALS: "GREATER_THAN_OR_EQUALS", +}; + +const Combiner = { + NOT: "NOT", + AND: "AND", + OR: "OR", +}; + +const TokenTypes = { + Literal: Literal, + Parenthesis: Parenthesis, + Operator: Operator, + Combiner: Combiner, +}; + +const TOKEN_SPEC = { + Parenthesis_BEGIN: "\\(", + Parenthesis_END: "\\)", + Operator_GREATER_THAN_OR_EQUALS: "<=", + Operator_GREATER_THAN: "<", + Operator_LESS_THAN_OR_EQUALS: ">=", + Operator_LESS_THAN: ">", + Operator_EQUALS: "=", + Operator_IN: ":", + Combiner_NOT: "\\bnot\\b", + Combiner_AND: "\\band\\b", + Combiner_OR: "\\bor\\b", + LexOnly_WHITESPACE: "\\s+", + LexOnly_LITERAL: `'[^']*'|"[^"]*"|[^\\s\\(\\):=<>]+`, +}; const TOKEN_REGEX = RegExp( Array.from( - TOKEN_SPEC.entries().map((pair) => { + Object.entries(TOKEN_SPEC).map((pair) => { return `(?<${pair[0]}>${pair[1]})`; - }) + }), ).join("|"), - "ig" + "ig", ); // Lex input string into tokens. @@ -78,24 +96,25 @@ function lexer(query) { throw new SyntaxError("Query did not consist of valid tokens."); } let [kind, value] = Object.entries(match.groups).filter( - (pair) => pair[1] !== undefined + (pair) => pair[1] !== undefined, )[0]; - switch (kind) { case "LexOnly_WHITESPACE": + // Skip whitespace. continue; - case "LexOnly_FACET": - const facet_name = value.replace(/\s*:$/, ""); - tokens.push(new Facet(facet_name)); - break; case "LexOnly_LITERAL": + // Unquote and store value for literals. if (/^".*"$|^'.+'$/.test(value)) { value = value.slice(1, -1); } tokens.push(new Literal(value)); break; default: - tokens.push(kind); + // Tokens for Operators and Combiners. + const kind_split_position = kind.indexOf("_"); + const kind_type = kind.slice(0, kind_split_position); + const kind_value = kind.slice(kind_split_position + 1); + tokens.push(TokenTypes[kind_type][kind_value]); break; } } @@ -103,54 +122,44 @@ function lexer(query) { } class Condition { - constructor(value, facet = new Facet("title"), operator = "Operator_IN") { - if (typeof value == "function") { + constructor(value, facet, operator) { + // Allow constructing a Condition from a Condition, e.g: (((Condition))) + if (typeof value === "function") { this.func = value; return; } - - const v = value.value; + const v = value.value.toLowerCase(); const f = facet.value; let cond_func; switch (operator) { - case "Operator_IN": + case Operator.IN: cond_func = function cond_in(d) { - return f in d && d[f].includes(v); - }; - break; - case "Operator_NOT_IN": - cond_func = function cond_nin(d) { - return f in d && !d[f].includes(v); + return f in d && d[f].toLowerCase().includes(v); }; break; - case "Operator_EQUALS": + case Operator.EQUALS: cond_func = function cond_eq(d) { - return f in d && v == d[f]; + return f in d && v === d[f].toLowerCase(); }; break; - case "Operator_NOT_EQUALS": - cond_func = function cond_neq(d) { - return f in d && v != d[f]; - }; - break; - case "Operator_GREATER_THAN": + case Operator.GREATER_THAN: cond_func = function cond_gt(d) { - return f in d && v > d[f]; + return f in d && v > d[f].toLowerCase(); }; break; - case "Operator_GREATER_THAN_OR_EQUALS": + case Operator.GREATER_THAN_OR_EQUALS: cond_func = function cond_gte(d) { - return f in d && v >= d[f]; + return f in d && v >= d[f].toLowerCase(); }; break; - case "Operator_LESS_THAN": + case Operator.LESS_THAN: cond_func = function cond_lt(d) { - return f in d && v < d[f]; + return f in d && v < d[f].toLowerCase(); }; break; - case "Operator_LESS_THAN_OR_EQUALS": + case Operator.LESS_THAN_OR_EQUALS: cond_func = function cond_lte(d) { - return f in d && v <= d[f]; + return f in d && v <= d[f].toLowerCase(); }; break; default: @@ -163,17 +172,14 @@ class Condition { return this.func(d); } - // Implement self & other. and(other) { return new Condition((d) => this.test(d) && other.test(d)); } - // Implement self | other. or(other) { return new Condition((d) => this.test(d) || other.test(d)); } - // Implement ~self. invert() { return new Condition((d) => !this.test(d)); } @@ -181,23 +187,23 @@ class Condition { // Parse a grouped expression from a stream of tokens. function parse_grouped_expression(tokens) { - if (tokens.length < 2 || tokens[0] !== "Parenthesis_BEGIN") { + if (tokens.length < 2 || tokens[0] !== Parenthesis.BEGIN) { return [0, null]; } let offset = 1; let depth = 1; while (depth > 0 && offset < tokens.length) { switch (tokens[offset]) { - case "Parenthesis_BEGIN": + case Parenthesis.BEGIN: depth += 1; break; - case "Parenthesis_END": + case Parenthesis.END: depth -= 1; break; } offset += 1; } - if (depth != 0) { + if (depth !== 0) { throw new Error("Unmatched parenthesis."); } // Recursively parse the grouped expression. @@ -207,31 +213,23 @@ function parse_grouped_expression(tokens) { // Parse a condition from a stream of tokens. function parse_condition(tokens) { - if (tokens[0] instanceof Literal) { - // Just a value to search for. - const lt = tokens[0]; - return [1, new Condition(lt)]; - } else if ( - typeof tokens[0] === "string" && - tokens[0].startsWith("Operator.") && - tokens[1] instanceof Literal - ) { - // Value to search for with operator. - const op = tokens[0]; - const lt = tokens[1]; - return [2, new Condition(lt, op)]; - } else if (tokens[0] instanceof Facet && tokens[1] instanceof Literal) { - // Value to search for in facet. - const fc = tokens[0]; - const lt = tokens[1]; - return [2, new Condition(lt, (facet = fc))]; - } else if ( - tokens[0] instanceof Facet && - tokens[1].startsWith("Operator.") && - tokens[1] instanceof Literal + if ( + tokens[0] instanceof Literal && + tokens[1] in Operator && + tokens[2] instanceof Literal ) { // Value to search for in facet with operator. - return [3, new Condition(lt, (facet = fc), (operator = op))]; + return [3, new Condition(tokens[2], (facet = tokens[0]), (operator = tokens[1]))]; + } else if (tokens[0] instanceof Literal) { + // Just a value to search for. + return [ + 1, + new Condition( + tokens[0], + (facet = new Literal("title")), + (operator = Operator.IN), + ), + ]; } else { // Not matched as a condition. return [0, null]; @@ -243,20 +241,17 @@ function evaluate_not(conditions) { const negated_conditions = []; let index = 0; while (index < conditions.length) { - if ( - conditions[index] == "Combiner_NOT" && - conditions[index + 1] == "Combiner_NOT" - ) { + if (conditions[index] === Combiner.NOT && conditions[index + 1] === Combiner.NOT) { // Skip double NOTs, as they negate each other. index += 2; } else if ( - conditions[index] == "Combiner_NOT" && + conditions[index] === Combiner.NOT && conditions[index + 1] instanceof Condition ) { const right = conditions[index + 1]; negated_conditions.push(right.invert()); index += 2; - } else if (conditions[index] != "Combiner_NOT") { + } else if (conditions[index] !== Combiner.NOT) { negated_conditions.push(conditions[index]); index += 1; } else { @@ -271,16 +266,13 @@ function evaluate_and(conditions) { const anded_conditions = []; let index = 0; while (index < conditions.length) { - let left; + let left = null; if (anded_conditions.length) { left = anded_conditions.pop(); - } else { - left = null; } - if ( left instanceof Condition && - conditions[index] == "Combiner_AND" && + conditions[index] === Combiner.AND && conditions[index + 1] instanceof Condition ) { const right = conditions[index + 1]; @@ -290,7 +282,7 @@ function evaluate_and(conditions) { const right = conditions[index]; anded_conditions.push(left.and(right)); index += 2; - } else if (conditions[index] != "Combiner_AND") { + } else if (conditions[index] !== Combiner.AND) { if (left !== null) { anded_conditions.push(left); } @@ -311,14 +303,14 @@ function evaluate_or(conditions) { while (index < conditions.length) { if ( conditions[index] instanceof Condition && - conditions[index + 1] === "Combiner_OR" && + conditions[index + 1] === Combiner.OR && conditions[index + 2] ) { const left = conditions[index]; const right = conditions[index + 2]; ored_conditions.push(left.or(right)); index += 3; - } else if (conditions[index] !== "Combiner_OR") { + } else if (conditions[index] !== Combiner.OR) { ored_conditions.push(conditions[index]); index += 1; } else { @@ -336,12 +328,11 @@ function parse_expression(tokens) { console.log("Conditions:", conditions); console.log("Token index:", index); // Accounts for AND/OR/NOT. - if (typeof tokens[index] === "string" && tokens[index].startsWith("Combiner_")) { + if (tokens[index] in Combiner) { conditions.push(tokens[index]); index += 1; continue; } - // Accounts for parentheses. let [offset, condition] = parse_grouped_expression(tokens.slice(index)); if (offset > 0 && condition !== null) { @@ -349,7 +340,6 @@ function parse_expression(tokens) { index += offset; continue; } - // Accounts for Facets, Operators, and Literals. [offset, condition] = parse_condition(tokens.slice(index)); if (offset > 0 && condition !== null) { @@ -357,28 +347,19 @@ function parse_expression(tokens) { index += offset; continue; } - console.error(tokens[index]); throw new Error(`Unexpected token in expression: ${tokens[index]}`); } - - // TODO: Investigate Pratt parsing for handling combiner precedence in a - // single pass. It should allow parsing them in the while loop above. - // Evaluate NOTs first, left to right. conditions = evaluate_not(conditions); - // Evaluate ANDs second, left to right. conditions = evaluate_and(conditions); - // Evaluate ORs third, left to right. conditions = evaluate_or(conditions); - // Verify we have collapsed down to a single condition at this point. - if (conditions.length !== 1 || !conditions[0] instanceof Condition) { - throw new Error("Collapse should produce a single condition."); + if (conditions.length !== 1 || !(conditions[0] instanceof Condition)) { + throw new Error("Collapse should produce a single Condition."); } - return conditions[0]; } @@ -387,6 +368,7 @@ function query2condition(query) { const tokens = lexer(query); console.log("Tokens:", tokens); if (tokens.length === 0) { + // If query is empty show everything. return new Condition((_) => true); } return parse_expression(tokens); @@ -409,7 +391,7 @@ function enforce_description_toggle() { } label: for (plot_frame of document.querySelectorAll("iframe")) { const description_container = plot_frame.contentDocument.getElementById( - "description-container" + "description-container", ); // Skip doing anything if plot not loaded. if (!description_container) { @@ -471,7 +453,7 @@ function add_to_sidebar(record, facet_values) { // Create card for diagnostic. const facets = document.createElement("dl"); for (const facet in record) { - if (facet != "title" && facet != "path") { + if (facet !== "title" && facet !== "path") { const facet_node = document.createElement("div"); const facet_name = document.createElement("dt"); const facet_value = document.createElement("dd"); @@ -508,12 +490,12 @@ function add_to_sidebar(record, facet_values) { button.addEventListener("click", (event) => { event.preventDefault(); // Open new window for popup. - if (position == "popup") { + if (position === "popup") { window.open(`${PLOTS_PATH}/${path}`, "_blank", "popup,width=800,height=600"); return; } // Set the appropriate frame layout. - position == "full" ? ensure_single_frame() : ensure_dual_frame(); + position === "full" ? ensure_single_frame() : ensure_dual_frame(); document.getElementById(`plot-frame-${position}`).src = `${PLOTS_PATH}/${path}`; }); @@ -570,7 +552,7 @@ function updateFacetQuery(e) { let new_query; // Construct regular expression matching facet condition. const pattern = RegExp(`${facet}:\\s*('[^']*'|"[^"]*"|[^ \\t\\(\\)]+)`, "i"); - if (value == "") { + if (value === "") { // Facet unselected, remove from query. new_query = query.replace(pattern, ""); } else if (pattern.test(query)) { @@ -701,7 +683,7 @@ function doSearch() { let searchTimeoutID = undefined; function debounce(e) { clearTimeout(searchTimeoutID); - if (e.data == " ") { + if (e.data === " ") { doSearch(); } else { searchTimeoutID = setTimeout(doSearch, 250); From afac20f43325324060f41977e72fe6e4a705f489 Mon Sep 17 00:00:00 2001 From: James Frost Date: Mon, 12 Jan 2026 16:12:28 +0000 Subject: [PATCH 3/8] Add initial filtering help --- .../app/finish_website/file/html/index.html | 46 ++++++++++++++++++- .../app/finish_website/file/html/style.css | 4 ++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/index.html b/src/CSET/cset_workflow/app/finish_website/file/html/index.html index c7a53c6cc..d917e86f2 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/index.html +++ b/src/CSET/cset_workflow/app/finish_website/file/html/index.html @@ -17,8 +17,52 @@

CSET

+
+ Filter help +

+ The filter consists of a space separated list of conditions. + A diagnostic is shown if all conditions are true. + Each condition takes the form: +

+ + [ facet ] [ operator ] value +

+ Where the facet is the name of the attribute of the diagnostic to test, and the operator is one of: +

+
+
:
+
A colon tests whether the value is in (is a substring of) the facet.
+
=
+
An equals sign tests whether the value is exactly equal to the facet.
+
<, >, <=, >=
+
A less-than, greater-than, less-than-or-equal or greater-than-or-equal sign tests whether the value sorts before or after the facet, allowing for ranges of dates and numbers.
+
+

+ If no facet is specified it defaults to checking the value is in the title, equivalent to title:value. + Values may be quoted to include special characters, such as spaces. +

+

Conditions may be combined with AND or OR, or prefixed with NOT to get the inverse.

+

Examples

+
+
histogram
+
"histogram" in the title.
+
title:"air temperature"
+
"air temperature" in the title.
+
field:temperature
+
"temperature" in facet "field".
+
field=temperature
+
"temperature" exactly matches field.
+
NOT temperature
+
Not "temperature" in title.
+
field=x_wind OR field=y_wind
+
field does not equal "x_wind" or "y_wind".
+
histogram AND field:temperature
+
"histogram" in title and "temperature in facet "field".
+
(histogram AND field:temperature) OR (time_series AND field:humidity)
+
Histograms of temperature and time series of humidity. Parenthesis indicate precedence.
+
+
-
Search facets
diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/style.css b/src/CSET/cset_workflow/app/finish_website/file/html/style.css index 66035ca13..b8aae4ab2 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/style.css +++ b/src/CSET/cset_workflow/app/finish_website/file/html/style.css @@ -42,6 +42,10 @@ nav { } >search { + dt { + font-family: monospace; + } + #filter-query { width: 100%; padding: 0.5em 1em; From eb3eb08c33d2f42889f00bc4245f48b61280428f Mon Sep 17 00:00:00 2001 From: James Frost Date: Wed, 21 Jan 2026 12:35:30 +0000 Subject: [PATCH 4/8] Filter plot_resolution --- .../cset_workflow/app/finish_website/bin/finish_website.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py index d970862fd..54edeb004 100755 --- a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py +++ b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py @@ -73,8 +73,9 @@ def construct_index(www_content: Path): plot_metadata = json.load(plot_fp) plot_metadata["path"] = str(metadata_file.parent.relative_to(plots_dir)) # Remove keys that are not useful for the index. - plot_metadata.pop("description", None) - plot_metadata.pop("plots", None) + removed_index_keys = ["description", "plots", "plot_resolution"] + for key in removed_index_keys: + plot_metadata.pop(key, None) # Sort plot metadata. plot_metadata = sort_dict(plot_metadata) # Write metadata into website index. From bc07f67079da27e7ba61cab10d2cb8adcf55c1b4 Mon Sep 17 00:00:00 2001 From: James Frost Date: Wed, 21 Jan 2026 14:42:02 +0000 Subject: [PATCH 5/8] Add recipe variables as metadata for filtering --- src/CSET/recipes/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/CSET/recipes/__init__.py b/src/CSET/recipes/__init__.py index 3c92d83fb..b459aa020 100644 --- a/src/CSET/recipes/__init__.py +++ b/src/CSET/recipes/__init__.py @@ -214,9 +214,15 @@ def parbake(self, ROSE_DATAC: Path, SHARE_DIR: Path) -> None: # Add input paths to recipe variables. self.variables["INPUT_PATHS"] = data_dirs - # Parbake this recipe, saving into recipe_dir. + # Parbake this recipe. recipe = parse_recipe(Path(self.recipe), self.variables) + # Add variables as extra metadata to filter on. + for key, value in self.variables.items(): + # Don't overwrite existing keys. + if key != "INPUT_PATHS" and key not in recipe: + recipe[key] = value + # Serialise into memory, as we use the serialised value twice. with StringIO() as s: with YAML(pure=True, output=s) as yaml: @@ -226,6 +232,7 @@ def parbake(self, ROSE_DATAC: Path, SHARE_DIR: Path) -> None: # with the same title. digest = hashlib.sha256(serialised_recipe).hexdigest() output_filename = recipe_dir / f"{slugify(recipe['title'])}_{digest[:12]}.yaml" + # Save into recipe_dir. with open(output_filename, "wb") as fp: fp.write(serialised_recipe) From ecfaa975e30e399d484ad77dbffd949f9ba27e60 Mon Sep 17 00:00:00 2001 From: James Frost Date: Wed, 28 Jan 2026 17:38:28 +0000 Subject: [PATCH 6/8] Remove more undesired facets from the plot index --- .../app/finish_website/bin/finish_website.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py index 54edeb004..799ee6370 100755 --- a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py +++ b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py @@ -73,7 +73,14 @@ def construct_index(www_content: Path): plot_metadata = json.load(plot_fp) plot_metadata["path"] = str(metadata_file.parent.relative_to(plots_dir)) # Remove keys that are not useful for the index. - removed_index_keys = ["description", "plots", "plot_resolution"] + removed_index_keys = [ + "description", + "plot_resolution", + "plots", + "skip_write", + "SUBAREA_EXTENT", + "SUBAREA_TYPE", + ] for key in removed_index_keys: plot_metadata.pop(key, None) # Sort plot metadata. From f38097939ec4f42c6904b5fbcbc5bfb87b061650 Mon Sep 17 00:00:00 2001 From: James Frost Date: Wed, 28 Jan 2026 17:39:45 +0000 Subject: [PATCH 7/8] Improve wording of a couple filter examples --- .../cset_workflow/app/finish_website/file/html/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/index.html b/src/CSET/cset_workflow/app/finish_website/file/html/index.html index d917e86f2..04281a4be 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/index.html +++ b/src/CSET/cset_workflow/app/finish_website/file/html/index.html @@ -53,9 +53,9 @@

Examples

field=temperature
"temperature" exactly matches field.
NOT temperature
-
Not "temperature" in title.
+
"temperature" is not in title.
field=x_wind OR field=y_wind
-
field does not equal "x_wind" or "y_wind".
+
Field equals "x_wind" or "y_wind".
histogram AND field:temperature
"histogram" in title and "temperature in facet "field".
(histogram AND field:temperature) OR (time_series AND field:humidity)
From 99c28f145bcea4c3e6a3b5217d5449d7ce3e2138 Mon Sep 17 00:00:00 2001 From: James Frost Date: Wed, 28 Jan 2026 17:41:01 +0000 Subject: [PATCH 8/8] Improve styling of facet dropdowns and sidebar in web UI --- .../app/finish_website/file/html/index.html | 6 +- .../app/finish_website/file/html/script.js | 11 +-- .../app/finish_website/file/html/style.css | 72 ++++++++++++------- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/index.html b/src/CSET/cset_workflow/app/finish_website/file/html/index.html index 04281a4be..8dc4d481c 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/index.html +++ b/src/CSET/cset_workflow/app/finish_website/file/html/index.html @@ -62,10 +62,10 @@

Examples

Histograms of temperature and time series of humidity. Parenthesis indicate precedence.
+
+ Search facet dropdowns +
-
- Search facets -
    diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/script.js b/src/CSET/cset_workflow/app/finish_website/file/html/script.js index 243dbffc8..f18ad1e72 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/script.js +++ b/src/CSET/cset_workflow/app/finish_website/file/html/script.js @@ -465,8 +465,7 @@ function add_to_sidebar(record, facet_values) { if (!(facet in facet_values)) { facet_values[facet] = new Set(); } - const values = facet_values[facet]; - values.add(record[facet]); + facet_values[facet].add(record[facet]); } } @@ -514,7 +513,7 @@ function add_to_sidebar(record, facet_values) { } function add_facet_dropdowns(facet_values) { - const fieldset = document.getElementById("filter-facets"); + const facets_container = document.getElementById("filter-facets"); for (const facet in facet_values) { const label = document.createElement("label"); @@ -537,9 +536,11 @@ function add_facet_dropdowns(facet_values) { select.append(option); } select.addEventListener("change", updateFacetQuery); - + // Put label and select in row. + const facet_row = document.createElement("div"); + facet_row.append(label, select); // Add to DOM. - fieldset.append(label, select); + facets_container.append(facet_row); } } diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/style.css b/src/CSET/cset_workflow/app/finish_website/file/html/style.css index b8aae4ab2..051fb3041 100644 --- a/src/CSET/cset_workflow/app/finish_website/file/html/style.css +++ b/src/CSET/cset_workflow/app/finish_website/file/html/style.css @@ -1,3 +1,16 @@ +/* Colour definitions. */ +:root { + --primary-fg-colour: black; + --primary-bg-colour: white; + --secondary-fg-colour: oklab(42% 0 0); + --secondary-bg-colour: oklab(97% 0 0); + --alternate-bg-colour: oklab(90% 0 0); + --invalid-colour: red; + --hover-fg-colour: white; + --hover-bg-colour: oklab(22% 0 0); + --shadow-colour: darkgrey; +} + /* Inherit fonts for inputs and buttons */ input, button, @@ -26,22 +39,22 @@ nav { width: 35em; height: 100%; overflow-y: scroll; - background-color: whitesmoke; - border-right: 1px solid black; + background-color: var(--secondary-bg-colour); + border-right: 1px solid var(--secondary-fg-colour); - >header { + > header { margin: 8px; - >h1 { + > h1 { font-size: xx-large; margin: 0; } - >button { + > button { margin: 4px 0; } - >search { + > search { dt { font-family: monospace; } @@ -53,38 +66,49 @@ nav { border-radius: 3em; &:invalid { - box-shadow: 0px 0px 4px 4px red; + box-shadow: 0px 0px 4px 4px var(--invalid-colour); } } - fieldset select { - display: block; - margin-bottom: 4px; + #filter-facets div { + padding: 4px; + &:nth-child(even) { + background-color: var(--alternate-bg-colour); + } + label { + display: inline-block; + width: 50%; + } + select { + width: 50%; + } } } } - >ul { + > ul { list-style: none; padding: 0; margin-top: 8px; - >li { - background-color: lightgrey; - margin: 8px 0; - padding: 4px; - border-top: 1px solid black; - border-bottom: 1px solid black; + > li { + padding: 5px 4px 3px; overflow-wrap: break-word; + box-shadow: inset 0 4px 4px var(--shadow-colour); + + &:nth-child(odd) { + background-color: var(--alternate-bg-colour); + } - >h2 { + > h2 { margin: 0; font-size: medium; } - >dl { + > dl { font-size: small; margin: 4px 0; + columns: 16em auto; dt { display: inline; @@ -133,8 +157,8 @@ nav { &:focus, &:hover { - background-color: #1a1a1a; - color: white; + background-color: var(--hover-bg-colour); + color: var(--hover-fg-colour); } } } @@ -161,7 +185,7 @@ nav { .websplit-container { display: flex; flex-direction: column; - outline: solid 1px black; + outline: solid 1px var(--primary-fg-colour); } main { @@ -173,7 +197,7 @@ main article { height: 100%; } -main article>iframe { +main article > iframe { border: none; height: 100%; width: 100%; @@ -212,7 +236,7 @@ loading-throbber { margin: 48px auto; width: 48px; height: 48px; - border: 5px solid black; + border: 5px solid var(--secondary-fg-colour); border-bottom-color: transparent; border-radius: 50%; box-sizing: border-box;