Preparations for LDML 48 release

.github/workflows/build_spec.yml

@@ -0,0 +1,18 @@
+name: Build spec as HTML
+
+on:
+  pull_request:
+    paths:
+      - 'publish/**'
+      - 'spec/**'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - run: make dist/index.html
+        working-directory: ./publish

.github/workflows/deploy_pages.yml

@@ -0,0 +1,37 @@
+name: Deploy spec to GitHub Pages
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'publish/**'
+      - 'spec/**'
+  workflow_dispatch:
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/configure-pages@v5
+      - run: make
+        working-directory: ./publish
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: publish/dist/
+      - uses: actions/deploy-pages@v4
+        id: deployment

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "publish/cldr"]
+	path = publish/cldr
+	url = git@github.com:unicode-org/cldr.git
+	branch = main

publish/.gitignore

@@ -0,0 +1,4 @@
+/dist/
+/gtag.html
+/header.html
+/reports-v2.css

publish/Makefile

@@ -0,0 +1,49 @@
+ldml = cldr/docs/ldml
+mf-target = $(ldml)/tr35-messageFormat.md
+tr-archive = cldr/tools/scripts/tr-archive
+
+all: dist
+.PHONY: prepare dist clean
+prepare: cldr/.git $(tr-archive)/node_modules/.package-lock.json header.md
+dist: dist/index.html dist/css/reports-v2.css dist/css/tr35.css dist/js/anchor.min.js dist/js/tr35search.js
+clean:
+	rm -r dist gtag.html header.html
+	cd cldr && git restore .
+
+dist/index.html: $(mf-target) fix-archive.mjs gtag.html header.html
+	@mkdir -p dist
+	cp $(mf-target) ./dist/index.md
+	node $(tr-archive)/archive.js
+	node fix-archive.mjs
+	rm ./dist/index.md
+
+$(mf-target): cldr/.git $(tr-archive)/node_modules/.package-lock.json collect.mjs check-toc.mjs $(shell find ../spec -type f) | header.md
+	node collect.mjs > $(mf-target)
+	cd $(tr-archive) && node fix-tocs.js
+	node check-toc.mjs < $(mf-target)
+
+cldr/.git:
+	git submodule update --init --remote --rebase cldr
+$(tr-archive)/node_modules/.package-lock.json: $(tr-archive)/package-lock.json
+	cd $(tr-archive) && npm ci
+
+header.md: update-header-footer.mjs
+	node update-header-footer.mjs < $(mf-target)
+
+dist/css/reports-v2.css:
+	@mkdir -p dist/css
+	cd dist/css && curl -O 'https://www.unicode.org/reports/reports-v2.css'
+dist/css/tr35.css: $(tr-archive)/assets/css/tr35.css
+	@mkdir -p dist/css
+	cp $< $@
+dist/js/anchor.min.js: $(tr-archive)/node_modules/.package-lock.json $(tr-archive)/node_modules/anchor-js/anchor.min.js
+	@mkdir -p dist/js
+	cp $< $@
+dist/js/tr35search.js: $(tr-archive)/assets/js/tr35search.js
+	@mkdir -p dist/js
+	cp $< $@
+
+gtag.html: $(tr-archive)/gtag.html
+	ln -s $<
+header.html: $(tr-archive)/header.html
+	ln -s $<

publish/README.md

@@ -0,0 +1,26 @@
+# Tools for publication
+
+The contents of [Unicode MessageFormat Standard](./spec/) are published
+as a part of [Unicode Technical Standard #35](https://unicode.org/reports/tr35/).
+
+The contents of this directory are used to collect the spec parts into a single file for publication,
+and to fix internal and external links.
+
+The scripts require Node.js 20 or later, and expect to be run in a POSIX environment.
+
+To build a spec update:
+
+1. Use `make prepare` to:
+   1. Update the CLDR submodule
+   1. Install `tr-archive` tool dependencies
+   1. Update `header.md` and `footer.md`
+1. Apply any updates needed in `header.md`.
+1. Update `collect.mjs` as required for any structural changes or new links.
+1. Use `make dist` to:
+   1. Collect the current spec parts into `cldr/docs/ldml/tr35-messageFormat.md`.
+   1. Fix link targets.
+   1. Add the Table of Contents.
+   1. Check the Table of Contents for duplicate section title links.
+   1. Build a local copy of the rendered HTML at `dist/index.html`.
+1. Inspect the results; `git diff` in the `cldr/` submodule may be useful.
+1. Apply fixes and repeat as necessary until the output is clean.

publish/check-toc.mjs

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+import { readFileSync } from "node:fs";
+import { exit } from "node:process";
+
+const lines = readFileSync(0, "utf8").split("\n"); // 0: stdin
+
+let i = lines.findIndex((line) => line.startsWith('## <a name="Contents"'));
+if (i < 0) throw new Error("ToC start not found");
+while (lines[++i] === "");
+
+/** @type {Record<string, string[]>} */
+const links = {};
+let match;
+while ((match = /\(#(.+?)\)$/.exec(lines[i]))) {
+  const target = match[1];
+  if (target in links) links[target].push(i);
+  else links[target] = [i];
+  ++i;
+}
+
+const n = Object.keys(links).length;
+if (n < 20) throw new Error(`ToC too small: ${n} entries`);
+
+let ok = true;
+for (const a of Object.values(links)) {
+  if (a.length > 1) {
+    if (ok) console.error("Duplicate ToC links:");
+    for (const i of a) console.error(`\tline ${i + 1}: ${lines[i]}`);
+    ok = false;
+  }
+}
+exit(ok ? 0 : 1);

publish/collect.mjs

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+
+import { readFileSync } from "node:fs";
+
+// order matters here
+const parts = {
+  header: "header.md",
+  intro: "../spec/intro.md",
+  syntax: "../spec/syntax.md",
+  abnf: "../spec/message.abnf",
+  formatting: "../spec/formatting.md",
+  errors: "../spec/errors.md",
+  functions: "../spec/functions/README.md",
+  stringFunctions: "../spec/functions/string.md",
+  numberFunctions: "../spec/functions/number.md",
+  datetimeFunctions: "../spec/functions/datetime.md",
+  uNamespace: "../spec/u-namespace.md",
+  datamodel: "../spec/data-model/README.md",
+  json: "../spec/data-model/message.json",
+  appendices: "../spec/appendices.md",
+  footer: "footer.md",
+};
+
+for (const [name, path] of Object.entries(parts)) {
+  parts[name] = readFileSync(path, "utf8").trim();
+}
+
+parts.abnf = "## message.abnf\n\n```abnf\n" + parts.abnf + "\n```";
+parts.json = "## message.json\n\n```json\n" + parts.json + "\n```";
+
+// Strip title + table of contents
+const introStart = parts.intro.indexOf("## Introduction");
+if (introStart < 0) throw new Error("Intro start not found");
+parts.intro = parts.intro.substring(introStart);
+
+parts.functions = parts.functions.replace(
+  /^#+ Table of Contents\n\n(.*?)\n\n/ms,
+  ""
+);
+
+const result = Object.values(parts)
+  .join("\n\n")
+  .replace(/ +$/g, "")
+  .replace(/\n{3,}/g, "\n\n")
+  .replace(/\[(.+?)\]\((.+?)\)/g, fixLink);
+console.log(result);
+
+/**
+ * @param {string} link
+ * @param {string} label
+ * @param {string} target
+ */
+function fixLink(link, label, target) {
+  if (target === "../docs/why_mf_next.md") return label;
+  if (target.endsWith("message.abnf")) return `[${label}](#messageabnf)`;
+  if (target.endsWith("message.abnf")) return `[${label}](#messageabnf)`;
+  if (target.endsWith("syntax.md")) return `[${label}](#syntax)`;
+  const local = /^(?:\.\/)?[\w./]+\.md(#.+)/.exec(target);
+  if (local) return `[${label}](${local[1]})`;
+  const tr35 = /\/(tr35(?:-\w+)?)\.html(#.+)?$/.exec(target);
+  if (tr35) return `[${label}](${tr35[1]}.md${tr35[2]})`;
+  return link;
+}

publish/fix-archive.mjs

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+import { readFileSync, writeFileSync } from "node:fs";
+
+const path = "dist/index.html";
+
+const html = readFileSync(path, "utf8")
+  .replace(/<script .*googletagmanager\.com.*\s*/, '')
+  .replace('href="../reports-v2.css"', 'href="css/reports-v2.css"')
+  .replaceAll(
+    '<a href="tr35',
+    '<a href="https://www.unicode.org/reports/tr35/dev/tr35'
+  );
+
+writeFileSync(path, html)

publish/footer.md

@@ -0,0 +1,15 @@
+* * *
+
+© 2001–2025 Unicode, Inc.
+This publication is protected by copyright, and permission must be obtained from Unicode, Inc.
+prior to any reproduction, modification, or other use not permitted by the [Terms of Use](https://www.unicode.org/copyright.html).
+Specifically, you may make copies of this publication and may annotate and translate it solely for personal or internal business purposes and not for public distribution,
+provided that any such permitted copies and modifications fully reproduce all copyright and other legal notices contained in the original.
+You may not make copies of or modifications to this publication for public distribution, or incorporate it in whole or in part into any product or publication without the express written permission of Unicode.
+
+Use of all Unicode Products, including this publication, is governed by the Unicode [Terms of Use](https://www.unicode.org/copyright.html).
+The authors, contributors, and publishers have taken care in the preparation of this publication,
+but make no express or implied representation or warranty of any kind and assume no responsibility or liability for errors or omissions or for consequential or incidental damages that may arise therefrom.
+This publication is provided “AS-IS” without charge as a convenience to users.
+
+Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries.