global: Refactor data parsing bootstrap code

It only took a year, but finally figured out that vite provides a plugin/hook
(configResolved()) that allows us to execute code AFTER the configuration has
been fully parsed and BEFORE the page transformations start.

This is the correct place to force full loading of the various data files,
since they need to pre-scan all the files for, e.g., frontmatter data needed
to do proper inter-page linking.

This allows us to use core javascript async/await functionality, rather than
the somewhat hackish previous method of using "import-sync" module.

Author: slusarz

.vitepress/config.js

@@ -4,6 +4,7 @@ import { pagefindPlugin } from 'vitepress-plugin-pagefind'
 import { generateSidebar } from 'vitepress-sidebar'
 import { dovecotMdExtend } from '../lib/markdown.js'
 import { getExcludes } from '../lib/utility.js'
+import dovecotVitepressInit from '../lib/dovecot_vitepress_init.js'
 
 const base = '/2.4'
 const base_url = 'https://doc.dovecot.org'
@@ -62,7 +63,8 @@ export default defineConfig({
 			chunkSizeWarningLimit: 1000,
 		},
 		plugins: [
-			pagefindPlugin()
+			pagefindPlugin(),
+			dovecotVitepressInit()
 		],
 	},
 

lib/data/doveadm.data.js

@@ -154,7 +154,7 @@ async function normalizeDoveadm(doveadm) {
 export default addWatchPaths({
 	async load() {
 		return await normalizeDoveadm(
-			structuredClone(loadData('doveadm').doveadm)
+			structuredClone((await loadData('doveadm')).doveadm)
 		)
 	}
 })

lib/data/event_categories.data.js

@@ -14,7 +14,7 @@ async function normalizeEventCategories(categories) {
 export default addWatchPaths({
 	async load() {
 		return await normalizeEventCategories(
-			structuredClone(loadData('event_categories').categories)
+			structuredClone((await loadData('event_categories')).categories)
 		)
 	}
 })

lib/data/event_reasons.data.js

@@ -14,7 +14,7 @@ async function normalizeEventReasons(reasons) {
 export default addWatchPaths({
 	async load() {
 		return await normalizeEventReasons(
-			structuredClone(loadData('event_reasons').reasons)
+			structuredClone((await loadData('event_reasons')).reasons)
 		)
 	}
 })

lib/data/lua.data.js

@@ -31,7 +31,7 @@ async function normalizeLuaFunctions(lua) {
 				/* Merge information from Dovecot settings. */
 				if (v2.dovecot_setting) {
 					if (!set) {
-						set = structuredClone(loadData('settings').settings)
+						set = structuredClone((await loadData('settings')).settings)
 					}
 
 					if (!v2.type) {
@@ -84,7 +84,7 @@ async function normalizeLuaVariables(lua) {
 
 export default addWatchPaths({
 	async load() {
-		const data = loadData('lua')
+		const data = await loadData('lua')
 
 		return {
 			constants: await normalizeLuaConstants(data.lua_constants),

lib/data/settings.data.js

@@ -131,7 +131,7 @@ async function normalizeSettings(settings) {
 export default addWatchPaths({
 	async load() {
 		return await normalizeSettings(
-			structuredClone(loadData('settings').settings)
+			structuredClone((await loadData('settings')).settings)
 		)
 	}
 })

lib/dovecot_vitepress_init.js

@@ -0,0 +1,16 @@
+import { dovecotMdInit } from './markdown.js'
+
+export default function dovecotVitepressInit() {
+	return {
+		name: 'dovecot-vitepress-init',
+		async configResolved(config) {
+			console.log('\n✅ Config resolved!')
+
+			/* We need to synchronously initialize markdown,
+			 * since we need to pre-populate various internal
+			 * tables (e.g. links). */
+			await dovecotMdInit()
+			console.log('\n✅ Markdown initialized!')
+		},
+	}
+}

lib/events.js

@@ -122,7 +122,7 @@ async function normalizeEvents(events, global_inherits, inherits) {
 }
 
 export async function loadEvents() {
-	const data = loadData('events')
+	const data = await loadData('events')
 	return await normalizeEvents(
 		structuredClone(data.events),
 		structuredClone(data.global_inherits),

lib/markdown.js

@@ -5,7 +5,56 @@ import path from 'path'
 import { createMarkdownRenderer } from 'vitepress'
 import { dovecotSetting, frontmatterIter, loadData } from './utility.js'
 
-export function dovecotMdExtend(md) {
+let md_conf = false
+export async function dovecotMdInit() {
+	if (md_conf) {
+		return md_conf
+	}
+
+	md_conf = {
+		doveadm: (await loadData('doveadm')).doveadm,
+		events: (await loadData('events')).events,
+		links: await (async () => {
+			const links = {}
+			const rewrites = globalThis.VITEPRESS_CONFIG.rewrites.map
+
+			frontmatterIter(Object.keys(rewrites), function (f, data) {
+				if (!data.dovecotlinks) {
+					return
+				}
+
+				for (const [k, v] of Object.entries(data.dovecotlinks)) {
+					if (links[k]) {
+						throw new Error("Duplicate Dovecot Link key: " + k)
+					}
+
+					links[k] = {
+						url: resolveURL(rewrites[f].substring(0, rewrites[f].lastIndexOf('.')) + '.html')
+					}
+
+					if ((typeof v) == 'object') {
+						links[k].text = v.text
+						if (v.hash) {
+							links[k].url += '#' + v.hash
+						}
+					} else {
+						links[k].text = v
+					}
+				}
+			})
+
+			return {
+				...links, ...((await loadData('links_overrides')).links_overrides)
+			}
+		})(),
+		settings: (await loadData('settings')).settings,
+		updates: (await loadData('updates')).updates
+	}
+
+	return md_conf
+}
+
+export async function dovecotMdExtend(md) {
 	md.use(containerPlugin, 'todo', {
 		render: function(tokens, idx) {
 			if (tokens[idx].nesting === 1) {
@@ -16,7 +65,7 @@ export function dovecotMdExtend(md) {
 		}
 	})
 	md.use(deflistPlugin)
-	md.use(dovecot_markdown)
+	md.use(dovecot_markdown, await dovecotMdInit())
 
 	return md
 }
@@ -26,7 +75,7 @@ export async function getVitepressMd() {
 	if (vitepress_md === null) {
 		const config = globalThis.VITEPRESS_CONFIG
 
-		vitepress_md = dovecotMdExtend(await createMarkdownRenderer(
+		vitepress_md = await dovecotMdExtend(await createMarkdownRenderer(
 			config.srcDir,
 			config.markdown,
 			config.site.base,
@@ -40,7 +89,7 @@ export async function getVitepressMd() {
 /* This is a dovecot markdown extension to support the "[[...]]" syntax.
  * Much of this is copied from existing markdown-it plugins. See, e.g.,
  * https://github.com/markdown-it/markdown-it-sub/blob/master/index.mjs */
-function dovecot_markdown(md) {
+function dovecot_markdown(md, opts) {
 	function process_brackets(state, silent) {
 		const max = state.posMax
 		const start = state.pos
@@ -130,8 +179,6 @@ function dovecot_markdown(md) {
 			let page = mode
 			switch (mode) {
 			case 'doveadm':
-				initDoveadm()
-
 				if (!opts.doveadm[env.inner]) {
 					if (!Object.values(opts.doveadm).find((x) => (x.man == 'doveadm-' + env.inner))) {
 						handle_error('doveadm link missing: ' + env.inner)
@@ -141,8 +188,6 @@ function dovecot_markdown(md) {
 				break
 
 			case 'event':
-				initEvents()
-
 				if (!opts.events[env.inner]) {
 					handle_error('event link missing: ' + env.inner)
 					return '<code><a>'
@@ -152,8 +197,6 @@ function dovecot_markdown(md) {
 
 			case 'setting':
 			case 'setting_text':
-				initSettings()
-
 				/* Settings names can have brackets, so we need to unescape
 				 * input for purposes of searching settings keys. */
 				const search_str = env.inner.replaceAll('&gt;', '>')
@@ -175,14 +218,12 @@ function dovecot_markdown(md) {
 			let url = '#'
 			env.inner = false
 
-			initDovecotLinks()
-
-			if (!opts.dovecotlinks[parts[1]]) {
+			if (!opts.links[parts[1]]) {
 				handle_error('Dovecot link missing: ' + parts[1])
 				return '<a>'
 			}
 
-			const d = opts.dovecotlinks[parts[1]]
+			const d = opts.links[parts[1]]
 			env.inner = parts[2] ? parts[2] : (d.text ? d.text : false)
 
 			return '<a href="' + d.url + '">'
@@ -287,8 +328,6 @@ function dovecot_markdown(md) {
 		case 'changed':
 		case 'deprecated':
 		case 'removed':
-			initUpdates()
-
 			if (!opts.updates[env.args]) {
 				handle_error('Missing updates entry for: ' + env.args)
 				return env.args
@@ -369,56 +408,6 @@ function dovecot_markdown(md) {
 		console.error(msg)
 	}
 
-	function initDoveadm() {
-		if (!opts.doveadm) {
-			opts.doveadm = loadData('doveadm').doveadm
-		}
-	}
-
-	function initDovecotLinks() {
-		if (opts.dovecotlinks) {
-			return
-		}
-
-		const links = {}
-		const rewrites = globalThis.VITEPRESS_CONFIG.rewrites.map
-
-		frontmatterIter(Object.keys(rewrites), function (f, data) {
-			if (!data.dovecotlinks) {
-				return
-			}
-
-			for (const [k, v] of Object.entries(data.dovecotlinks)) {
-				if (links[k]) {
-					throw new Error("Duplicate Dovecot Link key: " + k)
-				}
-
-				links[k] = {
-					url: resolveURL(rewrites[f].substring(0, rewrites[f].lastIndexOf('.')) + '.html')
-				}
-
-				if ((typeof v) == 'object') {
-					links[k].text = v.text
-					if (v.hash) {
-						links[k].url += '#' + v.hash
-					}
-				} else {
-					links[k].text = v
-				}
-			}
-		})
-
-		opts.dovecotlinks = {
-			...links, ...(loadData('links_overrides').links_overrides)
-		}
-	}
-
-	function initEvents() {
-		if (!opts.events) {
-			opts.events = loadData('events').events
-		}
-	}
-
 	function initManFiles() {
 		if (!opts.man) {
 			opts.man = dovecotSetting('man_paths').flatMap((x) => {
@@ -444,37 +433,17 @@ function dovecot_markdown(md) {
 		}
 	}
 
-	function initSettings() {
-		if (!opts.settings) {
-			opts.settings = loadData('settings').settings
-		}
-	}
-
-	function initUpdates() {
-		if (!opts.updates) {
-			opts.updates = loadData('updates').updates
-		}
-	}
-
-	function resolveURL(url) {
-		if (!('url_rewrite' in opts)) {
-			opts.url_rewrite = dovecotSetting('url_rewrite')
-		}
-
-		const new_url =
-			(opts.base.endsWith('/') ? opts.base.slice(0, -1) : opts.base) +
-			'/' + url
-		return (opts.url_rewrite) ? opts.url_rewrite(new_url) : new_url
-	}
-
-	const opts = {
-		base: globalThis.VITEPRESS_CONFIG.site.base
-	}
-
 	md.inline.ruler.after('emphasis', 'dovecot_brackets', process_brackets)
 	md.renderer.rules.dovecot_open = dovecot_open
 	md.renderer.rules.dovecot_body = dovecot_body
 	md.renderer.rules.dovecot_close = dovecot_close
+}
 
-	opts.resolveURL = resolveURL
+export function resolveURL(url) {
+	const base = globalThis.VITEPRESS_CONFIG.site.base
+	const url_rewrite = dovecotSetting('url_rewrite')
+	const new_url =
+		(base.endsWith('/') ? base.slice(0, -1) : base) +
+		'/' + url
+	return (url_rewrite) ? url_rewrite(new_url) : new_url
 }

lib/utility.js

@@ -3,7 +3,6 @@
 import fg from 'fast-glob'
 import fs from 'fs'
 import matter from 'gray-matter'
-import importSync from 'import-sync'
 import { dirname } from 'path'
 import { fileURLToPath } from 'url'
 
@@ -27,16 +26,21 @@ export function normalizeArrayData(data, keys) {
 	return data
 }
 
-export function loadData(id) {
-	const path = globalThis.VITEPRESS_CONFIG.userConfig.themeConfig.dovecot?.data_paths?.[id]
-		?? ('../data/' + id + '.js')
+const dataOb = {}
+export async function loadData(id) {
+	if (!dataOb[id]) {
+		const path = globalThis.VITEPRESS_CONFIG.userConfig.themeConfig.dovecot?.data_paths?.[id]
+			?? ('../data/' + id + '.js')
 
-	try {
-		return importSync(__dirname + '/' + path)
-	} catch (e) {
-		throw new Error('Unable to import module (' + __dirname + '/' +
-			path + '):' + e)
+		try {
+			dataOb[id] = await import(__dirname + '/' + path)
+		} catch (e) {
+			throw new Error('Unable to import module (' + __dirname + '/' +
+				path + '):' + e)
+		}
 	}
+
+	return dataOb[id]
 }
 
 function _dovecotSetting(name, setting) {