From 80a6ac103fb59cc501a4410a8bfa74ccff929730 Mon Sep 17 00:00:00 2001 From: Gabor Melis Date: Fri, 6 Jun 2025 08:59:35 +0200 Subject: [PATCH 1/2] math: support output to markdown Also, parameterize the THML output markers. --- math.lisp | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/math.lisp b/math.lisp index 99326a5..c77cffb 100644 --- a/math.lisp +++ b/math.lisp @@ -11,14 +11,30 @@ ; $$ ; \sum_{i=0}^{10} (u_{i} x_{i})^2 ; $$ +; +; Note that this departs from normal TeX syntax, which uses a single $ +; for inline math, and double for display math. By using double $, the +; need for escaping $ is much less. Also, although it's not +; documented, GitHub Flavored Markdown supports $$-delimited _inline_ +; math, too. +; ;------------------------------------------------------------------------------- (defpackage #:3bmd-math (:use #:cl #:esrap #:3bmd-ext) - (:export #:*math*)) + (:export #:*math* + #:*html-inline-start-marker* + #:*html-inline-end-marker* + #:*html-block-start-marker* + #:*html-block-end-marker*)) (in-package #:3bmd-math) +(defvar *html-inline-start-marker* "\\(") +(defvar *html-inline-end-marker* "\\)") +(defvar *html-block-start-marker* "\\[") +(defvar *html-block-end-marker* "\\]") + (defrule math-content (* (and (! "$$") character)) (:text t)) @@ -35,10 +51,20 @@ (list :math-block c))) (defmethod print-tagged-element ((tag (eql :math-inline)) stream rest) - (format stream "\\(~a\\)" (car rest))) + (format stream "~a~a~a" *html-inline-start-marker* (car rest) + *html-inline-end-marker*)) (defmethod print-tagged-element ((tag (eql :math-block)) stream rest) - (format stream "\\[~a\\]" (car rest))) + (format stream "~a~a~a" *html-block-start-marker* (car rest) + *html-block-end-marker*)) + +(defmethod print-md-tagged-element ((tag (eql :math-inline)) stream rest) + (format stream "$$~a$$" (car rest))) + +(defmethod print-md-tagged-element ((tag (eql :math-block)) stream rest) + (3bmd::ensure-block stream) + (3bmd::print-md (format nil "$$~a$$" (car rest)) stream) + (3bmd::end-block stream)) #++ (let ((3bmd-math:*math* t)) From 075b7877f83e94486c651cd139b5b4af6abf6e47 Mon Sep 17 00:00:00 2001 From: Gabor Melis Date: Fri, 6 Jun 2025 10:37:34 +0200 Subject: [PATCH 2/2] math: support alternative math syntaxes Works with inline math: $x_0$ $`x_0`$ $$x_0$$ text and block (display) math: $$x_0$$ - To avoid rendering "between $5 and $6" with inline math, both the opening and the closing $ character must be followed / preceded by a non-space character. This agrees with Pandoc. The other forms do not have such restriction. - In the block format, the opening $$ can only be preceded by spaces, and the closing $$ can only be followed by spaces on its own line. --- 3bmd-ext-math.asd | 20 +++-- extensions.lisp | 26 ++++++- markdown-printer.lisp | 38 ++++++++-- math.lisp | 111 ++++++++++++++++++--------- tests/extensions/math.lisp | 151 +++++++++++++++++++++++++++++++++++++ 5 files changed, 294 insertions(+), 52 deletions(-) create mode 100644 tests/extensions/math.lisp diff --git a/3bmd-ext-math.asd b/3bmd-ext-math.asd index fe2bf88..1a08d00 100644 --- a/3bmd-ext-math.asd +++ b/3bmd-ext-math.asd @@ -1,9 +1,19 @@ -(in-package #:asdf-user) - -(defsystem 3bmd-ext-math +(asdf:defsystem "3bmd-ext-math" :description "An extension for 3bmd for handling math markup" - :depends-on (3bmd esrap) + :depends-on ("3bmd" "esrap") :serial t :license "MIT" :author "Lukasz Janyst " - :components ((:file "math"))) + :components ((:file "math")) + :in-order-to ((test-op (test-op 3bmd-ext-math/tests)))) + +(asdf:defsystem "3bmd-ext-math/tests" + :depends-on ("3bmd-ext-math" "3bmd-tests" "fiasco") + :serial t + :components ((:module "tests" + :components ((:module "extensions" + :components ((:file "math")))))) + :perform (asdf:test-op (o s) + (or (uiop:symbol-call '#:fiasco '#:run-package-tests + :package '#:3bmd-ext-math-tests) + (error "tests failed")))) diff --git a/extensions.lisp b/extensions.lisp index b61e72a..c044edb 100644 --- a/extensions.lisp +++ b/extensions.lisp @@ -41,9 +41,11 @@ (push new (cdr (nthcdr (1- min) list))) list))))) -(defun %make-definer (extension-flag name expression options var rule exp) +(defun %make-definer (extension-flag name expression options var rule exp + extension-to-md-chars-to-escape) (let ((characters (cdr (assoc :character-rule options))) (escapes (cdr (assoc :escape-char-rule options))) + (md-chars-to-escapes (cdr (assoc :md-chars-to-escape options))) (after (cdr (assoc :after options))) (before (cdr (assoc :before options)))) `(progn @@ -76,6 +78,7 @@ ,@(remove-if (lambda (a) (member (car a) '(:character-rule :escape-char-rule + :md-chars-to-escape :after :before))) options)) (setf ,var @@ -83,13 +86,28 @@ ,var ,@(when before `(:before ',before)) ,@(when after `(:after ',after)) )) - (esrap:change-rule ',rule ,exp)))) + (esrap:change-rule ',rule ,exp) + (add-to-extension-to-md-chars-to-escape ,extension-to-md-chars-to-escape + ',extension-flag + ',md-chars-to-escapes)))) (defmacro define-extension-inline (extension-flag name expression &body options) (%make-definer extension-flag name expression options '%inline-rules% - '%inline '(cons 'or %inline-rules%))) + '%inline '(cons 'or %inline-rules%) + '3bmd::*extension-to-md-inline-chars-to-escape*)) (defmacro define-extension-block (extension-flag name expression &body options) (%make-definer extension-flag name expression options '%block-rules% '%block - '`(and (* blank-line) (or ,@%block-rules%)))) + '`(and (* blank-line) (or ,@%block-rules%)) + '3bmd::*extension-to-md-block-chars-to-escape*)) +;;; EXTENSION-FLAG to list of character hash tables where +;;; DEFINE-EXTENSION-INLINE and DEFINE-EXTENSION-BLOCK register the +;;; characters to escape when printing to Markdown with the +;;; corresponding extension enabled at the time of printing. +(defvar 3bmd::*extension-to-md-inline-chars-to-escape* (make-hash-table)) +(defvar 3bmd::*extension-to-md-block-chars-to-escape* (make-hash-table)) + +(defun add-to-extension-to-md-chars-to-escape (ht extension-flag chars) + (setf (gethash extension-flag ht) + (append (gethash extension-flag ht) chars))) diff --git a/markdown-printer.lisp b/markdown-printer.lisp index 45fe687..d71e949 100644 --- a/markdown-printer.lisp +++ b/markdown-printer.lisp @@ -35,16 +35,37 @@ ;;; These are some of the 3BMD-GRAMMAR::SPECIAL-CHARs. The ! character ;;; is not necessary to escape if [ is. Similary, no need to escape > ;;; if < is. Backslash should never be escaped. -(defparameter *block-chars-to-escape* "*_`&[]<#") -(defparameter *inline-chars-to-escape* (remove #\# *block-chars-to-escape*)) +(defparameter *md-default-block-chars-to-escape* "#") +(defparameter *md-default-inline-chars-to-escape* "*_`&[]<") + +(defvar *md-block-chars-to-escape*) +(defvar *md-inline-chars-to-escape*) + +(defun chars-to-escape-with-extensions (default extension-to-md-chars-to-escape) + (let ((strings (list default))) + (maphash (lambda (extension-flag chars) + (when (symbol-value extension-flag) + (push (coerce chars 'string) strings))) + extension-to-md-chars-to-escape) + (apply #'concatenate 'string strings))) + +(defmacro with-md-escapes (&body body) + `(let ((*md-block-chars-to-escape* + (chars-to-escape-with-extensions + *md-default-block-chars-to-escape* + *extension-to-md-block-chars-to-escape*)) + (*md-inline-chars-to-escape* + (chars-to-escape-with-extensions + *md-default-inline-chars-to-escape* + *extension-to-md-inline-chars-to-escape*))) + ,@body)) (defun print-md-escaped (string stream) (loop for char across string do (when (and (not *in-code*) - (find char - (if (eq *md-in-block* :right-after-indent) - *block-chars-to-escape* - *inline-chars-to-escape*))) + (or (find char *md-inline-chars-to-escape*) + (and (eq *md-in-block* :right-after-indent) + (find char *md-block-chars-to-escape*)))) ;; TODO: The escaping is overeager. For example, there is ;; no need for the escapes in "\\<->" and "\\&KEY " due ;; to how the parser works, but this needs information @@ -247,8 +268,9 @@ (*md-prefix* "") (*md-in-block* nil) (*md-block-seen-p* nil)) - (dolist (element doc) - (print-md-element element stream)))) + (with-md-escapes + (dolist (element doc) + (print-md-element element stream))))) #| diff --git a/math.lisp b/math.lisp index c77cffb..182cd18 100644 --- a/math.lisp +++ b/math.lisp @@ -2,26 +2,36 @@ ; Support math markup using libraries like MathJax ; Author: Lukasz Janyst ; -; Works both with inline math: +; Works with inline math: ; -; Begining of the paragraph $$ \sum_{i=0}^{10} (u_{i} x_{i})^2 $$ blah blah +; $x_0$ +; $`x_0`$ +; $$x_0$$ text ; -; and with blocks: +; and block (display) math: ; -; $$ -; \sum_{i=0}^{10} (u_{i} x_{i})^2 -; $$ +; $$x_0$$ ; -; Note that this departs from normal TeX syntax, which uses a single $ -; for inline math, and double for display math. By using double $, the -; need for escaping $ is much less. Also, although it's not -; documented, GitHub Flavored Markdown supports $$-delimited _inline_ -; math, too. +; - To avoid rendering "between $5 and $6" with inline math, both the +; opening and the closing $ character must be followed / preceded by +; a non-space character. This agrees with Pandoc. The other forms do +; not have such restriction. +; +; - In the block format, the opening $$ can only be preceded by +; spaces, and the closing $$ can only be followed by spaces on its +; own line. +; +; TODO: +; +; - Escaping within math (of e.g. $ characters) is not implemented. ; ;------------------------------------------------------------------------------- (defpackage #:3bmd-math (:use #:cl #:esrap #:3bmd-ext) + (:import-from #:3bmd #:ensure-block #:end-block #:print-md) + (:import-from #:3bmd-grammar #:eof #:escaped-character #:newline + #:sp #:space-char) (:export #:*math* #:*html-inline-start-marker* #:*html-inline-end-marker* @@ -35,22 +45,59 @@ (defvar *html-block-start-marker* "\\[") (defvar *html-block-end-marker* "\\]") -(defrule math-content (* (and (! "$$") character)) +(define-extension-inline *math* math-inline-1 + (and "$" inline-math-content-1 "$") + (:character-rule math-extended-chars #\$) + (:escape-char-rule math-escaped-characters #\$) + (:md-chars-to-escape #\$) + (:after escaped-character) + (:destructure (s c e) + (declare (ignore s e)) + (list :math-inline-1 c))) + +(defrule inline-math-content-1 + (and (! space-char) + (* (and (* (and (! space-char) (! "$") character)) + space-char)) + (+ (and (! (or space-char "$")) character))) (:text t)) -(define-extension-inline *math* math-inline - (and "$$" math-content "$$") +(define-extension-inline *math* math-inline-2 + (and "$`" inline-math-content-2 "`$") (:destructure (s c e) (declare (ignore s e)) - (list :math-inline c))) + (list :math-inline-2 c))) -(define-extension-block *math* math-block - (and "$$" math-content "$$") - (:destructure (s c e) +(defrule inline-math-content-2 (* (and (! "`$") character)) + (:text t)) + +(define-extension-inline *math* math-inline-3 + (and "$$" inline-math-content-3 "$$") + (:destructure (s c e) (declare (ignore s e)) + (list :math-inline-3 c))) + +(defrule inline-math-content-3 (* (and (! "$$") character)) + (:text t)) + +(define-extension-block *math* math-block + (and "$$" block-math-content "$$" sp (or newline eof)) + (:destructure (s c e sp l) + (declare (ignore s e sp l)) (list :math-block c))) -(defmethod print-tagged-element ((tag (eql :math-inline)) stream rest) +(defrule block-math-content (* (and (! "$$") character)) + (:text t)) + +(defmethod print-tagged-element ((tag (eql :math-inline-1)) stream rest) + (format stream "~a~a~a" *html-inline-start-marker* (car rest) + *html-inline-end-marker*)) + +(defmethod print-tagged-element ((tag (eql :math-inline-2)) stream rest) + (format stream "~a~a~a" *html-inline-start-marker* (car rest) + *html-inline-end-marker*)) + +(defmethod print-tagged-element ((tag (eql :math-inline-3)) stream rest) (format stream "~a~a~a" *html-inline-start-marker* (car rest) *html-inline-end-marker*)) @@ -58,22 +105,16 @@ (format stream "~a~a~a" *html-block-start-marker* (car rest) *html-block-end-marker*)) -(defmethod print-md-tagged-element ((tag (eql :math-inline)) stream rest) - (format stream "$$~a$$" (car rest))) - -(defmethod print-md-tagged-element ((tag (eql :math-block)) stream rest) - (3bmd::ensure-block stream) - (3bmd::print-md (format nil "$$~a$$" (car rest)) stream) - (3bmd::end-block stream)) +(defmethod print-md-tagged-element ((tag (eql :math-inline-1)) stream rest) + (format stream "$~a$" (car rest))) -#++ -(let ((3bmd-math:*math* t)) - (esrap:parse '%inline "$$ \sum_{i=0}^{10} (u_{i} x_{i})^2 $$")) +(defmethod print-md-tagged-element ((tag (eql :math-inline-2)) stream rest) + (format stream "$`~a`$" (car rest))) -#++(let ((3bmd-math:*math* t)) - (with-output-to-string (s) - (3bmd:parse-string-and-print-to-stream "test $$ \sum_{i=0}^{10} (u_{i} x_{i})^2 $$ test" s))) +(defmethod print-md-tagged-element ((tag (eql :math-inline-3)) stream rest) + (format stream "$$~a$$" (car rest))) -#++(let ((3bmd-math:*math* t)) - (with-output-to-string (s) - (3bmd:parse-string-and-print-to-stream "$$ \sum_{i=0}^{10} (u_{i} x_{i})^2 $$" s))) +(defmethod print-md-tagged-element ((tag (eql :math-block)) stream rest) + (ensure-block stream) + (print-md (format nil "$$~a$$" (car rest)) stream) + (end-block stream)) diff --git a/tests/extensions/math.lisp b/tests/extensions/math.lisp new file mode 100644 index 0000000..a3b870f --- /dev/null +++ b/tests/extensions/math.lisp @@ -0,0 +1,151 @@ +(fiasco:define-test-package #:3bmd-ext-math-tests + (:import-from #:3bmd-tests #:def-grammar-test #:def-print-test) + (:use #:3bmd-math)) + +(in-package #:3bmd-ext-math-tests) + +(def-grammar-test math-inline-1 + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$x_0$" + :expected '(:plain (:math-inline-1 "x_0"))) + +(def-print-test print-math-inline-1 + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "$x_0$" + :expected "$x_0$") + +(def-grammar-test math-inline-1/space-after-open + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$ x_0$" + :expected '(:plain "$" " " "x_0" "$")) + +(def-print-test print-math-inline-1/space-after-open + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "$ x_0$" + :expected "\\$ x\\_0\\$") + +(def-grammar-test math-inline-1/space-before-close + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$x_0 $" + :expected '(:plain "$" "x_0" " " "$")) + +(def-print-test print-math-inline-1/space-before-close + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "$x_0 $" + :expected "\\$x\\_0 \\$") + +(def-grammar-test math-inline-1/escaped + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "\\$x_0$" + :expected '(:plain "$" "x_0" "$")) + +(def-grammar-test math-inline-1/both-escaped + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "\\$x_0\\$" + :expected '(:plain "$" "x_0" "$")) + +(def-grammar-test math-inline-2 + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$`x_0`$" + :expected '(:plain (:math-inline-2 "x_0"))) + +(def-print-test print-math-inline-2 + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "$`x_0`$" + :expected "$`x_0`$") + +(def-grammar-test math-inline-2/escaped + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "\\$`x_0`$" + :expected '(:plain "$" (:code "x_0") "$")) + +(def-print-test print-math-inline-2/escaped + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "\\$`x_0`$" + :expected "\\$`x_0`\\$") + +(def-grammar-test math-inline-2/both-escaped + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "\\$`x_0`\\$" + :expected '(:plain "$" (:code "x_0") "$")) + +(def-print-test print-math-inline-2/both-escaped + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "\\$`x_0`\\$" + :expected "\\$`x_0`\\$") + +(def-grammar-test math-inline-3 + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$$x_0$$ x" + :expected '(:plain (:math-inline-3 "x_0") " " "x")) + +(def-print-test print-math-inline-3 + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "$$x_0$$ x" + :expected "$$x_0$$ x") + +(def-grammar-test math-inline-3/one-escaped + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "\\$$x_0$$ x" + :expected '(:plain "$" (:math-inline-1 "x_0") "$" " " "x")) + +(def-print-test print-math-inline-3/one-escaped + :enable-extensions 3bmd-math:*math* + :format :markdown + :text "\\$$x_0$$" + :expected "\\$$x_0$\\$") + +(def-grammar-test math-inline-3/two-escaped + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "\\$\\$x_0$$ x" + :expected '(:plain "$" "$" "x_0" "$" "$" " " "x")) + +(def-print-test print-math-inline-3/two-escaped + :enable-extensions *math* + :format :markdown + :text "\\$\\$x_0$$ x" + :expected "\\$\\$x\\_0\\$\\$ x") + +(def-grammar-test math-block + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$$x_0$$" + :expected '(:math-block "x_0")) + +(def-print-test print-math-block + :enable-extensions *math* + :format :markdown + :text "$$x_0$$" + :expected "$$x_0$$ +") + +(def-grammar-test math-block/trailing-whitespace + :enable-extensions *math* + :rule 3bmd-grammar::%block + :text "$$x_0$$ " + :expected '(:math-block "x_0")) + +(def-print-test print-math-block/trailing-whitespace + :enable-extensions *math* + :format :markdown + :text "$$x_0$$ " + :expected "$$x_0$$ +")