feat: regex plugins can have functions, and skip code blocks

hatton · hatton · commit 6fdc509933ec · 2023-01-21T09:00:10.000-07:00
diff --git a/src/config/configuration.ts b/src/config/configuration.ts
@@ -45,13 +45,20 @@ export type IPlugin = {
   // simple regex replacements on the markdown output
   regexMarkdownModifications?: IRegexMarkdownModification[];
 
-  // allow a plugin to perform an async operation before it can deliver its operations
+  // Allow a plugin to perform an async operation at the start of docu-notion.
+  // Notice that the plugin itself is given, so you can add things to it.
   init?(plugin: IPlugin): Promise<void>;
 };
 
 export type IRegexMarkdownModification = {
+  // Should match on markdown that you want to replace
   regex: RegExp;
-  output: string;
+  // Based on that regex, the outputPattern will be used to replace the matched text
+  replacementPattern?: string;
+  // Instead of a pattern, you can use this if you have to ask a server somewhere for help in getting the new markdown
+  getReplacement?(s: string): Promise<string>;
+
+  // If the output is creating things like react elements, you can import their definitions here
   imports?: string[];
 };
 
diff --git a/src/plugins/embedTweaks.spec.ts b/src/plugins/embedTweaks.spec.ts
@@ -1,4 +1,4 @@
-import { NotionBlock } from "../config/configuration";
+import { IPlugin, NotionBlock } from "../config/configuration";
 import { setLogLevel } from "../log";
 import { blocksToMarkdown } from "../TestRun";
 import {
@@ -93,3 +93,148 @@ test("gif", async () => {
     `![](https://en.wikipedia.org/wiki/GIF#/media/File:Rotating_earth_(large).gif)`
   );
 });
+
+test("tweaks are not applied inside code blocks", async () => {
+  setLogLevel("verbose");
+  const p: IPlugin = {
+    name: "test",
+    regexMarkdownModifications: [
+      {
+        regex: /find/,
+        replacementPattern: `found`,
+      },
+    ],
+  };
+  const config = { plugins: [p] };
+  const result = await blocksToMarkdown(config, [
+    {
+      type: "code",
+      code: {
+        caption: [],
+        rich_text: [
+          {
+            type: "text",
+            text: {
+              content: "don't find me",
+              link: null,
+            },
+            annotations: {
+              bold: false,
+              italic: false,
+              strikethrough: false,
+              underline: false,
+              code: false,
+              color: "default",
+            },
+            plain_text: "don't find me",
+            href: null,
+          },
+        ],
+        language: "",
+      },
+    } as unknown as NotionBlock,
+    {
+      type: "paragraph",
+      paragraph: {
+        rich_text: [
+          {
+            type: "text",
+            text: { content: "find this", link: null },
+            annotations: {
+              bold: false,
+              italic: false,
+              strikethrough: false,
+              underline: false,
+              code: true,
+              color: "default",
+            },
+            plain_text: "find this",
+            href: null,
+          },
+        ],
+      },
+    } as unknown as NotionBlock,
+  ]);
+  // we should not change the code one
+  expect(result.trim()).toContain("don't find me");
+  // but we should change the non-code block one
+  expect(result.trim()).toContain("found this");
+});
+
+test("simplest possible", async () => {
+  setLogLevel("verbose");
+  const p: IPlugin = {
+    name: "test",
+    regexMarkdownModifications: [
+      {
+        regex: /find/,
+        replacementPattern: `found`,
+      },
+    ],
+  };
+  const config = { plugins: [p] };
+  const result = await blocksToMarkdown(config, [
+    {
+      type: "paragraph",
+      paragraph: {
+        rich_text: [
+          {
+            type: "text",
+            text: { content: "find this", link: null },
+            annotations: {
+              bold: false,
+              italic: false,
+              strikethrough: false,
+              underline: false,
+              code: true,
+              color: "default",
+            },
+            plain_text: "find this",
+            href: null,
+          },
+        ],
+      },
+    } as unknown as NotionBlock,
+  ]);
+
+  expect(result.trim()).toContain("found this");
+});
+
+test("use match in output", async () => {
+  setLogLevel("verbose");
+  const p: IPlugin = {
+    name: "test",
+    regexMarkdownModifications: [
+      {
+        regex: /(find)/,
+        replacementPattern: `found $1`,
+      },
+    ],
+  };
+  const config = { plugins: [p] };
+  const result = await blocksToMarkdown(config, [
+    {
+      type: "paragraph",
+      paragraph: {
+        rich_text: [
+          {
+            type: "text",
+            text: { content: "find this", link: null },
+            annotations: {
+              bold: false,
+              italic: false,
+              strikethrough: false,
+              underline: false,
+              code: true,
+              color: "default",
+            },
+            plain_text: "find this",
+            href: null,
+          },
+        ],
+      },
+    } as unknown as NotionBlock,
+  ]);
+
+  expect(result.trim()).toContain("found find");
+});
diff --git a/src/plugins/embedTweaks.ts b/src/plugins/embedTweaks.ts
@@ -6,8 +6,8 @@ export const gifEmbed: IPlugin = {
     {
       // I once saw a gif coming from Notion that wasn't a full
       // url, which wouldn't work, hence the "http" requirement
-      regex: /\[.*\]\((http.*(\.(gif|GIF)))\)/gm,
-      output: `![]($1)`,
+      regex: /\[.*\]\((http.*(\.(gif|GIF)))\)/,
+      replacementPattern: `![]($1)`,
     },
   ],
 };
@@ -16,32 +16,32 @@ export const imgurGifEmbed: IPlugin = {
   name: "imgur",
   regexMarkdownModifications: [
     {
-      regex: /\[.*\]\((.*imgur\.com\/.*)\)/gm, // imgur.com
+      regex: /\[.*\]\((.*imgur\.com\/.*)\)/, // imgur.com
       // imgur links to gifs need a .gif at the end, but the url they give you doesn't have one.
-      output: `![]($1.gif)`,
+      replacementPattern: `![]($1.gif)`,
     },
   ],
 };
 export const youtubeEmbed: IPlugin = {
   name: "youtube",
   regexMarkdownModifications: [
     {
-      regex: /\[.*\]\((.*youtube\.com\/watch.*)\)/gm, //youtube.com/watch
+      regex: /\[.*\]\((.*youtube\.com\/watch.*)\)/, //youtube.com/watch
       imports: [`import ReactPlayer from "react-player";`],
-      output: `<ReactPlayer controls url="$1" />`,
+      replacementPattern: `<ReactPlayer controls url="$1" />`,
     },
   ],
 };
 export const vimeoEmbed: IPlugin = {
   name: "vimeo",
   regexMarkdownModifications: [
     {
-      regex: /\[.*\]\((https:\/\/.*vimeo.*)\)/gm,
+      regex: /\[.*\]\((https:\/\/.*vimeo.*)\)/,
       // we use to have the following, but the above should handle both the player an not-player urls.
       //regex: /\[.*\]\((.*player\.vimeo.*)\)/gm, // player.vimeo
 
       imports: [`import ReactPlayer from "react-player";`],
-      output: `<ReactPlayer controls url="$1" />`,
+      replacementPattern: `<ReactPlayer controls url="$1" />`,
     },
   ],
 };
diff --git a/src/transform.ts b/src/transform.ts
@@ -56,7 +56,7 @@ export async function getMarkdownFromNotionBlocks(
   //console.log("markdown after link fixes", markdown);
 
   // simple regex-based tweaks. These are usually related to docusaurus
-  const { imports, body } = doTransformsOnMarkdown(config, markdown);
+  const { imports, body } = await doTransformsOnMarkdown(config, markdown);
 
   // console.log("markdown after regex fixes", markdown);
   // console.log("body after regex", body);
@@ -81,7 +81,10 @@ function doNotionBlockTransforms(
   }
 }
 
-function doTransformsOnMarkdown(config: IDocuNotionConfig, input: string) {
+async function doTransformsOnMarkdown(
+  config: IDocuNotionConfig,
+  input: string
+) {
   const regexMods: IRegexMarkdownModification[] = config.plugins
     .filter(plugin => !!plugin.regexMarkdownModifications)
     .map(plugin => {
@@ -92,26 +95,50 @@ function doTransformsOnMarkdown(config: IDocuNotionConfig, input: string) {
     })
     .flat();
 
+  // regex that matches markdown code blocks
+  const codeBlocks = /```.*\n[\s\S]*?\n```/;
+
   let body = input;
   //console.log("body before regex: " + body);
   let match;
   const imports = new Set<string>();
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  regexMods.forEach(mod => {
-    //verbose(`Trying [${mod.name}]`);
-    while ((match = mod.regex.exec(input)) !== null) {
-      const string = match[0];
-      const url = match[1];
-      verbose(
-        `[${(mod as any).name}] ${string} --> ${mod.output.replace("$1", url)}`
-      );
-      body = body.replace(string, mod.output.replace("$1", url));
-      // add any library imports
-      mod.imports?.forEach(imp => imports.add(imp));
+  for (const mod of regexMods) {
+    let replacement = undefined;
+    // regex.exec is stateful, so we don't want to mess up the plugin's use of its own regex, so we clone it.
+    // we also add the "g" flag to make sure we get all matches
+    const regex = new RegExp(`${codeBlocks.source}|(${mod.regex.source})`, "g");
+    let count = 0;
+    while ((match = regex.exec(input)) !== null) {
+      if (match[0]) {
+        const original = match[0];
+        if (original.startsWith("```") && original.endsWith("```")) {
+          continue; // code block
+        }
+        if (mod.getReplacement) {
+          replacement = await mod.getReplacement(original);
+        } else if (mod.replacementPattern) {
+          console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
+          replacement = mod.replacementPattern.replace("$1", match[2]);
+        }
+        if (replacement !== undefined) {
+          verbose(`[${(mod as any).name}] ${original} --> ${replacement}`);
+
+          const precedingPart = body.substring(0, match.index); // ?
+          const partStartingFromThisMatch = body.substring(match.index); // ?
+          body =
+            precedingPart +
+            partStartingFromThisMatch.replace(original, replacement);
+          // add any library imports
+          mod.imports?.forEach(imp => imports.add(imp));
+        }
+      }
     }
-  });
-  return { body, imports: [...imports].join("\n") };
+  }
+  console.log("body after regex: " + body);
+  const uniqueImports = [...new Set(imports)];
+  return { body, imports: [...uniqueImports].join("\n") };
 }
 
 async function doNotionToMarkdown(
@@ -148,7 +175,7 @@ function doLinkFixes(
   // The key to understanding this `while` is that linkRegExp actually has state, and
   // it gives you a new one each time. https://stackoverflow.com/a/1520853/723299
   while ((match = linkRegExp.exec(markdownToSearch)) !== null) {
-    const originalLinkMarkdown = match[0]; // ?
+    const originalLinkMarkdown = match[0];
 
     verbose(
       `Checking to see if a plugin wants to modify "${originalLinkMarkdown}" `
