convert all former "Docusaurus Tweaks" to plugins with minimal unit tests

Author: hatton

package.json

@@ -79,6 +79,7 @@
     "eslint-plugin-prettier": "^3.4.0",
     "jest": "^28.1.3",
     "lint-staged": "^10.5.4",
+    "patch-package": "^6.5.1",
     "prettier": "^2.2.1",
     "semantic-release": "^19.0.2",
     "ts-jest": "^28.0.7",

src/DocusaurusTweaks.ts

@@ -1,4 +1,9 @@
-import { imgurGifTweak } from "../DocusaurusTweaks";
+import {
+  gifEmbed,
+  imgurGifEmbed,
+  vimeoEmbed,
+  youtubeEmbed,
+} from "../DocusaurusEmbeds";
 import { standardImageTransformer } from "../images";
 import { standardLinkConversion } from "../plugins/internalLinks";
 import { standardCalloutTransformer } from "../plugins/CalloutTransformer";
@@ -16,20 +21,23 @@ const defaultConfig: IDocuNotionConfig = {
     escapeHtmlBlockModifier,
     standardHeadingTransformer, // does operations on both the Notion JSON and then later, on the notion to markdown transform
 
-    // Notion to Markdown transformers
+    // Notion to Markdown transformers. Most things get transformed correctly by the notion-to-markdown library,
+    // but some things need special handling.
     standardColumnTransformer,
     standardColumnListTransformer,
     standardImageTransformer,
     standardCalloutTransformer,
     standardTableTransformer,
     standardNumberedListTransformer,
 
-    // Link modifiers, which are special because they can read all the pages
+    // Link modifiers, which are special because they can read metadata from all the pages in order to figure out the correct url
     standardLinkConversion,
 
-    // Regexps that operate on the Markdown output
-    imgurGifTweak,
-    // TODO: add the rest of the existing ones
+    // Regexps plus javascript `import`s that operate on the Markdown output
+    imgurGifEmbed,
+    gifEmbed,
+    youtubeEmbed,
+    vimeoEmbed,
   ],
 };
 

src/config/default.docunotion.config.ts

@@ -0,0 +1,95 @@
+import { NotionBlock } from "../config/configuration";
+import { setLogLevel } from "../log";
+import { blocksToMarkdown } from "../TestRun";
+import {
+  gifEmbed,
+  imgurGifEmbed,
+  vimeoEmbed,
+  youtubeEmbed,
+} from "./StandardEmbeddings";
+
+test("youtube", async () => {
+  const config = { plugins: [youtubeEmbed] };
+  const result = await blocksToMarkdown(config, [
+    {
+      object: "block",
+      id: "e6ddd1d4-36d4-4925-94c1-5dff4662c1f3",
+      has_children: false,
+      archived: false,
+      type: "video",
+      video: {
+        caption: [
+          {
+            type: "text",
+            text: {
+              content: "A video about editing in Notion",
+              link: null,
+            },
+            plain_text: "A video about editing in Notion",
+            href: null,
+          },
+        ],
+        type: "external",
+        external: { url: "https://www.youtube.com/watch?v=FXIrojSK3Jo" },
+      },
+    } as unknown as NotionBlock,
+  ]);
+  expect(result).toContain(`import ReactPlayer from "react-player";`);
+  expect(result).toContain(
+    `<ReactPlayer controls url="https://www.youtube.com/watch?v=FXIrojSK3Jo" />`
+  );
+});
+
+test("vimeo", async () => {
+  setLogLevel("verbose");
+  const config = { plugins: [vimeoEmbed] };
+  const result = await blocksToMarkdown(config, [
+    {
+      object: "block",
+      id: "39ff83a3-2fb5-4411-a715-960656a177ff",
+      type: "video",
+      video: {
+        caption: [],
+        type: "external",
+        external: { url: "https://vimeo.com/4613611xx" },
+      },
+    } as unknown as NotionBlock,
+  ]);
+  expect(result).toContain(`import ReactPlayer from "react-player";`);
+  expect(result).toContain(
+    `<ReactPlayer controls url="https://vimeo.com/4613611xx" />`
+  );
+});
+
+test("imgur", async () => {
+  setLogLevel("verbose");
+  const config = { plugins: [imgurGifEmbed] };
+  const result = await blocksToMarkdown(config, [
+    {
+      object: "block",
+      id: "e36710d8-98ad-40dc-b41b-b376ebdd6894",
+      type: "bookmark",
+      bookmark: { caption: [], url: "https://imgur.com/gallery/U8TTNuI" },
+    } as unknown as NotionBlock,
+  ]);
+  expect(result.trim()).toBe(`![](https://imgur.com/gallery/U8TTNuI.gif)`);
+});
+
+test("gif", async () => {
+  setLogLevel("verbose");
+  const config = { plugins: [gifEmbed] };
+  const result = await blocksToMarkdown(config, [
+    {
+      object: "block",
+      id: "e36710d8-98ad-40dc-b41b-b376ebdd6894",
+      type: "bookmark",
+      bookmark: {
+        caption: [],
+        url: "https://en.wikipedia.org/wiki/GIF#/media/File:Rotating_earth_(large).gif",
+      },
+    } as unknown as NotionBlock,
+  ]);
+  expect(result.trim()).toBe(
+    `![](https://en.wikipedia.org/wiki/GIF#/media/File:Rotating_earth_(large).gif)`
+  );
+});

src/plugins/StandardEmbeddings.spec.ts

@@ -0,0 +1,51 @@
+import { IPlugin } from "../config/configuration";
+
+export const gifEmbed: IPlugin = {
+  name: "gif",
+  regexMarkdownModifications: [
+    {
+      label: "gif",
+      // I once saw a gif coming from Notion that wasn't a full
+      // url, which wouldn't work, hence the "http" requirement
+      regex: /\[.*\]\((http.*(\.(gif|GIF)))\)/gm,
+      output: `![]($1)`,
+    },
+  ],
+};
+
+export const imgurGifEmbed: IPlugin = {
+  name: "imgur",
+  regexMarkdownModifications: [
+    {
+      label: "imgur",
+      regex: /\[.*\]\((.*imgur\.com\/.*)\)/gm, // imgur.com
+      // imgur links to gifs need a .gif at the end, but the url they give you doesn't have one.
+      output: `![]($1.gif)`,
+    },
+  ],
+};
+export const youtubeEmbed: IPlugin = {
+  name: "youtube",
+  regexMarkdownModifications: [
+    {
+      label: "youtube",
+      regex: /\[.*\]\((.*youtube\.com\/watch.*)\)/gm, //youtube.com/watch
+      imports: [`import ReactPlayer from "react-player";`],
+      output: `<ReactPlayer controls url="$1" />`,
+    },
+  ],
+};
+export const vimeoEmbed: IPlugin = {
+  name: "vimeo",
+  regexMarkdownModifications: [
+    {
+      label: "vimeo",
+      regex: /\[.*\]\((https:\/\/.*vimeo.*)\)/gm,
+      // we use to have the following, but the above should handle both the player an not-player urls.
+      //regex: /\[.*\]\((.*player\.vimeo.*)\)/gm, // player.vimeo
+
+      imports: [`import ReactPlayer from "react-player";`],
+      output: `<ReactPlayer controls url="$1" />`,
+    },
+  ],
+};

src/plugins/StandardEmbeddings.ts

@@ -56,7 +56,7 @@ export async function getMarkdownFromNotionBlocks(
   //console.log("markdown after link fixes", markdown);
 
   // simple regex-based tweaks. These are usually related to docusaurus
-  const { imports, body } = doRegexMarkdownTransforms(config, markdown);
+  const { imports, body } = doTransformsOnMarkdown(config, markdown);
 
   // console.log("markdown after regex fixes", markdown);
   // console.log("body after regex", body);
@@ -81,26 +81,24 @@ function doNotionBlockTransforms(
   }
 }
 
-// simple regex replacements on the markdown output
-function doRegexMarkdownTransforms(config: IDocuNotionConfig, input: string) {
+function doTransformsOnMarkdown(config: IDocuNotionConfig, input: string) {
   const regexMods: IRegexMarkdownModification[] = config.plugins
     .filter(plugin => !!plugin.regexMarkdownModifications)
     .map(plugin => plugin.regexMarkdownModifications!)
     .flat();
 
   let body = input;
+  //console.log("body before regex: " + body);
   let match;
   const imports = new Set<string>();
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   regexMods.forEach(mod => {
+    //verbose(`Trying [${mod.label}]`);
     while ((match = mod.regex.exec(input)) !== null) {
       const string = match[0];
       const url = match[1];
-      logDebug(
-        "DocusaurusTweaks",
-        `${string} --> ${mod.output.replace("$1", url)}`
-      );
+      verbose(`[${mod.label}] ${string} --> ${mod.output.replace("$1", url)}`);
       body = body.replace(string, mod.output.replace("$1", url));
       // add any library imports
       mod.imports?.forEach(imp => imports.add(imp));

src/transform.ts

@@ -6211,11 +6211,6 @@ please-upgrade-node@^3.2.0:
   dependencies:
     semver-compare "^1.0.0"
 
-postinstall-postinstall@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/postinstall-postinstall/-/postinstall-postinstall-2.1.0.tgz#4f7f77441ef539d1512c40bd04c71b06a4704ca3"
-  integrity sha512-7hQX6ZlZXIoRiWNrbMQaLzUUfH+sSx39u8EJ9HYuDc1kLo9IXKWjM5RSquZN1ad5GnH8CGFM78fsAAQi3OKEEQ==
-
 prelude-ls@^1.2.1:
   version "1.2.1"
   resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz"