Merge branch 'linkConversionFromPlugin' into main

Author: hatton

.vscode/settings.json

@@ -7,6 +7,7 @@
     "callouts",
     "DOCU_NOTION",
     "docu-notion",
+    "docunotion",
     "Greenshot",
     "imgur",
     "kanban",
@@ -17,4 +18,4 @@
     "statusBar.noFolderBackground": "#d649ca",
     "statussBar.prominentBackground": "#d649ca"
   }
-}
+}

package.json

@@ -7,7 +7,7 @@
     "dist/**/*"
   ],
   "scripts": {
-    "test": "jest",
+    "test": "vitest",
     "build": "yarn test && tsc && cp ./src/css/*.css dist/",
     "build-only": "tsc && cp ./src/css/*.css dist/",
     "clean": "rimraf ./dist/",
@@ -68,25 +68,25 @@
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.13",
-    "@types/jest": "^28.1.6",
     "@types/markdown-table": "^2.0.0",
     "@types/node": "^12.20.11",
     "@typescript-eslint/eslint-plugin": "^4.22.0",
     "@typescript-eslint/parser": "^4.22.0",
+    "@vitest/ui": "^0.30.1",
     "cross-var": "^1.1.0",
     "cz-conventional-changelog": "^3.3.0",
     "eslint": "^7.25.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prettier": "^3.4.0",
-    "jest": "^28.1.3",
     "lint-staged": "^10.5.4",
     "prettier": "^2.2.1",
     "rimraf": "^4.1.2",
     "semantic-release": "^19.0.2",
-    "ts-jest": "^28.0.7",
     "ts-node": "^10.2.1",
-    "typescript": "^4.6.4"
+    "typescript": "^4.6.4",
+    "vite": "^4.2.1",
+    "vitest": "^0.30.1"
   },
   "release": {
     "branches": [

src/plugins/internalLinks.spec.ts

@@ -458,6 +458,50 @@ Callouts inline [great page](/hello-world).
   );
 });
 
+test("internal link inside codeblock ignored", async () => {
+  const targetPageId = "123";
+  const targetPage: NotionPage = makeSamplePageObject({
+    slug: "hello-world",
+    name: "Hello World",
+    id: targetPageId,
+  });
+
+  const results = await getMarkdown(
+    {
+      type: "code",
+      code: {
+        caption: [],
+        rich_text: [
+          {
+            type: "text",
+            text: {
+              content:
+                "this should not change [link](https://www.notion.so/native/metapages/mypage)",
+              link: null,
+            },
+            annotations: {
+              bold: false,
+              italic: false,
+              strikethrough: false,
+              underline: false,
+              code: false,
+              color: "default",
+            },
+            plain_text:
+              "this should not change [link](https://www.notion.so/native/metapages/mypage)",
+            href: null,
+          },
+        ],
+        language: "javascript", // notion assumed javascript in my test in which I didn't specify a language
+      },
+    },
+    targetPage
+  );
+  expect(results.trim()).toContain(
+    "this should not change [link](https://www.notion.so/native/metapages/mypage)"
+  );
+});
+
 async function getMarkdown(block: object, targetPage?: NotionPage) {
   const config = {
     plugins: [

src/plugins/internalLinks.ts

@@ -2,6 +2,40 @@ import { IDocuNotionContext, IPlugin } from "./pluginTypes";
 import { error, warning } from "../log";
 import { NotionPage } from "../NotionPage";
 
+// converts a url to a local link, if it is a link to a page in the Notion site
+// only here for plugins, notion won't normally be giving us raw urls (at least not that I've noticed)
+// If it finds a URL but can't find the page it points to, it will return undefined.
+// If it doesn't find a match at all, it returns undefined.
+export function convertInternalUrl(
+  context: IDocuNotionContext,
+  url: string
+): string | undefined {
+  const kGetIDFromNotionURL = /https:\/\/www\.notion\.so\S+-([a-z,0-9]+)+.*/;
+  const match = kGetIDFromNotionURL.exec(url);
+  if (match === null) {
+    warning(
+      `[standardInternalLinkConversion] Could not parse link ${url} as a Notion URL`
+    );
+    return undefined;
+  }
+  const id = match[1];
+  const pages = context.pages;
+  // find the page where pageId matches hrefFromNotion
+  const targetPage = pages.find(p => {
+    return p.matchesLinkId(id);
+  });
+
+  if (!targetPage) {
+    // About this situation. See https://github.com/sillsdev/docu-notion/issues/9
+    warning(
+      `[standardInternalLinkConversion] Could not find the target of this link. Note that links to outline sections are not supported. ${url}. https://github.com/sillsdev/docu-notion/issues/9`
+    );
+    return undefined;
+  }
+  return convertLinkHref(context, targetPage, url);
+}
+
+// handles the whole markdown link, including the label
 function convertInternalLink(
   context: IDocuNotionContext,
   markdownLink: string

src/plugins/mermaidLinkPlugin.spec.ts

@@ -0,0 +1,69 @@
+import { NotionPage } from "../NotionPage";
+import { makeSamplePageObject, oneBlockToMarkdown } from "./pluginTestRun";
+import { IDocuNotionContext, IPlugin } from "./pluginTypes";
+
+test("raw url inside a mermaid codeblock gets converted to path using slug of that page", async () => {
+  const targetPageId = "123";
+  const targetPage: NotionPage = makeSamplePageObject({
+    slug: "slug-of-target",
+    name: "My Target Page",
+    id: targetPageId,
+  });
+
+  const input = {
+    type: "code",
+    code: {
+      caption: [],
+      rich_text: [
+        {
+          type: "text",
+          text: {
+            content: `click A "https://www.notion.so/native/metapages/A-Page-${targetPageId}"`,
+            link: null,
+          },
+          annotations: {
+            bold: false,
+            italic: false,
+            strikethrough: false,
+            underline: false,
+            code: false,
+            color: "default",
+          },
+          plain_text: `click A "https://www.notion.so/native/metapages/A-Page-${targetPageId}"`,
+          href: null,
+        },
+      ],
+      language: "mermaid", // notion assumed javascript in my test in which I didn't specify a language
+    },
+  };
+
+  const mermaidLinks: IPlugin = {
+    name: "mermaidLinks",
+    regexMarkdownModifications: [
+      {
+        regex: /```mermaid\n.*"(https:\/\/www\.notion\.so\S+)"/,
+        includeCodeBlocks: true,
+        getReplacement: async (
+          context: IDocuNotionContext,
+          match: RegExpExecArray
+        ) => {
+          const url = match[1];
+          const docusaurusUrl =
+            context.convertNotionLinkToLocalDocusaurusLink(url);
+          // eslint-disable-next-line @typescript-eslint/await-thenable
+          return await match[0].replace(url, docusaurusUrl);
+        },
+      },
+    ],
+  };
+
+  const config = {
+    plugins: [
+      // standardInternalLinkConversion,
+      // standardExternalLinkConversion,
+      mermaidLinks,
+    ],
+  };
+  const results = await oneBlockToMarkdown(config, input, targetPage);
+  expect(results.trim()).toContain(`click A "/slug-of-target"`);
+});

src/plugins/pluginTestRun.ts

@@ -1,14 +1,13 @@
 import { Client } from "@notionhq/client";
 import { GetPageResponse } from "@notionhq/client/build/src/api-endpoints";
-import { info } from "console";
 import { NotionToMarkdown } from "notion-to-md";
 import { IDocuNotionContext } from "./pluginTypes";
 import { HierarchicalNamedLayoutStrategy } from "../HierarchicalNamedLayoutStrategy";
-import { error, logDebug, verbose, warning } from "../log";
 import { NotionPage } from "../NotionPage";
 import { getMarkdownFromNotionBlocks } from "../transform";
 import { IDocuNotionConfig } from "../config/configuration";
 import { NotionBlock } from "../types";
+import { convertInternalUrl } from "./internalLinks";
 
 export async function blocksToMarkdown(
   config: IDocuNotionConfig,
@@ -34,6 +33,10 @@ export async function blocksToMarkdown(
         resolve([]);
       });
     },
+    convertNotionLinkToLocalDocusaurusLink: (url: string) => {
+      return convertInternalUrl(docunotionContext, url);
+    },
+
     //TODO might be needed for some tests, e.g. the image transformer...
     directoryContainingMarkdown: "not yet",
     relativeFilePathToFolderContainingPage: "not yet",

src/plugins/pluginTypes.ts

@@ -46,7 +46,12 @@ export type IRegexMarkdownModification = {
   // Based on that regex, the outputPattern will be used to replace the matched text
   replacementPattern?: string;
   // Instead of a pattern, you can use this if you have to ask a server somewhere for help in getting the new markdown
-  getReplacement?(s: string): Promise<string>;
+  getReplacement?(
+    context: IDocuNotionContext,
+    match: RegExpExecArray
+  ): Promise<string>;
+  // normally, anything in code blocks is will be ignored. If you want to make changes inside of code blocks, set this to true.
+  includeCodeBlocks?: boolean;
 
   // If the output is creating things like react elements, you can import their definitions here
   imports?: string[];
@@ -66,6 +71,7 @@ export type IDocuNotionContext = {
   notionToMarkdown: NotionToMarkdown;
   directoryContainingMarkdown: string;
   relativeFilePathToFolderContainingPage: string;
+  convertNotionLinkToLocalDocusaurusLink: (url: string) => string;
   pages: NotionPage[];
   counts: ICounts;
 };

src/pull.ts

@@ -28,6 +28,7 @@ import { Client, isFullBlock } from "@notionhq/client";
 import { exit } from "process";
 import { IDocuNotionConfig, loadConfigAsync } from "./config/configuration";
 import { NotionBlock } from "./types";
+import { convertInternalUrl } from "./plugins/internalLinks";
 
 export type DocuNotionOptions = {
   notionToken: string;
@@ -110,6 +111,8 @@ async function outputPages(
     options: options,
     pages: pages,
     counts: counts, // review will this get copied or pointed to?
+    convertNotionLinkToLocalDocusaurusLink: (url: string) =>
+      convertInternalUrl(context, url),
   };
   for (const page of pages) {
     layoutStrategy.pageWasSeen(page);

src/transform.ts

@@ -56,7 +56,11 @@ export async function getMarkdownFromNotionBlocks(
   //console.log("markdown after link fixes", markdown);
 
   // simple regex-based tweaks. These are usually related to docusaurus
-  const { imports, body } = await doTransformsOnMarkdown(config, markdown);
+  const { imports, body } = await doTransformsOnMarkdown(
+    context,
+    config,
+    markdown
+  );
 
   // console.log("markdown after regex fixes", markdown);
   // console.log("body after regex", body);
@@ -82,6 +86,7 @@ function doNotionBlockTransforms(
 }
 
 async function doTransformsOnMarkdown(
+  context: IDocuNotionContext,
   config: IDocuNotionConfig,
   input: string
 ) {
@@ -109,15 +114,24 @@ async function doTransformsOnMarkdown(
     // regex.exec is stateful, so we don't want to mess up the plugin's use of its own regex, so we clone it.
     // we also add the "g" flag to make sure we get all matches
     const regex = new RegExp(`${codeBlocks.source}|(${mod.regex.source})`, "g");
-    let count = 0;
     while ((match = regex.exec(input)) !== null) {
       if (match[0]) {
         const original = match[0];
-        if (original.startsWith("```") && original.endsWith("```")) {
-          continue; // code block
+        if (
+          original.startsWith("```") &&
+          original.endsWith("```") &&
+          !mod.includeCodeBlocks
+        ) {
+          continue; // code block, and they didn't say to include them
         }
         if (mod.getReplacement) {
-          replacement = await mod.getReplacement(original);
+          // our match here has an extra group, which is an implementation detail
+          // that shouldn't be made visible to the plugin
+          const matchAsThePluginWouldExpectIt = mod.regex.exec(match[0])!;
+          replacement = await mod.getReplacement(
+            context,
+            matchAsThePluginWouldExpectIt
+          );
         } else if (mod.replacementPattern) {
           console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
           replacement = mod.replacementPattern.replace("$1", match[2]);
@@ -149,7 +163,8 @@ async function doNotionToMarkdown(
     blocks
   );
 
-  let markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks);
+  const markdown =
+    docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks);
   return markdown;
 }
 
@@ -187,6 +202,7 @@ function doLinkFixes(
     config.plugins.some(plugin => {
       if (!plugin.linkModifier) return false;
       if (plugin.linkModifier.match.exec(originalLinkMarkdown) === null) {
+        verbose(`plugin "${plugin.name}" did not match this url`);
         return false;
       }
       const newMarkdown = plugin.linkModifier.convert(

tsconfig.json

@@ -46,7 +46,9 @@
     // "paths": {},                                 /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
     // "rootDirs": [],                              /* List of root folders whose combined content represents the structure of the project at runtime. */
     // "typeRoots": [],                             /* List of folders to include type definitions from. */
-    // "types": [],                                 /* Type declaration files to be included in compilation. */
+    "types": [
+      "vitest/globals"
+    ],
     // "allowSyntheticDefaultImports": true,        /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
     "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
     // "preserveSymlinks": true,                    /* Do not resolve the real path of symlinks. */