Skip to content

feat(core): backport 4 improvements to serializer from prosemirror-markdown upstream #1046

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
d3m1d0v wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(core): backport 4 improvements to serializer from prosemirror-markdown upstream #1046

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 194 additions & 12 deletions packages/editor/src/core/markdown/Markdown.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ const serializer = new MarkdownSerializer(
}) as SerializerNodeToken,
heading: ((state, node) => {
state.write(state.repeat('#', node.attrs.level) + ' ');
state.renderInline(node);
state.renderInline(node, false);
state.closeBlock(node);
}) as SerializerNodeToken,
list_item: ((state, node) => {
Expand Down Expand Up @@ -94,11 +94,12 @@ const {em, strong, code} = builder;

const {same, parse, serialize} = createMarkupChecker({parser, serializer});

describe('markdown', () => {
it('parses a paragraph', () => same('hello!', doc(p('hello!')))); // TODO: move test to extensions?
// Tests ported from prosemirror-markdown
describe('markdown (from prosemirror-markdown)', () => {
it('parses a paragraph', () => same('hello!', doc(p('hello!'))));

it('parses headings', () => {
same('# one\n\n## two\n\nthree', doc(h1('one'), h2('two'), p('three'))); // TODO: move test to extensions?
same('# one\n\n## two\n\nthree', doc(h1('one'), h2('two'), p('three')));
});

// FIXME bring back testing for preserving bullets and tight attrs
Expand All @@ -118,6 +119,9 @@ describe('markdown', () => {
),
));

// todo: parser adds empty paragraph before heading in list_item
it.skip('can parse a heading in a list', () => same('* # Foo', doc(ul(li(h1('Foo'))))));

it('parses inline marks', () =>
same(
'Hello. Some *em* text, some **strong** text, and some `code`',
Expand Down Expand Up @@ -147,19 +151,23 @@ describe('markdown', () => {
it('parses code mark inside strong text', () =>
same('**`code` is bold**', doc(p(strong(code('code'), ' is bold')))));

// tood
// todo: requires dynamic backtick fencing in code mark serializer
it.skip('parses code mark containing backticks', () =>
same(
'``` one backtick: ` two backticks: `` ```',
doc(p(code('one backtick: ` two backticks: ``'))),
));

// todo
// todo: requires code mark to preserve whitespace-only content
it.skip('parses code mark containing only whitespace', () =>
serialize(doc(p('Three spaces: ', code(' '))), 'Three spaces: ` `'));

it('parses a line break', () =>
same('line one\\\nline two', doc(p('line one', br(), 'line two'))));
it('parses hard breaks', () => {
same('line one\\\nline two', doc(p('line one', br(), 'line two')));
});

it('parses hard breaks inside emphasis', () =>
same('*foo\\\nbar*', doc(p(em('foo', br(), 'bar')))));

it('ignores HTML tags', () => parse('Foo < img> bar', doc(p('Foo < img> bar'))));

Expand All @@ -170,11 +178,11 @@ describe('markdown', () => {

it('drops trailing hard breaks', () => serialize(doc(p('a', br(), br())), 'a'));

it('should remove marks from edge break (before)', () =>
serialize(doc(p('text', strong(br(), 'text2'))), 'text\\\n**text2**'));
it('properly expels whitespace before a hard break', () =>
serialize(doc(p(strong('foo ', br()), 'bar')), '**foo** \\\nbar'));

it('should remove marks from edge break (after)', () =>
serialize(doc(p(strong('text', br()), 'text2')), '**text**\\\ntext2'));
it("doesn't crash when a block ends in a hard break", () =>
serialize(doc(p(strong('foo', br()))), '**foo**'));

it('expels enclosing whitespace from inside emphasis', () =>
serialize(
Expand Down Expand Up @@ -204,6 +212,48 @@ describe('markdown', () => {

it("doesn't escape characters in code", () => same('foo`*`', doc(p('foo', code('*')))));

// todo: requires smarter startOfLine escape – don't escape `\d+.` without trailing space
it.skip('does not escape list markers without space after them', () =>
same('1.2kg', doc(p('1.2kg'))));

// todo: requires removing `+` from defaultEsc in esc()
it.skip("doesn't escape +++", () => same('+++', doc(p('+++'))));

it('escapes list markers inside lists', () => {
same('* 1\\. hi\n\n* x', doc(ul(li(p('1. hi')), li(p('x')))));
});

it("doesn't escape block-start characters in heading content", () => {
same('# 1. foo', doc(h1('1. foo')));
});

it('escapes ATX heading markers with space after them', () => {
same('\\### text', doc(p('### text')));
});

it('escapes ATX heading markers followed by end of line', () => {
same('\\###', doc(p('###')));
});

// todo: requires smarter startOfLine escape for # – only escape #{1,6} followed by space or EOL
it.skip('does not escape ATX heading markers without space after them', () => {
same('#hashtag', doc(p('#hashtag')));
});

// todo: requires smarter startOfLine escape for # – only escape #{1,6} followed by space or EOL
it.skip('does not escape ATX heading markers consisting of more than 6 in a sequence', () => {
same('#######', doc(p('#######')));
});
});

// Tests specific to the fork's extensions and customizations
describe('markdown (fork-specific)', () => {
it('should remove marks from edge break (before)', () =>
serialize(doc(p('text', strong(br(), 'text2'))), 'text\\\n**text2**'));

it('should remove marks from edge break (after)', () =>
serialize(doc(p(strong('text', br()), 'text2')), '**text**\\\ntext2'));

it('escapes special characters in a text', () => {
same(
'Markdown special characters: \\_underscore\\_, \\*asterisk\\*, \\`backtick\\`, \\$dollar\\$, \\{curly\\} brace, \\[square\\] bracket, and a \\|vertical\\| bar.',
Expand Down Expand Up @@ -238,4 +288,136 @@ describe('markdown', () => {
same('trailing\\_', doc(p('trailing_')));
same('space \\_ space', doc(p('space _ space')));
});

describe('expelEnclosingWhitespace with mark continuation', () => {
it('keeps trailing whitespace inside mark when mark continues', () => {
// strong("hello ") + strong(em("world")) — strong continues
// trailing space stays inside strong, before em opens
serialize(doc(p(strong('hello '), strong(em('world')))), '**hello *world***');
});

it('expels trailing whitespace when mark does not continue', () => {
serialize(doc(p(strong('hello '), 'world')), '**hello** world');
});

it('keeps leading whitespace inside mark when mark is already active', () => {
// em("hello") + em(code("x")) + em(" world") — em continues throughout
// leading space in " world" stays inside em
serialize(doc(p(em('hello'), em(code('x')), em(' world'))), '*hello`x` world*');
});

it('expels leading whitespace when mark is opening', () => {
serialize(doc(p('before', strong(' hello'))), 'before **hello**');
});
});

describe('atBlockStart – startOfLine escaping precision', () => {
it('does not escape # after line break inside inline content', () => {
serialize(doc(p(strong('text1\n#text2'))), '**text1\n#text2**');
});

it('does not escape - after line break inside inline content', () => {
serialize(doc(p(strong('line1\n-line2'))), '**line1\n-line2**');
});

// todo: > is in defaultEsc so it's always escaped, not just at startOfLine; needs esc() change
it.skip('does not escape > after line break inside inline content', () => {
serialize(doc(p(strong('line1\n>quote'))), '**line1\n>quote**');
});

it('does not escape numbered list after line break inside inline content', () => {
serialize(doc(p(strong('line1\n1. item'))), '**line1\n1. item**');
});

it('still escapes # at actual block start', () => {
same('\\# not a heading', doc(p('# not a heading')));
});

it('still escapes - at actual block start', () => {
same('\\- not a list', doc(p('- not a list')));
});

it('still escapes > at actual block start', () => {
same('\\>not a quote', doc(p('>not a quote')));
});

it('still escapes numbered list at actual block start', () => {
same('1\\. not a list', doc(p('1. not a list')));
});
});

describe('render (strict mode for nodes)', () => {
const nodeStrictSerializer = new MarkdownSerializer(
{
text: ((state, node) => {
state.text(node.text ?? '');
}) as SerializerNodeToken,
paragraph: ((state, node) => {
state.renderInline(node);
state.closeBlock(node);
}) as SerializerNodeToken,
// 'heading' is NOT registered
},
{
em: {open: '*', close: '*', mixable: true, expelEnclosingWhitespace: true},
strong: {open: '**', close: '**', mixable: true, expelEnclosingWhitespace: true},
},
);

it('throws on unknown node in strict mode (default)', () => {
expect(() => nodeStrictSerializer.serialize(doc(h1('text')))).toThrow(
/Token type `heading` not supported by Markdown renderer/,
);
});

it('renders inline content of unknown node in non-strict mode', () => {
expect(nodeStrictSerializer.serialize(doc(h1('text')), {strict: false})).toBe('text');
});

it('renders inline content with marks of unknown node in non-strict mode', () => {
expect(
nodeStrictSerializer.serialize(doc(h1('hello ', strong('world'))), {strict: false}),
).toBe('hello **world**');
});
});

describe('getMark (strict mode)', () => {
const strictSerializer = new MarkdownSerializer(
{
text: ((state, node) => {
state.text(node.text ?? '');
}) as SerializerNodeToken,
paragraph: ((state, node) => {
state.renderInline(node);
state.closeBlock(node);
}) as SerializerNodeToken,
},
{
// only 'strong' is registered, 'em' and 'code' are missing
strong: {open: '**', close: '**', mixable: true, expelEnclosingWhitespace: true},
},
);

it('throws on unknown mark in strict mode (default)', () => {
expect(() => strictSerializer.serialize(doc(p(em('text'))))).toThrow(
/Mark type `em` not supported by Markdown renderer/,
);
});

it('silently ignores unknown mark in non-strict mode', () => {
expect(strictSerializer.serialize(doc(p(em('text'))), {strict: false})).toBe('text');
});

it('serializes known marks normally in non-strict mode', () => {
expect(strictSerializer.serialize(doc(p(strong('text'))), {strict: false})).toBe(
'**text**',
);
});

it('silently ignores unknown mark mixed with known marks in non-strict mode', () => {
expect(strictSerializer.serialize(doc(p(strong(em('text')))), {strict: false})).toBe(
'**text**',
);
});
});
});
Loading
Loading