From 2c07a42bdcc91019cc88a77c2b929bd55d4b90cc Mon Sep 17 00:00:00 2001 From: Stephan Meijer Date: Sat, 15 Nov 2025 16:29:43 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Import=20of=20documents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Import of (markdown) documents. See #1567 #1569. --- Makefile | 1 + compose.yml | 5 + docs/env.md | 1 + env.d/development/common | 2 + env.d/development/common.e2e | 1 + src/backend/core/api/serializers.py | 11 +- src/backend/core/api/viewsets.py | 39 +++-- .../core/services/converter_services.py | 71 +++++++- src/backend/core/services/mime_types.py | 6 + src/backend/impress/settings.py | 6 + .../docs/docs-grid/api/useImportDoc.tsx | 43 +++++ .../docs/docs-grid/components/DocsGrid.tsx | 39 ++++- .../y-provider/__tests__/convert.test.ts | 107 ++++++++---- .../y-provider/src/handlers/convertHandler.ts | 162 ++++++++++++------ 14 files changed, 385 insertions(+), 109 deletions(-) create mode 100644 src/backend/core/services/mime_types.py create mode 100644 src/frontend/apps/impress/src/features/docs/docs-grid/api/useImportDoc.tsx diff --git a/Makefile b/Makefile index 2655167912..142b99cc54 100644 --- a/Makefile +++ b/Makefile @@ -213,6 +213,7 @@ logs: ## display app-dev logs (follow mode) .PHONY: logs run-backend: ## Start only the backend application and all needed services + @$(COMPOSE) up --force-recreate -d docspec @$(COMPOSE) up --force-recreate -d celery-dev @$(COMPOSE) up --force-recreate -d y-provider-development @$(COMPOSE) up --force-recreate -d nginx diff --git a/compose.yml b/compose.yml index a774f11e07..55e2a8a87a 100644 --- a/compose.yml +++ b/compose.yml @@ -217,3 +217,8 @@ services: kc_postgresql: condition: service_healthy restart: true + + docspec: + image: ghcr.io/docspecio/api:2.0.0 + ports: + - "4000:4000" \ No newline at end of file diff --git a/docs/env.md b/docs/env.md index 0b3f9b3bf6..7292791828 100644 --- a/docs/env.md +++ b/docs/env.md @@ -103,6 +103,7 @@ These are the environment variables you can set for the `impress-backend` contai | USER_OIDC_ESSENTIAL_CLAIMS | Essential claims in OIDC token | [] | | Y_PROVIDER_API_BASE_URL | Y Provider url | | | Y_PROVIDER_API_KEY | Y provider API key | | +| DOCSPEC_API_URL | URL to endpoint of DocSpec conversion API | | ## impress-frontend image diff --git a/env.d/development/common b/env.d/development/common index de857d5b2a..39ce1a7fa3 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -67,5 +67,7 @@ DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token Y_PROVIDER_API_BASE_URL=http://y-provider-development:4444/api/ Y_PROVIDER_API_KEY=yprovider-api-key +DOCSPEC_API_URL=http://docspec:4000/conversion + # Theme customization THEME_CUSTOMIZATION_CACHE_TIMEOUT=15 \ No newline at end of file diff --git a/env.d/development/common.e2e b/env.d/development/common.e2e index 15434a6811..44f56f9b33 100644 --- a/env.d/development/common.e2e +++ b/env.d/development/common.e2e @@ -3,6 +3,7 @@ BURST_THROTTLE_RATES="200/minute" COLLABORATION_API_URL=http://y-provider:4444/collaboration/api/ SUSTAINED_THROTTLE_RATES="200/hour" Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/ +DOCSPEC_API_URL=http://docspec:4000/conversion # Throttle API_DOCUMENT_THROTTLE_RATE=1000/min diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 81b26d5e80..35e3c2dc0c 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -10,6 +10,7 @@ from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ +from core.services import mime_types import magic from rest_framework import serializers @@ -17,7 +18,7 @@ from core.services.ai_services import AI_ACTIONS from core.services.converter_services import ( ConversionError, - YdocConverter, + Converter, ) @@ -187,6 +188,7 @@ class DocumentSerializer(ListDocumentSerializer): content = serializers.CharField(required=False) websocket = serializers.BooleanField(required=False, write_only=True) + file = serializers.FileField(required=False, write_only=True, allow_null=True) class Meta: model = models.Document @@ -203,6 +205,7 @@ class Meta: "deleted_at", "depth", "excerpt", + "file", "is_favorite", "link_role", "link_reach", @@ -460,7 +463,11 @@ def create(self, validated_data): language = user.language or language try: - document_content = YdocConverter().convert(validated_data["content"]) + document_content = Converter().convert( + validated_data["content"], + mime_types.MARKDOWN, + mime_types.YJS + ) except ConversionError as err: raise serializers.ValidationError( {"content": ["Could not convert content"]} diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 84402ceaae..1759a5f4a8 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -39,14 +39,12 @@ from core.services.ai_services import AIService from core.services.collaboration_services import CollaborationService from core.services.converter_services import ( + ConversionError, ServiceUnavailableError as YProviderServiceUnavailableError, -) -from core.services.converter_services import ( ValidationError as YProviderValidationError, + Converter, ) -from core.services.converter_services import ( - YdocConverter, -) +from core.services import mime_types from core.tasks.mail import send_ask_for_access_mail from core.utils import extract_attachments, filter_descendants @@ -503,6 +501,27 @@ def perform_create(self, serializer): "IN SHARE ROW EXCLUSIVE MODE;" ) + # Remove file from validated_data as it's not a model field + # Process it if present + uploaded_file = serializer.validated_data.pop("file", None) + + # If a file is uploaded, convert it to Yjs format and set as content + if uploaded_file: + try: + file_content = uploaded_file.read() + + converter = Converter() + converted_content = converter.convert( + file_content, + content_type=uploaded_file.content_type, + accept=mime_types.YJS + ) + serializer.validated_data["content"] = converted_content + except ConversionError as err: + raise drf.exceptions.ValidationError( + {"file": ["Could not convert file content"]} + ) from err + obj = models.Document.add_root( creator=self.request.user, **serializer.validated_data, @@ -1602,14 +1621,14 @@ def content(self, request, pk=None): if base64_content is not None: # Convert using the y-provider service try: - yprovider = YdocConverter() + yprovider = Converter() result = yprovider.convert( base64.b64decode(base64_content), - "application/vnd.yjs.doc", + mime_types.YJS, { - "markdown": "text/markdown", - "html": "text/html", - "json": "application/json", + "markdown": mime_types.MARKDOWN, + "html": mime_types.HTML, + "json": mime_types.JSON, }[content_format], ) content = result diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py index 9c79a7192d..8790bf9ad6 100644 --- a/src/backend/core/services/converter_services.py +++ b/src/backend/core/services/converter_services.py @@ -5,7 +5,9 @@ from django.conf import settings import requests +import typing +from core.services import mime_types class ConversionError(Exception): """Base exception for conversion-related errors.""" @@ -19,8 +21,65 @@ class ServiceUnavailableError(ConversionError): """Raised when the conversion service is unavailable.""" +class ConverterProtocol(typing.Protocol): + def convert(self, text, content_type, accept): ... + + +class Converter: + docspec: ConverterProtocol + ydoc: ConverterProtocol + + def __init__(self): + self.docspec = DocSpecConverter() + self.ydoc = YdocConverter() + + def convert(self, input, content_type, accept): + """Convert input into other formats using external microservices.""" + + if content_type == mime_types.DOCX and accept == mime_types.YJS: + return self.convert( + self.docspec.convert(input, mime_types.DOCX, mime_types.BLOCKNOTE), + mime_types.BLOCKNOTE, + mime_types.YJS + ) + + return self.ydoc.convert(input, content_type, accept) + + +class DocSpecConverter: + """Service class for DocSpec conversion-related operations.""" + + def _request(self, url, data, content_type): + """Make a request to the DocSpec API.""" + + response = requests.post( + url, + headers={"Accept": mime_types.BLOCKNOTE}, + files={"file": ("document.docx", data, content_type)}, + timeout=settings.CONVERSION_API_TIMEOUT, + verify=settings.CONVERSION_API_SECURE, + ) + response.raise_for_status() + return response + + def convert(self, data, content_type, accept): + """Convert a Document to BlockNote.""" + if not data: + raise ValidationError("Input data cannot be empty") + + if content_type != mime_types.DOCX or accept != mime_types.BLOCKNOTE: + raise ValidationError(f"Conversion from {content_type} to {accept} is not supported.") + + try: + return self._request(settings.DOCSPEC_API_URL, data, content_type).content + except requests.RequestException as err: + raise ServiceUnavailableError( + "Failed to connect to DocSpec conversion service", + ) from err + + class YdocConverter: - """Service class for conversion-related operations.""" + """Service class for YDoc conversion-related operations.""" @property def auth_header(self): @@ -45,7 +104,7 @@ def _request(self, url, data, content_type, accept): return response def convert( - self, text, content_type="text/markdown", accept="application/vnd.yjs.doc" + self, text, content_type=mime_types.MARKDOWN, accept=mime_types.YJS ): """Convert a Markdown text into our internal format using an external microservice.""" @@ -59,14 +118,14 @@ def convert( content_type, accept, ) - if accept == "application/vnd.yjs.doc": + if accept == mime_types.YJS: return b64encode(response.content).decode("utf-8") - if accept in {"text/markdown", "text/html"}: + if accept in {mime_types.MARKDOWN, "text/html"}: return response.text - if accept == "application/json": + if accept == mime_types.JSON: return response.json() raise ValidationError("Unsupported format") except requests.RequestException as err: raise ServiceUnavailableError( - "Failed to connect to conversion service", + f"Failed to connect to YDoc conversion service {content_type}, {accept}", ) from err diff --git a/src/backend/core/services/mime_types.py b/src/backend/core/services/mime_types.py new file mode 100644 index 0000000000..84714e7f8f --- /dev/null +++ b/src/backend/core/services/mime_types.py @@ -0,0 +1,6 @@ +BLOCKNOTE = "application/vnd.blocknote+json" +YJS = "application/vnd.yjs.doc" +MARKDOWN = "text/markdown" +JSON = "application/json" +DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +HTML = "text/html" diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 2229036c8a..6d2e653472 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -680,6 +680,12 @@ class Base(Configuration): environ_prefix=None, ) + # DocSpec API microservice + DOCSPEC_API_URL = values.Value( + environ_name="DOCSPEC_API_URL", + environ_prefix=None + ) + # Conversion endpoint CONVERSION_API_ENDPOINT = values.Value( default="convert", diff --git a/src/frontend/apps/impress/src/features/docs/docs-grid/api/useImportDoc.tsx b/src/frontend/apps/impress/src/features/docs/docs-grid/api/useImportDoc.tsx new file mode 100644 index 0000000000..f76c7bebf9 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/docs-grid/api/useImportDoc.tsx @@ -0,0 +1,43 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query'; + +import { APIError, errorCauses, fetchAPI } from '@/api'; + +import { Doc, KEY_LIST_DOC } from '../../doc-management'; + +export const importDoc = async (file: File): Promise => { + const form = new FormData(); + form.append('file', file); + + const response = await fetchAPI(`documents/`, { + method: 'POST', + body: form, + withoutContentType: true, + }); + + if (!response.ok) { + throw new APIError('Failed to import the doc', await errorCauses(response)); + } + + return response.json() as Promise; +}; + +interface ImportDocProps { + onSuccess?: (data: Doc) => void; + onError?: (error: APIError) => void; +} + +export function useImportDoc({ onSuccess, onError }: ImportDocProps) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: importDoc, + onSuccess: (data) => { + void queryClient.resetQueries({ + queryKey: [KEY_LIST_DOC], + }); + onSuccess?.(data); + }, + onError: (error) => { + onError?.(error); + }, + }); +} diff --git a/src/frontend/apps/impress/src/features/docs/docs-grid/components/DocsGrid.tsx b/src/frontend/apps/impress/src/features/docs/docs-grid/components/DocsGrid.tsx index 51df82b47f..178fa6c8ed 100644 --- a/src/frontend/apps/impress/src/features/docs/docs-grid/components/DocsGrid.tsx +++ b/src/frontend/apps/impress/src/features/docs/docs-grid/components/DocsGrid.tsx @@ -1,5 +1,5 @@ import { Button } from '@openfun/cunningham-react'; -import { useMemo } from 'react'; +import { useMemo, useRef } from 'react'; import { useTranslation } from 'react-i18next'; import { InView } from 'react-intersection-observer'; import { css } from 'styled-components'; @@ -9,6 +9,7 @@ import { DocDefaultFilter, useInfiniteDocs } from '@/docs/doc-management'; import { useResponsiveStore } from '@/stores'; import { useInfiniteDocsTrashbin } from '../api'; +import { useImportDoc } from '../api/useImportDoc'; import { useResponsiveDocGrid } from '../hooks/useResponsiveDocGrid'; import { @@ -27,6 +28,7 @@ export const DocsGrid = ({ const { isDesktop } = useResponsiveStore(); const { flexLeft, flexRight } = useResponsiveDocGrid(); + const importInputRef = useRef(null); const { data, @@ -75,6 +77,33 @@ export const DocsGrid = ({ title = t('All docs'); } + const resetImportInput = () => { + if (!importInputRef.current) { + return; + } + + importInputRef.current.value = ''; + }; + + const { mutate: importDoc } = useImportDoc({ + onSuccess: resetImportInput, + onError: resetImportInput, + }); + + const handleImport = (event: React.ChangeEvent) => { + console.log(event); + + if (!event.target.files || event.target.files.length === 0) { + return; + } + + const file = event.target.files[0]; + + console.log(file); + + importDoc(file); + }; + return ( + + {!hasDocs && !loading && ( diff --git a/src/frontend/servers/y-provider/__tests__/convert.test.ts b/src/frontend/servers/y-provider/__tests__/convert.test.ts index 44c21c2870..a31f4b5893 100644 --- a/src/frontend/servers/y-provider/__tests__/convert.test.ts +++ b/src/frontend/servers/y-provider/__tests__/convert.test.ts @@ -69,7 +69,7 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', 'wrong-api-key') + .set('authorization', `Bearer wrong-api-key`) .set('content-type', 'application/json'); expect(response.status).toBe(401); @@ -99,7 +99,7 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'application/json'); expect(response.status).toBe(400); @@ -114,7 +114,7 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'application/json') .send(''); @@ -129,9 +129,10 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'image/png') .send('randomdata'); + expect(response.status).toBe(415); expect(response.body).toStrictEqual({ error: 'Unsupported Content-Type' }); }); @@ -141,38 +142,73 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'text/markdown') .set('accept', 'image/png') .send('# Header'); + expect(response.status).toBe(406); expect(response.body).toStrictEqual({ error: 'Unsupported format' }); }); - test.each([[apiKey], [`Bearer ${apiKey}`]])( - 'POST /api/convert with correct content with Authorization: %s', - async (authHeader) => { - const app = initApp(); + test('POST /api/convert BlockNote to Markdown', async () => { + const app = initApp(); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', `Bearer ${apiKey}`) + .set('content-type', 'application/vnd.blocknote+json') + .set('accept', 'text/markdown') + .send(expectedBlocks); - const response = await request(app) - .post('/api/convert') - .set('Origin', origin) - .set('Authorization', authHeader) - .set('content-type', 'text/markdown') - .set('accept', 'application/vnd.yjs.doc') - .send(expectedMarkdown); + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe( + 'text/markdown; charset=utf-8', + ); + expect(typeof response.text).toBe('string'); + expect(response.text.trim()).toBe(expectedMarkdown); + }); - expect(response.status).toBe(200); - expect(response.body).toBeInstanceOf(Buffer); + test('POST /api/convert BlockNote to Yjs', async () => { + const app = initApp(); + const editor = ServerBlockNoteEditor.create(); + const blocks = await editor.tryParseMarkdownToBlocks(expectedMarkdown); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', `Bearer ${apiKey}`) + .set('content-type', 'application/vnd.blocknote+json') + .set('accept', 'application/vnd.yjs.doc') + .send(blocks) + .responseType('blob'); - const editor = ServerBlockNoteEditor.create(); - const doc = new Y.Doc(); - Y.applyUpdate(doc, response.body); - const blocks = editor.yDocToBlocks(doc, 'document-store'); + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe('application/vnd.yjs.doc'); - expect(blocks).toStrictEqual(expectedBlocks); - }, - ); + // Decode the Yjs response and verify it contains the correct blocks + const responseBuffer = Buffer.from(response.body as Buffer); + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, responseBuffer); + const decodedBlocks = editor.yDocToBlocks(ydoc, 'document-store'); + + expect(decodedBlocks).toStrictEqual(expectedBlocks); + }); + + test('POST /api/convert BlockNote to HTML', async () => { + const app = initApp(); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', `Bearer ${apiKey}`) + .set('content-type', 'application/vnd.blocknote+json') + .set('accept', 'text/html') + .send(expectedBlocks); + + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe('text/html; charset=utf-8'); + expect(typeof response.text).toBe('string'); + expect(response.text).toBe(expectedHTML); + }); test('POST /api/convert Yjs to HTML', async () => { const app = initApp(); @@ -183,10 +219,11 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'application/vnd.yjs.doc') .set('accept', 'text/html') .send(Buffer.from(yjsUpdate)); + expect(response.status).toBe(200); expect(response.header['content-type']).toBe('text/html; charset=utf-8'); expect(typeof response.text).toBe('string'); @@ -202,10 +239,11 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'application/vnd.yjs.doc') .set('accept', 'text/markdown') .send(Buffer.from(yjsUpdate)); + expect(response.status).toBe(200); expect(response.header['content-type']).toBe( 'text/markdown; charset=utf-8', @@ -223,15 +261,16 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'application/vnd.yjs.doc') .set('accept', 'application/json') .send(Buffer.from(yjsUpdate)); + expect(response.status).toBe(200); expect(response.header['content-type']).toBe( 'application/json; charset=utf-8', ); - expect(Array.isArray(response.body)).toBe(true); + expect(response.body).toBeInstanceOf(Array); expect(response.body).toStrictEqual(expectedBlocks); }); @@ -240,15 +279,16 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'text/markdown') .set('accept', 'application/json') .send(expectedMarkdown); + expect(response.status).toBe(200); expect(response.header['content-type']).toBe( 'application/json; charset=utf-8', ); - expect(Array.isArray(response.body)).toBe(true); + expect(response.body).toBeInstanceOf(Array); expect(response.body).toStrictEqual(expectedBlocks); }); @@ -257,11 +297,12 @@ describe('Server Tests', () => { const response = await request(app) .post('/api/convert') .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'application/vnd.yjs.doc') .set('accept', 'application/json') .send(Buffer.from('notvalidyjs')); + expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ error: 'Invalid Yjs content' }); + expect(response.body).toStrictEqual({ error: 'Invalid content' }); }); }); diff --git a/src/frontend/servers/y-provider/src/handlers/convertHandler.ts b/src/frontend/servers/y-provider/src/handlers/convertHandler.ts index bdfbd2c8a9..0452724c54 100644 --- a/src/frontend/servers/y-provider/src/handlers/convertHandler.ts +++ b/src/frontend/servers/y-provider/src/handlers/convertHandler.ts @@ -14,27 +14,115 @@ interface ErrorResponse { error: string; } +type ConversionResponseBody = Uint8Array | string | object | ErrorResponse; + +interface InputReader { + supportedContentTypes: string[]; + read(data: Buffer): Promise; +} + +interface OutputWriter { + supportedContentTypes: string[]; + write(blocks: PartialBlock[]): Promise; +} + const editor = ServerBlockNoteEditor.create< DefaultBlockSchema, DefaultInlineContentSchema, DefaultStyleSchema >(); +const ContentTypes = { + XMarkdown: 'text/x-markdown', + Markdown: 'text/markdown', + YJS: 'application/vnd.yjs.doc', + FormUrlEncoded: 'application/x-www-form-urlencoded', + OctetStream: 'application/octet-stream', + HTML: 'text/html', + BlockNote: 'application/vnd.blocknote+json', + JSON: 'application/json', +} as const; + +const createYDocument = (blocks: PartialBlock[]) => + editor.blocksToYDoc(blocks, 'document-store'); + +const readers: InputReader[] = [ + { + // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility + supportedContentTypes: [ + ContentTypes.Markdown, + ContentTypes.XMarkdown, + ContentTypes.FormUrlEncoded, + ], + read: (data) => editor.tryParseMarkdownToBlocks(data.toString()), + }, + { + supportedContentTypes: [ContentTypes.YJS, ContentTypes.OctetStream], + read: async (data) => { + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, data); + return editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[]; + }, + }, + { + supportedContentTypes: [ContentTypes.BlockNote], + read: async (data) => JSON.parse(data.toString()), + }, +]; + +const writers: OutputWriter[] = [ + { + supportedContentTypes: [ContentTypes.BlockNote, ContentTypes.JSON], + write: async (blocks) => blocks, + }, + { + supportedContentTypes: [ContentTypes.YJS, ContentTypes.OctetStream], + write: async (blocks) => Y.encodeStateAsUpdate(createYDocument(blocks)), + }, + { + supportedContentTypes: [ContentTypes.Markdown, ContentTypes.XMarkdown], + write: (blocks) => editor.blocksToMarkdownLossy(blocks), + }, + { + supportedContentTypes: [ContentTypes.HTML], + write: (blocks) => editor.blocksToHTMLLossy(blocks), + }, +]; + +const normalizeContentType = (value: string) => value.split(';')[0]; + export const convertHandler = async ( req: Request, - res: Response, + res: Response, ) => { if (!req.body || req.body.length === 0) { res.status(400).json({ error: 'Invalid request: missing content' }); return; } - const contentType = (req.header('content-type') || 'text/markdown').split( - ';', - )[0]; - const accept = (req.header('accept') || 'application/vnd.yjs.doc').split( - ';', - )[0]; + const contentType = normalizeContentType( + req.header('content-type') || ContentTypes.Markdown, + ); + + const reader = readers.find((reader) => + reader.supportedContentTypes.includes(contentType), + ); + + if (!reader) { + res.status(415).json({ error: 'Unsupported Content-Type' }); + return; + } + + const accept = normalizeContentType(req.header('accept') || ContentTypes.YJS); + + const writer = writers.find((writer) => + writer.supportedContentTypes.includes(accept), + ); + + if (!writer) { + res.status(406).json({ error: 'Unsupported format' }); + return; + } let blocks: | PartialBlock< @@ -44,63 +132,23 @@ export const convertHandler = async ( >[] | null = null; try { - // First, convert from the input format to blocks - // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility - if ( - contentType === 'text/markdown' || - contentType === 'application/x-www-form-urlencoded' - ) { - blocks = await editor.tryParseMarkdownToBlocks(req.body.toString()); - } else if ( - contentType === 'application/vnd.yjs.doc' || - contentType === 'application/octet-stream' - ) { - try { - const ydoc = new Y.Doc(); - Y.applyUpdate(ydoc, req.body); - blocks = editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[]; - } catch (e) { - logger('Invalid Yjs content:', e); - res.status(400).json({ error: 'Invalid Yjs content' }); - return; - } - } else { - res.status(415).json({ error: 'Unsupported Content-Type' }); + try { + blocks = await reader.read(req.body); + } catch (e) { + logger('Invalid content:', e); + res.status(400).json({ error: 'Invalid content' }); return; } + if (!blocks || blocks.length === 0) { res.status(500).json({ error: 'No valid blocks were generated' }); return; } - // Then, convert from blocks to the output format - if (accept === 'application/json') { - res.status(200).json(blocks); - } else { - const yDocument = editor.blocksToYDoc(blocks, 'document-store'); - - if ( - accept === 'application/vnd.yjs.doc' || - accept === 'application/octet-stream' - ) { - res - .status(200) - .setHeader('content-type', 'application/octet-stream') - .send(Y.encodeStateAsUpdate(yDocument)); - } else if (accept === 'text/markdown') { - res - .status(200) - .setHeader('content-type', 'text/markdown') - .send(await editor.blocksToMarkdownLossy(blocks)); - } else if (accept === 'text/html') { - res - .status(200) - .setHeader('content-type', 'text/html') - .send(await editor.blocksToHTMLLossy(blocks)); - } else { - res.status(406).json({ error: 'Unsupported format' }); - } - } + res + .status(200) + .setHeader('content-type', accept) + .send(await writer.write(blocks)); } catch (e) { logger('conversion failed:', e); res.status(500).json({ error: 'An error occurred' });