✨ Import of documents

Import of (markdown) documents. See #1567 #1569.

Author: StephanMeijer

Makefile

@@ -213,6 +213,7 @@ logs: ## display app-dev logs (follow mode)
 .PHONY: logs
 
 run-backend: ## Start only the backend application and all needed services
+	@$(COMPOSE) up --force-recreate -d docspec
 	@$(COMPOSE) up --force-recreate -d celery-dev
 	@$(COMPOSE) up --force-recreate -d y-provider-development
 	@$(COMPOSE) up --force-recreate -d nginx

compose.yml

@@ -217,3 +217,8 @@ services:
       kc_postgresql:
         condition: service_healthy
         restart: true
+
+  docspec:
+    image: ghcr.io/docspecio/api:0.2.1
+    ports:
+      - "4000:4000"

src/backend/core/api/serializers.py

@@ -187,6 +187,7 @@ class DocumentSerializer(ListDocumentSerializer):
 
     content = serializers.CharField(required=False)
     websocket = serializers.BooleanField(required=False, write_only=True)
+    file = serializers.FileField(required=False, write_only=True, allow_null=True)
 
     class Meta:
         model = models.Document
@@ -203,6 +204,7 @@ class Meta:
             "deleted_at",
             "depth",
             "excerpt",
+            "file",
             "is_favorite",
             "link_role",
             "link_reach",

src/backend/core/api/viewsets.py

@@ -39,12 +39,9 @@
 from core.services.ai_services import AIService
 from core.services.collaboration_services import CollaborationService
 from core.services.converter_services import (
+    ConversionError,
     ServiceUnavailableError as YProviderServiceUnavailableError,
-)
-from core.services.converter_services import (
     ValidationError as YProviderValidationError,
-)
-from core.services.converter_services import (
     YdocConverter,
 )
 from core.tasks.mail import send_ask_for_access_mail
@@ -503,6 +500,30 @@ def perform_create(self, serializer):
                 "IN SHARE ROW EXCLUSIVE MODE;"
             )
 
+        # Remove file from validated_data as it's not a model field
+        # Process it if present
+        uploaded_file = serializer.validated_data.pop("file", None)
+
+        # If a file is uploaded, convert it to Yjs format and set as content
+        if uploaded_file:
+            try:
+                # Read file content
+                file_content = uploaded_file.read()
+                if isinstance(file_content, bytes):
+                    file_content = file_content.decode("utf-8")
+
+
+                # Convert to Yjs format using the file's content type
+                converter = YdocConverter()
+                converted_content = converter.convert(
+                    file_content, content_type=uploaded_file.content_type
+                )
+                serializer.validated_data["content"] = converted_content
+            except (ConversionError, UnicodeDecodeError) as err:
+                raise drf.exceptions.ValidationError(
+                    {"file": ["Could not convert file content"]}
+                ) from err
+
         obj = models.Document.add_root(
             creator=self.request.user,
             **serializer.validated_data,

src/frontend/apps/impress/src/features/docs/docs-grid/api/useImportDoc.tsx

@@ -0,0 +1,43 @@
+import { useMutation, useQueryClient } from '@tanstack/react-query';
+
+import { APIError, errorCauses, fetchAPI } from '@/api';
+
+import { Doc, KEY_LIST_DOC } from '../../doc-management';
+
+export const importDoc = async (file: File): Promise<Doc> => {
+  const form = new FormData();
+  form.append('file', file);
+
+  const response = await fetchAPI(`documents/`, {
+    method: 'POST',
+    body: form,
+    withoutContentType: true,
+  });
+
+  if (!response.ok) {
+    throw new APIError('Failed to import the doc', await errorCauses(response));
+  }
+
+  return response.json() as Promise<Doc>;
+};
+
+interface ImportDocProps {
+  onSuccess?: (data: Doc) => void;
+  onError?: (error: APIError) => void;
+}
+
+export function useImportDoc({ onSuccess, onError }: ImportDocProps) {
+  const queryClient = useQueryClient();
+  return useMutation<Doc, APIError, File>({
+    mutationFn: importDoc,
+    onSuccess: (data) => {
+      void queryClient.resetQueries({
+        queryKey: [KEY_LIST_DOC],
+      });
+      onSuccess?.(data);
+    },
+    onError: (error) => {
+      onError?.(error);
+    },
+  });
+}

src/frontend/apps/impress/src/features/docs/docs-grid/components/DocsGrid.tsx

@@ -1,5 +1,5 @@
 import { Button } from '@openfun/cunningham-react';
-import { useMemo } from 'react';
+import { useMemo, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
 import { InView } from 'react-intersection-observer';
 import { css } from 'styled-components';
@@ -9,6 +9,7 @@ import { DocDefaultFilter, useInfiniteDocs } from '@/docs/doc-management';
 import { useResponsiveStore } from '@/stores';
 
 import { useInfiniteDocsTrashbin } from '../api';
+import { useImportDoc } from '../api/useImportDoc';
 import { useResponsiveDocGrid } from '../hooks/useResponsiveDocGrid';
 
 import {
@@ -27,6 +28,7 @@ export const DocsGrid = ({
 
   const { isDesktop } = useResponsiveStore();
   const { flexLeft, flexRight } = useResponsiveDocGrid();
+  const importInputRef = useRef<HTMLInputElement>(null);
 
   const {
     data,
@@ -75,6 +77,33 @@ export const DocsGrid = ({
       title = t('All docs');
   }
 
+  const resetImportInput = () => {
+    if (!importInputRef.current) {
+      return;
+    }
+
+    importInputRef.current.value = '';
+  };
+
+  const { mutate: importDoc } = useImportDoc({
+    onSuccess: resetImportInput,
+    onError: resetImportInput,
+  });
+
+  const handleImport = (event: React.ChangeEvent<HTMLInputElement>) => {
+    console.log(event);
+
+    if (!event.target.files || event.target.files.length === 0) {
+      return;
+    }
+
+    const file = event.target.files[0];
+
+    console.log(file);
+
+    importDoc(file);
+  };
+
   return (
     <Box
       $position="relative"
@@ -107,6 +136,14 @@ export const DocsGrid = ({
           {title}
         </Text>
 
+        <input
+          type="file"
+          name="doc"
+          accept=".md"
+          onChange={handleImport}
+          ref={importInputRef}
+        ></input>
+
         {!hasDocs && !loading && (
           <Box $padding={{ vertical: 'sm' }} $align="center" $justify="center">
             <Text $size="sm" $variation="600" $weight="700">

src/frontend/servers/y-provider/src/handlers/convertHandler.ts

@@ -48,6 +48,7 @@ export const convertHandler = async (
     // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility
     if (
       contentType === 'text/markdown' ||
+      contentType === 'text/x-markdown' ||
       contentType === 'application/x-www-form-urlencoded'
     ) {
       blocks = await editor.tryParseMarkdownToBlocks(req.body.toString());