convert backend to typescript

.gitignore

@@ -166,3 +166,69 @@ cython_debug/
 #.idea/
 links.csv
 *.xcuserstate
+node_modules/
+.node_modules/
+built/*
+tests/cases/rwc/*
+tests/cases/perf/*
+!tests/cases/webharness/compilerToString.js
+test-args.txt
+~*.docx
+\#*\#
+.\#*
+tests/baselines/local/*
+tests/baselines/local.old/*
+tests/services/baselines/local/*
+tests/baselines/prototyping/local/*
+tests/baselines/rwc/*
+tests/baselines/reference/projectOutput/*
+tests/baselines/local/projectOutput/*
+tests/baselines/reference/testresults.tap
+tests/baselines/symlinks/*
+tests/services/baselines/prototyping/local/*
+tests/services/browser/typescriptServices.js
+src/harness/*.js
+src/compiler/diagnosticInformationMap.generated.ts
+src/compiler/diagnosticMessages.generated.json
+src/parser/diagnosticInformationMap.generated.ts
+src/parser/diagnosticMessages.generated.json
+rwc-report.html
+*.swp
+build.json
+*.actual
+tests/webTestServer.js
+tests/webTestServer.js.map
+tests/webhost/*.d.ts
+tests/webhost/webtsc.js
+tests/cases/**/*.js
+tests/cases/**/*.js.map
+*.config
+scripts/eslint/built/
+scripts/debug.bat
+scripts/run.bat
+scripts/**/*.js
+scripts/**/*.js.map
+coverage/
+internal/
+**/.DS_Store
+.settings
+**/.vs
+**/.vscode/*
+!**/.vscode/tasks.json
+!**/.vscode/settings.template.json
+!**/.vscode/launch.template.json
+!**/.vscode/extensions.json
+!tests/cases/projects/projectOption/**/node_modules
+!tests/cases/projects/NodeModulesSearch/**/*
+!tests/baselines/reference/project/nodeModules*/**/*
+.idea
+yarn.lock
+yarn-error.log
+.parallelperf.*
+tests/baselines/reference/dt
+.failed-tests
+TEST-results.xml
+package-lock.json
+.eslintcache
+*v8.log
+/lib/

server/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+# Keep environment variables out of version control
+.env

server/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "server",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/marked": "^6.0.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.4"
+  },
+  "dependencies": {
+    "@prisma/client": "^5.19.1",
+    "axios": "^1.7.7",
+    "dotenv": "^16.4.5",
+    "express": "^4.19.2",
+    "google-auth-library": "^9.14.1",
+    "googleapis": "^144.0.0",
+    "gpt-3-encoder": "^1.1.4",
+    "inversify": "^6.0.2",
+    "marked": "^14.1.2",
+    "openai": "^4.58.1",
+    "pdf-lib": "^1.17.1",
+    "pdf2pic": "^3.1.3",
+    "pgvector": "^0.2.0",
+    "prisma": "^5.19.1",
+    "tesseract.js": "^5.1.1"
+  },
+  "scripts": {
+    "start": "ts-node src/server.ts",
+    "generate": "yarn prisma generate",
+    "ingest": "ts-node src/scripts/ingest.ts"
+  }
+}

server/prisma/migrations/20240909154453_init/migration.sql

@@ -0,0 +1,63 @@
+-- CreateExtension
+CREATE EXTENSION IF NOT EXISTS "vector";
+
+-- CreateEnum
+CREATE TYPE "ContentType" AS ENUM ('EMAIL', 'FILE', 'LINK');
+
+-- CreateTable
+CREATE TABLE "Email" (
+    "id" UUID NOT NULL,
+    "sender" TEXT NOT NULL,
+    "subject" TEXT NOT NULL,
+    "body" TEXT NOT NULL,
+    "messageId" TEXT NOT NULL,
+
+    CONSTRAINT "Email_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "File" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "contentHash" VARCHAR(64) NOT NULL,
+
+    CONSTRAINT "File_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Link" (
+    "id" UUID NOT NULL,
+    "url" TEXT NOT NULL,
+    "title" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+
+    CONSTRAINT "Link_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Embedding" (
+    "id" UUID NOT NULL,
+    "contentType" "ContentType" NOT NULL,
+    "emailId" UUID,
+    "fileId" UUID,
+    "linkId" UUID,
+
+    CONSTRAINT "Embedding_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Email_messageId_key" ON "Email"("messageId");
+
+-- CreateIndex
+CREATE INDEX "File_contentHash_idx" ON "File"("contentHash");
+
+-- AddForeignKey
+ALTER TABLE "Embedding" ADD CONSTRAINT "Embedding_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Embedding" ADD CONSTRAINT "Embedding_emailId_fkey" FOREIGN KEY ("emailId") REFERENCES "Email"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Embedding" ADD CONSTRAINT "Embedding_linkId_fkey" FOREIGN KEY ("linkId") REFERENCES "Link"("id") ON DELETE CASCADE ON UPDATE CASCADE;

server/prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"

server/prisma/schema.prisma

@@ -0,0 +1,61 @@
+// This is your Prisma schema file,
+// learn more about it in the docs: https://pris.ly/d/prisma-schema
+
+// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions?
+// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init
+
+generator client {
+    provider        = "prisma-client-js"
+    previewFeatures = ["postgresqlExtensions"] // Enable the postgresqlExtensions. Currently in preview
+}
+
+datasource db {
+    provider   = "postgresql"
+    url        = env("DATABASE_URL")
+    extensions = [vector]
+}
+
+enum ContentType {
+    EMAIL
+    FILE
+    LINK
+}
+
+model Email {
+    id         String      @id @default(uuid()) @db.Uuid
+    sender     String
+    subject    String
+    body       String
+    messageId  String      @unique
+    embeddings Embedding[]
+}
+
+model File {
+    id          String      @id @default(uuid()) @db.Uuid
+    name        String
+    path        String
+    content     String
+    contentHash String      @db.VarChar(64)
+    embeddings  Embedding[]
+
+    @@index([contentHash])
+}
+
+model Link {
+    id         String      @id @default(uuid()) @db.Uuid
+    url        String
+    title      String
+    content    String
+    embeddings Embedding[]
+}
+
+model Embedding {
+    id          String      @id @default(uuid()) @db.Uuid
+    contentType ContentType
+    emailId     String?     @db.Uuid
+    fileId      String?     @db.Uuid
+    linkId      String?     @db.Uuid
+    file        File?       @relation(fields: [fileId], references: [id], onDelete: Cascade)
+    email       Email?      @relation(fields: [emailId], references: [id])
+    link        Link?       @relation(fields: [linkId], references: [id], onDelete: Cascade)
+}

server/src/dao/emailDao.ts

@@ -0,0 +1,22 @@
+import { PrismaClient, Prisma } from '@prisma/client';
+import { container } from '../utils/container';
+
+export async function createEmail(data: Prisma.EmailCreateInput) {
+  const prisma = container.get(PrismaClient);
+  return prisma.email.create({ data });
+}
+
+export async function getEmail(id: string) {
+  const prisma = container.get(PrismaClient);
+  return prisma.email.findUnique({ where: { id } });
+}
+
+export async function updateEmail(id: string, data: Prisma.EmailUpdateInput) {
+  const prisma = container.get(PrismaClient);
+  return prisma.email.update({ where: { id }, data });
+}
+
+export async function deleteEmail(id: string) {
+  const prisma = container.get(PrismaClient);
+  return prisma.email.delete({ where: { id } });
+}

server/src/dao/embeddingDao.ts

@@ -0,0 +1,59 @@
+import { PrismaClient, ContentType } from '@prisma/client';
+import { container } from '../utils/container';
+import pgvector from 'pgvector';
+
+// Prisma doesn't support pgvector types, so we need to use raw SQL and make our own create input types
+type EmbeddingCreateInput = {
+  embedding: number[];
+  contentType: ContentType;
+  emailId?: string;
+  fileId?: string;
+  linkId?: string;
+};
+
+export async function createEmbedding(data: EmbeddingCreateInput) {
+  const prisma = container.get(PrismaClient);
+  const embedding = pgvector.toSql(data.embedding);
+  return prisma.$executeRaw`
+    INSERT INTO "Embedding" (id, embedding, "contentType", "emailId", "fileId", "linkId")
+    VALUES (gen_random_uuid(), ${embedding}::vector, ${data.contentType}, ${data.emailId}, ${data.fileId}, ${data.linkId})
+  `;
+}
+
+export type SimilaritySearchResult = {
+  content_type: ContentType;
+  source: string;
+  title: string;
+  content: string;
+  distance: number;
+};
+
+export async function performSimilaritySearch(queryEmbedding: number[], limit: number): Promise<SimilaritySearchResult[]> {
+  const prisma = container.get(PrismaClient);
+  const embedding = pgvector.toSql(queryEmbedding);
+  const results = await prisma.$queryRaw<SimilaritySearchResult[]>`
+    SELECT 
+      e."contentType" as content_type,
+      COALESCE(em."messageId", f.path, l.url) AS source,
+      COALESCE(em.subject, f.name, l.title) AS title,
+      COALESCE(em.body, f.content, l.content) AS content,
+      e.embedding <-> ${embedding}::vector AS distance
+    FROM 
+      "Embedding" e
+      LEFT JOIN "Email" em ON e."emailId" = em.id
+      LEFT JOIN "File" f ON e."fileId" = f.id
+      LEFT JOIN "Link" l ON e."linkId" = l.id
+    ORDER BY distance
+    LIMIT ${limit}
+  `;
+
+  return results;
+}
+
+export async function clearAllTables() {
+  const prisma = container.get(PrismaClient);
+  await prisma.embedding.deleteMany();
+  await prisma.email.deleteMany();
+  await prisma.file.deleteMany();
+  await prisma.link.deleteMany();
+}

server/src/dao/fileDao.ts

@@ -0,0 +1,22 @@
+import { PrismaClient, Prisma } from '@prisma/client';
+import { container } from '../utils/container';
+
+export async function createFile(data: Prisma.FileCreateInput) {
+  const prisma = container.get(PrismaClient);
+  return prisma.file.create({ data });
+}
+
+export async function getFile(id: string) {
+  const prisma = container.get(PrismaClient);
+  return prisma.file.findUnique({ where: { id } });
+}
+
+export async function updateFile(id: string, data: Prisma.FileUpdateInput) {
+  const prisma = container.get(PrismaClient);
+  return prisma.file.update({ where: { id }, data });
+}
+
+export async function deleteFile(id: string) {
+  const prisma = container.get(PrismaClient);
+  return prisma.file.delete({ where: { id } });
+}

server/src/dao/linkDao.ts

@@ -0,0 +1,22 @@
+import { PrismaClient, Prisma } from '@prisma/client';
+import { container } from '../utils/container';
+
+export async function createLink(data: Prisma.LinkCreateInput) {
+  const prisma = container.get(PrismaClient);
+  return prisma.link.create({ data });
+}
+
+export async function getLink(id: string) {
+  const prisma = container.get(PrismaClient);
+  return prisma.link.findUnique({ where: { id } });
+}
+
+export async function updateLink(id: string, data: Prisma.LinkUpdateInput) {
+  const prisma = container.get(PrismaClient);
+  return prisma.link.update({ where: { id }, data });
+}
+
+export async function deleteLink(id: string) {
+  const prisma = container.get(PrismaClient);
+  return prisma.link.delete({ where: { id } });
+}