feat: htmx test app (#383)

* feat: app-htmx

* chore: respond with from-to in extract api

* add environment variable APP_HTMX_PORT

---------

Co-authored-by: Ivan Ivic <ivanivic842@gmail.com>

Author: dejan-crocoder

apps/app-htmx/build.mjs

@@ -0,0 +1,29 @@
+import * as esbuild from "esbuild";
+
+await esbuild.build({
+    keepNames: true,
+    bundle: true,
+    platform: "node",
+    format: "esm",
+    target: "esnext",
+    metafile: true,
+    outdir: "out",
+    entryPoints: ["src/index.ts"],
+    mainFields: ["module", "main"],
+    external: [
+        '@clerk/fastify',
+        '@fastify/formbody',
+        '@fastify/view',
+        'fastify',
+        'nunjucks',
+    ],
+    banner: {
+        js: [
+            `import { createRequire as topLevelCreateRequire } from 'module';`,
+            `const require = topLevelCreateRequire(import.meta.url);`,
+            `import { fileURLToPath as topLevelFileUrlToPath, URL as topLevelURL } from "url"`,
+            `const __dirname = topLevelFileUrlToPath(new topLevelURL(".", import.meta.url))`,
+        ].join("\n"),
+    },
+
+});

apps/app-htmx/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@acme/app-htmx",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "esbuild src/index.ts --bundle --platform=node | npm run with-env node",
+    "dev": "node build.mjs && npm run with-env node ./out/index.js",
+    "with-env": "dotenv -e ../../.env --"
+  },
+  "devDependencies": {
+    "@types/nunjucks": "3.2.6",
+    "dotenv-cli": "^7.3.0",
+    "esbuild": "^0.19.9"
+  },
+  "dependencies": {
+    "@acme/source-control": "*",
+    "@acme/super-schema": "*",
+    "@clerk/fastify": "0.6.30",
+    "@fastify/formbody": "7.4.0",
+    "@fastify/view": "8.2.0",
+    "date-fns": "3.3.1",
+    "fastify": "4.25.2",
+    "nunjucks": "3.2.4"
+  }
+}

apps/app-htmx/src/app-config.ts

@@ -0,0 +1,37 @@
+import { z } from "zod"
+
+const ENVSchema = z.object({
+  SUPER_DATABASE_URL: z.string(),
+  SUPER_DATABASE_AUTH_TOKEN: z.string().optional(),
+  TENANT_DATABASE_AUTH_TOKEN: z.string().optional(),
+
+  // TODO: remove after removing next
+  NEXT_PUBLIC_EXTRACT_API_URL: z.string(),
+  NEXT_PUBLIC_TRANSFORM_API_URL: z.string(),
+
+  CLERK_PUBLISHABLE_KEY: z.string(),
+  CLERK_SECRET_KEY: z.string(),
+  CLERK_DOMAIN: z.string(),
+  APP_HTMX_PORT: z.coerce.number().optional(),
+});
+
+const parsedEnv = ENVSchema.safeParse(process.env);
+if (!parsedEnv.success) throw new Error(`Invalid environment: ${parsedEnv.error}`);
+
+export const AppConfig = {
+  port: parsedEnv.data.APP_HTMX_PORT,
+  superDatabase: {
+    url: parsedEnv.data.SUPER_DATABASE_URL,
+    authToken: parsedEnv.data.SUPER_DATABASE_AUTH_TOKEN,
+  },
+  tenantDatabaseAuthToken: parsedEnv.data.TENANT_DATABASE_AUTH_TOKEN,
+  apis: {
+    extractStart: parsedEnv.data.NEXT_PUBLIC_EXTRACT_API_URL,
+    transformStart: parsedEnv.data.NEXT_PUBLIC_TRANSFORM_API_URL,
+  },
+  clerk: {
+    domain: parsedEnv.data.CLERK_DOMAIN,
+    publishableKey: parsedEnv.data.CLERK_PUBLISHABLE_KEY,
+    secretKey: "", // should be left in ENV
+  }
+}

apps/app-htmx/src/context/htmx.context.ts

@@ -0,0 +1,7 @@
+import type { IncomingHttpHeaders } from "http";
+
+export const htmxContext = (headers: IncomingHttpHeaders) => ({
+  htmx: {
+    boosted: 'hx-boosted' in headers,
+  }
+});

apps/app-htmx/src/context/page.context.ts

@@ -0,0 +1,5 @@
+export const pageContext = (title: string) => ({
+  page: {
+    title
+  },
+});

apps/app-htmx/src/context/tenant-list.context.ts

@@ -0,0 +1,17 @@
+import { getTenants } from "@acme/super-schema";
+import { createClient } from "@libsql/client";
+import { drizzle } from "drizzle-orm/libsql";
+import { AppConfig } from "src/app-config";
+
+const loadTenants = async ()=> {
+  const superDb = drizzle(createClient(AppConfig.superDatabase));
+  
+  const tenants = await getTenants(superDb);
+
+  return tenants;
+}
+const TENANT_LIST = await loadTenants();
+
+export const tenantListContext = () => ({
+  tenantList: TENANT_LIST
+})

apps/app-htmx/src/functions/fetch-repository.ts

@@ -0,0 +1,42 @@
+import { GitHubSourceControl, GitlabSourceControl, type SourceControl } from "@acme/source-control";
+import { clerkClient } from "@clerk/fastify";
+
+const getUserForgeAccessToken = async (userId: string, forge: "github" | "gitlab") => {
+  const userTokens = await clerkClient.users.getUserOauthAccessToken(userId, `oauth_${forge}`);
+  if (userTokens[0] === undefined) throw new Error("no token");
+  return userTokens[0].token;
+}
+
+type Props = {
+  forge: "github" | "gitlab";
+  userId: string;
+  repositoryId: number;
+  repositoryName: string;
+  namespaceName: string;
+}
+export const tryFetchRepository = async ({
+  userId,
+  forge,
+  namespaceName,
+  repositoryId,
+  repositoryName,
+}: Props) => {
+  const token = await getUserForgeAccessToken(userId, forge);
+
+  let sc: SourceControl;
+  if (forge === 'github') sc = new GitHubSourceControl(token);
+  else sc = new GitlabSourceControl(token);
+
+  try {
+    const { repository, namespace } = await sc.fetchRepository(repositoryId, namespaceName, repositoryName);
+    return { repository, namespace };
+  } catch (error) {
+    console.log("FAILED TO FETCH REPOSITORY");
+    console.log(error);
+  }
+
+  return {
+    repository: undefined,
+    namespace: undefined
+  }
+}

apps/app-htmx/src/functions/get-crawl-stats.ts

@@ -0,0 +1,53 @@
+import type { Tenant } from "@acme/super-schema";
+import { type Client, createClient } from "@libsql/client";
+import { fromUnixTime } from "date-fns/fromUnixTime";
+import { formatDistanceToNow } from "date-fns/formatDistanceToNow";
+import { AppConfig } from "src/app-config";
+import { z } from "zod";
+
+const formatDuration = (s: number) => {
+  const seconds = s % 60;
+  if (s < 60) return `${seconds}s`
+  const m = (s - seconds) / 60;
+  const minutes = m % 60;
+  if (m < 60) return `${minutes}m ${seconds.toString().padStart(2,'0')}s`
+  const h = (m - minutes) / 60;
+  return `${h}h ${minutes.toString().padStart(2, '0')}m ${seconds.toString().padStart(2, '0')}s`
+}
+
+const rowSchema = z.object({
+  crawl_instance: z.number(),
+  event_count: z.number(),
+  crawl_duration_sec: z.number(),
+  crawl_started_at: z.number()
+});
+
+const readCrawlStats =async (client:Client) => {
+  const x = await client.execute(`
+  SELECT
+    ce.instance_id as crawl_instance,
+    COUNT(ce.instance_id) as event_count,
+    MAX(ce.timestamp) - ci.started_at AS crawl_duration_sec,
+    ci.started_at as crawl_started_at
+FROM crawl_events ce
+JOIN crawl_instances ci ON ce.instance_id = ci.id
+GROUP BY ce.instance_id;
+  `);
+  const rawStats = x.rows.map(row => rowSchema.parse(row));
+
+  return rawStats.map((rawStat)=>({
+    instanceId: rawStat.crawl_instance,
+    eventCount: rawStat.event_count,
+    duration: formatDuration(rawStat.crawl_duration_sec),
+    startedAt: formatDistanceToNow(fromUnixTime(rawStat.crawl_started_at))
+  }))
+}
+
+export const getCrawlStats = async (tenant: Tenant) => {
+  const client = createClient({
+    url: tenant.dbUrl,
+    authToken: AppConfig.tenantDatabaseAuthToken
+  });
+
+  return await readCrawlStats(client);
+}

apps/app-htmx/src/functions/get-repositories.ts

@@ -0,0 +1,14 @@
+import { namespaces, repositories } from "@acme/extract-schema";
+import { eq } from "drizzle-orm";
+import type { LibSQLDatabase } from "drizzle-orm/libsql";
+
+export const getRepositories = async (db: LibSQLDatabase) => {
+  const repos = await db.select({
+    forge: repositories.forgeType,
+    name: repositories.name,
+    org: namespaces.name,
+    projectId: repositories.externalId,
+  }).from(repositories).innerJoin(namespaces, eq(repositories.namespaceId, namespaces.id)).all()
+  
+  return repos;
+}

apps/app-htmx/src/index.ts

@@ -0,0 +1,40 @@
+import Fastify from "fastify";
+import nunjucks from "nunjucks";
+import { fastifyView } from "@fastify/view";
+import path from "path";
+import { fileURLToPath } from "url";
+import { clerkPlugin } from "@clerk/fastify";
+import { Home } from "./pages/page.home.js";
+import fastifyFormbody from "@fastify/formbody";
+import { SignIn } from "./pages/page.sign-in.js";
+import { ExtractRepository } from "./pages/repository/extract.js";
+import { RegisterRepository } from "./pages/repository/register.js";
+import { AppConfig } from "./app-config.js";
+import { StartTransform } from "./pages/start-transform.js";
+
+AppConfig; // ensure loaded before starting server
+
+const TEMPLATE_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "../", "src", "views");
+
+const fastify = Fastify({ logger: true });
+
+
+await fastify.register(fastifyView, {
+  engine: {
+    nunjucks: nunjucks,
+  },
+  templates: TEMPLATE_DIR
+});
+await fastify.register(clerkPlugin);
+await fastify.register(fastifyFormbody);
+
+fastify.get('/', Home);
+fastify.get('/sign-in',SignIn);
+fastify.post('/repository/extract', ExtractRepository); 
+fastify.post('/repository/register', RegisterRepository);
+fastify.post('/transform', StartTransform);
+
+const PORT = AppConfig.port || 3001;
+
+await fastify.listen({ port: PORT, host: "127.0.0.1" })
+console.log(`Server started on http://127.0.0.1:${PORT}/`);