dynamic entity creation from xlsx

src/api/http.ts

@@ -51,6 +51,7 @@ import {
   setEventEndpointsUpdater,
   setRelationshipEndpointsUpdater,
   UnauthorisedError,
+  updateEndpoints,
 } from '../runtime/defs.js';
 import { evaluate } from '../runtime/interpreter.js';
 import { Config } from '../runtime/state.js';
@@ -286,6 +287,10 @@ export async function createApp(appSpec: ApplicationSpec, config?: Config): Prom
       handleFileUpload(req, res, config);
     });
 
+    app.post('/excelUpload', upload.single('file'), (req: Request, res: Response) => {
+      handleExcelUpload(req, res, uploadDir!, config);
+    });
+
     app.get('/downloadFile/:filename', (req: Request, res: Response) => {
       handleFileDownload(req, res, uploadDir!, config);
     });
@@ -298,6 +303,10 @@ export async function createApp(appSpec: ApplicationSpec, config?: Config): Prom
       res.status(501).send({ error: 'File upload is only supported in Node.js environment' });
     });
 
+    app.post('/excelUpload', (req: Request, res: Response) => {
+      res.status(501).send({ error: 'Excel upload is only supported in Node.js environment' });
+    });
+
     app.get('/downloadFile/:filename', (req: Request, res: Response) => {
       res.status(501).send({ error: 'File download is only supported in Node.js environment' });
     });
@@ -1324,6 +1333,73 @@ async function handleFileUpload(
   }
 }
 
+async function handleExcelUpload(
+  req: Request & { file?: Express.Multer.File },
+  res: Response,
+  uploadDir: string,
+  config?: Config
+): Promise<void> {
+  try {
+    if (!isNodeEnv) {
+      res.status(501).send({ error: 'Excel upload is only supported in Node.js environment' });
+      return;
+    }
+
+    if (!config?.service?.httpFileHandling) {
+      res
+        .status(403)
+        .send({ error: 'File handling is not enabled. Set httpFileHandling: true in config.' });
+      return;
+    }
+
+    const sessionInfo = await verifyAuth('', '', req.headers.authorization);
+
+    if (isNoSession(sessionInfo)) {
+      res.status(401).send({ error: 'Authorization required' });
+      return;
+    }
+
+    if (!req.file) {
+      res.status(400).send({ error: 'No file uploaded' });
+      return;
+    }
+
+    const file = req.file;
+    const ext = path.extname(file.originalname).toLowerCase();
+    if (!['.xlsx', '.xls'].includes(ext)) {
+      res.status(400).send({ error: 'Only Excel files (.xlsx, .xls) are supported' });
+      return;
+    }
+
+    const entityName = req.body?.entityName as string | undefined;
+    const sheetName = req.body?.sheetName as string | undefined;
+
+    const filePath = path.join(uploadDir, file.filename);
+    const { createEntityFromExcelFile } = await import('../runtime/excel.js');
+    const result = await createEntityFromExcelFile(filePath, {
+      entityName: entityName || undefined,
+      sheetName: sheetName || undefined,
+    });
+
+    updateEndpoints(result.moduleName);
+
+    logger.info(
+      `Excel entity created: ${result.fqName} from ${file.originalname} (uploaded by ${sessionInfo.userId})`
+    );
+
+    res.contentType('application/json');
+    res.send({
+      success: true,
+      ...result,
+      originalFilename: file.originalname,
+      uploadedAt: new Date().toISOString(),
+    });
+  } catch (err: any) {
+    logger.error(`Excel upload error: ${err}`);
+    res.status(500).send({ error: err.message || 'Excel upload failed' });
+  }
+}
+
 async function handleFileDownload(
   req: Request,
   res: Response,

src/runtime/excel-resolver.ts

@@ -0,0 +1,113 @@
+/**
+ * Excel file resolver - query and create rows in Excel sheets.
+ * Used by entities created from Excel config (agentlang.excel).
+ */
+import { makeInstance, newInstanceAttributes } from './module.js';
+import { Instance } from './module.js';
+
+function isUrl(path: string): boolean {
+  return typeof path === 'string' && /^https?:\/\//i.test(path);
+}
+
+async function loadWorkbook(excelPath: string, XLSX: any): Promise<any> {
+  if (isUrl(excelPath)) {
+    const res = await fetch(excelPath);
+    if (!res.ok) throw new Error(`Failed to fetch: ${res.status} ${res.statusText}`);
+    const buf = await res.arrayBuffer();
+    return XLSX.read(buf, { type: 'arraybuffer' });
+  }
+  return XLSX.readFile(excelPath);
+}
+
+export async function querySheet(
+  _resolver: any,
+  inst: Instance,
+  _queryAll?: boolean
+): Promise<Instance[]> {
+  const filePath = inst.record.getMeta('excel_path');
+  const sheetName = inst.record.getMeta('sheet_name');
+  const excelHeaders: string[] | undefined = inst.record.getMeta('excel_headers');
+
+  if (!filePath) {
+    throw new Error('Excel file path is required for querying sheets');
+  }
+
+  const XLSXModule = await import('xlsx');
+  const XLSX = (XLSXModule as any).default || XLSXModule;
+  const workbook = await loadWorkbook(filePath, XLSX);
+  const sheetNames = workbook.SheetNames || [];
+
+  const targetSheet = sheetName && sheetNames.includes(sheetName) ? sheetName : sheetNames[0];
+  const ws = workbook.Sheets[targetSheet];
+  const rows = (XLSX.utils as any).sheet_to_json(ws, { raw: false, defval: null });
+
+  const schema = inst.record.schema;
+  const attrNames = Array.from(schema.keys()).filter(k => !k.startsWith('__') && k !== 'id');
+  const headers = excelHeaders || attrNames;
+
+  const result: Instance[] = [];
+  for (const row of rows) {
+    const attrs = newInstanceAttributes();
+    for (let i = 0; i < attrNames.length; i++) {
+      const attrName = attrNames[i];
+      const headerKey = headers[i] ?? attrName;
+      const val = (row as any)[headerKey];
+      attrs.set(attrName, val === '' || val === null || val === undefined ? null : val);
+    }
+    const id = (row as any).id ?? `${filePath}|${targetSheet}|${Date.now()}-${Math.random()}`;
+    attrs.set('id', id);
+    result.push(makeInstance(inst.moduleName, inst.name, attrs));
+  }
+  return result;
+}
+
+export async function createSheetRow(_resolver: any, inst: Instance): Promise<Instance> {
+  const meta = inst.record?.meta;
+  if (!meta) {
+    throw new Error('Meta with excel_path and sheet_name is required');
+  }
+  const filePath = meta.get('excel_path');
+  if (!filePath) {
+    throw new Error('excel_path is required in meta');
+  }
+  if (isUrl(filePath)) {
+    throw new Error('createSheetRow requires a local file path; URLs are read-only');
+  }
+
+  const XLSXModule = await import('xlsx');
+  const XLSX = (XLSXModule as any).default || XLSXModule;
+  const workbook = (XLSX as any).readFile(filePath);
+  const sheetName = meta.get('sheet_name') || workbook.SheetNames[0];
+
+  const ws = workbook.Sheets[sheetName];
+  if (!ws) {
+    throw new Error(`Sheet not found: ${sheetName}`);
+  }
+
+  const headerRows = (XLSX.utils as any).sheet_to_json(ws, { header: 1 });
+  const headers = (headerRows[0] || []) as string[];
+
+  const row: Record<string, string> = {};
+  inst.attributes.forEach((value, key) => {
+    if (!key.startsWith('__')) {
+      row[key] = value != null ? String(value) : '';
+    }
+  });
+
+  (XLSX.utils as any).sheet_add_json(ws, [row], {
+    skipHeader: true,
+    header: headers.length ? headers : Object.keys(row),
+    origin: -1,
+  });
+
+  (XLSX as any).writeFile(workbook, filePath);
+
+  const created = newInstanceAttributes();
+  inst.attributes.forEach((v, k) => {
+    if (!k.startsWith('__')) created.set(k, v);
+  });
+  const id = inst.attributes.get('id') ?? `${filePath}|${sheetName}|${Date.now()}`;
+  created.set('id', id);
+
+  return makeInstance(inst.moduleName, inst.name, created);
+}

src/runtime/excel.ts

@@ -0,0 +1,162 @@
+/**
+ * Excel-based entity creation.
+ * Reads Excel files specified in config and creates entities with columns as String @optional fields.
+ * Each entity is connected to a file resolver that reads/writes the Excel file on demand (no data loading).
+ */
+import * as XLSX from 'xlsx';
+import * as nodePath from 'node:path';
+import { getFileSystem, readFileBuffer } from '../utils/fs-utils.js';
+import { addModule, getEntity, isModule } from './module.js';
+import { parseAndIntern } from './loader.js';
+import { makeFqName } from './util.js';
+import { logger } from './logger.js';
+
+const ExcelModuleName = 'Excel';
+
+export type ExcelConfigEntry = {
+  url: string;
+  sheet?: string;
+  entityName?: string;
+};
+
+function toEntityName(filePath: string, explicitName?: string): string {
+  let name: string;
+  if (explicitName) {
+    name = explicitName;
+  } else {
+    const base = nodePath.basename(filePath, nodePath.extname(filePath));
+    // Strip multer-style suffix: basename-timestamp-random (e.g. Regions-1772039648356-726164924)
+    const multerSuffix = /-\d{10,}-\d+$/;
+    name = base.replace(multerSuffix, '') || base;
+    name = name.charAt(0).toUpperCase() + name.slice(1);
+  }
+  // Sanitize for valid identifier: only letters, digits, underscore (hyphens parse as minus)
+  const sanitized = name.replace(/[^a-zA-Z0-9_]/g, '_').replace(/^(\d)/, '_$1') || 'Entity';
+  return sanitized.charAt(0).toUpperCase() + sanitized.slice(1);
+}
+
+function toAttrName(header: string): string {
+  const s = header
+    .trim()
+    .replace(/\s+/g, '_')
+    .replace(/[^a-zA-Z0-9_]/g, '')
+    .replace(/^(\d)/, '_$1');
+  return s || 'field';
+}
+
+export type CreateEntityFromExcelResult = {
+  entityName: string;
+  moduleName: string;
+  fqName: string;
+  headers: string[];
+  sheetName: string;
+  filePath: string;
+};
+
+/**
+ * Create an entity and file resolver from an Excel file at the given path.
+ * Used by config (processExcelConfig) and by the /excelUpload HTTP endpoint.
+ */
+export async function createEntityFromExcelFile(
+  filePath: string,
+  options?: { entityName?: string; sheetName?: string }
+): Promise<CreateEntityFromExcelResult> {
+  const buffer = await readFileBuffer(filePath);
+  const workbook = (XLSX as any).read(buffer, { type: 'buffer' });
+
+  const sheetName = options?.sheetName ?? workbook.SheetNames[0];
+  const sheet = workbook.Sheets[sheetName];
+
+  if (!sheet) {
+    throw new Error(`Sheet "${sheetName}" not found in ${filePath}`);
+  }
+
+  const rows = (XLSX.utils as any).sheet_to_json(sheet, { header: 1 });
+  if (rows.length === 0) {
+    throw new Error(`Sheet "${sheetName}" is empty in ${filePath}`);
+  }
+
+  const headers = (rows[0] as any[]).map((h: any) => String(h ?? '')).filter(Boolean);
+  if (headers.length === 0) {
+    throw new Error(`No headers found in sheet "${sheetName}" of ${filePath}`);
+  }
+
+  const entityName = toEntityName(filePath, options?.entityName);
+  const attrSpecs = headers
+    .map(h => toAttrName(String(h)))
+    .filter(name => name !== 'id')
+    .map(name => `${name} String @optional`);
+
+  const entityDef = `entity ${entityName} {
+  id String @id,
+  ${attrSpecs.join(',\n  ')}
+}`;
+
+  if (!isModule(ExcelModuleName)) {
+    addModule(ExcelModuleName);
+  }
+
+  await parseAndIntern(entityDef, ExcelModuleName);
+  const entity = getEntity(entityName, ExcelModuleName);
+  if (entity) {
+    entity.addMeta('excel_path', nodePath.resolve(filePath));
+    entity.addMeta('sheet_name', sheetName);
+    entity.addMeta('excel_headers', headers);
+  }
+  const fqName = makeFqName(ExcelModuleName, entityName);
+
+  const [{ GenericResolver }, { querySheet, createSheetRow }, { registerResolver, setResolver }] =
+    await Promise.all([
+      import('./resolvers/interface.js'),
+      import('./excel-resolver.js'),
+      import('./resolvers/registry.js'),
+    ]);
+  const resolverName = `excel/${entityName}`;
+  const resolver = new GenericResolver(resolverName, {
+    query: querySheet,
+    create: createSheetRow,
+    upsert: undefined,
+    update: undefined,
+    delete: undefined,
+    startTransaction: undefined,
+    commitTransaction: undefined,
+    rollbackTransaction: undefined,
+  });
+  registerResolver(resolverName, () => resolver);
+  setResolver(fqName, resolverName);
+
+  logger.info(`Created entity ${fqName} from ${filePath} (file resolver)`);
+  return {
+    entityName,
+    moduleName: ExcelModuleName,
+    fqName,
+    headers,
+    sheetName,
+    filePath: nodePath.resolve(filePath),
+  };
+}
+
+export async function processExcelConfig(
+  entries: ExcelConfigEntry[],
+  basePath: string
+): Promise<void> {
+  if (!entries || entries.length === 0) return;
+
+  const fs = await getFileSystem();
+
+  for (const entry of entries) {
+    const resolvedPath = nodePath.isAbsolute(entry.url)
+      ? entry.url
+      : nodePath.resolve(basePath, entry.url);
+
+    if (!(await fs.exists(resolvedPath))) {
+      logger.warn(`Excel file not found: ${resolvedPath}`);
+      continue;
+    }
+
+    await createEntityFromExcelFile(resolvedPath, {
+      entityName: entry.entityName,
+      sheetName: entry.sheet,
+    });
+  }
+}