diff --git a/src/api/http.ts b/src/api/http.ts index 25379e12..85376e70 100644 --- a/src/api/http.ts +++ b/src/api/http.ts @@ -51,6 +51,7 @@ import { setEventEndpointsUpdater, setRelationshipEndpointsUpdater, UnauthorisedError, + updateEndpoints, } from '../runtime/defs.js'; import { evaluate } from '../runtime/interpreter.js'; import { Config } from '../runtime/state.js'; @@ -286,6 +287,10 @@ export async function createApp(appSpec: ApplicationSpec, config?: Config): Prom handleFileUpload(req, res, config); }); + app.post('/excelUpload', upload.single('file'), (req: Request, res: Response) => { + handleExcelUpload(req, res, uploadDir!, config); + }); + app.get('/downloadFile/:filename', (req: Request, res: Response) => { handleFileDownload(req, res, uploadDir!, config); }); @@ -298,6 +303,10 @@ export async function createApp(appSpec: ApplicationSpec, config?: Config): Prom res.status(501).send({ error: 'File upload is only supported in Node.js environment' }); }); + app.post('/excelUpload', (req: Request, res: Response) => { + res.status(501).send({ error: 'Excel upload is only supported in Node.js environment' }); + }); + app.get('/downloadFile/:filename', (req: Request, res: Response) => { res.status(501).send({ error: 'File download is only supported in Node.js environment' }); }); @@ -1324,6 +1333,73 @@ async function handleFileUpload( } } +async function handleExcelUpload( + req: Request & { file?: Express.Multer.File }, + res: Response, + uploadDir: string, + config?: Config +): Promise { + try { + if (!isNodeEnv) { + res.status(501).send({ error: 'Excel upload is only supported in Node.js environment' }); + return; + } + + if (!config?.service?.httpFileHandling) { + res + .status(403) + .send({ error: 'File handling is not enabled. Set httpFileHandling: true in config.' }); + return; + } + + const sessionInfo = await verifyAuth('', '', req.headers.authorization); + + if (isNoSession(sessionInfo)) { + res.status(401).send({ error: 'Authorization required' }); + return; + } + + if (!req.file) { + res.status(400).send({ error: 'No file uploaded' }); + return; + } + + const file = req.file; + const ext = path.extname(file.originalname).toLowerCase(); + if (!['.xlsx', '.xls'].includes(ext)) { + res.status(400).send({ error: 'Only Excel files (.xlsx, .xls) are supported' }); + return; + } + + const entityName = req.body?.entityName as string | undefined; + const sheetName = req.body?.sheetName as string | undefined; + + const filePath = path.join(uploadDir, file.filename); + const { createEntityFromExcelFile } = await import('../runtime/excel.js'); + const result = await createEntityFromExcelFile(filePath, { + entityName: entityName || undefined, + sheetName: sheetName || undefined, + }); + + updateEndpoints(result.moduleName); + + logger.info( + `Excel entity created: ${result.fqName} from ${file.originalname} (uploaded by ${sessionInfo.userId})` + ); + + res.contentType('application/json'); + res.send({ + success: true, + ...result, + originalFilename: file.originalname, + uploadedAt: new Date().toISOString(), + }); + } catch (err: any) { + logger.error(`Excel upload error: ${err}`); + res.status(500).send({ error: err.message || 'Excel upload failed' }); + } +} + async function handleFileDownload( req: Request, res: Response, diff --git a/src/runtime/excel-resolver.ts b/src/runtime/excel-resolver.ts new file mode 100644 index 00000000..a56f698c --- /dev/null +++ b/src/runtime/excel-resolver.ts @@ -0,0 +1,113 @@ +/** + * Excel file resolver - query and create rows in Excel sheets. + * Used by entities created from Excel config (agentlang.excel). + */ +import { makeInstance, newInstanceAttributes } from './module.js'; +import { Instance } from './module.js'; + +function isUrl(path: string): boolean { + return typeof path === 'string' && /^https?:\/\//i.test(path); +} + +async function loadWorkbook(excelPath: string, XLSX: any): Promise { + if (isUrl(excelPath)) { + const res = await fetch(excelPath); + if (!res.ok) throw new Error(`Failed to fetch: ${res.status} ${res.statusText}`); + const buf = await res.arrayBuffer(); + return XLSX.read(buf, { type: 'arraybuffer' }); + } + return XLSX.readFile(excelPath); +} + +export async function querySheet( + _resolver: any, + inst: Instance, + _queryAll?: boolean +): Promise { + const filePath = inst.record.getMeta('excel_path'); + const sheetName = inst.record.getMeta('sheet_name'); + const excelHeaders: string[] | undefined = inst.record.getMeta('excel_headers'); + + if (!filePath) { + throw new Error('Excel file path is required for querying sheets'); + } + + const XLSXModule = await import('xlsx'); + const XLSX = (XLSXModule as any).default || XLSXModule; + const workbook = await loadWorkbook(filePath, XLSX); + const sheetNames = workbook.SheetNames || []; + + const targetSheet = sheetName && sheetNames.includes(sheetName) ? sheetName : sheetNames[0]; + const ws = workbook.Sheets[targetSheet]; + const rows = (XLSX.utils as any).sheet_to_json(ws, { raw: false, defval: null }); + + const schema = inst.record.schema; + const attrNames = Array.from(schema.keys()).filter(k => !k.startsWith('__') && k !== 'id'); + const headers = excelHeaders || attrNames; + + const result: Instance[] = []; + for (const row of rows) { + const attrs = newInstanceAttributes(); + for (let i = 0; i < attrNames.length; i++) { + const attrName = attrNames[i]; + const headerKey = headers[i] ?? attrName; + const val = (row as any)[headerKey]; + attrs.set(attrName, val === '' || val === null || val === undefined ? null : val); + } + const id = (row as any).id ?? `${filePath}|${targetSheet}|${Date.now()}-${Math.random()}`; + attrs.set('id', id); + result.push(makeInstance(inst.moduleName, inst.name, attrs)); + } + return result; +} + +export async function createSheetRow(_resolver: any, inst: Instance): Promise { + const meta = inst.record?.meta; + if (!meta) { + throw new Error('Meta with excel_path and sheet_name is required'); + } + const filePath = meta.get('excel_path'); + if (!filePath) { + throw new Error('excel_path is required in meta'); + } + if (isUrl(filePath)) { + throw new Error('createSheetRow requires a local file path; URLs are read-only'); + } + + const XLSXModule = await import('xlsx'); + const XLSX = (XLSXModule as any).default || XLSXModule; + const workbook = (XLSX as any).readFile(filePath); + const sheetName = meta.get('sheet_name') || workbook.SheetNames[0]; + + const ws = workbook.Sheets[sheetName]; + if (!ws) { + throw new Error(`Sheet not found: ${sheetName}`); + } + + const headerRows = (XLSX.utils as any).sheet_to_json(ws, { header: 1 }); + const headers = (headerRows[0] || []) as string[]; + + const row: Record = {}; + inst.attributes.forEach((value, key) => { + if (!key.startsWith('__')) { + row[key] = value != null ? String(value) : ''; + } + }); + + (XLSX.utils as any).sheet_add_json(ws, [row], { + skipHeader: true, + header: headers.length ? headers : Object.keys(row), + origin: -1, + }); + + (XLSX as any).writeFile(workbook, filePath); + + const created = newInstanceAttributes(); + inst.attributes.forEach((v, k) => { + if (!k.startsWith('__')) created.set(k, v); + }); + const id = inst.attributes.get('id') ?? `${filePath}|${sheetName}|${Date.now()}`; + created.set('id', id); + + return makeInstance(inst.moduleName, inst.name, created); +} diff --git a/src/runtime/excel.ts b/src/runtime/excel.ts new file mode 100644 index 00000000..df0eccb8 --- /dev/null +++ b/src/runtime/excel.ts @@ -0,0 +1,162 @@ +/** + * Excel-based entity creation. + * Reads Excel files specified in config and creates entities with columns as String @optional fields. + * Each entity is connected to a file resolver that reads/writes the Excel file on demand (no data loading). + */ +import * as XLSX from 'xlsx'; +import * as nodePath from 'node:path'; +import { getFileSystem, readFileBuffer } from '../utils/fs-utils.js'; +import { addModule, getEntity, isModule } from './module.js'; +import { parseAndIntern } from './loader.js'; +import { makeFqName } from './util.js'; +import { logger } from './logger.js'; + +const ExcelModuleName = 'Excel'; + +export type ExcelConfigEntry = { + url: string; + sheet?: string; + entityName?: string; +}; + +function toEntityName(filePath: string, explicitName?: string): string { + let name: string; + if (explicitName) { + name = explicitName; + } else { + const base = nodePath.basename(filePath, nodePath.extname(filePath)); + // Strip multer-style suffix: basename-timestamp-random (e.g. Regions-1772039648356-726164924) + const multerSuffix = /-\d{10,}-\d+$/; + name = base.replace(multerSuffix, '') || base; + name = name.charAt(0).toUpperCase() + name.slice(1); + } + // Sanitize for valid identifier: only letters, digits, underscore (hyphens parse as minus) + const sanitized = name.replace(/[^a-zA-Z0-9_]/g, '_').replace(/^(\d)/, '_$1') || 'Entity'; + return sanitized.charAt(0).toUpperCase() + sanitized.slice(1); +} + +function toAttrName(header: string): string { + const s = header + .trim() + .replace(/\s+/g, '_') + .replace(/[^a-zA-Z0-9_]/g, '') + .replace(/^(\d)/, '_$1'); + return s || 'field'; +} + +export type CreateEntityFromExcelResult = { + entityName: string; + moduleName: string; + fqName: string; + headers: string[]; + sheetName: string; + filePath: string; +}; + +/** + * Create an entity and file resolver from an Excel file at the given path. + * Used by config (processExcelConfig) and by the /excelUpload HTTP endpoint. + */ +export async function createEntityFromExcelFile( + filePath: string, + options?: { entityName?: string; sheetName?: string } +): Promise { + const buffer = await readFileBuffer(filePath); + const workbook = (XLSX as any).read(buffer, { type: 'buffer' }); + + const sheetName = options?.sheetName ?? workbook.SheetNames[0]; + const sheet = workbook.Sheets[sheetName]; + + if (!sheet) { + throw new Error(`Sheet "${sheetName}" not found in ${filePath}`); + } + + const rows = (XLSX.utils as any).sheet_to_json(sheet, { header: 1 }); + if (rows.length === 0) { + throw new Error(`Sheet "${sheetName}" is empty in ${filePath}`); + } + + const headers = (rows[0] as any[]).map((h: any) => String(h ?? '')).filter(Boolean); + if (headers.length === 0) { + throw new Error(`No headers found in sheet "${sheetName}" of ${filePath}`); + } + + const entityName = toEntityName(filePath, options?.entityName); + const attrSpecs = headers + .map(h => toAttrName(String(h))) + .filter(name => name !== 'id') + .map(name => `${name} String @optional`); + + const entityDef = `entity ${entityName} { + id String @id, + ${attrSpecs.join(',\n ')} +}`; + + if (!isModule(ExcelModuleName)) { + addModule(ExcelModuleName); + } + + await parseAndIntern(entityDef, ExcelModuleName); + const entity = getEntity(entityName, ExcelModuleName); + if (entity) { + entity.addMeta('excel_path', nodePath.resolve(filePath)); + entity.addMeta('sheet_name', sheetName); + entity.addMeta('excel_headers', headers); + } + const fqName = makeFqName(ExcelModuleName, entityName); + + const [{ GenericResolver }, { querySheet, createSheetRow }, { registerResolver, setResolver }] = + await Promise.all([ + import('./resolvers/interface.js'), + import('./excel-resolver.js'), + import('./resolvers/registry.js'), + ]); + const resolverName = `excel/${entityName}`; + const resolver = new GenericResolver(resolverName, { + query: querySheet, + create: createSheetRow, + upsert: undefined, + update: undefined, + delete: undefined, + startTransaction: undefined, + commitTransaction: undefined, + rollbackTransaction: undefined, + }); + registerResolver(resolverName, () => resolver); + setResolver(fqName, resolverName); + + logger.info(`Created entity ${fqName} from ${filePath} (file resolver)`); + return { + entityName, + moduleName: ExcelModuleName, + fqName, + headers, + sheetName, + filePath: nodePath.resolve(filePath), + }; +} + +export async function processExcelConfig( + entries: ExcelConfigEntry[], + basePath: string +): Promise { + if (!entries || entries.length === 0) return; + + const fs = await getFileSystem(); + + for (const entry of entries) { + const resolvedPath = nodePath.isAbsolute(entry.url) + ? entry.url + : nodePath.resolve(basePath, entry.url); + + if (!(await fs.exists(resolvedPath))) { + logger.warn(`Excel file not found: ${resolvedPath}`); + continue; + } + + await createEntityFromExcelFile(resolvedPath, { + entityName: entry.entityName, + sheetName: entry.sheet, + }); + } +} diff --git a/src/runtime/loader.ts b/src/runtime/loader.ts index 1f7f4efb..7cfc7a1b 100644 --- a/src/runtime/loader.ts +++ b/src/runtime/loader.ts @@ -90,6 +90,7 @@ import { ScratchModuleName, splitRefs, } from './util.js'; +import { processExcelConfig } from './excel.js'; import { getFileSystem, toFsPath, readFile, readdir, exists } from '../utils/fs-utils.js'; import { URI } from 'vscode-uri'; import { AstNode, LangiumCoreServices, LangiumDocument } from 'langium'; @@ -407,17 +408,27 @@ function isStringContent(content: string): boolean { return content.includes('{'); } -export async function loadAppConfig(configDirOrContent: string): Promise { +export async function loadAppConfig( + configDirOrContent: string, + options?: { basePath?: string } +): Promise { const stringContent = isStringContent(configDirOrContent); + let basePath = options?.basePath; + let configDir: string | undefined; let cfgObj: any = undefined; if (!stringContent) { + const configPath = configDirOrContent; const fs = await getFileSystem(); - const alCfgFile = `${configDirOrContent}${path.sep}config.al`; + configDir = configPath.endsWith('.al') ? path.dirname(configPath) : configPath; + basePath = basePath ?? configDir; + const alCfgFile = `${configDir}${path.sep}config.al`; if (await fs.exists(alCfgFile)) { configDirOrContent = await fs.readFile(alCfgFile); } + } else { + basePath = basePath ?? (isNodeEnv ? process.cwd() : '/'); } if (canParse(configDirOrContent)) { @@ -426,8 +437,10 @@ export async function loadAppConfig(configDirOrContent: string): Promise try { let cfg = cfgObj - ? await configFromObject(cfgObj) - : await loadRawConfig(`${configDirOrContent}${path.sep}app.config.json`); + ? await configFromObject(cfgObj, true, basePath) + : await loadRawConfig( + configDir ? `${configDir}${path.sep}app.config.json` : 'app.config.json' + ); const envAppConfig = typeof process !== 'undefined' ? process.env.APP_CONFIG : undefined; if (envAppConfig) { @@ -1328,9 +1341,10 @@ export async function loadRawConfig( } } -function filterConfigEntityInstances(rawConfig: any): [any, Array] { +function filterConfigEntityInstances(rawConfig: any): [any, Array, any[]] { let cfg: any = undefined; const insts = new Array(); + let excelEntries: Array<{ url: string; sheet?: string; entityName?: string }> = []; const oldFormat = Object.keys(rawConfig).some((k: string) => { return k === 'store' || k === 'service'; }); @@ -1339,6 +1353,12 @@ function filterConfigEntityInstances(rawConfig: any): [any, Array] { Object.entries(rawConfig).forEach(([key, value]: [string, any]) => { if (key === 'agentlang') { cfg = value; + const excel = value?.excel; + if (Array.isArray(excel)) { + excelEntries = excel; + } + } else if (key === 'excel' && Array.isArray(value)) { + excelEntries = value; } else { if (value instanceof Array) { value.forEach((v: any) => { @@ -1350,16 +1370,24 @@ function filterConfigEntityInstances(rawConfig: any): [any, Array] { } }); if (cfg === undefined) cfg = {}; - return [cfg, insts]; + return [cfg, insts, excelEntries]; } else { - return [rawConfig, insts]; + return [rawConfig, insts, excelEntries]; } } -async function configFromObject(cfgObj: any, validate: boolean = true): Promise { +async function configFromObject( + cfgObj: any, + validate: boolean = true, + basePath?: string +): Promise { const rawConfig = preprocessRawConfig(cfgObj); if (validate && rawConfig) { - const [cfg, insts] = filterConfigEntityInstances(rawConfig); + const [cfg, insts, excelEntries] = filterConfigEntityInstances(rawConfig); + if (excelEntries.length > 0 && basePath) { + await processExcelConfig(excelEntries, basePath); + if (cfg?.excel) delete cfg.excel; + } const pats = new Array(); insts.forEach((v: any) => { const n = Object.keys(v)[0]; diff --git a/test/runtime/loader.test.ts b/test/runtime/loader.test.ts index d34410d8..5a959ead 100644 --- a/test/runtime/loader.test.ts +++ b/test/runtime/loader.test.ts @@ -20,9 +20,8 @@ describe('loadAppConfig', () => { afterEach(() => { if (tempDir && fs.existsSync(tempDir)) { - const configFile = path.join(tempDir, 'config.al'); - if (fs.existsSync(configFile)) { - fs.unlinkSync(configFile); + for (const name of fs.readdirSync(tempDir)) { + fs.unlinkSync(path.join(tempDir, name)); } fs.rmdirSync(tempDir); tempDir = null; @@ -139,4 +138,44 @@ describe('loadAppConfig', () => { const config = await loadAppConfig(tempDir); assert(config !== undefined, 'Should handle AgentLang pattern format'); }); + + test('should create entities from Excel files in config', async () => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agentlang-test-excel-')); + const XLSX = await import('xlsx'); + const wb = (XLSX as any).utils.book_new(); + const ws = (XLSX as any).utils.aoa_to_sheet([ + ['name', 'country', 'birthdate'], + ['Alice', 'USA', '1990-01-15'], + ['Bob', 'UK', '1985-06-20'], + ]); + (XLSX as any).utils.book_append_sheet(wb, ws, 'Sheet1'); + (XLSX as any).writeFile(wb, path.join(tempDir, 'Person.xlsx')); + + const configContent = JSON.stringify({ + agentlang: { + service: { port: 8080 }, + store: { type: 'sqlite', dbname: 'excel_test.db' }, + excel: [{ url: './Person.xlsx' }], + }, + }); + const configFile = path.join(tempDir, 'config.al'); + fs.writeFileSync(configFile, configContent); + + const config = await loadAppConfig(tempDir, { basePath: tempDir }); + assert(config !== undefined, 'Config should load with excel entries'); + + const { getEntity } = await import('../../src/runtime/module.js'); + const personEntity = getEntity('Person', 'Excel'); + assert(personEntity !== undefined, 'Person entity should be created from Person.xlsx'); + const schema = personEntity.schema; + assert(schema.has('name'), 'Person should have name field'); + assert(schema.has('country'), 'Person should have country field'); + assert(schema.has('birthdate'), 'Person should have birthdate field'); + assert(personEntity.getMeta('excel_path'), 'Person should have excel_path in meta'); + assert(personEntity.getMeta('sheet_name'), 'Person should have sheet_name in meta'); + + const { getResolverNameForPath } = await import('../../src/runtime/resolvers/registry.js'); + const resolverName = getResolverNameForPath('Excel/Person'); + assert(resolverName !== undefined, 'Person entity should have a file resolver registered'); + }); });