From bc39ab141d506b519a65fa42f918ccf3b07ef559 Mon Sep 17 00:00:00 2001 From: Lewechi Date: Thu, 16 Apr 2026 11:04:22 +0100 Subject: [PATCH 1/6] Feat: Add fail-fast environment variable validation with Zod and unit tests --- node_modules/@watchtower/shared/src/index.ts | 1 + packages/shared/src/__tests__/config.test.ts | 50 +++++++++++++++++++ packages/shared/src/config.ts | 51 ++++++++++++++++++++ packages/shared/src/index.ts | 1 + 4 files changed, 103 insertions(+) create mode 100644 packages/shared/src/__tests__/config.test.ts create mode 100644 packages/shared/src/config.ts diff --git a/node_modules/@watchtower/shared/src/index.ts b/node_modules/@watchtower/shared/src/index.ts index d309b92..9b90898 100644 --- a/node_modules/@watchtower/shared/src/index.ts +++ b/node_modules/@watchtower/shared/src/index.ts @@ -1,3 +1,4 @@ export * from './utils/xdr.js'; export * from './constants/errors.js'; export * from './schemas/index.js'; +export * from './config.js'; diff --git a/packages/shared/src/__tests__/config.test.ts b/packages/shared/src/__tests__/config.test.ts new file mode 100644 index 0000000..d48477b --- /dev/null +++ b/packages/shared/src/__tests__/config.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi } from 'vitest'; +import { validateConfig } from '../config.js'; + +describe('Environment Config Loader', () => { + const validEnv = { + DATABASE_URL: 'postgresql://postgres:password@localhost:5432/watchtower', + REDIS_URL: 'redis://localhost:6379', + SOROBAN_RPC_MAINNET: 'https://mainnet.stellar.org', + SOROBAN_RPC_TESTNET: 'https://testnet.stellar.org', + JWT_SECRET: 'a'.repeat(32), + }; + + it('should validate and return config for valid environment', () => { + const config = validateConfig(validEnv); + expect(config.DATABASE_URL).toBe(validEnv.DATABASE_URL); + expect(config.JWT_SECRET).toBe(validEnv.JWT_SECRET); + expect(config.MERCURY_API_KEY).toBeNull(); // Optional defaults to null + }); + + it('should throw error for missing required variables', () => { + const invalidEnv = { ...validEnv }; + delete (invalidEnv as any).DATABASE_URL; + + expect(() => validateConfig(invalidEnv)).toThrow(/Invalid environment configuration/); + expect(() => validateConfig(invalidEnv)).toThrow(/DATABASE_URL: Required/); + }); + + it('should throw error for invalid URL', () => { + const invalidEnv = { ...validEnv, REDIS_URL: 'not-a-url' }; + + expect(() => validateConfig(invalidEnv)).toThrow(/REDIS_URL: REDIS_URL must be a valid URL/); + }); + + it('should throw error for short JWT_SECRET', () => { + const invalidEnv = { ...validEnv, JWT_SECRET: 'short' }; + + expect(() => validateConfig(invalidEnv)).toThrow(/JWT_SECRET: JWT_SECRET must be at least 32 characters long/); + }); + + it('should pass if optional Mercury variables are present', () => { + const mercuryEnv = { + ...validEnv, + MERCURY_API_KEY: 'test-key', + MERCURY_API_URL: 'https://mercury.api', + }; + const config = validateConfig(mercuryEnv); + expect(config.MERCURY_API_KEY).toBe('test-key'); + expect(config.MERCURY_API_URL).toBe('https://mercury.api'); + }); +}); diff --git a/packages/shared/src/config.ts b/packages/shared/src/config.ts new file mode 100644 index 0000000..2208ac4 --- /dev/null +++ b/packages/shared/src/config.ts @@ -0,0 +1,51 @@ +import { z } from 'zod'; + +/** + * Environment Configuration Schema + */ +const ConfigSchema = z.object({ + // Required Database and Cache + DATABASE_URL: z.string().url({ message: 'DATABASE_URL must be a valid URL' }), + REDIS_URL: z.string().url({ message: 'REDIS_URL must be a valid URL' }), + + // Required Soroban RPC Endpoints + SOROBAN_RPC_MAINNET: z.string().url({ message: 'SOROBAN_RPC_MAINNET must be a valid URL' }), + SOROBAN_RPC_TESTNET: z.string().url({ message: 'SOROBAN_RPC_TESTNET must be a valid URL' }), + + // Security + JWT_SECRET: z.string().min(32, { message: 'JWT_SECRET must be at least 32 characters long' }), + + // Optional Mercury API + MERCURY_API_KEY: z.string().nullable().default(null), + MERCURY_API_URL: z.string().url().nullable().default(null), + + // Environment + NODE_ENV: z.enum(['development', 'production', 'test']).default('development'), +}); + +/** + * Validates and returns the environment configuration. + * Throws a detailed error if validation fails. + */ +export function validateConfig(env: Record = process.env) { + const result = ConfigSchema.safeParse(env); + + if (!result.success) { + const errors = result.error.issues + .map((issue) => ` - ${issue.path.join('.')}: ${issue.message}`) + .join('\n'); + + throw new Error( + `❌ Invalid environment configuration:\n${errors}\n\nPlease check your .env file or environment variables.` + ); + } + + return result.data; +} + +/** + * Exported config object for shared use. + * Note: Components should usually call validateConfig() at startup to ensure fail-fast behavior. + */ +export type Config = z.infer; +export const config = (typeof process !== 'undefined') ? validateConfig() : {} as Config; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index d309b92..9b90898 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,3 +1,4 @@ export * from './utils/xdr.js'; export * from './constants/errors.js'; export * from './schemas/index.js'; +export * from './config.js'; From a71e079f38e0a171e1f1e27ca3d641c3b5ab152a Mon Sep 17 00:00:00 2001 From: Lewechi Date: Thu, 16 Apr 2026 11:09:05 +0100 Subject: [PATCH 2/6] feat(api): implement initial database migrations for auth and users --- apps/api/knexfile.ts | 21 ++ .../20240416110000_create_auth_tables.ts | 57 ++++ apps/api/package.json | 7 +- node_modules/.package-lock.json | 308 +++++++++++++++++- node_modules/@watchtower/api/package.json | 7 +- package-lock.json | 308 +++++++++++++++++- 6 files changed, 696 insertions(+), 12 deletions(-) create mode 100644 apps/api/knexfile.ts create mode 100644 apps/api/migrations/20240416110000_create_auth_tables.ts diff --git a/apps/api/knexfile.ts b/apps/api/knexfile.ts new file mode 100644 index 0000000..78dd47c --- /dev/null +++ b/apps/api/knexfile.ts @@ -0,0 +1,21 @@ +import type { Knex } from 'knex'; + +const config: { [key: string]: Knex.Config } = { + development: { + client: 'pg', + connection: process.env.DATABASE_URL || 'postgresql://postgres:password@localhost:5432/watchtower', + migrations: { + directory: './migrations', + loadExtensions: ['.ts', '.js'], + }, + }, + production: { + client: 'pg', + connection: process.env.DATABASE_URL, + migrations: { + directory: './migrations', + }, + }, +}; + +export default config; diff --git a/apps/api/migrations/20240416110000_create_auth_tables.ts b/apps/api/migrations/20240416110000_create_auth_tables.ts new file mode 100644 index 0000000..6450d3a --- /dev/null +++ b/apps/api/migrations/20240416110000_create_auth_tables.ts @@ -0,0 +1,57 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + // users table + await knex.schema.createTable('users', (table) => { + table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid()')); + table.string('email').unique().notNullable(); + table.string('password_hash').nullable(); // Nullable for OAuth-only users + table.string('github_id').unique().nullable(); + table.string('display_name').notNullable(); + table.timestamps(true, true); // Adds created_at and updated_at + + table.index('email'); + table.index('github_id'); + }); + + // sessions table + await knex.schema.createTable('sessions', (table) => { + table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid()')); + table + .uuid('user_id') + .notNullable() + .references('id') + .inTable('users') + .onDelete('CASCADE'); + table.string('token_hash').notNullable(); + table.timestamp('expires_at').notNullable(); + table.timestamp('created_at').defaultTo(knex.fn.now()); + + table.index('token_hash'); + }); + + // api_keys table + await knex.schema.createTable('api_keys', (table) => { + table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid()')); + table + .uuid('user_id') + .notNullable() + .references('id') + .inTable('users') + .onDelete('CASCADE'); + table.string('key_hash').notNullable(); + table.string('label').notNullable(); + table.timestamp('last_used_at').nullable(); + table.timestamp('created_at').defaultTo(knex.fn.now()); + table.timestamp('revoked_at').nullable(); + + table.index('key_hash'); + }); +} + +export async function down(knex: Knex): Promise { + // Drop in reverse order of foreign keys + await knex.schema.dropTableIfExists('api_keys'); + await knex.schema.dropTableIfExists('sessions'); + await knex.schema.dropTableIfExists('users'); +} diff --git a/apps/api/package.json b/apps/api/package.json index 348e14e..3559c24 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -10,7 +10,10 @@ "lint": "eslint src --ext .ts", "test": "vitest run unit", "test:integration": "vitest run integration", - "type-check": "tsc --noEmit" + "type-check": "tsc --noEmit", + "migrate:latest": "knex migrate:latest --knexfile knexfile.ts", + "migrate:rollback": "knex migrate:rollback --knexfile knexfile.ts", + "migrate:make": "knex migrate:make --knexfile knexfile.ts" }, "dependencies": { "@fastify/cors": "^9.0.0", @@ -22,6 +25,8 @@ "@fastify/websocket": "^9.0.0", "@watchtower/shared": "*", "fastify": "^4.24.0", + "knex": "^3.2.9", + "pg": "^8.20.0", "zod": "^3.22.0" }, "devDependencies": { diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index 9dbe959..5a0693e 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -16,6 +16,8 @@ "@fastify/websocket": "^9.0.0", "@watchtower/shared": "*", "fastify": "^4.24.0", + "knex": "^3.2.9", + "pg": "^8.20.0", "zod": "^3.22.0" }, "devDependencies": { @@ -2370,6 +2372,12 @@ "dev": true, "license": "MIT" }, + "node_modules/colorette": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==", + "license": "MIT" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2908,7 +2916,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3069,6 +3076,15 @@ "node": "*" } }, + "node_modules/esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -3620,6 +3636,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -3658,6 +3683,12 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/getopts": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==", + "license": "MIT" + }, "node_modules/glob": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", @@ -3917,6 +3948,15 @@ "node": ">=12" } }, + "node_modules/interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ioredis": { "version": "5.10.0", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.0.tgz", @@ -3979,7 +4019,6 @@ "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -4206,6 +4245,104 @@ "json-buffer": "3.0.1" } }, + "node_modules/knex": { + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/knex/-/knex-3.2.9.tgz", + "integrity": "sha512-dtAILTjBMaG8YloP5oBxohDIKyIsdQ/TkcVvSjhsksvsjeH63Y0PADyuMDfNZKbVT3Rlx3vEYVBlecbPT/KerA==", + "license": "MIT", + "dependencies": { + "colorette": "2.0.19", + "commander": "^10.0.0", + "debug": "4.3.4", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.6.2", + "rechoir": "^0.8.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.2", + "tildify": "2.0.0" + }, + "bin": { + "knex": "bin/cli.js" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "pg-query-stream": "^4.14.0" + }, + "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + }, + "mysql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-native": { + "optional": true + }, + "pg-query-stream": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "tedious": { + "optional": true + } + } + }, + "node_modules/knex/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/knex/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "license": "MIT", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/knex/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "license": "MIT" + }, + "node_modules/knex/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -4805,7 +4942,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, "license": "MIT" }, "node_modules/path-type": { @@ -4835,6 +4971,101 @@ "node": "*" } }, + "node_modules/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "^1.13.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pg/node_modules/pg-connection-string": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", + "license": "MIT" + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -5120,6 +5351,45 @@ "dev": true, "license": "MIT" }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -5445,6 +5715,18 @@ "decimal.js-light": "^2.4.1" } }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "license": "MIT", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -5492,7 +5774,6 @@ "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.1", @@ -6062,7 +6343,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6119,6 +6399,15 @@ "node": ">=14.0.0" } }, + "node_modules/tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -6158,6 +6447,15 @@ "real-require": "^0.2.0" } }, + "node_modules/tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", diff --git a/node_modules/@watchtower/api/package.json b/node_modules/@watchtower/api/package.json index 348e14e..3559c24 100644 --- a/node_modules/@watchtower/api/package.json +++ b/node_modules/@watchtower/api/package.json @@ -10,7 +10,10 @@ "lint": "eslint src --ext .ts", "test": "vitest run unit", "test:integration": "vitest run integration", - "type-check": "tsc --noEmit" + "type-check": "tsc --noEmit", + "migrate:latest": "knex migrate:latest --knexfile knexfile.ts", + "migrate:rollback": "knex migrate:rollback --knexfile knexfile.ts", + "migrate:make": "knex migrate:make --knexfile knexfile.ts" }, "dependencies": { "@fastify/cors": "^9.0.0", @@ -22,6 +25,8 @@ "@fastify/websocket": "^9.0.0", "@watchtower/shared": "*", "fastify": "^4.24.0", + "knex": "^3.2.9", + "pg": "^8.20.0", "zod": "^3.22.0" }, "devDependencies": { diff --git a/package-lock.json b/package-lock.json index a316cff..e2081b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,8 @@ "@fastify/websocket": "^9.0.0", "@watchtower/shared": "*", "fastify": "^4.24.0", + "knex": "^3.2.9", + "pg": "^8.20.0", "zod": "^3.22.0" }, "devDependencies": { @@ -3244,6 +3246,12 @@ "dev": true, "license": "MIT" }, + "node_modules/colorette": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==", + "license": "MIT" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -3782,7 +3790,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3943,6 +3950,15 @@ "node": "*" } }, + "node_modules/esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -4508,6 +4524,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -4546,6 +4571,12 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/getopts": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==", + "license": "MIT" + }, "node_modules/glob": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", @@ -4805,6 +4836,15 @@ "node": ">=12" } }, + "node_modules/interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ioredis": { "version": "5.10.0", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.0.tgz", @@ -4867,7 +4907,6 @@ "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -5094,6 +5133,104 @@ "json-buffer": "3.0.1" } }, + "node_modules/knex": { + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/knex/-/knex-3.2.9.tgz", + "integrity": "sha512-dtAILTjBMaG8YloP5oBxohDIKyIsdQ/TkcVvSjhsksvsjeH63Y0PADyuMDfNZKbVT3Rlx3vEYVBlecbPT/KerA==", + "license": "MIT", + "dependencies": { + "colorette": "2.0.19", + "commander": "^10.0.0", + "debug": "4.3.4", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.6.2", + "rechoir": "^0.8.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.2", + "tildify": "2.0.0" + }, + "bin": { + "knex": "bin/cli.js" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "pg-query-stream": "^4.14.0" + }, + "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + }, + "mysql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-native": { + "optional": true + }, + "pg-query-stream": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "tedious": { + "optional": true + } + } + }, + "node_modules/knex/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/knex/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "license": "MIT", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/knex/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "license": "MIT" + }, + "node_modules/knex/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5693,7 +5830,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, "license": "MIT" }, "node_modules/path-type": { @@ -5723,6 +5859,101 @@ "node": "*" } }, + "node_modules/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "^1.13.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pg/node_modules/pg-connection-string": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", + "license": "MIT" + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -6008,6 +6239,45 @@ "dev": true, "license": "MIT" }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -6333,6 +6603,18 @@ "decimal.js-light": "^2.4.1" } }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "license": "MIT", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -6380,7 +6662,6 @@ "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.1", @@ -6950,7 +7231,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7007,6 +7287,15 @@ "node": ">=14.0.0" } }, + "node_modules/tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -7046,6 +7335,15 @@ "real-require": "^0.2.0" } }, + "node_modules/tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", From 4d391de98fab7550fe2613f56a4eb4c60a87b5ae Mon Sep 17 00:00:00 2001 From: Lewechi Date: Thu, 16 Apr 2026 11:20:20 +0100 Subject: [PATCH 3/6] feat(api): implement database schema for account tiers and contract tracking --- .../20240416120000_create_contract_tables.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 apps/api/migrations/20240416120000_create_contract_tables.ts diff --git a/apps/api/migrations/20240416120000_create_contract_tables.ts b/apps/api/migrations/20240416120000_create_contract_tables.ts new file mode 100644 index 0000000..a0a6eb6 --- /dev/null +++ b/apps/api/migrations/20240416120000_create_contract_tables.ts @@ -0,0 +1,62 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + // Create native ENUM types for PG + await knex.raw("CREATE TYPE account_tier AS ENUM ('free', 'pro', 'team', 'enterprise')"); + await knex.raw("CREATE TYPE network_type AS ENUM ('mainnet', 'testnet')"); + + // accounts table + await knex.schema.createTable('accounts', (table) => { + table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid()')); + table + .uuid('user_id') + .unique() + .notNullable() + .references('id') + .inTable('users') + .onDelete('CASCADE'); + table + .specificType('tier', 'account_tier') + .notNullable() + .defaultTo('free'); + table.integer('max_contracts').notNullable().defaultTo(2); + table.integer('retention_days').notNullable().defaultTo(7); + table.timestamps(true, true); + }); + + // contracts table + await knex.schema.createTable('contracts', (table) => { + table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid()')); + table + .uuid('account_id') + .notNullable() + .references('id') + .inTable('accounts') + .onDelete('CASCADE'); + table.string('contract_address', 56).notNullable(); + table.specificType('network', 'network_type').notNullable(); + table.string('label', 100).nullable(); + table.specificType('tags', 'text[]').nullable(); + table.string('deployer_address', 56).nullable(); + table.string('wasm_hash', 64).nullable(); + table.integer('creation_ledger').nullable(); + table.boolean('is_active').notNullable().defaultTo(true); + table.timestamps(true, true); + + // Unique constraint on account_id + address + network + table.unique(['account_id', 'contract_address', 'network']); + + // Index for fast address lookups + table.index('contract_address'); + }); +} + +export async function down(knex: Knex): Promise { + // Drop in reverse order + await knex.schema.dropTableIfExists('contracts'); + await knex.schema.dropTableIfExists('accounts'); + + // Clean up types + await knex.raw('DROP TYPE IF EXISTS network_type'); + await knex.raw('DROP TYPE IF EXISTS account_tier'); +} From 13d062e76597b704662b58e0b804df799edc8784 Mon Sep 17 00:00:00 2001 From: Lewechi Date: Thu, 16 Apr 2026 11:29:09 +0100 Subject: [PATCH 4/6] feat(api): implement TimescaleDB hypertable for invocation events --- .gitignore | 1 + ...000_create_invocation_events_hypertable.ts | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 apps/api/migrations/20240416130000_create_invocation_events_hypertable.ts diff --git a/.gitignore b/.gitignore index deed335..4ff272a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ dist/ .env +.agents/ diff --git a/apps/api/migrations/20240416130000_create_invocation_events_hypertable.ts b/apps/api/migrations/20240416130000_create_invocation_events_hypertable.ts new file mode 100644 index 0000000..9c77bb5 --- /dev/null +++ b/apps/api/migrations/20240416130000_create_invocation_events_hypertable.ts @@ -0,0 +1,58 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + // 1. Create the base table + await knex.schema.createTable('invocation_events', (table) => { + table.bigIncrements('id').notNullable(); + table.timestamp('time').notNullable(); + table + .uuid('contract_id') + .notNullable() + .references('id') + .inTable('contracts') + .onDelete('CASCADE'); + table.integer('ledger_sequence').notNullable(); + table.string('tx_hash', 64).notNullable(); + table.string('function_name', 256).nullable(); + table.string('invoker_address', 56).nullable(); + table.boolean('success').notNullable(); + + // Error details + table.string('error_type', 64).nullable(); + table.string('error_code', 64).nullable(); + table.text('error_raw').nullable(); + + // Resource metrics + table.bigInteger('cpu_instructions').nullable(); + table.bigInteger('memory_bytes').nullable(); + table.integer('ledger_reads').nullable(); + table.integer('ledger_writes').nullable(); + table.bigInteger('read_bytes').nullable(); + table.bigInteger('write_bytes').nullable(); + table.integer('tx_size_bytes').nullable(); + table.integer('events_emitted').nullable(); + table.bigInteger('fee_charged').nullable(); + + // Cursor for pagination/tracking + table.string('event_cursor', 128).notNullable(); + }); + + // 2. Convert to Hypertable (TimescaleDB specific) + // Partitioned by 'time' with 1-day chunks + await knex.raw("SELECT create_hypertable('invocation_events', 'time', chunk_time_interval => INTERVAL '1 day')"); + + // 3. Create optimized indexes + // We use knex.raw to ensure correct DESC order on the time column + await knex.raw('CREATE INDEX idx_invocation_contract_time ON invocation_events (contract_id, time DESC)'); + await knex.raw('CREATE INDEX idx_invocation_contract_func_time ON invocation_events (contract_id, function_name, time DESC)'); + await knex.raw('CREATE INDEX idx_invocation_tx_hash ON invocation_events (tx_hash)'); + + // 4. Configure default retention policy + // Automatically drop chunks older than 7 days + await knex.raw("SELECT add_retention_policy('invocation_events', INTERVAL '7 days')"); +} + +export async function down(knex: Knex): Promise { + // Note: In TimescaleDB, dropping the table also cleans up chunks and policies. + await knex.schema.dropTableIfExists('invocation_events'); +} From 9feff674328db73ab2fdfd6b5ce5efa3da417b98 Mon Sep 17 00:00:00 2001 From: Lewechi Date: Thu, 16 Apr 2026 11:43:39 +0100 Subject: [PATCH 5/6] Feat: Implement cursor_checkpoints table for Event Poller progress tracking --- ...0240416140000_create_cursor_checkpoints.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 apps/api/migrations/20240416140000_create_cursor_checkpoints.ts diff --git a/apps/api/migrations/20240416140000_create_cursor_checkpoints.ts b/apps/api/migrations/20240416140000_create_cursor_checkpoints.ts new file mode 100644 index 0000000..b8e375d --- /dev/null +++ b/apps/api/migrations/20240416140000_create_cursor_checkpoints.ts @@ -0,0 +1,29 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.createTable('cursor_checkpoints', (table) => { + table.increments('id').primary(); + table + .uuid('contract_id') + .notNullable() + .references('id') + .inTable('contracts') + .onDelete('CASCADE'); + + // Use the network_type enum created in 20240416120000_create_contract_tables.ts + table.specificType('network', 'network_type').notNullable(); + + table.string('last_cursor', 128).notNullable(); + table.integer('last_ledger_sequence').notNullable(); + table.timestamp('last_processed_at').notNullable(); + table.boolean('gap_detected').defaultTo(false); + table.timestamps(true, true); + + // Unique constraint to enable easy upserts for the Poller + table.unique(['contract_id', 'network']); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('cursor_checkpoints'); +} From 270212bc49788543790de561cde8f0c8fe7b4282 Mon Sep 17 00:00:00 2001 From: Lewechi Date: Thu, 16 Apr 2026 11:46:25 +0100 Subject: [PATCH 6/6] Feat: Add 1m and 1h materialized rollups for optimized dashboard performance --- ...0416150000_create_continuous_aggregates.ts | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 apps/api/migrations/20240416150000_create_continuous_aggregates.ts diff --git a/apps/api/migrations/20240416150000_create_continuous_aggregates.ts b/apps/api/migrations/20240416150000_create_continuous_aggregates.ts new file mode 100644 index 0000000..87031cb --- /dev/null +++ b/apps/api/migrations/20240416150000_create_continuous_aggregates.ts @@ -0,0 +1,75 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + // 1. Ensure TimescaleDB Toolkit is enabled (optional but recommended for percentile_agg) + // Note: This might fail if the shared library is not in the image, + // but most modern Timescale images include it. + await knex.raw('CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit CASCADE').catch(() => { + console.warn('TimescaleDB Toolkit extension not found. Percentile approximations might not work.'); + }); + + // 2. Create 1-minute rollup + await knex.raw(` + CREATE MATERIALIZED VIEW invocations_1m + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 minute', "time") AS bucket, + contract_id, + function_name, + count(*) AS total_count, + count(*) FILTER (WHERE success = true) AS success_count, + count(*) FILTER (WHERE success = false) AS error_count, + avg(cpu_instructions) AS avg_cpu, + avg(memory_bytes) AS avg_memory, + stats_agg(cpu_instructions) AS cpu_stats, -- For mean, variance, etc. + percentile_agg(cpu_instructions) AS cpu_percentiles -- For P95 + FROM invocation_events + GROUP BY bucket, contract_id, function_name; + `); + + // 3. Create 1-hour rollup + await knex.raw(` + CREATE MATERIALIZED VIEW invocations_1h + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 hour', "time") AS bucket, + contract_id, + function_name, + count(*) AS total_count, + count(*) FILTER (WHERE success = true) AS success_count, + count(*) FILTER (WHERE success = false) AS error_count, + avg(cpu_instructions) AS avg_cpu, + avg(memory_bytes) AS avg_memory, + stats_agg(cpu_instructions) AS cpu_stats, + percentile_agg(cpu_instructions) AS cpu_percentiles + FROM invocation_events + GROUP BY bucket, contract_id, function_name; + `); + + // 4. Set Refresh Policies + // 1m aggregate refreshes every 1 minute with a 5-minute lag window + await knex.raw(` + SELECT add_continuous_aggregate_policy('invocations_1m', + start_offset => INTERVAL '5 minutes', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + `); + + // 1h aggregate refreshes every 10 minutes with a 1-hour lag window + await knex.raw(` + SELECT add_continuous_aggregate_policy('invocations_1h', + start_offset => INTERVAL '2 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '10 minutes'); + `); +} + +export async function down(knex: Knex): Promise { + // Drop policies first + await knex.raw("SELECT remove_continuous_aggregate_policy('invocations_1h')").catch(() => {}); + await knex.raw("SELECT remove_continuous_aggregate_policy('invocations_1m')").catch(() => {}); + + // Drop materialized views + await knex.raw('DROP MATERIALIZED VIEW IF EXISTS invocations_1h'); + await knex.raw('DROP MATERIALIZED VIEW IF EXISTS invocations_1m'); +}