Begin migration back to Postgres

.env

@@ -1,4 +1,12 @@
+PGHOST=localhost
+PGPORT=5432
+PGDATABASE=travis_ci_test
+PGUSER=postgres
+PGPASSWORD=postgres
+PGSSL=false
 JWT_SECRET=dummysecret
 VHOST=api.destinyitemmanager.com
 STATELY_STORE_ID=4691621389625154
 STATELY_REGION=us-west-2
+VHOST=api.destinyitemmanager.com
+STATELY_ACCESS_KEY="CAISRzBFAiEAhBwlEIYOXbLFzWyqZsTn3iLbyBUCjOVL8HzaAwDz6WkCIBf1QSVzt5VLV0VckmSsv2D3OHvnzHnctfTDDPifUOsDGgkI0sHq9_DVqAM"

.github/workflows/deploy.yml

@@ -50,6 +50,31 @@ jobs:
       - name: Save DigitalOcean kubeconfig with short-lived credentials
         run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{secrets.K8S_CLUSTER}}
 
+      - name: Add IP address to trusted source (managed database)
+        uses: GarreauArthur/manage-digital-ocean-managed-database-trusted-sources-gh-action@main
+        with:
+          action: 'add'
+          database_id: ${{ secrets.DATABASE_ID }}
+          digitalocean_token: ${{ secrets.DIGITALOCEAN_TOKEN }}
+
+      - name: Run DB migrations
+        run: cd api && npx db-migrate up -e prod
+        env:
+          DATABASE_USER: ${{ secrets.DATABASE_USER }}
+          DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
+          DATABASE_HOST: ${{ secrets.DATABASE_HOST }}
+          DATABASE_NAME: ${{ secrets.DATABASE_NAME }}
+          DATABASE_PORT: ${{ secrets.DATABASE_PORT }}
+
+      - name: Remove IP address to trusted source (managed database)
+        if: always()
+        continue-on-error: true
+        uses: GarreauArthur/manage-digital-ocean-managed-database-trusted-sources-gh-action@main
+        with:
+          action: 'remove'
+          database_id: ${{ secrets.DATABASE_ID }}
+          digitalocean_token: ${{ secrets.DIGITALOCEAN_TOKEN }}
+
       - name: Build and deploy
         run: pnpm run deploy
 

.github/workflows/pr-build.yml

@@ -7,7 +7,7 @@ on: pull_request
 jobs:
   build:
     runs-on: ubuntu-latest
-    environment: 'test'
+    environment: "test"
 
     steps:
       - uses: actions/checkout@v4
@@ -17,7 +17,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version-file: '.nvmrc'
+          node-version-file: ".nvmrc"
           cache: pnpm
 
       - name: Install

api/apps/index.ts

@@ -1,10 +1,11 @@
 import * as Sentry from '@sentry/node';
-import { ListToken } from '@stately-cloud/client';
 import { keyBy } from 'es-toolkit';
 import { RequestHandler } from 'express';
+import { addAllApps, getAllApps as getAllAppsPostgres } from '../db/apps-queries.js';
+import { pool } from '../db/index.js';
 import { metrics } from '../metrics/index.js';
 import { ApiApp } from '../shapes/app.js';
-import { getAllApps, updateApps } from '../stately/apps-queries.js';
+import { getAllApps as getAllAppsStately } from '../stately/apps-queries.js';
 
 /**
  * Express middleware that requires an API key be provided in a header
@@ -42,7 +43,6 @@ let apps: ApiApp[] = [];
 let appsByApiKey: { [apiKey: string]: ApiApp };
 let origins = new Set<string>();
 let appsInterval: NodeJS.Timeout | null = null;
-let token: ListToken | undefined;
 
 export function stopAppsRefresh() {
   if (appsInterval) {
@@ -70,21 +70,27 @@ export async function refreshApps(): Promise<void> {
   stopAppsRefresh();
 
   try {
-    if (!token) {
-      // First time, get 'em all
-      const [appsFromStately, newToken] = await getAllApps();
+    if (apps.length === 0) {
+      // Start off with a copy from StatelyDB, just in case postgres is having
+      // problems.
+      const [appsFromStately] = await getAllAppsStately();
       if (appsFromStately.length > 0) {
         apps = appsFromStately;
         digestApps();
-        token = newToken;
       }
+    }
+
+    const appsFromPostgres = await fetchAppsFromPostgres();
+
+    if (appsFromPostgres.length > 0) {
+      apps = appsFromPostgres;
+      digestApps();
     } else {
-      // After that, use a sync to update them
-      const [appsFromStately, newToken] = await updateApps(token, apps);
-      if (appsFromStately.length > 0) {
-        apps = appsFromStately;
-        digestApps();
-        token = newToken;
+      // import them into Postgres
+      try {
+        await addAllApps(apps);
+      } catch (e) {
+        console.error('Error importing apps into Postgres', e);
       }
     }
     metrics.increment('apps.refresh.success.count');
@@ -101,6 +107,16 @@ export async function refreshApps(): Promise<void> {
   }
 }
 
+async function fetchAppsFromPostgres() {
+  const client = await pool.connect();
+  try {
+    const appsFromPostgres = await getAllAppsPostgres(client);
+    return appsFromPostgres;
+  } finally {
+    client.release();
+  }
+}
+
 function digestApps() {
   appsByApiKey = keyBy(apps, (a) => a.dimApiKey.toLowerCase());
   origins = new Set<string>();

api/database.json

@@ -4,7 +4,8 @@
     "user": "postgres",
     "password": "postgres",
     "host": "localhost",
-    "database": "travis_ci_test"
+    "database": "travis_ci_test",
+    "port": 5432
   },
   "dev": {
     "driver": "pg",
@@ -16,14 +17,24 @@
   },
   "prod": {
     "driver": "pg",
-    "user": { "ENV": "DATABASE_USER" },
-    "password": { "ENV": "DATABASE_PASSWORD" },
-    "host": { "ENV": "DATABASE_HOST" },
-    "database": { "ENV": "DATABASE_NAME" },
-    "port": { "ENV": "DATABASE_PORT" },
+    "user": {
+      "ENV": "DATABASE_USER"
+    },
+    "password": {
+      "ENV": "DATABASE_PASSWORD"
+    },
+    "host": {
+      "ENV": "DATABASE_HOST"
+    },
+    "database": {
+      "ENV": "DATABASE_NAME"
+    },
+    "port": {
+      "ENV": "DATABASE_PORT"
+    },
     "ssl": {
       "rejectUnauthorized": false,
       "ca": "./ca-certificate.crt"
     }
   }
-}
+}

api/db/apps-queries.test.ts

@@ -0,0 +1,52 @@
+import { DatabaseError } from 'pg-protocol';
+import { v4 as uuid } from 'uuid';
+import { ApiApp } from '../shapes/app.js';
+import { getAllApps, getAppById, insertApp } from './apps-queries.js';
+import { closeDbPool, pool, transaction } from './index.js';
+
+const appId = 'apps-queries-test-app';
+const app: ApiApp = {
+  id: 'apps-queries-test-app',
+  bungieApiKey: 'foo',
+  origin: 'https://localhost',
+  dimApiKey: uuid(),
+};
+
+beforeEach(async () => pool.query({ text: 'delete from apps where id = $1', values: [appId] }));
+
+afterAll(async () => closeDbPool());
+
+it('can create a new app', async () => {
+  await transaction(async (client) => {
+    expect(await getAppById(client, appId)).toBeNull();
+
+    await insertApp(client, app);
+
+    const fetchedApp = await getAppById(client, appId);
+    expect(fetchedApp?.dimApiKey).toEqual(app.dimApiKey);
+  });
+});
+
+it('cannot create a new app with the same name as an existing one', async () => {
+  await transaction(async (client) => {
+    await insertApp(client, app);
+    try {
+      await insertApp(client, app);
+    } catch (e) {
+      if (!(e instanceof DatabaseError)) {
+        fail('should have thrown a DatabaseError');
+      }
+      expect(e.code).toBe('23505');
+    }
+  });
+});
+
+it('can get all apps', async () => {
+  await transaction(async (client) => {
+    await insertApp(client, app);
+
+    const apps = await getAllApps(client);
+    expect(apps.length).toBeGreaterThanOrEqual(1);
+    expect(apps.find((a) => a.id === appId)?.dimApiKey).toBe(app.dimApiKey);
+  });
+});

api/db/apps-queries.ts

@@ -0,0 +1,51 @@
+import { ClientBase, QueryResult } from 'pg';
+import { ApiApp } from '../shapes/app.js';
+import { camelize, KeysToSnakeCase, TypesForKeys } from '../utils.js';
+import { transaction } from './index.js';
+
+/**
+ * Get all registered apps.
+ */
+export async function getAllApps(client: ClientBase): Promise<ApiApp[]> {
+  const results = await client.query<KeysToSnakeCase<ApiApp>>({
+    name: 'get_all_apps',
+    text: 'SELECT * FROM apps',
+  });
+  return results.rows.map((row) => camelize(row));
+}
+
+export async function addAllApps(apps: ApiApp[]): Promise<void> {
+  await transaction(async (client) => {
+    for (const app of apps) {
+      await insertApp(client, app);
+    }
+  });
+}
+
+/**
+ * Get an app by its ID.
+ */
+export async function getAppById(client: ClientBase, id: string): Promise<ApiApp | null> {
+  const results = await client.query<KeysToSnakeCase<ApiApp>>({
+    name: 'get_apps',
+    text: 'SELECT * FROM apps where id = $1',
+    values: [id],
+  });
+  if (results.rows.length > 0) {
+    return camelize(results.rows[0]);
+  } else {
+    return null;
+  }
+}
+
+/**
+ * Insert a new app into the list of registered apps.
+ */
+export async function insertApp(client: ClientBase, app: ApiApp): Promise<QueryResult> {
+  return client.query<any, TypesForKeys<ApiApp, ['id', 'bungieApiKey', 'dimApiKey', 'origin']>>({
+    name: 'insert_app',
+    text: `insert into apps (id, bungie_api_key, dim_api_key, origin)
+values ($1, $2, $3, $4)`,
+    values: [app.id, app.bungieApiKey, app.dimApiKey, app.origin],
+  });
+}

api/db/dbmigrate-permissions.sql

@@ -0,0 +1,12 @@
+-- Grant permissions to dbmigrate user for defaultdb schema
+GRANT USAGE ON SCHEMA public TO dbmigrate;
+
+-- Grant CREATE privilege on schema to allow table creation
+GRANT CREATE ON SCHEMA public TO dbmigrate;
+
+-- Set default privileges for future tables
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO dbmigrate;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO dimapi;
+
+-- Set default privileges for future sequences
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO dbmigrate;

api/db/global-settings-queries.ts

@@ -0,0 +1,23 @@
+import { GlobalSettings } from '../shapes/global-settings.js';
+import { pool } from './index.js';
+
+export async function getGlobalSettingsQuery(flavor: string) {
+  return pool.query<{ settings: GlobalSettings }>({
+    name: 'get_global_settings',
+    text: 'SELECT * FROM global_settings where flavor = $1 LIMIT 1',
+    values: [flavor],
+  });
+}
+
+export async function setGlobalSettings(flavor: string, settings: Partial<GlobalSettings>) {
+  return pool.query({
+    name: 'set_global_settings',
+    text: `
+      INSERT INTO global_settings (flavor, settings, updated_at)
+      VALUES ($1, $2, NOW())
+      ON CONFLICT (flavor)
+      DO UPDATE SET settings = (global_settings.settings || $2)
+    `,
+    values: [flavor, settings],
+  });
+}