Feature/library etl

.github/workflows/deploy-base.yml

@@ -26,8 +26,8 @@ jobs:
         run: |
           TARGET_INPUT="${{ inputs.target }}"
 
-          if [ ! -d "apps/$TARGET_INPUT" ]; then
-            echo "Error: The target directory 'apps/$TARGET_INPUT' does not exist." >&2
+          if [ ! -d "apps/$TARGET_INPUT" ] && [ ! -d "jobs/$TARGET_INPUT" ]; then
+            echo "Error: The target directory 'apps/$TARGET_INPUT' or 'jobs/$TARGET_INPUT' does not exist." >&2
             exit 1
           fi
           echo "target input '$TARGET_INPUT' is valid."
@@ -75,7 +75,7 @@ jobs:
             TARGETS="${{ inputs.target }}"
           else
             echo "Detecting targets from git diff..."
-            TARGETS=$(git diff --name-only HEAD~1..HEAD | grep '^apps' | cut -d '/' -f 2 | sort -u)
+            TARGETS=$(git diff --name-only HEAD~1..HEAD | awk -F/ '/^(apps|jobs)\//{print $2}' | sort -u)
           fi
 
           if [ -z "$TARGETS" ] ; then
@@ -309,11 +309,24 @@ jobs:
         id: deploy_vars
         run: |
           TARGET_APP=${{ matrix.target }}
-          PUBLISH_PORT=$(yq .publishPort apps/$TARGET_APP/package.json)
+          if [ -f "jobs/$TARGET_APP/package.json" ]; then
+            TARGET_TYPE="job"
+            CRON_SCHEDULE=$(yq '.["cron-schedule"]' jobs/$TARGET_APP/package.json)
+            if [ -z "$CRON_SCHEDULE" ] || [ "$CRON_SCHEDULE" = "null" ]; then
+              echo "Missing cron-schedule in jobs/$TARGET_APP/package.json" >&2
+              exit 1
+            fi
+            echo "cron_schedule=$CRON_SCHEDULE" >> $GITHUB_OUTPUT
+          else
+            TARGET_TYPE="app"
+            PUBLISH_PORT=$(yq .publishPort apps/$TARGET_APP/package.json)
+            echo "publish_port=$PUBLISH_PORT" >> $GITHUB_OUTPUT
+          fi
 
-          echo "publish_port=$PUBLISH_PORT" >> $GITHUB_OUTPUT
+          echo "target_type=$TARGET_TYPE" >> $GITHUB_OUTPUT
 
       - name: Deploy Service
+        if: steps.deploy_vars.outputs.target_type == 'app'
         uses: ./.github/actions/deploy-service
         with:
           target_app: ${{ matrix.target }}
@@ -327,7 +340,39 @@ jobs:
           aws_region: ${{ secrets.AWS_REGION }}
           aws_ecr_uri: ${{ secrets.AWS_ECR_URI }}
 
+      - name: Deploy Cron Job
+        if: steps.deploy_vars.outputs.target_type == 'job'
+        uses: appleboy/ssh-action@v1
+        with:
+          host: ${{ secrets.EC2_HOST }}
+          username: ${{ secrets.EC2_USER }}
+          key: ${{ secrets.EC2_SSH_KEY }}
+          script: |
+            set -e
+            export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
+            export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+            TARGET_APP=${{ matrix.target }}
+            DEPLOY_TAG=${{ steps.determine_version.outputs.deploy_version }}
+            CRON_SCHEDULE='${{ steps.deploy_vars.outputs.cron_schedule }}'
+            AWS_ECR_URI=${{ secrets.AWS_ECR_URI }}
+            AWS_REGION=${{ secrets.AWS_REGION }}
+
+            CRON_ID="wxyc_${TARGET_APP}"
+            echo "Logging into AWS ECR..."
+            aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ECR_URI
+
+            echo "Pulling Docker image for deploy..."
+            docker pull $AWS_ECR_URI/$TARGET_APP:$DEPLOY_TAG
+
+            CRON_CMD="docker rm -f ${TARGET_APP}-cron >/dev/null 2>&1 || true; docker run --rm --name ${TARGET_APP}-cron --env-file .env $AWS_ECR_URI/$TARGET_APP:$DEPLOY_TAG"
+
+            (crontab -l 2>/dev/null | grep -v "$CRON_ID"; echo "$CRON_SCHEDULE $CRON_CMD # $CRON_ID") | crontab -
+
+            echo "Cron schedule updated."
+
       - name: Confirm server is up
+        if: steps.deploy_vars.outputs.target_type == 'app'
         uses: appleboy/ssh-action@v1
         with:
           host: ${{ secrets.EC2_HOST }}
@@ -344,3 +389,23 @@ jobs:
               echo "Server is not running. Deployment failed." >&2
               exit 1
             fi
+
+      - name: Confirm cron is updated
+        if: steps.deploy_vars.outputs.target_type == 'job'
+        uses: appleboy/ssh-action@v1
+        with:
+          host: ${{ secrets.EC2_HOST }}
+          username: ${{ secrets.EC2_USER }}
+          key: ${{ secrets.EC2_SSH_KEY }}
+          script: |
+            set -e
+            TARGET_APP=${{ matrix.target }}
+            CRON_ID="wxyc_${TARGET_APP}"
+
+            echo "Verifying crontab entry..."
+            if crontab -l 2>/dev/null | grep -F "$CRON_ID" > /dev/null; then
+              echo "Cronjob entry verified."
+            else
+              echo "Cronjob not updated properly. Deployment failed" >&2
+              exit 1
+            fi

Dockerfile.library-etl

@@ -0,0 +1,27 @@
+#Build stage
+FROM node:22-alpine AS builder
+
+WORKDIR /library-etl-builder
+
+COPY ./package.json ./
+COPY ./tsconfig.base.json ./
+COPY ./shared/database ./shared/database
+COPY ./jobs/library-etl ./jobs/library-etl
+
+RUN npm install && npm run build --workspace=@wxyc/database --workspace=@wxyc/library-etl
+
+#Production stage
+FROM node:22-alpine AS prod
+
+WORKDIR /library-etl
+
+COPY ./package* ./
+COPY ./jobs/library-etl/package* ./jobs/library-etl/
+COPY ./shared/database/package* ./shared/database/
+
+RUN npm install --omit=dev
+
+COPY --from=builder ./library-etl-builder/jobs/library-etl/dist ./jobs/library-etl/dist
+COPY --from=builder ./library-etl-builder/shared/database/dist ./shared/database/dist
+
+CMD ["npm", "start", "--workspace=@wxyc/library-etl"]

apps/backend/app.yaml

@@ -907,10 +907,32 @@ paths:
                   type: string
                 genre_id:
                   type: integer
+                code_number:
+                  type: integer
       responses:
         '200':
           description: Successfully added artist
 
+  /library/artists/peek-code:
+    get:
+      summary: Peek next artist code number for a given genre and code letters
+      security:
+        - BearerAuth: ['station-management']
+      parameters:
+        - in: query
+          name: code_letters
+          required: true
+          schema:
+            type: string
+        - in: query
+          name: genre_id
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          description: Next available artist code number
+
   /library/formats:
     get:
       summary: Get format list

apps/backend/controllers/library.controller.ts

@@ -120,30 +120,47 @@ export const searchForAlbum: RequestHandler = async (
 
 type NewArtistRequest = {
   artist_name: string;
+  alphabetical_name?: string;
   code_letters: string;
   genre_id: number;
+  code_number: number;
 };
 
 export const addArtist: RequestHandler = async (req: Request<object, object, NewArtistRequest>, res, next) => {
   const { body } = req;
-  //TODO auto_generate artist code letters and make it an optional parameter
-  if (body.artist_name === undefined || body.code_letters === undefined || body.genre_id === undefined) {
+  if (
+    body.artist_name === undefined ||
+    body.code_letters === undefined ||
+    body.genre_id === undefined ||
+    body.code_number === undefined
+  ) {
     res.status(400);
-    res.send('Missing Request Parameters: artist_name, code_letters, or genre_id');
+    res.send('Missing Request Parameters: artist_name, code_letters, genre_id, or code_number');
   } else {
     try {
-      const generatedArtistNumber = await libraryService.generateArtistNumber(body.code_letters, body.genre_id);
+      const existingArtist = await libraryService.getArtistByCode(body.code_letters, body.genre_id, body.code_number);
+      if (existingArtist) {
+        res.status(409).json({
+          message: 'Artist code already exists for that genre and code letters.',
+          artist: existingArtist,
+        });
+        return;
+      }
 
       const new_artist: NewArtist = {
         artist_name: body.artist_name,
+        alphabetical_name: body.alphabetical_name ?? body.artist_name,
         code_letters: body.code_letters,
-        code_artist_number: generatedArtistNumber,
-        genre_id: body.genre_id,
       };
 
       const response: Artist = await libraryService.insertArtist(new_artist);
+      await libraryService.insertArtistGenreCrossreference(response.id, body.genre_id, body.code_number);
       res.status(200);
-      res.send(response);
+      res.json({
+        ...response,
+        code_number: body.code_number,
+        genre_id: body.genre_id,
+      });
     } catch (e) {
       console.error('Error: Failed to add new artist');
       console.error(e);
@@ -152,6 +169,40 @@ export const addArtist: RequestHandler = async (req: Request<object, object, New
   }
 };
 
+type ArtistNumberPeekQuery = {
+  code_letters?: string;
+  genre_id?: string;
+};
+
+export const peekArtistNumber: RequestHandler = async (
+  req: Request<object, object, object, ArtistNumberPeekQuery>,
+  res,
+  next
+) => {
+  const { query } = req;
+  if (!query.code_letters || !query.genre_id) {
+    res.status(400);
+    res.send('Missing query parameters: code_letters and genre_id');
+    return;
+  }
+
+  const genreId = Number(query.genre_id);
+  if (!Number.isFinite(genreId)) {
+    res.status(400);
+    res.send('Invalid genre_id');
+    return;
+  }
+
+  try {
+    const nextCode = await libraryService.generateArtistNumber(query.code_letters, genreId);
+    res.status(200).json({ next_code_number: nextCode });
+  } catch (e) {
+    console.error('Error: Failed to generate artist number');
+    console.error(e);
+    next(e);
+  }
+};
+
 export const getRotation: RequestHandler = async (req, res, next) => {
   try {
     const rotation = await libraryService.getRotationFromDB();

apps/backend/middleware/legacy/commandqueue.mirror.ts

@@ -3,7 +3,7 @@ import { EventData, MirrorEvents, serverEventsMgr } from '../../utils/serverEven
 import { promises } from 'fs';
 import { EventEmitter } from 'node:events';
 import path from 'path';
-import { MirrorSQL } from './sql.mirror';
+import { MirrorSQL } from '@wxyc/database';
 import { cryptoRandomId, expBackoffMs } from './utilities.mirror';
 
 const CommandQueueEvents = {

apps/backend/routes/events.route.ts

@@ -4,9 +4,11 @@ import * as serverEvents from '../controllers/events.conroller.js';
 
 export const events_route = Router();
 
-//TODO: secure - mgmt & individual dj
+//TODO: You shouldn't have to be authenticated to register an event client.
+// Each topic has it's own permissions, some of which are public.
 events_route.post('/register', requirePermissions({ flowsheet: ['read'] }), serverEvents.registerEventClient);
 
+//TODO: You shouldn't have to be authenticated to subscribe to a topic
 events_route.put('/subscribe', requirePermissions({ flowsheet: ['read'] }), serverEvents.subscribeToTopic);
 
-events_route.get('/test', requirePermissions({ flowsheet: ['read'] }), serverEvents.testTrigger);
+events_route.get('/test', serverEvents.testTrigger);

apps/backend/routes/library.route.ts

@@ -22,6 +22,8 @@ library_route.patch('/rotation', requirePermissions({ catalog: ['write'] }), lib
 
 library_route.post('/artists', requirePermissions({ catalog: ['write'] }), libraryController.addArtist);
 
+library_route.get('/artists/peek-code', requirePermissions({ catalog: ['write'] }), libraryController.peekArtistNumber);
+
 library_route.get('/formats', requirePermissions({ catalog: ['read'] }), libraryController.getFormats);
 
 library_route.post('/formats', requirePermissions({ catalog: ['write'] }), libraryController.addFormat);

apps/backend/services/djs.service.ts

@@ -7,6 +7,7 @@ import {
   db,
   flowsheet,
   format,
+  genre_artist_crossreference,
   genres,
   library,
   show_djs,
@@ -35,9 +36,10 @@ export const getBinFromDB = async (dj_id: string) => {
       album_id: bins.album_id,
       album_title: library.album_title,
       artist_name: artists.artist_name,
+      alphabetical_name: artists.alphabetical_name,
       label: library.label,
       code_letters: artists.code_letters,
-      code_artist_number: artists.code_artist_number,
+      code_artist_number: genre_artist_crossreference.artist_genre_code,
       code_number: library.code_number,
       format_name: format.format_name,
       genre_name: genres.genre_name,
@@ -47,6 +49,13 @@ export const getBinFromDB = async (dj_id: string) => {
     .innerJoin(artists, eq(library.artist_id, artists.id))
     .innerJoin(format, eq(format.id, library.format_id))
     .innerJoin(genres, eq(genres.id, library.genre_id))
+    .innerJoin(
+      genre_artist_crossreference,
+      and(
+        eq(genre_artist_crossreference.artist_id, library.artist_id),
+        eq(genre_artist_crossreference.genre_id, library.genre_id)
+      )
+    )
     .where(eq(bins.dj_id, dj_id));
 
   return dj_bin;