feat(programs): Add unique hashed ID for requirements

apps/api/src/graphql/schema/programs.ts

@@ -30,24 +30,28 @@ type SpecializationPreview implements ProgramPreview @cacheControl(maxAge: 86400
 
 interface ProgramRequirementBase {
     label: String!
+    requirementId: String!
 }
 
 type ProgramCourseRequirement implements ProgramRequirementBase @cacheControl(maxAge: 86400) {
     label: String!
+    requirementId: String!
     requirementType: String!
     courseCount: Int!
     courses: [String!]!
 }
 
 type ProgramUnitRequirement implements ProgramRequirementBase @cacheControl(maxAge: 86400) {
     label: String!
+    requirementId: String!
     requirementType: String!
     unitCount: Int!
     courses: [String!]!
 }
 
 type ProgramGroupRequirement implements ProgramRequirementBase @cacheControl(maxAge: 86400) {
     label: String!
+    requirementId: String!
     requirementType: String!
     requirementCount: Int!
     # circular
@@ -56,6 +60,7 @@ type ProgramGroupRequirement implements ProgramRequirementBase @cacheControl(max
 
 type ProgramMarkerRequirement implements ProgramRequirementBase @cacheControl(maxAge: 86400) {
     label: String!
+    requirementId: String!
 }
 
 union ProgramRequirement = ProgramCourseRequirement | ProgramUnitRequirement | ProgramGroupRequirement | ProgramMarkerRequirement

apps/api/src/schema/programs.ts

@@ -56,6 +56,9 @@ export const programRequirementBaseSchema = z.object({
   label: z.string().openapi({
     description: "Human description of this requirement",
   }),
+  requirementId: z.string().openapi({
+    description: "Identifier for this requirement block",
+  }),
 });
 
 export const programCourseRequirementSchema = programRequirementBaseSchema
@@ -74,6 +77,7 @@ export const programCourseRequirementSchema = programRequirementBaseSchema
     example: {
       requirementType: "Course",
       label: "I&CSci 6N or Math 3A",
+      requirementId: "n5q3hZb2-T",
       courseCount: 1,
       courses: ["I&CSCI6N", "MATH3A"],
     },
@@ -96,6 +100,7 @@ export const programUnitRequirementSchema = programRequirementBaseSchema
       "A unit requirement; a requirement for some number of units earned from a set of courses.",
     example: {
       label: "8 Units Of DRAMA 101",
+      requirementId: "Q7yJf8lR2w",
       requirementType: "Unit",
       unitCount: 8,
       courses: ["DRAMA101A", "DRAMA101B", "DRAMA101C", "DRAMA101D", "DRAMA101E", "DRAMA101S"],
@@ -130,17 +135,20 @@ export const programGroupRequirementSchema: z.ZodType<
     description: "A group requirement; a requirement to fulfill some number of sub-requirements.",
     example: {
       label: "Select I&CSCI 31-32-33 or I&CSCI H32-33",
+      requirementId: "kP3uGZ1s0_",
       requirementType: "Group",
       requirementCount: 1,
       requirements: [
         {
           label: "I&CSCI 31, 32, 33",
+          requirementId: "xA9mVt4Dce",
           requirementType: "Course",
           courseCount: 3,
           courses: ["I&CSCI31", "I&CSCI32", "I&CSCI33"],
         },
         {
           label: "I&CSCI H32, 33",
+          requirementId: "b2R_M7pQwL",
           requirementType: "Course",
           courseCount: 2,
           courses: ["I&CSCIH32", "I&CSCI33"],
@@ -158,6 +166,7 @@ export const programMarkerRequirementSchema = programRequirementBaseSchema
       "A rule which must be marked as complete, e.g the fulfillment of GE VIII (foreign language) via high school credit",
     example: {
       label: "Entry Level Writing",
+      requirementId: "hX7q9N3b_s",
       requirementType: "Marker",
     },
   });

apps/data-pipeline/degreeworks-scraper/src/components/AuditParser.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import type { Block, Rule } from "$types";
 import type { database } from "@packages/db";
 import { eq } from "@packages/db/drizzle";
@@ -13,6 +14,7 @@ export class AuditParser {
   private static readonly electiveMatcher = /ELECTIVE @+/;
   private static readonly wildcardMatcher = /\w@/;
   private static readonly rangeMatcher = /-\w+/;
+  private requirementIdMap = new Map<string, string>();
 
   constructor(private readonly db: ReturnType<typeof database>) {
     console.log("[AuditParser.new] AuditParser initialized");
@@ -99,6 +101,28 @@ export class AuditParser {
       .limit(1);
   }
 
+  generateRequirementId(requirementType: string, contentsSalt: string): string {
+    const requirementObjectStr = JSON.stringify({
+      requirementType,
+      contentsSalt,
+    });
+
+    const requirementId = createHash("md5")
+      .update(requirementObjectStr)
+      .digest("base64url")
+      .slice(0, 10);
+
+    const existingRequirementObjectStr = this.requirementIdMap.has(requirementId)
+      ? this.requirementIdMap.get(requirementId)
+      : null;
+    if (existingRequirementObjectStr && existingRequirementObjectStr !== requirementObjectStr) {
+      console.error("Collision detected between two requirementIds");
+    }
+    this.requirementIdMap.set(requirementId, requirementObjectStr);
+
+    return requirementId;
+  }
+
   /**
    * Certain requirements change label depending on whether they've been fulfilled.
    * This is undesirable for archival so we will quash these.
@@ -143,40 +167,62 @@ export class AuditParser {
             )
             .map(([x]) => x);
           if (rule.requirement.classesBegin) {
+            const label = AuditParser.suppressLabelPolymorphism(rule.label);
+            const requirementType = "Course";
+            const contentsSalt = courses.join("_");
+            const requirementId = this.generateRequirementId(requirementType, contentsSalt);
             ret.push({
-              label: AuditParser.suppressLabelPolymorphism(rule.label),
-              requirementType: "Course",
+              label,
+              requirementId,
+              requirementType,
               courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
               courses,
             });
           } else if (rule.requirement.creditsBegin) {
+            const label = AuditParser.suppressLabelPolymorphism(rule.label);
+            const requirementType = "Unit";
+            const contentsSalt = courses.join("_");
+            const requirementId = this.generateRequirementId(requirementType, contentsSalt);
             ret.push({
-              label: AuditParser.suppressLabelPolymorphism(rule.label),
-              requirementType: "Unit",
+              label,
+              requirementId,
+              requirementType,
               unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
               courses,
             });
           }
           break;
         }
         case "Group": {
+          const label = AuditParser.suppressLabelPolymorphism(rule.label);
+          const requirementType = "Group";
+          const requirements = await this.ruleArrayToRequirements(rule.ruleArray);
+          const contentsSalt = requirements.map((req) => req.requirementId).join("_");
+          const requirementId = this.generateRequirementId(requirementType, contentsSalt);
           ret.push({
-            label: AuditParser.suppressLabelPolymorphism(rule.label),
-            requirementType: "Group",
+            label,
+            requirementId,
+            requirementType,
             requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
-            requirements: await this.ruleArrayToRequirements(rule.ruleArray),
+            requirements,
           });
           break;
         }
         case "IfStmt": {
           const rules = this.flattenIfStmt([rule]);
           if (!rules.some((x) => x.ruleType === "Block")) {
             if (rules.length > 1) {
+              const label = "Select 1 of the following";
+              const requirementType = "Group";
+              const requirements = await this.ruleArrayToRequirements(rules);
+              const contentsSalt = requirements.map((req) => req.requirementId).join("_");
+              const requirementId = this.generateRequirementId(requirementType, contentsSalt);
               ret.push({
-                label: "Select 1 of the following",
-                requirementType: "Group",
+                label,
+                requirementId,
+                requirementType,
                 requirementCount: 1,
-                requirements: await this.ruleArrayToRequirements(rules),
+                requirements,
               });
             } else if (rules.length === 1) {
               ret.push(...(await this.ruleArrayToRequirements(rules)));
@@ -185,17 +231,28 @@ export class AuditParser {
           break;
         }
         case "Complete":
-        case "Incomplete":
+        case "Incomplete": {
+          const label = AuditParser.suppressLabelPolymorphism(rule.label);
+          const requirementType = "Marker";
+          const contentsSalt = label;
+          const requirementId = this.generateRequirementId(requirementType, contentsSalt);
           ret.push({
-            label: AuditParser.suppressLabelPolymorphism(rule.label),
-            requirementType: "Marker",
+            label,
+            requirementId,
+            requirementType,
           });
           break;
+        }
         case "Subset": {
+          const label = AuditParser.suppressLabelPolymorphism(rule.label);
+          const requirementType = "Group";
           const requirements = await this.ruleArrayToRequirements(rule.ruleArray);
+          const contentsSalt = requirements.map((req) => req.requirementId).join("_");
+          const requirementId = this.generateRequirementId(requirementType, contentsSalt);
           ret.push({
-            label: AuditParser.suppressLabelPolymorphism(rule.label),
-            requirementType: "Group",
+            label,
+            requirementId,
+            requirementType,
             requirementCount: Object.keys(requirements).length,
             requirements,
           });

apps/data-pipeline/degreeworks-scraper/src/index.ts

@@ -136,7 +136,7 @@ async function main() {
       .insert(collegeRequirement)
       .values(collegeBlocks)
       .onConflictDoUpdate({
-        target: collegeRequirement.requirements,
+        target: collegeRequirement.requirementsHash,
         set: conflictUpdateSetAllCols(collegeRequirement),
       })
       .returning({ id: collegeRequirement.id })

apps/data-pipeline/degreeworks-scraper/src/types.ts

@@ -3,7 +3,9 @@ import type { DegreeWorksProgramId } from "@packages/db/schema";
 /**
  * The base type for all `Rule` objects.
  */
-export type RuleBase = { label: string };
+export type RuleBase = {
+  label: string;
+};
 /**
  * A group of `numberOfRules` rules,
  * of which `numberOfGroups` must be satisfied

packages/db/migrations/0022_college_requirements_hash_unique.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "college_requirement" DROP CONSTRAINT "college_requirement_requirements_unique";--> statement-breakpoint
+ALTER TABLE "college_requirement" ADD COLUMN "requirements_hash" bigint GENERATED ALWAYS AS (jsonb_hash_extended(requirements, 0)) STORED;--> statement-breakpoint
+ALTER TABLE "college_requirement" ADD CONSTRAINT "college_requirement_requirements_hash_unique" UNIQUE("requirements_hash");