diff --git a/kanban-dashboard/package.json b/kanban-dashboard/package.json index 2784fbd..314ddd2 100644 --- a/kanban-dashboard/package.json +++ b/kanban-dashboard/package.json @@ -4,7 +4,7 @@ "private": true, "scripts": { "build": "next build", - "dev": "next dev", + "dev": "npx prisma db push && next dev", "postinstall": "prisma generate", "lint": "next lint", "start": "next start", @@ -12,7 +12,7 @@ }, "dependencies": { "@headlessui/react": "^1.7.15", - "@prisma/client": "^4.11.0", + "@prisma/client": "^4.14.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-alert-dialog": "^1.0.4", "@radix-ui/react-avatar": "^1.0.3", @@ -46,6 +46,7 @@ "clsx": "^1.2.1", "cmdk": "^0.2.0", "eslint-config-next": "^13.4.2", + "fetch-retry": "^5.0.6", "framer-motion": "^10.12.16", "lucide-react": "^0.233.0", "moment": "^2.29.4", diff --git a/kanban-dashboard/prisma/schema.prisma b/kanban-dashboard/prisma/schema.prisma index d73d42a..10715bf 100644 --- a/kanban-dashboard/prisma/schema.prisma +++ b/kanban-dashboard/prisma/schema.prisma @@ -1,164 +1,212 @@ // This is your Prisma schema file, // learn more about it in the docs: https://pris.ly/d/prisma-schema +// NOTE: models have either id or code fields +// Models whose primary key is not explicitly used by cal poly have an id field although it may be something aquired from cal poly such as +// the degree id which is determined from the degree page url path on the catalog site +// Models (such as subject) whose primary key is used by cal poly have a code field + generator client { provider = "prisma-client-js" } datasource db { - provider = "sqlite" - url = env("DATABASE_URL") -} - -model Department { - id Int @id @default(autoincrement()) - long_name String - short_name String @unique - Program Program[] - Instructor Instructor[] + provider = "mysql" + url = env("DATABASE_URL") + relationMode = "prisma" } model Course { - id Int @id @default(autoincrement()) - // TODO: make department table and convert this to a foreign key - department String - number Int + code String @id + subjectCode String + subject Subject @relation(fields: [subjectCode], references: [code]) + title String + number Int + description String @db.LongText // string because it can be range or single number and we only need // to display it either way - units String + minUnits Int + maxUnits Int // string of comma separated terms - terms String - - fulfillsRequirements Requirement[] - - prerequisites Prerequisite[] @relation("Prereq") - prerequisiteFor Prerequisite[] @relation("PrereqFor") - sections Section[] - - @@unique([department, number]) -} - -model Prerequisite { - id Int @id @default(autoincrement()) - requiredCourse Course @relation(name: "Prereq", fields: [requiredCourseId], references: [id]) - forCourse Course @relation(name: "PrereqFor", fields: [forCourseId], references: [id]) - forCourseId Int - requiredCourseId Int - - @@unique([requiredCourseId, forCourseId]) -} - -model Requirement { - // TODO: figure out how to make unique ids based on the requirement - id Int @id @default(autoincrement()) - course Course @relation(fields: [courseId], references: [id]) - program Program @relation(fields: [programId], references: [id]) - // Ge, major, tech elective, etc. - // TODO: make this an enum - kind String - - courseId Int - programId Int -} - -model Program { - id Int @id @default(autoincrement()) - department Department @relation(fields: [departmentId], references: [id]) - // BS, BA, etc. - type String - requirements Requirement[] - departmentId Int - catolog Catolog @relation(fields: [catologId], references: [id]) - catologId Int - User User[] - - @@unique([departmentId, type]) -} - -model Catolog { - id Int @id @default(autoincrement()) - startYear Int - endYear Int - programs Program[] -} - -model Instructor { - id Int @id @default(autoincrement()) - name String @unique - department Department @relation(fields: [departmentId], references: [id]) - departmentId Int - officeHours String - Section Section[] -} - -// TODO: make sure this is the best way we could store this -model Meeting { - id Int @id @default(autoincrement()) - startTime DateTime - endTime DateTime - section Section? @relation(fields: [sectionClassNumber], references: [classNumber]) - sectionClassNumber Int? - location String -} - -model Availability { - id Int @id @default(autoincrement()) - seats Int - capacity Int - waitlist Int - waitlistCapacity Int - time DateTime @default(now()) - section Section @relation(fields: [sectionClassNumber], references: [classNumber]) - sectionClassNumber Int -} - -model Section { - classNumber Int @id - course Course @relation(fields: [courseId], references: [id]) - courseId Int - quarter Quarter @relation(fields: [quarterId], references: [id]) - quarterId Int - status String - session String - intstructionMode String - career String - // TODO: make sure this aligns with a quarters start and end - // dates MeetingTime - instructor Instructor @relation(fields: [instructorId], references: [id]) - instructorId Int - meetings Meeting[] - grading String - location String - campus String - classComponents String - availability Availability[] -} - -model Quarter { - id Int @id @default(autoincrement()) - term String - year Int - classes Section[] - user User? @relation(fields: [userCalPolyUsername], references: [calPolyUsername]) - userCalPolyUsername String? - UserFlowchart UserFlowchart? @relation(fields: [userFlowchartId], references: [id]) - userFlowchartId Int? - startDate DateTime - endDate DateTime + // TODO: make this a bitmask + termsTypicallyOffered String + + fullfillsGERequirements GEAreaFullfillmentCourse[] + fullfillsCourseRequirements CourseRequirement[] + USCP Boolean @default(false) + GWR Boolean @default(false) + + @@index([code, USCP, GWR]) +} + +model Subject { + code String @id + name String + courses Course[] +} + +// TODO: make this a type +model RecommendedCompletion { + year Int + term Term + GERequirement GERequirement[] + CourseRequirement CourseRequirement[] + + @@id([year, term]) +} + +enum Term { + F + W + SP + SU +} + +enum GEArea { + A + B + C + D + E + F + ELECTIVE +} + +enum GESubArea { + LowerDivision + UpperDivision + LowerDivisionElective + UpperDivisionElective + F + E + Elective + A1 + A2 + A3 + A4 + B1 + B2 + B3 + B4 + C1 + C2 + C3 + C4 + D1 + D2 + D3 + D4 +} + +model GERequirement { + area GEArea + subArea GESubArea + units Int + recommendedCompletion RecommendedCompletion? @relation(fields: [recommendedCompletionYear, recommendedCompletionTerm], references: [year, term]) + recommendedCompletionYear Int? + recommendedCompletionTerm Term? + degree Degree @relation(fields: [degreeId], references: [id]) + degreeId String + // TODO: connect to course requirement + + @@id([area, subArea, degreeId]) +} + +enum RequirementKind { + major + support + elective +} + +model CourseRequirement { + course Course @relation(fields: [courseCode], references: [code]) + courseCode String + kind RequirementKind + recommendedCompletion RecommendedCompletion? @relation(fields: [recommendedCompletionYear, recommendedCompletionTerm], references: [year, term]) + recommendedCompletionYear Int? + recommendedCompletionTerm Term? + RequirementGroup CourseRequirementGroup @relation(fields: [requirementGroupId], references: [id]) + requirementGroupId Int + + @@id([courseCode, requirementGroupId]) +} + +enum RequirementGroupKind { + or + and +} + +// represents a series (such as PHYS 1,2,3) or a list of choices (such as an elective) +model CourseRequirementGroup { + id Int @id @default(autoincrement()) + groupKind RequirementGroupKind + coursesKind RequirementKind + // e.x. Life Science Support where coursesKind is elective + courseKindInfo String? + courses CourseRequirement[] + // child and parent are swapped in the relation name because it is referring to the reference to itself + // i.e. the reference to this groups parent views this group as the child + parentGroup CourseRequirementGroup? @relation("toParent", fields: [parentId], references: [id], onDelete: NoAction, onUpdate: NoAction) + parentId Int? + childGroups CourseRequirementGroup[] @relation("toParent") + // should be total units for and group or total required for or group + unitsOf Int? + degree Degree? @relation(fields: [degreeId], references: [id]) + concentration Concentration? @relation(fields: [concentrationId], references: [id]) + degreeId String? + concentrationId String? + + @@index([id]) +} + +enum RequirementsKind { + degree + concentration +} + +model Concentration { + id String @id + name String + courseRequirements CourseRequirementGroup[] + degree Degree @relation(fields: [degreeId], references: [id]) + degreeId String + + @@unique([degreeId, name]) +} + +model Degree { + id String @id + name String + link String + // TODO: make this an enum of BS, BA, etc. + kind String + requirements CourseRequirementGroup[] + concentrations Concentration[] + GERequirement GERequirement[] + + @@unique([name, kind]) + @@index([id]) +} + +model GEAreaFullfillmentCourse { + id Int @id @default(autoincrement()) + area GEArea + subArea GESubArea + courseId String + course Course @relation(fields: [courseId], references: [code]) + + @@unique([area, subArea, courseId]) } model User { calPolyUsername String @id - program Program @relation(fields: [programId], references: [id]) year Int - quarters Quarter[] - programId Int flowcharts UserFlowchart[] } model UserFlowchart { - id Int @id @default(autoincrement()) - user User @relation(fields: [userCalPolyUsername], references: [calPolyUsername]) - quarters Quarter[] + id Int @id @default(autoincrement()) + user User @relation(fields: [userCalPolyUsername], references: [calPolyUsername]) + startYear Int userCalPolyUsername String } diff --git a/kanban-dashboard/src/components/buttons/index.tsx b/kanban-dashboard/src/components/buttons/index.tsx new file mode 100644 index 0000000..1712a3f --- /dev/null +++ b/kanban-dashboard/src/components/buttons/index.tsx @@ -0,0 +1,103 @@ +import React, { type ReactNode, type HTMLAttributes } from "react"; +import IconButton, { + type IconButtonProps as MuiIconButtonProps, +} from "@material-ui/core/IconButton"; +import AddIcon from "@material-ui/icons/Add"; +import MoreHorizIcon from "@material-ui/icons/MoreHoriz"; +import DeleteIcon from "@material-ui/icons/DeleteForever"; +import EditIcon from "@material-ui/icons/Edit"; +import CloseIcon from "@material-ui/icons/Close"; +import CheckIcon from "@material-ui/icons/Check"; +import { type OverridableComponent } from "@material-ui/core/OverridableComponent"; +import { type SvgIconTypeMap } from "@material-ui/core/SvgIcon/SvgIcon"; +import Grid from "@material-ui/core/Grid"; +import Button from "@material-ui/core/Button"; + +import { useStyles } from "./styles"; + +export type IconButtonProps = MuiIconButtonProps & { + iconProps?: OverridableComponent; +}; + +export const AddButton = ({ iconProps, ...props }: IconButtonProps) => ( + + + +); + +export const DeleteButton = ({ iconProps, ...props }: IconButtonProps) => ( + + + +); + +export const OptionsButton = ({ iconProps, ...props }: IconButtonProps) => ( + + + +); + +export const EditButton = ({ iconProps, ...props }: IconButtonProps) => ( + + + +); + +export const CloseButton = ({ iconProps, ...props }: IconButtonProps) => ( + + + +); + +export const CheckButton = ({ iconProps, ...props }: IconButtonProps) => ( + + + +); + +export interface TextButtonProps extends HTMLAttributes { + children: ReactNode; +} +export const TextButton = ({ children, ...props }: TextButtonProps) => { + const classNames = useStyles(); + return ( + + ); +}; + +export interface ConfirmationButtonsProps { + onCancel: () => void; + onConfirm: () => void; + cancelLabel?: string; + confirmLabel?: string; + confirmColor?: "primary" | "secondary"; +} + +export function ConfirmationButtons({ + onCancel, + onConfirm, + cancelLabel = "Cancel", + confirmLabel = "Done", + confirmColor = "primary", +}: ConfirmationButtonsProps) { + const classes = useStyles(); + + return ( +
+ + + + + + + + +
+ ); +} diff --git a/kanban-dashboard/src/dashboard/CourseCard.tsx b/kanban-dashboard/src/dashboard/CourseCard.tsx index eabaf58..a9a8ceb 100644 --- a/kanban-dashboard/src/dashboard/CourseCard.tsx +++ b/kanban-dashboard/src/dashboard/CourseCard.tsx @@ -78,7 +78,7 @@ export default function CourseCard({ requirement, index, collapsed }: Props) { }; const [completeStatus, setCompleteStatus] = useState("incomplete"); - const { data: currentQuarter } = api.currentQuarterId.useQuery(undefined, { + const { data: currentQuarter } = api.quarters.current.useQuery(undefined, { staleTime: Infinity, // don't refresh until the user refreshes }); diff --git a/kanban-dashboard/src/dashboard/CourseEditorForm.tsx b/kanban-dashboard/src/dashboard/CourseEditorForm.tsx new file mode 100644 index 0000000..4a7af59 --- /dev/null +++ b/kanban-dashboard/src/dashboard/CourseEditorForm.tsx @@ -0,0 +1,55 @@ +import React, { useState } from "react"; +import TextField from "@material-ui/core/TextField"; + +import { ConfirmationButtons } from "../components/buttons"; + +export interface Props { + title?: string; + description?: string; + onSubmit: (title: string, description: string) => void; + onCancel: () => void; +} + +export default function CourseEditorForm({ + onSubmit, + onCancel, + title: initialTitle = "", + description: initialDesc = "", +}: Props) { + const [title, setTitle] = useState(initialTitle); + const [description, setDesc] = useState(initialDesc); + + const handleSubmit = () => { + if (title && description) { + onSubmit(title, description); + setTitle(""); + } + }; + + const handleCancel = () => { + onCancel(); + setTitle(""); + }; + + return ( +
+ setTitle(e.target.value)} + /> + setDesc(e.target.value)} + /> + + +
+ ); +} diff --git a/kanban-dashboard/src/dashboard/Flowchart.tsx b/kanban-dashboard/src/dashboard/Flowchart.tsx index b9cfe36..25657ac 100644 --- a/kanban-dashboard/src/dashboard/Flowchart.tsx +++ b/kanban-dashboard/src/dashboard/Flowchart.tsx @@ -15,7 +15,7 @@ export default function Flowchart() { const { startYear, selectedRequirements, setSelectedRequirements } = React.useContext(FlowchartState); const moveRequirement = useMoveRequirement(); - const quartersQuery = api.quarters.useQuery({ startYear }); + const quartersQuery = api.quarters.all.useQuery({ startYear }); const classNames = useBoardStyles(); @@ -50,7 +50,8 @@ export default function Flowchart() { return; } if (source && destination) { - const requirementId = parseInt(draggableId); + // moveRequirement: (requirementId: number, quarterId: number) => void; + const requirementId = draggableId; const quarterId = parseInt(destination.droppableId); moveRequirement(requirementId, quarterId); } diff --git a/kanban-dashboard/src/dashboard/FlowchartSelectingMenu.tsx b/kanban-dashboard/src/dashboard/FlowchartSelectingMenu.tsx new file mode 100644 index 0000000..9caba8b --- /dev/null +++ b/kanban-dashboard/src/dashboard/FlowchartSelectingMenu.tsx @@ -0,0 +1,260 @@ +import React, { type MouseEventHandler, useState } from "react"; +import { makeStyles } from "@material-ui/core/styles"; +import List from "@material-ui/core/List"; +import ListItem from "@material-ui/core/ListItem"; +import ListItemText from "@material-ui/core/ListItemText"; +import ListItemSecondaryAction from "@material-ui/core/ListItemSecondaryAction"; +import IconButton from "@material-ui/core/IconButton"; +import DeleteIcon from "@material-ui/icons/Delete"; +import AddIcon from "@material-ui/icons/Add"; +import StarIcon from "@material-ui/icons/Star"; +import DuplicateIcon from "@material-ui/icons/FileCopy"; +import FavoriteIcon from "@material-ui/icons/Favorite"; +import MoreVertIcon from "@material-ui/icons/MoreVert"; +import Button from "@material-ui/core/Button"; +import Typography from "@material-ui/core/Typography"; +import Dialog from "@material-ui/core/Dialog"; +import DialogActions from "@material-ui/core/DialogActions"; +import DialogContent from "@material-ui/core/DialogContent"; +import DialogTitle from "@material-ui/core/DialogTitle"; +import TextField from "@material-ui/core/TextField"; +import MenuItem from "@material-ui/core/MenuItem"; +import Menu from "@material-ui/core/Menu"; +import Fade from "@material-ui/core/Fade"; +import { yellow } from "@material-ui/core/colors"; +import { DialogContentText } from "@material-ui/core"; + +const useStyles = makeStyles((theme) => ({ + root: { + width: "300px", + display: "flex", + flexDirection: "column", + height: "100vh", + }, + title: { + margin: theme.spacing(2), + }, + listItem: { + padding: theme.spacing(1), + }, + createButton: { + margin: theme.spacing(2), + marginTop: "auto", + }, + selectedItem: { + border: "2px solid", + borderColor: theme.palette.primary.main, + transition: "border-color 0.3s ease", + "&:hover": { + borderColor: theme.palette.secondary.main, + }, + }, + formControl: { + margin: theme.spacing(1), + minWidth: 120, + }, + starIcon: { + color: yellow[800], + }, + actionMenu: { + marginLeft: theme.spacing(2), + }, + itemText: { + flex: "initial", + }, +})); + +export default function FlowchartSelectingMenu() { + const classes = useStyles(); + // Replace this with your own state management logic + const [flowcharts, setFlowcharts] = useState([ + "Flowchart 1", + "Flowchart 2", + "Flowchart 3", + ]); + const [openCreateModal, setOpenCreateModal] = useState(false); + const [openDeleteModal, setOpenDeleteModal] = useState(false); + const [newFlowchartName, setNewFlowchartName] = useState(""); + const [newFlowchartConcentration, setNewFlowchartConcentration] = + useState(""); + const [selectedFlowchart, setSelectedFlowchart] = useState(""); + const [favoritedFlowcharts, setFavoritedFlowcharts] = useState( + [] as string[] + ); + const [anchorEl, setAnchorEl] = useState(null as HTMLAnchorElement | null); + + const handleClick: MouseEventHandler = (event: Event) => { + setAnchorEl(event.currentTarget as HTMLAnchorElement); + }; + + const handleClose = () => { + setAnchorEl(null); + }; + const createFlowchart = () => { + // Add flowchart creation logic here + console.log( + "Create flowchart: ", + newFlowchartName, + newFlowchartConcentration + ); + setFlowcharts([...flowcharts, newFlowchartName]); + setOpenCreateModal(false); + }; + + const deleteFlowchart = () => { + // Add flowchart deletion logic here + console.log("Delete flowchart: ", selectedFlowchart); + setFlowcharts(flowcharts.filter((f) => f !== selectedFlowchart)); + setOpenDeleteModal(false); + }; + + const duplicateFlowchart = (flowchart: string) => { + // Add your logic here + console.log(`Duplicate flowchart: ${flowchart}`); + }; + + const favoriteFlowchart = (flowchart: string) => { + // Add your logic here + console.log(`Favorite flowchart: ${flowchart}`); + if (!favoritedFlowcharts.includes(flowchart)) { + setFavoritedFlowcharts([...favoritedFlowcharts, flowchart]); + } else { + setFavoritedFlowcharts( + favoritedFlowcharts.filter((fav) => fav !== flowchart) + ); + } + }; + + return ( +
+ + Saved Flowcharts + + + {flowcharts.map((flowchart, index) => ( + setSelectedFlowchart(flowchart)} + > + + {favoritedFlowcharts.includes(flowchart) && ( + + )} + + + + + + { + favoriteFlowchart(flowchart); + handleClose(); + }} + > + + + + Favorite + + { + duplicateFlowchart(flowchart); + handleClose(); + }} + > + + + + Duplicate + + { + setOpenDeleteModal(true); + setSelectedFlowchart(flowchart); + handleClose(); + }} + > + + + + Delete + + + + + ))} + + + + setOpenCreateModal(false)}> + Create New Flowchart + + setNewFlowchartName(e.target.value)} + /> + setNewFlowchartConcentration(e.target.value)} + > + {/* Add your concentration options here */} + Concentration 1 + Concentration 2 + + + + + + + + + + setOpenDeleteModal(false)}> + Delete Flowchart + + + Are you sure you want to delete the flowchart "{selectedFlowchart}"? + + + + + + + +
+ ); +} diff --git a/kanban-dashboard/src/dashboard/Menubar.tsx b/kanban-dashboard/src/dashboard/Menubar.tsx index 7378215..1aa68df 100644 --- a/kanban-dashboard/src/dashboard/Menubar.tsx +++ b/kanban-dashboard/src/dashboard/Menubar.tsx @@ -95,7 +95,7 @@ export default function Menubar({}: MenubarProps) { const handleClose = () => { setAnchorEl(null); }; - const degreesQuery = api.degrees.useQuery(undefined, { + const degreesQuery = api.degrees.all.useQuery(undefined, { staleTime: Infinity, // don't refresh until the user refreshes }); @@ -109,7 +109,7 @@ export default function Menubar({}: MenubarProps) { // TODO: create record of string id: Degree for faster lookup if (degree.name === name) { console.log("fetching degree requirements for:", degree); - trpcClient.degreeRequirements.prefetch({ degree, startYear }); + trpcClient.degrees.requirements.prefetch({ degree, startYear }); setDegree(degree); setSelectedDegreeDisplayName(degree.name); break; @@ -295,7 +295,7 @@ export function FlowchartSwitcher({ className }: TeamSwitcherProps) { // TODO: create record of string id: Degree for faster lookup if (degree.name === name) { console.log("fetching degree requirements for:", degree); - trpcClient.degreeRequirements.prefetch({ degree, startYear }); + trpcClient.degrees.requirements.prefetch({ degreeId: degree.id, startYear }); setDegree(degree); break; } @@ -304,7 +304,7 @@ export function FlowchartSwitcher({ className }: TeamSwitcherProps) { const [value, setValue] = React.useState(""); const [openDegree, setOpenDegree] = React.useState(false); - const degreesQuery = api.degrees.useQuery(undefined, { + const degreesQuery = api.degrees.all.useQuery(undefined, { staleTime: Infinity, // don't refresh until the user refreshes }); diff --git a/kanban-dashboard/src/dashboard/state.tsx b/kanban-dashboard/src/dashboard/state.tsx index af58789..f4e7da1 100644 --- a/kanban-dashboard/src/dashboard/state.tsx +++ b/kanban-dashboard/src/dashboard/state.tsx @@ -8,16 +8,18 @@ import React, { type Dispatch, } from "react"; -import { type Requirement, type Degree } from "~/server/api/root"; +import { type Requirement, type Degree} from "~/server/api/root"; import { api } from "~/utils/api"; type Setter = React.Dispatch>; +export type PartialDegree = Pick; + type FlowchartStateType = { requirements: Requirement[]; setRequirements: Setter; - degree: Degree | null; - setDegree: Setter; + degree: PartialDegree | null; + setDegree: Setter; startYear: number; setStartYear: Setter; selectedRequirements: number[]; @@ -35,7 +37,7 @@ export const FlowchartStateProvider: FC<{ children: React.ReactNode }> = ({ // TODO: remove StoreProvider and replace with trpc quarters query in flowchart // TODO: merge dashboard and flowhcart components // TODO: make moveRequirement a backend mutation - const [degree, setDegree] = useState(null); + const [degree, setDegree] = useState(null); const [requirements, setRequirements] = useState([]); const [selectedRequirements, setSelectedRequirements] = useState( @@ -47,8 +49,8 @@ export const FlowchartStateProvider: FC<{ children: React.ReactNode }> = ({ useEffect(() => { console.log("updating requirements!"); }, [requirements]); - const _requirementsQuery = api.degreeRequirements.useQuery( - { degree, startYear }, + const _requirementsQuery = api.degrees.requirements.useQuery( + { degreeId: degree?.id ?? null, startYear }, { enabled: false, onSuccess: (data) => setRequirements(data) } ); const flowchartContext = { @@ -74,9 +76,9 @@ export const FlowchartStateProvider: FC<{ children: React.ReactNode }> = ({ export const useMoveRequirement = () => { const { degree, startYear } = useContext(FlowchartState); const trpcClient = api.useContext(); - const moveRequirement = (requirementId: number, quarterId: number) => { - trpcClient.degreeRequirements.setData( - { degree, startYear }, + const moveRequirement = (requirementId: string, quarterId: number) => { + trpcClient.degrees.requirements.setData( + { degreeId: degree?.id ?? null, startYear }, (requirements) => { if (!requirements) { console.error("No requirements found for degree:", degree); diff --git a/kanban-dashboard/src/pages/onboarding.tsx b/kanban-dashboard/src/pages/onboarding.tsx index 924ce12..5a02258 100644 --- a/kanban-dashboard/src/pages/onboarding.tsx +++ b/kanban-dashboard/src/pages/onboarding.tsx @@ -49,7 +49,7 @@ export default function OnboardingPage() { const [open, setOpen] = useState(false); const [value, setValue] = useState(""); const router = useRouter(); - const degreesQuery = api.degrees.useQuery(undefined, { + const degreesQuery = api.degrees.all.useQuery(undefined, { staleTime: Infinity, // don't refresh until the user refreshes }); const [selectedClasses, setSelectedClasses] = useState([]); @@ -73,7 +73,7 @@ export default function OnboardingPage() { // TODO: create record of string id: Degree for faster lookup if (degree.name === name) { console.log("fetching degree requirements for:", degree); - trpcClient.degreeRequirements.prefetch({ degree, startYear }); + trpcClient.degrees.requirements.prefetch({ degreeId: degree.id, startYear }); setDegree(degree); break; } diff --git a/kanban-dashboard/src/scraping/catalog.ts b/kanban-dashboard/src/scraping/catalog.ts index 7ac4cdf..dcc914e 100644 --- a/kanban-dashboard/src/scraping/catalog.ts +++ b/kanban-dashboard/src/scraping/catalog.ts @@ -1,10 +1,16 @@ +import { PrismaClient, Requirement } from "@prisma/client"; import assert from "assert"; +import { constants } from "buffer"; import * as cheerio from "cheerio/lib/slim"; +import { CheerioAPI } from "cheerio/lib/slim"; +import fetchRetry from "fetch-retry"; +const fetch = fetchRetry(global.fetch); import { z } from "zod"; const DOMAIN = "https://catalog.calpoly.edu"; export const DepartmentSchema = z.object({ + id: z.string(), name: z.string(), link: z.string().url(), }); @@ -34,10 +40,15 @@ export const scrapeCollegesAndDepartments = async () => { .next("ul") .find("li>a") .each((_i, elem) => { + const link = DOMAIN + $(elem).attr("href"); + const id = link + .split("/") + .findLast((s) => s.length > 0 && !s.startsWith("#")); departmentList.push( DepartmentSchema.parse({ name: $(elem).text().trim(), - link: DOMAIN + $(elem).attr("href"), + link, + id, }) ); }); @@ -52,27 +63,6 @@ export const scrapeCollegesAndDepartments = async () => { return colleges; }; -export const SubjectCodeSchema = z.string().regex(/[A-Z]+/); -export const SubjectSchema = z.object({ - subject: z.string(), - code: SubjectCodeSchema, -}); -export type Subject = z.infer; - -export const scrapeSubjects = async () => { - const subjectRE = /(.+)\s+\(([A-Z ]+)\)/; - - const URL = "https://catalog.calpoly.edu/coursesaz/"; - const $ = cheerio.load(await fetch(URL).then((res) => res.text())); - const subjects: Subject[] = []; - $("a.sitemaplink").each((_i, elem) => { - const txt = $(elem).text(); - const [_matched, subject, code] = txt.match(subjectRE) ?? []; - subjects.push(SubjectSchema.parse({ subject, code })); - }); - return subjects; -}; - export const BACHELOR_DEGREE_KINDS = [ "BA", "BFA", @@ -97,29 +87,37 @@ export const RequirementTypeSchema = z.enum([ ]); export type RequirementType = z.infer; -export const RequirementCourseCodeSchema = z.string(); +const stripCrosslistInfoFromCourseCode = (courseCode: string) => { + const match = courseCode.match(/([A-Z]+)(?:\/[A-Z]+)*\s+(\d+)(?:\s+\d+)*/); + if (!match) + throw new Error( + `course code: ${courseCode} did not match the course code regex` + ); + const [_, code, num] = match; + courseCode = code + " " + num; + if (!courseCode.match(/^[A-Z]+\s\d+$/)) + throw new Error( + `resulting course code: ${courseCode} from stripping crosslist info is not a valid course code` + ); + return courseCode; +}; -export const RequirementOneOfSchema: z.ZodType = z.object({ - kind: z.literal("oneof"), - oneof: z.array( - RequirementCourseCodeSchema.or(z.lazy(() => RequirementAllOfSchema)) - ), -}); -export const RequirementAllOfSchema = z.object({ - kind: z.literal("allof"), - allof: z.array(RequirementCourseCodeSchema.or(RequirementOneOfSchema)), -}); -export const RequirementSchema = z.object({ - kind: RequirementTypeSchema.or(z.string()), - fulfilledBy: z.array( - RequirementCourseCodeSchema.or(RequirementAllOfSchema).or( - RequirementOneOfSchema - ) - ), -}); +// NOTE: it is expected that information from crosslistings can be parsed in subject course +// lists and handled properly when using degree course requirements +const CourseCodeSchema = z.string().transform(stripCrosslistInfoFromCourseCode); // .regex(/^[A-Z]+\s\d+$/); + +export type CourseCode = z.infer; + +// TODO: decide whether this is usefull +// export const RequirementSchema = z.object({ +// kind: RequirementTypeSchema.or(z.string()), +// fulfilledBy: +// +// }); export const RequirementCourseSchema = z.object({ - code: z.string(), + kind: RequirementTypeSchema.or(z.string()).nullable(), + code: CourseCodeSchema, units: z.number(), title: z.string(), }); @@ -130,242 +128,513 @@ export const DegreeSchema = z.object({ name: z.string(), kind: z.enum(BACHELOR_DEGREE_KINDS), link: z.string().url(), - id: z.number().nonnegative(), + id: z.string(), + // departmentId: z.string(), }); export type Degree = z.infer; -export const DegreeWithRequirementsSchema = DegreeSchema.extend({ - requirements: z.array(RequirementSchema), - courses: z.map(z.string(), RequirementCourseSchema), + +const GEAreaCodeRE = /[ABCDEF][1234]/; +const GEDivisionCodeRE = /(Upper|Lower)-Division [ABCDEF]( Elective)?/; +const GeAreaRE = /Area [ABCDEF]( Elective)?/; + +const GEDivisionSubAreaRE = /(Upper|Lower)-Division( Elective)?/; +const GESubAreaVariantSchema = z.union([ + z.string().regex(GEAreaCodeRE), + z.string().regex(GEDivisionCodeRE), + z.literal("Elective"), +]); + +const GEAreasEnumSchema = z.enum(["A", "B", "C", "D", "E", "F", "ELECTIVE"]); + +type GEArea = z.infer; + +export const GeRequirementSchema = z.object({ + area: GEAreasEnumSchema, + subarea: GESubAreaVariantSchema, + units: z.number().nonnegative(), + // TODO: constraints (C1 or C2) / fullfilled by (B3 = lab w/ B1 or B2 course) }); -export type DegreeWithRequirements = z.infer< - typeof DegreeWithRequirementsSchema + +export type GeRequirement = z.infer; + +export type CourseRequirement = + | { kind: "or"; courses: CourseRequirement[]; units: number } + | { kind: "and"; courses: CourseRequirement[]; units: number } + | { kind: "course"; course: CourseCode; units: number }; + +const CourseRequirementSchema: z.ZodType = z.lazy(() => + z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("or"), + courses: z.array(CourseRequirementSchema), + units: z.number(), + }), + z.object({ + kind: z.literal("and"), + courses: z.array(CourseRequirementSchema), + units: z.number(), + }), + z.object({ + kind: z.literal("course"), + course: CourseCodeSchema, + units: z.number(), + }), + ]) +); + +const DegreeRequirementSectionSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("major"), + courses: z.array(CourseRequirementSchema), + }), + z.object({ + kind: z.literal("elective"), + electiveKind: z.string(), + courses: z.array(CourseRequirementSchema), + }), + z.object({ + kind: z.literal("support"), + supportKind: z.string(), + courses: z.array(CourseRequirementSchema), + }), + z.object({ kind: z.literal("ge") }), +]); +export type DegreeRequirementSection = z.infer< + typeof DegreeRequirementSectionSchema >; -export const scrapeDegreeRequirements = async ( - degreeWOReqs: Degree -): Promise => { - // select from the following - // If we're in a sftf block the courses are in one of the following formats: - // a) - // [course] - // or [course] - // or [course] - // ... until row not starting with "or" - // - // b) - // [course] - // or - // [course] - // or - // ... until there are two [course] rows not separated by "or" row - // - // c) - // [course] - // [course] - // [course] - // ... until another sftf block or areaheader block (because y tf not) - // - // To differentiate these cases the in_sftf flag signifies if we're in a - // stft block and the sftf_sep is either "or" (for cases a or b) or null (for case c) - // - // Note that or conditions can occur OUTSIDE of a sftf block in which - // case we get our FOURTH and (hopefullly) final option: - // d) - // [course] - // or [course] - const SFTF_RE = /\s*Select( one sequence)? from the following.*/; +export type DegreeRequirementSectionInfo = + | { kind: null; header: string } + | { kind: RequirementType } + | { kind: "elective"; electiveKind: string } + | { kind: "support"; supportKind: string | null }; - const degree: DegreeWithRequirements = { - ...degreeWOReqs, - requirements: [], - courses: new Map(), - }; - const $ = cheerio.load(await fetch(degree.link).then((res) => res.text())); +export const DegreeRequirementsSchema = z.lazy(() => + z.object({ + courses: z.array(DegreeRequirementSectionSchema), + ge: z.array(GeRequirementSchema), + concentrations: z.array( + z.object({ + id: z.string(), + name: z.string(), + link: z.string(), + courses: z.array(DegreeRequirementSectionSchema), + }) + ), + }) +); - const tables = $("table.sc_courselist"); +export type DegreeRequirements = z.infer; - const requirements = degree.requirements; - const courses = degree.courses; +const parseDegreeRequirementSectionHeader = ( + $: CheerioAPI, + headerElem: cheerio.Element +): DegreeRequirementSectionInfo => { + let kind: RequirementType | null = null; + let sectionTitle = $(headerElem) + .find("span") + .text() + .replace(/\(.*\)$/, "") + .trim(); + let electiveKind; + let supportKind = null; + if (sectionTitle.includes("MAJOR COURSES")) { + kind = "major"; + } else if (!!sectionTitle.match(/^[\w\/\s\-]+ electives?$/i)) { + // FIXME: include the type of elective + kind = "elective"; + electiveKind = sectionTitle.replace(/electives?/i, "").trim(); + } else if (sectionTitle.includes("GENERAL EDUCATION")) { + kind = "ge"; + } else if (sectionTitle.includes("SUPPORT COURSES")) { + kind = "support"; + } else { + kind = "support"; + supportKind = sectionTitle; + } + if (electiveKind) { + if (kind !== "elective") + throw new Error("found elective without elective kind: " + sectionTitle); + return { kind, electiveKind }; + } + if (kind === "support") { + return { kind, supportKind }; + } + if (!kind) { + return { kind, header: sectionTitle }; + } + return { kind }; +}; - tables.each((_ti, table) => { - const table_desc = $(table).prevAll().filter("h2").first().text(); - if (table_desc !== "Degree Requirements and Curriculum") { - console.warn("Parsing table of unrecognized kind:", table_desc); +// select from the following blocks: +// If we're in a sftf block the courses are in one of the following formats: +// a) +// [course] +// or [course] +// or [course] +// ... until row not starting with "or" +// +// b) +// [course] +// or +// [course] +// or +// ... until there are two [course] rows not separated by "or" row +// +// c) +// [course] +// [course] +// [course] +// ... until another sftf block or areaheader block (because y tf not) +// +// To differentiate these cases the in_sftf flag signifies if we're in a +// stft block and the sftf_sep is either "or" (for cases a or b) or null (for case c) +// +// Note that or conditions can occur OUTSIDE of a sftf block in which +// case we get our FOURTH and (hopefullly) final option: +// d) +// [course] +// or [course] +/// mapping of course code to meta information about the course (such as whether it is a support/major/etc) + +export const parseCourseRequirementsTable = ( + $: cheerio.CheerioAPI, + table: cheerio.Element, + ctx: { kind: "degree" | "concentration"; name: string; link: string } +) => { + const courses = new Map(); + const parsedClass = "parsed"; + + const parseRowUnits = (row: cheerio.Element) => { + // TODO: error checking + let unitsStr = $(row).find("td.hourscol").text(); + let units = unitsStr == "" ? 0 : parseInt(unitsStr); + return units; + }; + const markRowAsParsed = ( + rowOrIndex: number | cheerio.Element, + row?: cheerio.Element + ) => { + if (typeof rowOrIndex !== "number") { + row = rowOrIndex; + } + if (!row) throw new Error("trying to mark undefined row as parsed"); + $(row).addClass(parsedClass); + }; + const parseCourseRow = (row: cheerio.Element): CourseRequirement => { + let codeCol = $(row).find("td.codecol"); + if (codeCol.length !== 1) { + throw new Error( + `tried to parse row: ${$(row).text()} without codecol ${ctx.link}` + ); } - let cur_section: string | null = null; - // within a select from the following block - // see comment at top of file which explains these flags - let in_sftf = false; - let sftf_sep = null; - let prev_was_or = false; - const rows = $(table).find("tr"); - for (let i = 0; i < rows.length; i++) { - const tr = rows[i]; - // TODO: extracting 1 footnote tags - - if ($(tr).hasClass("areaheader")) { - in_sftf = false; - prev_was_or = false; - sftf_sep = null; - // new section - const cur_section_title = $(tr).text().trim(); - if (cur_section_title.includes("MAJOR COURSES")) { - cur_section = "major"; - } else if (cur_section_title.includes("Electives")) { - // FIXME: include the type of elective - cur_section = "elective"; - } else if (cur_section_title.includes("GENERAL EDUCATION")) { - cur_section = "ge"; - } else if (cur_section_title.includes("SUPPORT COURSES")) { - cur_section = "support"; - } else { - cur_section = cur_section_title as RequirementType; - // console.log("Unrecognized section kind for", cur_section_title); - } - if (!cur_section) { - console.error("no text in header:", $(tr)); - break; + let units = parseRowUnits(row); + + markRowAsParsed(row); + + let code = $(codeCol).find("a.code"); + let course; + // TODO: check for orclass + if (code.length > 1 && $(row).is(":has(span:contains(&))")) { + // and group + // TODO: check to make sure its an and row + let courses = $(code) + .map((_, c) => ({ kind: "course", course: $(c).text(), units: 0 })) + .get(); + course = { kind: "and", courses, units }; + } else if (codeCol.hasClass("orclass")) { + course = { + kind: "or", + courses: [{ kind: "course", course: $(code).text(), units: 0 }], + units, + }; + } else { + course = { kind: "course", course: $(code).text(), units }; + } + return CourseRequirementSchema.parse(course); + }; + const parseListOfCourseRows = (rows: cheerio.Cheerio) => { + let courses: CourseRequirement[] = []; + rows.each((_, row) => { + if ($(row).is(":has(span.courselistcomment)")) { + // TODO: parsing comments (especially unit counts) + console.log("comment:", $(row).text()); + return; + } + let course = parseCourseRow(row); + if (course.kind === "or") { + let prevCourse = courses.pop(); + if (!prevCourse) { + console.error( + "found or row:", + $(row).text(), + "with no preceeding row" + ); + return; } - } else if ($(tr).find("span.courselistcomment").length > 0) { - const comment = $(tr).find("span.courselistcomment").text().trim(); - if (comment.match(SFTF_RE)) { - in_sftf = true; - requirements.push({ or: [] }); - sftf_sep = null; - } else if (comment === "or") { - if (!in_sftf) { - console.warn( - "Found an \"or\" row outside of 'Select from the following' block. There's even more variations :/", - "in", - degree.name, - "(", - degree.link, - ")", - comment - ); + switch (prevCourse.kind) { + case "or": + prevCourse.courses.push(...course.courses); + course = prevCourse; + break; + case "course": + case "and": + course.units = prevCourse.units; + prevCourse.units = 0; + course.courses.push(prevCourse); break; - } - sftf_sep = "or"; - prev_was_or = true; - } else { - console.error("unrecognized comment:", comment); } - } else { - const course_elem = $(tr).find("td.codecol a[title]"); - let course: any; // TODO: type this - const is_and = course_elem.length > 1; - if (is_and) { - // course is actually courses plural - // make sure it's actually an '&' of the courses - if (!$(tr).find("span.blockindent").text().includes("&")) { - console.error( - "multiple elements found but no & to be found in:", - $(tr) - ); - } - const course_codes: string[] = []; - $(course_elem).each((_i, c) => { - course_codes.push($(c).text().trim()); - }); - const course_titles = []; - const titles = $(tr).find("td:not([class])"); - $(titles) - .contents() - .each((i, t) => { - if (t.type === "text" && i === 0) { - course_titles.push($(t).text().trim()); - } else if (t.type === "tag" && t.name === "span") { - course_titles.push($(t).text().trim().replace(/^and /, "")); - } - }); - if (course_codes.length !== course_titles.length) { - console.warn( - "found different length code,title lists in and block:", - { course_titles, course_codes }, - "in", - degree.name - ); - } else if (course_titles.length === 0) { - console.warn( - "didnt find any titles for course list:", - course_codes - ); + } + courses.push(course); + }); + return courses; + }; + // some of the concentrations have no top level header for the course list + // if that is the case we add one add one with the label Tech Electives + if ( + ctx.kind === "concentration" && + $(table).is( + ":has(tr.firstrow:has(td.codecol,span.courselistcomment:not(.areaheader)))" + ) + ) { + let txt = "Technical Electives"; + if ($(table).find("tr.firstrow").is(":has(td.codecol)")) { + txt = "MAJOR COURSES"; + } + $( + `${txt}` + ).insertBefore("tr.firstrow"); + } + // TODO: consider just reparsing the page when a non-header header is found instead of checking each comment for each page + // TODO: consider making sections not be the result of mapping but of pushing found sections to list and create way to modify the lists of + // headers/sftf comments then use if statements instead of the following replacements + let comments = $(table) + .find("tr:has(span.courselistcomment)") + .each((_, c) => { + const txt = $(c).find("span").text(); + let match; + // sometimes things that should be headers but are comments instead :/ + // see Mathematics/Statistics Elective on the csc game-development concentration page + if (!$(c).hasClass("areaheader") && !$(c).prev().hasClass("areaheader")) { + let header = parseDegreeRequirementSectionHeader($, c); + if ( + header.kind !== null && + header.kind === "support" && + !header.supportKind + ) { + console.log("making comment:", $(c).text(), "a header"); + $(c).addClass("areaheader"); + return; + } + } + const approvedBelowRE = + /up to (\d+) units may be taken from the approved ([\w\s]+) listed below/i; + if (!!(match = txt.match(approvedBelowRE))) { + const [_, units, sectionTitle] = match; + $(c).replaceWith(`${sectionTitle} + + Select from the following + ${units} + `); + } + }); + + const sectionHeaders = $(table).find("tr.areaheader"); + const SFTF_RE = /\s*Select( one sequence)? from the following.*/; + // TODO: check if table begins with course row (in concentrations) and figure out how to handle that + const sections = $(sectionHeaders) + .get() + .map((headerElem) => { + $(headerElem).addClass(parsedClass); + const sectionRows = $(headerElem).nextUntil("tr.areaheader,tr.listsum"); + const sftfBlocks: CourseRequirement[] = $(sectionRows) + .filter("tr:has(span.courselistcomment)") + .filter( + (_i, e) => !!$(e).find("span.courselistcomment").text().match(SFTF_RE) + ) + .get() + .map((sftf) => { + // see select from the following blocks comment above + let orRow = $(sftf).next().next(); + let courses; + if ($(orRow).is(":has(td.orclass)")) { + console.log("orclass"); + courses = $(orRow) + .nextUntil(":not(:has(td.orclass))") + .add(orRow) + .add($(orRow).prev()); + } else if ($(orRow).is(":has(span.courselistcomment:contains(or))")) { + console.log("dedicated or row"); + courses = $(orRow).prev().add($(orRow).next()); + orRow.addClass(parsedClass); + if ($(orRow).next().next().text() === "or") { + console.warn( + "found more than two courses in or list separated by dedicated or rows...skipping" + ); + } } else { - course_codes.map((code, i) => { - courses.set(code, { - code, - title: course_titles[i], - units: 0, // TODO: total units - }); - }); + courses = $(sftf).nextUntil( + "tr.areaheader,tr:has(span.courselistcomment:contains(Select)),tr.listsum" + ); } + // FIXME: remove this filterSelector and parse comments + courses = parseListOfCourseRows(courses.filter(":has(td.codecol)")); + $(sftf).addClass(parsedClass); + return { kind: "or", courses, units: parseRowUnits(sftf) }; + }); + const remainingRows = $(sectionRows).not(`tr.${parsedClass}`); + let remainingCourses: CourseRequirement[] = + parseListOfCourseRows(remainingRows); - course = { and: course_codes }; - } else if (course_elem.length !== 1) { - if ($(tr).hasClass("listsum")) { - continue; - } - // TODO: handle sections with references to other information on page - console.log("no title for:", $(tr).find("td.codecol").text().trim()); - } - if (!is_and && course_elem.length === 1) { - course = course_elem.text().trim(); + const section = { + ...parseDegreeRequirementSectionHeader($, headerElem), + courses: sftfBlocks.concat(remainingCourses), + }; + if (!section.kind) { + console.error("failed to parse header:", $(headerElem).text()); + } + return section; + }); + const unparsedRows = $(table) + .find("tr") + .not("." + parsedClass); + if (unparsedRows.length > 0) { + console.error( + "did not parse:", + unparsedRows.get().map((e) => $(e).text()) + ); + } + return sections; +}; + +const parseGeCourseRequirementsTable = ( + $: cheerio.CheerioAPI, + table: cheerio.Element +) => { + // FIXME: parse ge "constraints" (C1 or C2, three different prefixes, etc) + const requirements: GeRequirement[] = $(table) + .find("tr") + .filter(":not(:is(.areaheader,.listsum))") + .map((_i, tr) => { + const label = $(tr).find("td").first().text(); + let units = parseInt($(tr).find("td.hourscol").text().trim()); + if (isNaN(units)) { + let cSubjectPrefixesWarningStr = + "Lower-division courses in Area C must come from three different subject prefixes."; + if (label.includes(cSubjectPrefixesWarningStr)) { + return null; } - const has_orclass = $(tr).hasClass("orclass"); - const last = requirements.length - 1; - - if (in_sftf) { - const last_or = requirements[last].or; - if (last_or.length === 0) { - last_or.push(course); - } else if (last_or.length >= 1) { - if (last_or.length === 1 && has_orclass) { - sftf_sep = "or"; - } - if (has_orclass || prev_was_or || sftf_sep == null) { - requirements[last].or.push(course); - } else { - in_sftf = false; - sftf_sep = null; - } - } - prev_was_or = false; + } + let match; + let area = null; + let subarea = null; + if ((match = label.match(/^([ABCDEF])([1234])?$/))) { + let num; + [subarea, area, num] = match; + } else if ((match = label.match(/Area ([ABCDEF])( Elective)/))) { + let _, elective; + [_, area, elective] = match; + if (!!elective) { + subarea = elective.trim(); } - if (!in_sftf && has_orclass) { - // TODO: assert data.cur.last is not or already - // (multiple or's chained togehter outside of sftf) - requirements[last] = { - or: [requirements[last], course], - }; - } else if (!in_sftf) { - // Normal row - requirements.push(course); + } else if ( + (match = label.match( + /((?:Upper|Lower)-Division) ([A-F])( Elective)?s?/ + )) + ) { + let _, elective; + [_, subarea, area, elective] = match; + subarea = subarea.replace("-", "") + (elective ? elective.trim() : ""); + } + if (area === null) { + if (label.includes("Select courses from two different areas")) { + return null; + } else if (label.includes("GE Electives")) { + console.assert(isNaN(units), "electives does not have NaN units"); + } else { + console.error("unrecognized ge:", label); + return null; } - if (typeof course === "string") { - const title = $(tr).find("td:not([class])").text().trim(); - // TODO: more accurate units when in or/and block - const unitsStr = $(tr).find("td.hourscol").text().trim() || 0; - const units = parseInt(unitsStr); - const code = course; - const courseObj = RequirementCourseSchema.parse({ - title, - units, - code, - }); - courses.set(code, courseObj); + } + if (isNaN(units)) { + units = 0; + const rowText = $(tr).text(); + const b3OneLabWarningStr = + "One lab taken with either a B1 or B2 course"; + const twoAreasWarning = "Select courses from two different areas"; + if (!rowText.includes(twoAreasWarning) && area !== "B3") { + console.error("could not determine why units for:", label, "was NaN"); } } + return { area, subarea, units }; + }) + .get() + // FIXME: figure out why req?.area is sometimes null + .filter((req) => !!req && !!req.area); + return requirements; +}; + +export const scrapeDegreeRequirements = async (degree: Degree) => { + const $ = cheerio.load(await fetch(degree.link).then((res) => res.text())); + let requirements: DegreeRequirements = {} as DegreeRequirements; + + // TODO: Parse footers (sc_footnotes) + const tables = $("table.sc_courselist"); + + tables.each((_ti, table) => { + // page has flat structure and this is a way to find the previous h2 element (prevUntil is exclusive) + const titleElem = $(table).prevUntil("h2").last().prev(); + const title = $(titleElem).text().trim(); + if (title === "Degree Requirements and Curriculum") { + requirements.courses = parseCourseRequirementsTable($, table, degree); + } else if (title === "General Education (GE) Requirements") { + requirements.ge = parseGeCourseRequirementsTable($, table); + } else { + console.warn("Unrecognized table with title:", title); } - // TODO: Parse GE table - // (returning false prevents it from being parsed by ending the iteration) - return false; }); + // FIXME: get to the bottom of this + if (!requirements.courses) { + console.error("no courses found for degree:", degree); + requirements.courses = []; + } + const concentrationsList = $("h2:contains(Concentration)+ul>li a") + .get() + .map((elem) => ({ + name: $(elem).text(), + link: "https://catalog.calpoly.edu" + $(elem).attr("href"), + })); + const concentrations = await Promise.all( + concentrationsList.map(async (conc) => { + if (!conc.link) throw new Error(`Concentration ${conc.name} has no link`); + const $ = await fetch(conc.link) + .then((res) => res.text()) + .then(cheerio.load); + const tables = $("table.sc_courselist").get(); + // FIXME: uncomment this + const ctx: { kind: "concentration"; link: string; name: string } = { + ...conc, + kind: "concentration", + }; + conc.courses = tables + .map((table) => parseCourseRequirementsTable($, table, ctx)) + .flat(); + const linkSegments = conc.link + .split("/") + .filter((s) => s.length > 0 && !s.startsWith("#")); + conc.id = linkSegments.at(-1); + return conc; + }) + ); + requirements.concentrations = concentrations; + // FIXME: using unit counts to determine how many classes of sftf/or groups are required // TODO: check for correctness by counting the units and comparing to the degree's unit count + // could keep total units count in groups and check against stated unit totals - return degree; + return requirements; }; export const scrapeDegrees = async () => { @@ -379,7 +648,344 @@ export const scrapeDegrees = async () => { .each((i, elem) => { const [_matched, name, kind] = $(elem).text().match(majorRE) ?? []; const link = DOMAIN + $(elem).find("a").attr("href"); - degrees.push(DegreeSchema.parse({ name, kind, link, id: i })); + const linkSegments = link + .split("/") + .filter((s) => s.length > 0 && !s.startsWith("#")); + const id = linkSegments.at(-1); + const departmentId = linkSegments.at(-2); + degrees.push(DegreeSchema.parse({ name, kind, link, id, departmentId })); }); return degrees; }; + +// const TermSchema = z.enum(["F", "W", "SP", "SU", "TBD"]); + +const CourseSchema = z.object({ + code: CourseCodeSchema, + title: z.string(), + // subjectCode: z.string(), + number: z.number(), + description: z.string(), + // TODO: turn termsTypicallyOffered into bitmask based on 2,4,6,8 term codes + termsTypicallyOffered: z.string(), // z.array(TermSchema), + // if not range minUnits is maxUnits + minUnits: z.number(), + maxUnits: z.number(), +}); + +type Course = z.infer; + +export const scrapeSubjectCourses = async (subjectCode: string) => { + const COURSE_INFO_RE = /(([A-Z]+)\s+(\d+))\. (.*?)\.?$/; + const URL = `https://catalog.calpoly.edu/coursesaz/${subjectCode.toLowerCase()}/`; + const $ = cheerio.load(await fetch(URL).then((res) => res.text())); + + const courses = $(".courseblock"); + const scrapedCourses: Course[] = []; + courses.each((i, course) => { + const title_block = $(course).find(".courseblocktitle"); + const units_str = $(title_block).find("strong span").text().trim(); + let minUnits = 0, + maxUnits = 0; + const units_num = units_str.replace(" units", ""); + if (units_num.includes("-")) { + [minUnits, maxUnits] = units_num.split("-").map(Number); + } else { + let units = parseInt(units_num); + minUnits = units; + maxUnits = units; + } + let [_, code, subjectCode, numStr, title] = + $(title_block) + .find("strong") + .text() + .replace(units_str, "") + .trim() + .match(COURSE_INFO_RE) ?? []; + const number = parseInt(numStr); + const info_block = $(course).find(".courseextendedwrap"); + let termsTypicallyOffered = null; + $(info_block) + .find("p") + .each((i, info_field) => { + const field_text = $(info_field).text().trim(); + if (field_text.startsWith("Term Typically Offered:")) { + // normalize terms offered list so it is in csv format without extra spaces + termsTypicallyOffered = field_text + .replace("Term Typically Offered: ", "") + .split(/, ?/) + .join(","); + } + // TODO: "catolog:" field specifying the requirements it fulfills + // TODO: prerequisite field + // TODO: "CR/NC" field + // TODO: crosslisted as (+ field in db schema) + }); + const description = $(course).find(".courseblockdesc").text().trim(); + scrapedCourses.push( + CourseSchema.parse({ + number, + code, + title, + termsTypicallyOffered, + minUnits, + maxUnits, + description, + }) + ); + }); + return scrapedCourses; +}; + +export const SubjectSchema = z.object({ + name: z.string(), + code: z.string(), +}); + +export type Subject = z.infer; + +export const scrapeSubjects = async () => { + // TODO: scrape ge areas, gwr, uscp, etc from same page + const subjectRE = /(.+)\s+\(([A-Z ]+)\)/; + + const URL = "https://catalog.calpoly.edu/coursesaz/"; + const $ = cheerio.load(await fetch(URL).then((res) => res.text())); + const subjects: Subject[] = []; + $("a.sitemaplink").each((_i, elem) => { + const txt = $(elem).text(); + const [_matched, name, code] = txt.match(subjectRE) ?? []; + subjects.push(SubjectSchema.parse({ name, code })); + }); + return subjects; +}; + +const GESubAreaDataSchema = z + .object({ + constraints: z.array(z.string()).default([]), + name: z.string().nullable().default(null), + fullfilledBy: z.array(CourseCodeSchema).default([]), + description: z.string().nullable().default(null), + }) + .default({}); + +type GESubAreaData = z.infer; + +const GEAreaDataSchema = z + .object({ + name: z.string().default(""), + constraints: z.array(z.string()).default([]), + subareas: z.record(GESubAreaVariantSchema, GESubAreaDataSchema).default({}), + fullfilledBy: z.array(CourseCodeSchema).default([]), + }) + .default({}); + +type GEAreaData = z.infer; + +export const GEDataSchema = z.map( + GEAreasEnumSchema.or(z.literal("USCP")).or(z.literal("GWR")), + GEAreaDataSchema +); +export type GEData = z.infer; + +/** Returns courses that fulfill ge requirements for each area */ +export const scrapeCourseGEFullfillments = async () => { + const url = + "https://catalog.calpoly.edu/generalrequirementsbachelorsdegree/#GE-Requirements"; + const $ = await fetch(url) + .then((res) => res.text()) + .then(cheerio.load); + // TODO: use high-unit/standard info for verification against major ge requirement scraping + + const sections: GEData = new Map(); + const areaTables = $("table.tbl_transfercredits").filter( + (_i, table) => + // do not include info tables that are adjacent to sc_courselist tables + $(table).next(":not(.sc_courselist)").length > 0 && + $(table).prev(":not(.sc_courselist)").length > 0 + ); + areaTables.each((_i, table) => { + const info: GEAreaData = GEAreaDataSchema.parse({}); + const tbody = $(table).find("tbody"); + let areaLabel = $(tbody) + .find("tr.firstrow") + .find("td.column0") + .first() + .text() + .trim(); + let area: GEArea; + if (areaLabel.includes("GE ELECTIVES")) { + info.name = areaLabel; + area = "ELECTIVE"; + // TODO: include limit info (only area B C D) + } else { + let match = areaLabel.match(/\(AREA ([ABCDEF])\)/); + if (!match) + throw new Error( + "unrecognized ge area label for section: ".concat(areaLabel) + ); + area = GEAreasEnumSchema.parse(match[1]); + areaLabel = areaLabel.replace(match[0], "").trim(); + info.name = areaLabel; + } + + $(tbody) + .find("tr:not(.firstrow)") + .find("td.column0") + .get() + .map((labelElement) => $(labelElement).text().trim()) + .filter((label) => { + return !!label && !["Unit Sub-total", "GE TOTAL"].includes(label); + }) + .forEach((label) => { + let match; + let subarea; + let subareaInfo: GESubAreaData = GESubAreaDataSchema.parse({}); + if ( + (match = label.match( + /(?:-?(Writing Intensive))|(?:\s-\s((?:[\w \d,;-]|\(Standard\))+))/ + )) + ) { + let subconstraint = match[1] ?? match[2]; + if (match[1]) subareaInfo.constraints.push(subconstraint); + label = label.replace(match[0], "").replace("()", "").trim(); + // still push the label at the end + match = undefined; + } + if ((match = label.match(new RegExp(`\\((${area}([1234])?)\\)`)))) { + let [matched, _subarea, num] = match; + label = label.replace(matched, "").replace(/1$/, "").trim(); + if (num) { + subarea = _subarea; + subareaInfo.description = label; + } + } + if ( + (match = label.match( + /((?:Upper|Lower)-Division) [A-F]\s?(Elective)?s?/ + )) + ) { + let [matched, _subarea, elective] = match; + _subarea += elective ?? ""; + subarea = _subarea; + label = label.replace(matched, "").trim(); + subareaInfo.description = label ?? null; + } + if ((match = label.match(/(Area [A-F] Elective)/))) { + subarea = "Elective"; + } + if (!subarea) { + info.constraints.push(label); + if (subareaInfo.constraints.length > 0) { + console.warn( + "adding subconstraints:", + subareaInfo.constraints, + "to area:", + area, + "because no subarea was found" + ); + } + return; + } + info.subareas[subarea] = subareaInfo; + }); + sections.set(area, info); + }); + const courseTables = $("table.sc_courselist:not(div#uscptextcontainer *)"); + + courseTables.each((_i, table) => { + const headerElement = $(table) + .prev("table.sc_sctable") + .find("tbody") + .find("tr.lastrow") + .find("td.column0") + .get(); + + const headerInfo = headerElement.map((e) => $(e).text().trim()); + let [title, ...meta] = headerInfo; + const commentHeader = $(table) + .find("tr.firstrow span.courselistcomment.areaheader") + .first() + .text() + .trim(); + if (commentHeader && commentHeader !== "") title = commentHeader; + console.log("title:", title); + // title = commentHeader; + // TODO: updating title when "td>div.courselistcomment" is encountered + // TODO: parsing "ge" from title + + const courses = z.array(CourseCodeSchema).parse( + $(table) + .find("td.codecol") + .map((_i, codeCol) => $(codeCol).text().trim()) + .get() + ); + if (!title) { + console.error("found courses for table with no title", { courses }); + return; + } + let match; + if (title.includes("GE ELECTIVES")) { + sections.get("ELECTIVE")!.fullfilledBy = courses; + } else if ((match = title.match(/((?:Upper|Lower)-Division) ([A-F])/))) { + let [_, division, area] = match; + division = division.replace("-", ""); + const subarea = sections.get(GEAreasEnumSchema.parse(area))!.subareas[ + division + ]; + if (!subarea) { + console.warn( + "creating new found subarea:", + division, + "for", + area, + "because it was found in the course list" + ); + sections.get(GEAreasEnumSchema.parse(area))!.subareas[division] = + GESubAreaDataSchema.parse({ + fulfilledBy: courses, + }); + } else subarea.fullfilledBy = courses; + } else if ((match = title.match(/(([A-F])[1-4])/))) { + const [_, subarea, area] = match; + sections.get(GEAreasEnumSchema.parse(area))!.subareas[ + subarea + ].fullfilledBy = courses; + if (subarea === "B2" || subarea === "B1") { + const b3courses = $(table) + .find('td:contains("& B3")') + .map((_i, b3desc) => $(b3desc).prev().text().trim()) + .get(); + const b3 = sections.get(GEAreasEnumSchema.parse(area))!.subareas.B3; + b3.fullfilledBy = b3.fullfilledBy ?? []; + b3.fullfilledBy.push(...b3courses); + } + } + // FIXME: (B2 & B3) + + // sections.set(ge.ge, ge); + }); + const uscpCourses = $("div#uscptextcontainer table.sc_courselist") + .first() + .find("td.codecol") + .map((_i, codeCol) => $(codeCol).text().trim()) + .get(); + sections.set("USCP", GEAreaDataSchema.parse({ fullfilledBy: uscpCourses })); + const gwr$ = await fetch( + "https://catalog.calpoly.edu/coursesaz/#gwrcoursestext" + ) + .then((r) => r.text()) + .then(cheerio.load) + + const gwrCourses = gwr$("h2:contains(GWR)") + .next("table.sc_courselist") + .find("a.code") + .get() + .map((c) => $(c).text()); + sections.set("GWR", GEAreaDataSchema.parse({ fullfilledBy: gwrCourses })); + + // FIXME: extract parsing of these tables into separate functtions + // to allow all ge's, uscp, gwr, and subjects from https://catalog.calpoly.edu/coursesaz + + // return Array.from(sections.values()); + return sections; +}; diff --git a/kanban-dashboard/src/scraping/department_courses.ts b/kanban-dashboard/src/scraping/department_courses.ts deleted file mode 100644 index cc1ec54..0000000 --- a/kanban-dashboard/src/scraping/department_courses.ts +++ /dev/null @@ -1,52 +0,0 @@ -import cheerio from "cheerio"; - -const COURSE_INFO_RE = /([A-Z]+)\s+(\d+)\. (.*)$/; - -function scrape_department_courses(page) { - const $ = cheerio.load(page); - - const courses = $(".courseblock"); - courses.each((i, course) => { - const title_block = $(course).find(".courseblocktitle"); - const units_str = $(title_block).find("strong span").text().trim(); - const title_str = $(title_block) - .find("strong") - .text() - .replace(units_str, "") - .trim(); - const [_, major, num, name] = title_str.match(COURSE_INFO_RE); - let units = null; - const units_num = units_str.replace(" units", ""); - if (units_num.includes("-")) { - const [start, end] = units_num.split("-").map(Number); - units = Array.from({ length: end - start + 1 }, (_, i) => i + start); - } else { - units = parseInt(units_num); - } - const info_block = $(course).find(".courseextendedwrap"); - let terms; - $(info_block) - .find("p") - .each((i, info_field) => { - const field_text = $(info_field).text().trim(); - if (field_text.startsWith("Term Typically Offered:")) { - const terms_offered = field_text - .replace("Term Typically Offered: ", "") - .split(", "); - terms = terms_offered; - } - // TODO: "catolog:" field specifying the requirements it fulfills - // TODO: "CR/NC" field - }); - console.log( - `department=${major} num=${num} units=${units} name=${name} terms=${terms}` - ); - }); -} - -// TEST: -const DEPARTMENT = "csc"; -const page = await fetch( - `https://catalog.calpoly.edu/coursesaz/${DEPARTMENT}/` -).then((res) => res.text()); -scrape_department_courses(page); diff --git a/kanban-dashboard/src/server/api/db.ts b/kanban-dashboard/src/server/api/db.ts new file mode 100644 index 0000000..920615f --- /dev/null +++ b/kanban-dashboard/src/server/api/db.ts @@ -0,0 +1,393 @@ +import { Course, GESubArea, PrismaClient } from "@prisma/client"; +import { + scrapeDegrees, + scrapeSubjects, + scrapeSubjectCourses, + scrapeDegreeRequirements, + CourseRequirement, + DegreeRequirementSection, + GeRequirement, + CourseCode, + GEData, + scrapeCourseGEFullfillments, +} from "../../scraping/catalog"; + +const CREATE_COURSES = false; +const CREATE_DEGREES = false; +const CREATE_GE_REQUIREMENTS = false; +const CREATE_COURSE_REQUIREMENTS = false; +const CREATE_CONCENTRATIONS = false; +const REMOVE_DEGREES_AND_REQUIREMENTS = false; +const UPDATE_GE_FULLFILLMENTS = true; + +const skipDuplicates = true; + +const createCourses = async (prisma: PrismaClient) => { + const subjects = await scrapeSubjects(); + await prisma.subject.createMany({ data: subjects, skipDuplicates }); + const foundCourses = new Set(); + await Promise.all( + subjects.map(async (subject) => { + const courses = await scrapeSubjectCourses(subject.code); + console.log( + "found", + courses.length, + "courses for subject:", + subject.code + ); + for (const course of courses) { + foundCourses.add(course.code); + } + await prisma.subject.update({ + where: { code: subject.code }, + data: { + courses: { + createMany: { + data: courses, + skipDuplicates, + }, + }, + }, + }); + }) + ); +}; + +const createDegrees = async (prisma: PrismaClient) => { + // await prisma.degree.deleteMany(); + const degrees = await scrapeDegrees(); + return await Promise.all( + degrees.map( + async (degree) => + await prisma.degree.create({ + data: { + name: degree.name, + link: degree.link, + id: degree.id, + kind: degree.kind, + }, + select: { + id: true, + name: true, + link: true, + kind: true, + }, + }) + ) + ); +}; + +const createDegreeGERequirements = async ( + prisma: PrismaClient, + ges: GeRequirement[], + degreeId: string +) => { + return await Promise.all( + ges.map(async (geReq: GeRequirement) => { + await prisma.gERequirement.create({ + data: { + degree: { connect: { id: degreeId } }, + area: geReq.area, + subArea: geReq.subarea as GESubArea, + units: geReq.units, + }, + }); + }) + ); +}; + +const createRequirementGroup = async ( + prisma: PrismaClient, + group: CourseRequirement, + section: DegreeRequirementSection +): Promise => { + if (section.kind === "ge") { + throw new Error( + "cannot create course group for ge's:" + `${{ group, section }}` + ); + } + const courses: CourseCode[] = []; + const childGroups = []; + let kind; + if (group.kind === "course") { + throw new Error("cannot create group from only course"); + } + for (let subGroup of group.courses) { + if (subGroup.kind === "course") { + courses.push(subGroup.course); + continue; + } + let subGroupId = await createRequirementGroup(prisma, subGroup, section); + childGroups.push(subGroupId); + } + + let courseKindInfo = null; + if (section.kind === "elective") { + courseKindInfo = section.electiveKind; + } else if (section.kind === "support") { + courseKindInfo = section.supportKind; + } + + const createdGroup = await prisma.courseRequirementGroup + .create({ + data: { + groupKind: group.kind as "or" | "and", + coursesKind: section.kind, + unitsOf: group.units, + courseKindInfo, + courses: { + createMany: { + data: courses.map((c: CourseCode) => ({ + courseCode: c, + kind: section.kind, + })), + skipDuplicates, + }, + }, + ...(childGroups.length > 0 && { + childGroups: { + connect: childGroups.map((id) => ({ id })), + }, + }), + }, + select: { + id: true, + }, + }) + .catch((e) => { + console.error("failed to create group:", group); + throw e; + }); + return createdGroup.id; +}; +const createCourseRequirements = async ( + prisma: PrismaClient, + sections: DegreeRequirementSection[], + degreeId: string +) => { + for (const section of sections) { + if (section.kind === "ge") continue; + if (section.courses.length === 0) continue; + + let groupKind: "or" | "and" = section.kind === "elective" ? "or" : "and"; + let rootGroupId = await createRequirementGroup( + prisma, + { + kind: groupKind, + courses: section.courses, + // FIXME: scrape degree units + units: 0, + }, + section + ); + + await prisma.degree.update({ + where: { id: degreeId }, + data: { + requirements: { + connect: [{ id: rootGroupId }], + }, + }, + }); + } +}; + +const createConcentrations = async ( + prisma: PrismaClient, + concentrations: { + name: string; + link: string; + id: string; + courses: DegreeRequirementSection[]; + }[], + degreeId: string +) => { + await prisma.degree.update({ + where: { id: degreeId }, + data: { + concentrations: { + createMany: { + data: concentrations.map(({ name, id }) => ({ name, id })), + skipDuplicates, + }, + }, + }, + }); + + for (const concentration of concentrations) { + for (const section of concentration.courses) { + if (section.kind === "ge") + throw new Error( + "found ge section in concentration!" + `${{ concentration }}` + ); + let groupKind: "or" | "and" = section.kind === "elective" ? "or" : "and"; + let rootGroupId = await createRequirementGroup( + prisma, + { + kind: groupKind, + courses: section.courses, + // FIXME: scrape degree units + units: 0, + }, + section + ); + // TODO: make array of root group ids and update all with connnect{many} + await prisma.concentration.update({ + where: { id: concentration.id }, + data: { + courseRequirements: { + connect: [{ id: rootGroupId }], + }, + }, + }); + } + } +}; + +const createGEFullfillments = async (prisma: PrismaClient, ges: GEData) => { + await prisma.gEAreaFullfillmentCourse.deleteMany(); + for (let [area, info] of ges.entries()) { + for (let [subArea, subInfo] of Object.entries(info.subareas)) { + if (area === "USCP" || area === "GWR") { + await prisma.course.updateMany({ + where: { + code: { + in: subInfo.fullfilledBy, + }, + }, + data: { + [area]: true, + }, + }); + continue; + } + for (let course of subInfo.fullfilledBy) { + if (!area || !subArea || !course) { + console.error("missing data for ge fullfillment", { + area, + subArea, + course, + }); + continue; + } + try { + await prisma.course.findUniqueOrThrow({ + where: { + code: course, + }, + }); + } catch { + console.error("could not find course:", course); + continue; + } + await prisma.gEAreaFullfillmentCourse + .create({ + data: { + course: { + connect: { code: course }, + }, + courseId: course, + area, + subArea, + }, + }) + .catch((_) => + console.error("failed to create:", { + area, + subArea, + course, + }) + ); + } + } + } +}; + +const removeDegreesAndRequirements = async (prisma: PrismaClient) => { + await prisma.courseRequirement.deleteMany({}); + await prisma.courseRequirementGroup.updateMany({ + where: { + parentGroup: { + isNot: null, + }, + }, + data: { + parentId: null, + }, + }); + await prisma.courseRequirementGroup.updateMany({ + where: { + degree: { + isNot: null, + }, + }, + data: { + degreeId: null, + }, + }); + await prisma.gERequirement.deleteMany({}); + await prisma.concentration.deleteMany({}); + await prisma.courseRequirementGroup.deleteMany({}); + await prisma.degree.deleteMany({}); +}; + +export const updateCatalogDataInDB = async (prisma: PrismaClient) => { + if (REMOVE_DEGREES_AND_REQUIREMENTS) { + await removeDegreesAndRequirements(prisma); + return "removed degrees and requirements"; + } + if (CREATE_COURSES) await createCourses(prisma); + + let degrees; + if (CREATE_DEGREES) { + degrees = await createDegrees(prisma); + } else { + degrees = await prisma.degree.findMany({ + select: { + id: true, + name: true, + link: true, + kind: true, + }, + }); + } + if (UPDATE_GE_FULLFILLMENTS) { + const ges = await scrapeCourseGEFullfillments(); + await createGEFullfillments(prisma, ges); + } + if ( + CREATE_GE_REQUIREMENTS || + CREATE_COURSE_REQUIREMENTS || + CREATE_CONCENTRATIONS + ) { + for (let degree of degrees) { + const requirements = await scrapeDegreeRequirements(degree); + if (CREATE_GE_REQUIREMENTS) + await createDegreeGERequirements(prisma, requirements.ge, degree.id); + if (CREATE_COURSE_REQUIREMENTS) + await createCourseRequirements(prisma, requirements.courses, degree.id); + if (CREATE_CONCENTRATIONS) + await createConcentrations( + prisma, + requirements.concentrations, + degree.id + ); + } + } + // FIXME: use CourseCodeSchema for ALL course codes to catch weird edge cases and prevent the "not found" errors when connecting + // FIXME: implement retry logic as connection is very unstable +}; + +if (require.main === module) { + const prisma = new PrismaClient({ + log: + process.env.NODE_ENV === "development" + ? ["query", "error", "warn"] + : ["error"], + datasources: { + // set timeout to unlimited because adding degree requirements is very slow + db: { url: process.env.DATABASE_URL + "&pool_timeout=0" }, + }, + }); + (async () => await updateCatalogDataInDB(prisma).then(console.log))(); +} diff --git a/kanban-dashboard/src/server/api/root.ts b/kanban-dashboard/src/server/api/root.ts index 2f7cabd..c95f2ce 100644 --- a/kanban-dashboard/src/server/api/root.ts +++ b/kanban-dashboard/src/server/api/root.ts @@ -1,10 +1,12 @@ -import { createTRPCRouter, publicProcedure } from "~/server/api/trpc"; +import { t } from "~/server/api/trpc"; import { scrapeDegrees, scrapeDegreeRequirements, type RequirementCourse, DegreeSchema, RequirementTypeSchema, + RequirementType, + scrapeCourseGEFullfillments, } from "~/scraping/catalog"; export type { Degree, @@ -18,6 +20,7 @@ import { scrapeCurrentQuarter, termCode, } from "~/scraping/registrar"; +import { GEArea, GESubArea } from "@prisma/client"; const courseType_arr = RequirementTypeSchema.options; @@ -31,14 +34,31 @@ const SchoolYearTermSchema = z.union([ z.literal(8), ]); +export const GroupSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("ge"), + area: z.string(), + subArea: z.string(), + }), + z.object({ kind: z.literal("uscp"), degreeId: z.string() }), + z.object({ kind: z.literal("gwr"), degreeId: z.string() }), + z.object({ + kind: z.literal("elective"), + groupId: z.number(), + degreeId: z.string(), + }), +]); + const RequirementSchema = z.object({ code: z.string(), // TODO: validate course code - id: z.number(), + id: z.string(), title: z.string(), courseType: RequirementTypeSchema, units: z.number().nonnegative(), quarterId: z.number().gte(2000, { message: "term code < 2000" }), // see termCode function in scraping/registrar.ts for details + groupId: GroupSchema.optional(), }); + export type Requirement = z.infer; const QuarterSchema = z.object({ @@ -46,75 +66,316 @@ const QuarterSchema = z.object({ year: YearSchema, termNum: SchoolYearTermSchema, }); + export type Quarter = z.infer; + +const randomQuarter = (startYear: number) => { + return termCode( + Math.floor(Math.random() * 4) + startYear, + SchoolYearTermSchema.parse([2, 4, 8][Math.floor(Math.random() * 3)]) + ); +}; + /** * This is the primary router for your server. * * All routers added in /api/routers should be manually added here. */ -export const appRouter = createTRPCRouter({ - currentQuarterId: publicProcedure - .output(z.number().gt(2000)) - .query(async () => { +export const appRouter = t.router({ + quarters: t.router({ + current: t.procedure.output(z.number().gt(2000)).query(async () => { return await scrapeCurrentQuarter(); }), - quarters: publicProcedure - .input(z.object({ startYear: z.number().gte(2000) })) - .output(z.array(QuarterSchema)) - .query(({ input: { startYear } }) => { - const quarters = []; - - let calYear = startYear; - let schoolYear = 0; - - const q = (termSeason: Term) => ({ - id: termCode(calYear, termSeason), - termNum: TERM_NUMBER[termSeason] as z.infer< - typeof SchoolYearTermSchema - >, - year: schoolYear, - }); - while (schoolYear < 4) { - // winter/spring quarter will be in yeear 5 senior year but this is still 4th year - - quarters.push(q("fall")); - calYear++; - quarters.push(q("winter")); - quarters.push(q("spring")); - schoolYear++; - } - return quarters; - }), - degreeRequirements: publicProcedure - .input( - z.object({ - degree: DegreeSchema.nullable(), - startYear: z.number().gte(2000), - }) + all: t.procedure + .input(z.object({ startYear: z.number().gte(2000) })) + .query(({ input: { startYear } }) => { + const quarters = []; + + let calYear = startYear; + let schoolYear = 0; + + const q = (termSeason: Term) => ({ + id: termCode(calYear, termSeason), + termNum: TERM_NUMBER[termSeason] as z.infer< + typeof SchoolYearTermSchema + >, + year: schoolYear, + }); + while (schoolYear < 4) { + // winter/spring quarter will be in yeear 5 senior year but this is still 4th year + + quarters.push(q("fall")); + calYear++; + quarters.push(q("winter")); + quarters.push(q("spring")); + schoolYear++; + } + return quarters; + }), + }), + degrees: t.router({ + requirements: t.procedure + .input( + z.object({ + degreeId: z.string().nullable(), + startYear: z.number().gte(2000), + }) + ) + .output(z.array(RequirementSchema)) + .query(async ({ ctx, input }) => { + if (!input.degreeId) return []; + const reqGroups = await ctx.prisma.courseRequirementGroup.findMany({ + where: { + degreeId: input.degreeId, + }, + select: { + courses: { + select: { + courseCode: true, + course: { + select: { + title: true, + maxUnits: true, + minUnits: true, + }, + }, + }, + }, + childGroups: { + select: { + id: true, + unitsOf: true, + coursesKind: true, + courseKindInfo: true, + groupKind: true, + }, + }, + coursesKind: true, + courseKindInfo: true, + groupKind: true, + unitsOf: true, + }, + }); + const geReqs = await ctx.prisma.gERequirement.findMany({ + where: { + degreeId: input.degreeId, + }, + select: { + units: true, + area: true, + subArea: true, + }, + }); + let courses = []; + reqGroups.forEach((group) => { + courses = courses.concat( + group.courses.map((req) => ({ + code: req.courseCode, + id: `${group.groupKind}-${group.coursesKind}-${req.courseCode}`, + title: req.course.title, + courseType: group.coursesKind as RequirementType, + quarterId: randomQuarter(input.startYear), + units: req.course.maxUnits, + })) + ); + }); + + reqGroups.forEach((group) => { + group.childGroups.forEach((childGroup) => { + switch (childGroup.groupKind) { + case "or": + if (!childGroup.unitsOf || !group.unitsOf) + childGroup.unitsOf = 4; + let numCourses = (childGroup.unitsOf ?? group.unitsOf) / 4; + if (numCourses < 1) numCourses = 1; + let code; + let title; + if (group.coursesKind === "elective") { + code = group.courseKindInfo + ? group.courseKindInfo + " " + "Elective" + : "Elective"; + title = ""; + } else { + console.warn("skipping group:", { group }); + return; + } + for (let i = 0; i < numCourses; i++) { + courses.push({ + code, + title, + id: `${group.groupKind}-${group.coursesKind}-${group.courseKindInfo}-${i}`, + courseType: group.coursesKind as RequirementType, + quarterId: randomQuarter(input.startYear), + units: childGroup.unitsOf ?? group.unitsOf, + groupId: { + kind: "elective", + groupId: childGroup.id, + degreeId: input.degreeId, + }, + }); + } + break; + case "and": + // FIXME: fetch courses + + // let subGroupCourses = + // (await ctx.prisma.courseRequirementGroup.findUnique({ + // where: { + // id: childGroup.id, + // }, + // select: { + // courses: { + // select: { + // courseCode: true, + // course: { + // select: { + // title: true, + // }, + // }, + // }, + // }, + // }, + // })) ?? { courses: [] }; + // courses = courses.concat( + // subGroupCourses.courses.map((req) => ({ + // code: req.courseCode, + // title: req.course.title, + // id: `${group.groupKind}-${group.coursesKind}-${req.courseCode}`, + // courseType: group.coursesKind as RequirementType, + // quarterId: randomQuarter(input.startYear), + // units: group.unitsOf, + // })) + // ); + } + }); + }); + courses = courses.concat( + geReqs.map((req) => ({ + code: `GE Area ${req.area} ${req.subArea}`, + id: `${req.area}-${req.subArea}`, + title: `${req.subArea}`, + courseType: "ge", + quarterId: randomQuarter(input.startYear), + units: req.units, + groupId: { + kind: "ge", + area: req.area, + subArea: req.subArea, + degreeId: input.degreeId, + }, + })) + ); + courses.push({ + code: "GWR", + id: "GWR", + title: "Graduation Writing Requirement", + courseType: "GWR", + quarterId: randomQuarter(startYear), + units: 4, + groupId: { kind: "GWR", degreeId: input.degreeId }, + }); + courses.push({ + code: "USCP", + id: "USCP", + title: "United States Cultural Pluralism", + courseType: "USCP", + quarterId: randomQuarter(startYear), + units: 4, + groupId: { kind: "USCP", degreeId: input.degreeId }, + }); + console.dir(courses, { depth: null }); + let courseSet = new Set(courses.map((c) => c.id)); + if (courseSet.size !== courses.length) + throw new Error( + `only ${courseSet.size} unique ids in ${courses.length} courses` + ); + return courses; + }), + all: t.procedure + .output(z.array(z.object({ name: z.string(), id: z.string() }))) + .query(async ({ ctx }) => { + let degrees = await ctx.prisma.degree.findMany({ + select: { + id: true, + name: true, + }, + }); + return degrees; + }), + concentrations: t.procedure + .input(z.object({ degreeId: z.string() })) + .output(z.array(z.object({ name: z.string(), id: z.string() }))) + .query(async ({ ctx, input }) => { + return ( + (await ctx.prisma.concentration.findMany({ + where: { + degreeId: input.degreeId, + }, + select: { + name: true, + id: true, + }, + })) ?? [] + ); + }), + }), + fulllfillments: t.procedure + .input(z.object({ group: GroupSchema })) + .output( + z.array( + z.object({ code: z.string(), title: z.string(), units: z.number() }) + ) ) - .output(z.array(RequirementSchema)) - .query(async ({ input }) => { - if (input.degree === null) { - return []; + .query(async ({ ctx, input }) => { + switch (input.group.kind) { + case "uscp": + return await scrapeCourseGEFullfillments().then( + (reqs) => reqs.get("USCP")?.fullfilledBy + ); + break; + case "gwr": + return await scrapeCourseGEFullfillments().then( + (reqs) => reqs.get("GWR")?.fullfilledBy + ); + break; + case "ge": + return await scrapeCourseGEFullfillments().then( + (reqs) => + reqs.get(input.group.area)?.[input.group.subArea].fullfilledBy + ); + break; + case "elective": + return await ctx.prisma.courseRequirementGroup + .findUnique({ + where: { + id: input.group.groupId, + }, + select: { + courses: { + select: { + courseCode: true, + course: { + select: { + title: true, + maxUnits: true, + minUnits: true, + }, + }, + }, + }, + }, + }) + .then((courses) => { + if (!courses) return []; + + return courses.courses.map((course) => ({ + code: course.courseCode, + title: course.course.title, + units: course.course.maxUnits, + })); + }); } - const courses = await scrapeDegreeRequirements(input.degree); - // generate random info for the data that isn't being scraped yet - return Array.from(courses.courses.values()).map( - (course: RequirementCourse, i) => ({ - ...course, - courseType: - courseType_arr[Math.floor(Math.random() * courseType_arr.length)], // TODO: figure out course type from group - quarterId: termCode( - Math.floor(Math.random() * 4) + input.startYear, - SchoolYearTermSchema.parse([2, 4, 8][Math.floor(Math.random() * 3)]) - ), - id: i, - }) - ); }), - degrees: publicProcedure.query(async () => { - return scrapeDegrees(); - }), }); // export type definition of API diff --git a/kanban-dashboard/src/server/api/trpc.ts b/kanban-dashboard/src/server/api/trpc.ts index 08305da..1a358f8 100644 --- a/kanban-dashboard/src/server/api/trpc.ts +++ b/kanban-dashboard/src/server/api/trpc.ts @@ -57,7 +57,7 @@ import { initTRPC } from "@trpc/server"; import superjson from "superjson"; import { ZodError } from "zod"; -const t = initTRPC.context().create({ +export const t = initTRPC.context().create({ transformer: superjson, errorFormatter({ shape, error }) { return { @@ -70,26 +70,3 @@ const t = initTRPC.context().create({ }; }, }); - -/** - * 3. ROUTER & PROCEDURE (THE IMPORTANT BIT) - * - * These are the pieces you use to build your tRPC API. You should import these a lot in the - * "/src/server/api/routers" directory. - */ - -/** - * This is how you create new routers and sub-routers in your tRPC API. - * - * @see https://trpc.io/docs/router - */ -export const createTRPCRouter = t.router; - -/** - * Public (unauthenticated) procedure - * - * This is the base piece you use to build new queries and mutations on your tRPC API. It does not - * guarantee that a user querying is authorized, but you can still access user session data if they - * are logged in. - */ -export const publicProcedure = t.procedure; diff --git a/kanban-dashboard/update-db.sh b/kanban-dashboard/update-db.sh new file mode 100755 index 0000000..bdfc802 --- /dev/null +++ b/kanban-dashboard/update-db.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash + +ts-node --transpile-only --skip-project ./src/server/api/db.ts