From 62ab6521a208b551c911f03bc0992411e500b23d Mon Sep 17 00:00:00 2001 From: AssassinMagic Date: Thu, 23 Oct 2025 17:37:06 -0500 Subject: [PATCH 1/6] updated enhance to use asyncio for multithreading --- data-app/main.py | 3 +- data-app/src/enhance/abstract.py | 12 +-- data-app/src/enhance/courseInfo.py | 117 +++++++++++++++++------------ 3 files changed, 79 insertions(+), 53 deletions(-) diff --git a/data-app/main.py b/data-app/main.py index ba41497..68c696f 100644 --- a/data-app/main.py +++ b/data-app/main.py @@ -1,6 +1,7 @@ import argparse import pandas as pd import numpy as np +import asyncio from db.Models import Session, Professor, DepartmentDistribution, TermDistribution from src.generation.process import Process @@ -87,7 +88,7 @@ session = Session() dept_dists = session.query(DepartmentDistribution).all() session.close() - CourseInfoEnhance().enhance(dept_dists) + asyncio.run(CourseInfoEnhance().enhance(dept_dists)) print("[MAIN] Finished CourseInfo Updating") if not args.DisableRMP: diff --git a/data-app/src/enhance/abstract.py b/data-app/src/enhance/abstract.py index 7c02b84..5d6a5c0 100644 --- a/data-app/src/enhance/abstract.py +++ b/data-app/src/enhance/abstract.py @@ -1,10 +1,9 @@ from abc import ABC, abstractmethod from db.Models import DepartmentDistribution -from multiprocessing import Pool +import asyncio class EnhanceBase(ABC): """Base class for data enhancement operations.""" - def __init__(self): pass @@ -13,7 +12,10 @@ def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: """Abstract method to be implemented by subclasses for enhancing data.""" pass - def enhance(self, dept_dists: list[DepartmentDistribution]) -> None: + async def enhance(self, dept_dists: list[DepartmentDistribution]) -> None: """Enhance the data for a list of department distributions in a multiprocessing pool.""" - with Pool() as pool: - pool.map(self.enhance_helper, dept_dists) \ No newline at end of file + + semaphore = asyncio.Semaphore(10) # Limit concurrent tasks to 10 + + tasks = [self.enhance_helper(dept) for dept in dept_dists] + await asyncio.gather(*tasks) diff --git a/data-app/src/enhance/courseInfo.py b/data-app/src/enhance/courseInfo.py index 279a161..a16cfa4 100644 --- a/data-app/src/enhance/courseInfo.py +++ b/data-app/src/enhance/courseInfo.py @@ -3,10 +3,10 @@ from .abstract import EnhanceBase from db.Models import DepartmentDistribution, ClassDistribution, Libed, Session, and_ -import requests +import httpx import datetime from mapping.mappings import libed_mapping - +import asyncio class CourseInfoEnhance(EnhanceBase): @@ -29,53 +29,76 @@ def _calculate_current_term(self) -> str: sterm = f"{year - 1900}{semester_code}" return sterm - def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: - dept = dept_dist.dept_abbr - campus = dept_dist.campus - campus_str = str(campus) + def _process_course_data(self, courses: list[dict], dept: str, campus: str) -> None: + session = Session() + try: + for course in courses: + course_nbr = course["catalog_number"] + class_dist = session.query(ClassDistribution).filter(and_(ClassDistribution.dept_abbr == dept, ClassDistribution.course_num == course_nbr, ClassDistribution.campus == campus)).first() + if class_dist: + class_dist.libeds.clear() + + for attribute in course.get("course_attributes", []): + family = attribute.get("family", "") + attr_id = attribute.get("attribute_id", "") + api_key = f"{family}_{attr_id}" + + if api_key in libed_mapping: + libed_name = libed_mapping[api_key] + elif attr_id in libed_mapping: + libed_name = libed_mapping[attr_id] + else: + continue + # Removed logging for libed not found due to many irrelevant unmapped attributes making it hard to monitor + # [CI Enhanced] Libed mapping not found for attribute: ONL_ONLINE / ONLINE + # [CI Enhanced] Libed mapping not found for attribute: DELM_08 / 08 + # There are more examples like this but they are not useful to log + + libed_dist = session.query(Libed).filter(Libed.name == libed_name).first() + if libed_dist and class_dist not in libed_dist.class_dists: + libed_dist.class_dists.append(class_dist) + + print(f"[CI Enhance] Updated [{class_dist.campus}] {class_dist.dept_abbr} {class_dist.course_num} : Libeds: ({class_dist.libeds})") + session.commit() + except Exception as e: + print(f"[CI ERROR] Error processing course data for {dept} at {campus}: {e}") + session.rollback() + finally: + session.close() + + async def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: + + async with asyncio.Semaphore(10): + dept = dept_dist.dept_abbr + campus = dept_dist.campus + campus_str = str(campus) + + # Only process UMNTC and UMNRO campuses + if campus_str not in ["UMNTC", "UMNRO"]: + return + + current_term = self._calculate_current_term() + link = f"https://courses.umn.edu/campuses/{campus_str.lower()}/terms/{current_term}/courses.json?q=subject_id={dept}" - # Only process UMNTC and UMNRO campuses - if campus_str not in ["UMNTC", "UMNRO"]: - return + courses = [] + async with httpx.AsyncClient() as client: + try: + response = await client.get(link) + response.raise_for_status() + req = response.json() + courses = req.get("courses", []) + except (httpx.RequestError, ValueError) as e: + print(f"[CI ERROR] Failed to fetch or parse data for {dept} at {campus_str}: {e}") + return - current_term = self._calculate_current_term() - link = f"https://courses.umn.edu/campuses/{campus_str.lower()}/terms/{current_term}/courses.json?q=subject_id={dept}" + if not courses: + print(f"[CI INFO] No courses found for {dept} at {campus_str}") + return - with requests.get(link) as url: try: - req = url.json() - courses = req.get("courses", []) - except ValueError: - print("Json malformed, icky!") + await asyncio.to_thread(self._process_course_data, courses, dept, campus) + except Exception as e: + print(f"[CI ERROR] Error in processing thread for {dept}: {e}") return - - for course in courses: - course_nbr = course["catalog_number"] - session = Session() - class_dist = session.query(ClassDistribution).filter(and_(ClassDistribution.dept_abbr == dept, ClassDistribution.course_num == course_nbr, ClassDistribution.campus == campus)).first() - if class_dist: - class_dist.libeds.clear() - - for attribute in course.get("course_attributes", []): - family = attribute.get("family", "") - attr_id = attribute.get("attribute_id", "") - api_key = f"{family}_{attr_id}" - - if api_key in libed_mapping: - libed_name = libed_mapping[api_key] - elif attr_id in libed_mapping: - libed_name = libed_mapping[attr_id] - else: - continue - # Removed logging for libed not found due to many irrelevant unmapped attributes making it hard to monitor - # [CI Enhanced] Libed mapping not found for attribute: ONL_ONLINE / ONLINE - # [CI Enhanced] Libed mapping not found for attribute: DELM_08 / 08 - # There are more examples like this but they are not useful to log - - libed_dist = session.query(Libed).filter(Libed.name == libed_name).first() - if class_dist not in libed_dist.class_dists: - libed_dist.class_dists.append(class_dist) - - print(f"[CI Enhance] Updated [{class_dist.campus}] {class_dist.dept_abbr} {class_dist.course_num} : Libeds: ({class_dist.libeds})") - session.commit() - session.close() + + \ No newline at end of file From 54e16a1bb1e27097cc37ed3d4d0da74a7036ac1b Mon Sep 17 00:00:00 2001 From: AssassinMagic Date: Thu, 23 Oct 2025 20:42:29 -0500 Subject: [PATCH 2/6] fixed semaphore issue in enhance --- data-app/src/enhance/abstract.py | 4 ++-- data-app/src/enhance/courseInfo.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/data-app/src/enhance/abstract.py b/data-app/src/enhance/abstract.py index 5d6a5c0..b498130 100644 --- a/data-app/src/enhance/abstract.py +++ b/data-app/src/enhance/abstract.py @@ -15,7 +15,7 @@ def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: async def enhance(self, dept_dists: list[DepartmentDistribution]) -> None: """Enhance the data for a list of department distributions in a multiprocessing pool.""" - semaphore = asyncio.Semaphore(10) # Limit concurrent tasks to 10 + semaphore = asyncio.Semaphore(9) # Limit concurrent tasks to something under 5 due to rate limiting - tasks = [self.enhance_helper(dept) for dept in dept_dists] + tasks = [self.enhance_helper(dept, semaphore) for dept in dept_dists] await asyncio.gather(*tasks) diff --git a/data-app/src/enhance/courseInfo.py b/data-app/src/enhance/courseInfo.py index a16cfa4..450030b 100644 --- a/data-app/src/enhance/courseInfo.py +++ b/data-app/src/enhance/courseInfo.py @@ -5,6 +5,7 @@ from db.Models import DepartmentDistribution, ClassDistribution, Libed, Session, and_ import httpx import datetime +import json from mapping.mappings import libed_mapping import asyncio @@ -66,9 +67,9 @@ def _process_course_data(self, courses: list[dict], dept: str, campus: str) -> N finally: session.close() - async def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: + async def enhance_helper(self, dept_dist: DepartmentDistribution, semaphore: asyncio.Semaphore) -> None: - async with asyncio.Semaphore(10): + async with semaphore: dept = dept_dist.dept_abbr campus = dept_dist.campus campus_str = str(campus) @@ -87,8 +88,14 @@ async def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: response.raise_for_status() req = response.json() courses = req.get("courses", []) - except (httpx.RequestError, ValueError) as e: - print(f"[CI ERROR] Failed to fetch or parse data for {dept} at {campus_str}: {e}") + except httpx.HTTPStatusError as e: + print(f"[CI HTTP ERROR] Failed for {dept}. Status: {e.response.status_code}. URL: {e.request.url}") + return + except json.JSONDecodeError as e: + print(f"[CI PARSE ERROR] Failed to parse JSON for {dept}. URL: {link}. Error: {repr(e)}") + return + except httpx.RequestError as e: + print(f"[CI REQUEST ERROR] Failed to fetch data for {dept}. Error: {repr(e)}") return if not courses: From e4270a292b8da3a7c52ef7b1304326fd837eac4c Mon Sep 17 00:00:00 2001 From: AssassinMagic Date: Sun, 2 Nov 2025 22:44:11 -0600 Subject: [PATCH 3/6] Map issue fix; debug and error logging; documentation --- chrome-extension/plotter/apiapi.js | 134 ++++++--- chrome-extension/plotter/main.js | 424 +++++++++++++++++++--------- chrome-extension/plotter/util.js | 265 +++++++++++++++-- chrome-extension/sidebar/sidebar.js | 164 +++++++++-- 4 files changed, 769 insertions(+), 218 deletions(-) diff --git a/chrome-extension/plotter/apiapi.js b/chrome-extension/plotter/apiapi.js index ea91d25..5b61fe2 100644 --- a/chrome-extension/plotter/apiapi.js +++ b/chrome-extension/plotter/apiapi.js @@ -1,49 +1,101 @@ /** - * abstraction built around the Schedule Builder API to help form requests and - * form concise objects to represent data for use in the extension's UI - */ - -/** - * object to interface with the API and cache results + * Abstraction layer over the Schedule Builder API (schedulebuilder.umn.edu). + * + * Responsibilities: + * - form well-shaped requests for section information + * - cache results locally to avoid repeated network requests + * - provide small, well-typed SBSection objects to the plotter UI + * + * Important notes: + * - This module intentionally validates HTTP responses and JSON shape + * before assuming an array of sections is returned. + * - Errors are surfaced via `errorLog` and re-thrown so callers may + * apply backoff/guarding behavior. */ class SBAPI { /** - * private cache to store requested data so it can quickly be retrieved when - * needed (e.g. user flips through built schedules) - * - * @type {Map} + * Private in-memory cache for fetched SBSection objects. Keys are + * numeric section IDs. + * @type {Map} */ static #cache = new Map() /** - * @param sections{int[]} section numbers - * @param semesterStrm{int} strm number for user's current semester - * @returns {SBSection[]} SBSchedule object with information from all sections + * Fetch section details from the Schedule Builder API for the provided + * `sections` list and `semesterStrm` term. + * + * Behavior: + * - Uses an internal cache to avoid re-fetching previously requested + * sections. + * - Validates HTTP response status and JSON shape; if the response is + * not OK or not an array the function logs debug information and + * throws an error. + * - When new data is received, it is converted to `SBSection` objects + * and stored in the cache. + * + * @param {number[]} sections - list of class_nbrs to fetch + * @param {number|string} semesterStrm - Schedule Builder term code + * @returns {Promise} array of SBSection objects in the same order as `sections` */ static async fetchSectionInformation(sections, semesterStrm) { - // const cacheSections = sections.filter(s => SBAPI.#cache.has(s)) + // determine which sections still need to be fetched const fetchSections = sections.filter(s => !SBAPI.#cache.has(s)) - if (fetchSections.length !== 0) { - let requestNbrs = fetchSections.join("%2C") - //todo there's a proper way to form api requests using js objects - const data = await fetch("https://schedulebuilder.umn.edu/api.php" + - "?type=sections" + - "&institution=UMNTC" + - "&campus=UMNTC" + - // todo use the right term/semester - "&term=" + semesterStrm + - "&class_nbrs=" + requestNbrs) - .then((response) => response.json()) + try { + debug(`SBAPI.fetchSectionInformation: requested ${sections.length} sections; need fetch ${fetchSections.length}`) - //add fetchSections to cache - data.forEach(s => { - const section_data = this.#constructSectionFromData(s) - SBAPI.#cache.set(s.id, section_data) - }) - } + if (fetchSections.length !== 0) { + // class_nbrs are comma-encoded in the API; join with %2C to be safe + let requestNbrs = fetchSections.join("%2C") + // Construct URL; historically the API is a simple GET interface + const url = "https://schedulebuilder.umn.edu/api.php" + + "?type=sections" + + "&institution=UMNTC" + + "&campus=UMNTC" + + "&term=" + semesterStrm + + "&class_nbrs=" + requestNbrs - return sections.map(s => SBAPI.#cache.get(s)) + debug(`SBAPI fetching url: ${url}`) + const response = await fetch(url) + + // If HTTP status is not OK, capture the response body for debugging + if (!response.ok) { + let text = "" + try { text = await response.text() } catch (e) { /* ignore read errors */ } + debug(`SBAPI fetch failed: ${response.status} ${response.statusText} - ${text}`) + throw new Error(`SBAPI fetch failed: ${response.status} ${response.statusText}`) + } + + // Parse JSON safely and validate expected structure (array) + let data + try { + data = await response.json() + } catch (e) { + errorLog(e, 'SBAPI.parseJSON') + throw e + } + + if (!Array.isArray(data)) { + // API returned an unexpected shape (often an error object). + debug('SBAPI returned non-array response: ' + JSON.stringify(data).slice(0, 500)) + throw new TypeError('SBAPI returned non-array response') + } + + // Convert received raw objects into SBSection instances and cache + for (const s of data) { + const section_data = this.#constructSectionFromData(s) + SBAPI.#cache.set(s.id, section_data) + } + } + + // Return an array of SBSection objects in the original order + return sections.map(s => SBAPI.#cache.get(s)) + } catch (e) { + // Surface helpful debugging info then rethrow so callers can apply + // their own backoff/guard logic. + try { errorLog(e, 'SBAPI.fetchSectionInformation') } catch (ee) { console.error('[GG/plotter] error logging failed', ee) } + throw e + } } /** @@ -54,6 +106,10 @@ class SBAPI { * @returns {SBSection} */ static #constructSectionFromData(data) { + // Extract only the fields we care about for plotting. The API's + // `meetings` array may contain multiple meeting times; for the + // purposes of the map we take the first meeting entry (most common + // case: lecture meeting). const { id, campus, @@ -62,14 +118,16 @@ class SBAPI { meetings, } = data; - //todo replace with foreach for multiple meetings per section + // TODO: If sections with multiple meetings become important for the + // map, iterate `meetings` and return multiple SBSection-like entries. const { start_time, end_time, - // bools representing whether a class takes place on a given weekday + // booleans representing whether a class takes place on a given weekday monday, tuesday, wednesday, thursday, friday, saturday, sunday, } = meetings[0]; - //gather meeting info from meetings object + + // Normalize weekday booleans into an array indexed Monday..Sunday const days = [monday, tuesday, wednesday, thursday, friday, saturday, sunday] return new SBSection(id, days, start_time, end_time) @@ -88,10 +146,12 @@ class SBSection { * @param endTime{int} end time in seconds since midnight */ constructor(id, days, startTime, endTime) { + /** numeric section id (class_nbr) */ this.id = id + /** boolean[] Monday..Sunday indicating meeting days */ this.days = days - //only works for sections with a meeting time/location - // (the overwhelming majority of classes, but not all unfortunately) + // only works for sections with a meeting time/location + // (the overwhelming majority of classes, but not all) this.startTime = startTime this.endTime = endTime } diff --git a/chrome-extension/plotter/main.js b/chrome-extension/plotter/main.js index dabf935..2435542 100644 --- a/chrome-extension/plotter/main.js +++ b/chrome-extension/plotter/main.js @@ -6,65 +6,85 @@ let plotterPresented = false let daySelected = "Monday" /** - * updates the UI to match the current state of the page + * Update UI state in response to DOM mutations. * - * this is where all the decisions on when to refresh different components are - * made: - * UI creation - * UI updates + * Called by a debounced MutationObserver. Responsibilities: + * - create the plotter UI when the schedule appears + * - update the Map button when the toolbar changes + * - trigger map updates when attributes or structure change * - * @param mutations{MutationRecord[]} mutations from MutationObserver + * The function attempts to be resilient to transient DOM states and is + * wrapped in a try/catch so errors are surfaced via `errorLog` rather than + * throwing into the page. + * + * @param {MutationRecord[]} mutations - array of DOM mutation records */ async function onChange(mutations) { - if (!await plotterUIPresented) return; + try { + if (!await plotterUIPresented) return; - const schedule = document.querySelector("#schedule-main"); + const schedule = document.querySelector("#schedule-main"); - //create elements if change is detected - if (schedule !== schedulePresented) { - schedulePresented = schedule - if (schedule) { - log("schedule has just appeared; creating UI") - createUI() + //create elements if change is detected + if (schedule !== schedulePresented) { + schedulePresented = schedule + if (schedule) { + log("schedule has just appeared; creating UI") + try { createUI() } catch (e) { errorLog(e, 'createUI') } + } } - } - //update iff schedule view is present - if (!schedule) return; - - // update button only when necessary - // (changes to DOM structure, e.g. button is removed) - let doUpdateButton = mutations.find(m => m.type === "childList") - //update map when DOM changes or attributes change - // (for color hover """"feature"""") - let doUpdateMap = - doUpdateButton || mutations.find(m => m.type === "attributes"); - - if (doUpdateButton) updateButton(); - if (doUpdateMap && plotterPresented) await updateMap(); - - mutations.forEach(mutation => { - switch (mutation.type) { - case "childList": - break; - case "attributes": - switch (mutation.attributeName) { - case "status": - case "username": - default: - break; - } - break; + //update iff schedule view is present + if (!schedule) return; + + // update button only when necessary + // (changes to DOM structure, e.g. button is removed) + let doUpdateButton = mutations.find(m => m.type === "childList") + //update map when DOM changes or attributes change + // (for color hover """"feature"""") + let doUpdateMap = + doUpdateButton || mutations.find(m => m.type === "attributes"); + + if (doUpdateButton) updateButton(); + if (doUpdateMap && plotterPresented) { + // protect against runaway repeated updates when a persistent error exists + const gp = window.GGPlotter || {} + if (!gp.shouldRun || gp.shouldRun()) { + await updateMap() + } else { + debug('updateMap suppressed due to recent failures or ongoing update') + } } - }) + + mutations.forEach(mutation => { + switch (mutation.type) { + case "childList": + break; + case "attributes": + switch (mutation.attributeName) { + case "status": + case "username": + default: + break; + } + break; + } + }) + } catch (e) { + errorLog(e, 'onChange') + } } +/** + * Ensure the Map button is present in the right-side toolbar. + * This function is idempotent and will not duplicate the button. + */ function updateButton() { - debug('updating button') const group = document.querySelector("div#rightside div.btn-group") const buttonPresent = document.querySelector("#gg-map-btn") //button bar not loaded yet or button already exists if (!group || buttonPresent) return; + debug('updating button') const buttonTemplate = (active) => `
@@ -83,18 +103,27 @@ function updateButton() { }); } +/** + * Toggle the plotter panel visibility and update the Map button + * styling. Wrapped in a try/catch to avoid uncaught exceptions from the + * UI path. + */ function toggleMap() { - plotterPresented = !plotterPresented - const plotter = document.querySelector("#gg-plotter") - // const map = document.querySelector("#gg-plotter-map") - const button = document.querySelector("div#rightside div.btn-group") - .children[2].children[0] - if (plotterPresented) { - plotter.style.display = "block"; - button.className = "btn btn-default active" - } else { - plotter.style.display = "none"; - button.className = "btn btn-default" + try { + plotterPresented = !plotterPresented + const plotter = document.querySelector("#gg-plotter") + // const map = document.querySelector("#gg-plotter-map") + const button = document.querySelector("div#rightside div.btn-group") + .children[2].children[0] + if (plotterPresented) { + plotter.style.display = "block"; + button.className = "btn btn-default active" + } else { + plotter.style.display = "none"; + button.className = "btn btn-default" + } + } catch (e) { + errorLog(e, 'toggleMap') } //todo: consider map an option like the schedule and agenda? @@ -112,8 +141,16 @@ function toggleMap() { * - changing from agenda view to calendar view * - (buttons obliterate entire main content view and replace it) */ +/** + * Build and insert the plotter UI (day buttons, info panel, canvas). + * + * This function is tolerant of missing DOM pieces and wraps the main + * construction in try/catch to ensure errors are logged rather than + * interrupting page execution. + */ function createUI() { - debug("creating UI") + try { + debug("creating UI") const days = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]; const right = document.querySelector("#rightside"); @@ -164,20 +201,27 @@ function createUI() { //todo: if rightside's class is col-md-12, fullscreen is active // place plotter to the right of the schedule? - //add button functionality - days.forEach(day => { - const button = document.querySelector(`#gg-plotter-${day.toLowerCase()}-btn`) - if (day === "Monday") - button.className = "btn btn-default active"; - button.onclick = async function () { - setDayButtonSelected(day) - daySelected = day - //this technically doesn't need to happen due to setDayButtonSelected - // causing an attribute change to be signalled but just in case i ever - // need it i'm going to leave it here with this egregiously long note - // await updateMap() - } - }) + //add button functionality + days.forEach(day => { + const button = document.querySelector(`#gg-plotter-${day.toLowerCase()}-btn`) + if (day === "Monday") + button.className = "btn btn-default active"; + button.onclick = async function () { + try { + setDayButtonSelected(day) + daySelected = day + //this technically doesn't need to happen due to setDayButtonSelected + // causing an attribute change to be signalled but just in case i ever + // need it i'm going to leave it here with this egregiously long note + // await updateMap() + } catch (e) { + errorLog(e, 'day button onclick') + } + } + }) + } catch (e) { + errorLog(e, 'createUI') + } // const mapDiv = document.createElement("div") // mapDiv.id = "gg-plotter-slippy" // mapDiv.style.aspectRatio = "auto 2304/1296"; @@ -202,10 +246,14 @@ function createUI() { * * @param activeDay string of day to switch to active styling */ +/** + * Update the styling of the day selector buttons so only the active day + * is styled as selected. + * + * @param {string} activeDay - day name to set as active (e.g. 'Monday') + */ function setDayButtonSelected(activeDay) { - //maybe make global at this point const days = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]; - //reset button styling days.forEach(day => { const button = document.querySelector(`#gg-plotter-${day.toLowerCase()}-btn`) if (day === activeDay) button.className = "btn btn-default active"; @@ -218,9 +266,27 @@ function setDayButtonSelected(activeDay) { * plots on the map the representation of all the classes in a schedule from * information on the page (namely the section list in the left column) */ +/** + * Gather the current schedule, fetch section details from the SBAPI, + * and draw the map. Uses `window.GGPlotter` guards to avoid concurrent + * updates and to respect exponential backoff when errors occur. + */ async function updateMap() { debug("updating map") - const canvas = document.querySelector("#gg-plotter-map"); + // prevent concurrent updates and honor suppression set by recordFailure + const gp = window.GGPlotter || {} + if (gp._updating) { + debug('updateMap already running; skipping') + return + } + if (gp._suppressUntil && Date.now() < gp._suppressUntil) { + debug('updateMap suppressed until ' + new Date(gp._suppressUntil).toISOString()) + return + } + + gp._updating = true + try { + const canvas = document.querySelector("#gg-plotter-map"); // canvas not loaded yet if (!canvas) return @@ -228,8 +294,40 @@ async function updateMap() { const invalidSections = schedule.sections.filter(sec => sec.location === undefined) //use only sections that have a valid location schedule.sections = schedule.sections.filter(sec => sec.location) + // lightweight change-detection: skip work if schedule/day hasn't changed + try { + const gpLocal = window.GGPlotter || {} + const makeKey = (sched) => { + const ids = (sched.sections || []).map(s => `${s.id}:${s.location ? s.location.x+','+s.location.y : 'nil'}:${s.color || ''}`) + return ids.join('|') + '|' + daySelected + '|' + (plotterPresented ? '1' : '0') + } + const key = makeKey(schedule) + if (gpLocal._lastScheduleKey === key) { + // avoid noisy logs; use debug so it only appears at verbose level + debug('updateMap: no change in schedule since last draw; skipping') + // refresh small UI parts that are cheap (e.g., gMaps link) if needed + try { + const latLongs = pixelsToLatLong(schedule.sections || []) + const link = "https://www.google.com/maps/dir/" + (latLongs.map(c => c[0]+","+c[1]).reverse().join("/")) + const gMapsNode = document.querySelector("#gg-plotter-gmaps") + if (gMapsNode && gMapsNode.href !== link) gMapsNode.href = link + } catch (e) { /* ignore */ } + return + } + gpLocal._lastScheduleKey = key + gpLocal._lastScheduleTs = Date.now() + window.GGPlotter = gpLocal + } catch (e) { + // ignore change-detection errors + } const section_nbrs = schedule.sections.map(s => s.id) - const term = Number(getTermStrm()) + const termStrm = getTermStrm() + if (!termStrm) { + errorLog('No term/strm found on page; aborting map update', 'updateMap') + return + } + const term = Number(termStrm) + debug('updateMap using term: ' + term) // classes not loaded yet // there shouldn't be a case where the scheduler is loaded without sections @@ -295,10 +393,24 @@ async function updateMap() { //report if there are any sections that do not have a location const reportNode = document.querySelector("#gg-plotter-report") - log(invalidSections.length) + // avoid spamming the console when there are zero missing locations if (invalidSections.length > 0) { log("sections without locations: " + invalidSections.length) - reportNode.textContent = `Warning: ${invalidSections.length} sections do not have a location` + if (reportNode) reportNode.textContent = `Warning: ${invalidSections.length} sections do not have a location` + } else { + debug('invalidSections=0') + } + // success -- reset failure count so subsequent updates run immediately + try { if (gp.resetFailures) gp.resetFailures() } catch (e) { debug('resetFailures failed ' + e) } + } catch (e) { + try { + if (gp.recordFailure) gp.recordFailure(e, 'updateMap') + else errorLog(e, 'updateMap') + } catch (ee) { + errorLog(ee, 'updateMap.recordFailure') + } + } finally { + gp._updating = false } } @@ -310,69 +422,80 @@ async function updateMap() { * and color attributes */ function getScheduleSections() { - let currColor = null - const scheduleBody = document.querySelector( - "#schedule-courses > div > div > table > tbody"); - const scheduleList = Array.from(scheduleBody.children) - - let sections = [] - let selected = null - - scheduleList.forEach(element => { - const tds = element.children + try { + let currColor = null + const scheduleBody = document.querySelector( + "#schedule-courses > div > div > table > tbody"); + if (!scheduleBody) return new PlotterSchedule([], null) + const scheduleList = Array.from(scheduleBody.children) + + let sections = [] + let selected = null + + scheduleList.forEach(element => { + const tds = element.children + + //class header + if (tds.length === 3) { + currColor = tds[0].style.backgroundColor + return + } - //class header - if (tds.length === 3) { - currColor = tds[0].style.backgroundColor - return - } + //section (id/location) + let sectionNbr = tds[1] + //check this td for highlighting (could check any other than the first) + const highlighted = sectionNbr && sectionNbr.classList && sectionNbr.classList.contains("info") + if (sectionNbr) { + sectionNbr = sectionNbr.firstElementChild.textContent.trim(); + try { + sectionNbr = Number(sectionNbr) + } catch (err) { + debug("could not read section number " + sectionNbr) + sectionNbr = null + } + } else { + sectionNbr = null; + debug("could not obtain section no. (very weird)") + } + //this should only evaluate to true once, but shouldn't cause problems + // if this statement doesn't hold + if (highlighted) selected = sectionNbr; - //section (id/location) - let sectionNbr = tds[1] - //check this td for highlighting (could check any other than the first) - const highlighted = sectionNbr.classList.contains("info") - if (sectionNbr) { - sectionNbr = sectionNbr.firstElementChild.textContent.trim(); + //location (excl. room no. (for now)) + //a section can have multiple locations for different meeting times + let location = undefined try { - sectionNbr = Number(sectionNbr) - } catch { - debug("could not read section number " + sectionNbr) - sectionNbr = null + const locTd = tds[4] && tds[4].firstElementChild + location = locTd && locTd.textContent && locTd.textContent.trim() + } catch (e) { + debug('error reading location cell: ' + e) + location = undefined } - } else { - sectionNbr = null; - debug("could not obtain section no. (very weird)") - } - //this should only evaluate to true once, but shouldn't cause problems - // if this statement doesn't hold - if (highlighted) selected = sectionNbr; - - //location (excl. room no. (for now)) - //a section can have multiple locations for different meeting times - let location = tds[4].firstElementChild.textContent.trim(); - if (location === "No room listed.") { - location = undefined - } else if (location === "Remote Class" || location === "Online Only") { - debug("Remote Class detected don't acknowledge") - //todo: acknowledge - // when it says remote class that means mandatory attendance which would - // be important to tell the user about whatever none of this matters - location = undefined - } else { - const locationObject = locations.find(loc => loc.location === location); - if (!locationObject) { - debug("could not find location " + location) + + if (location === "No room listed.") { location = undefined - } else { - location = locationObject + } else if (location === "Remote Class" || location === "Online Only") { + debug("Remote Class detected don't acknowledge") + location = undefined + } else if (location) { + const locationObject = locations.find(loc => loc.location === location); + if (!locationObject) { + debug("could not find location " + location) + location = undefined + } else { + location = locationObject + } } - } - const newSection = new PlotterSection(sectionNbr, location, currColor); - sections.push(newSection) - }); + const newSection = new PlotterSection(sectionNbr, location, currColor); + sections.push(newSection) + }); - return new PlotterSchedule(sections, selected) + return new PlotterSchedule(sections, selected) + } catch (e) { + errorLog(e, 'getScheduleSections') + return new PlotterSchedule([], null) + } } //load map image as an img tag @@ -391,13 +514,40 @@ function loadMapImage() { } // initialization; keep out of global scope -{ - new MutationObserver(onChange).observe(document, { - childList: true, - attributes: true, - subtree: true - }); +(function () { + try { + // Debounced MutationObserver: batch rapid DOM changes to avoid tight loops + let mutationTimer = null + let mutationQueue = [] + const MUTATION_DEBOUNCE_MS = 1000 + + const observer = new MutationObserver((mutationsList) => { + // accumulate mutations and schedule a single onChange call + mutationQueue.push(...mutationsList) + if (mutationTimer) return + mutationTimer = setTimeout(() => { + const queued = mutationQueue.slice() + mutationQueue.length = 0 + mutationTimer = null + try { + onChange(queued) + } catch (e) { + errorLog(e, 'onChange(debounced)') + } + }, MUTATION_DEBOUNCE_MS) + }) + + observer.observe(document, { + childList: true, + attributes: true, + subtree: true + }) + + debug('plotter debounced mutation observer installed') + } catch (e) { + errorLog(e, 'mutationObserver') + } //load only once loadMapImage() -} +})() diff --git a/chrome-extension/plotter/util.js b/chrome-extension/plotter/util.js index 73947b8..786a22b 100644 --- a/chrome-extension/plotter/util.js +++ b/chrome-extension/plotter/util.js @@ -1,20 +1,198 @@ /** - * imagine print() being a function to print to a printer + * Lightweight logger for informational messages. + * Routes through a rate-limited logger to avoid console spam when the + * extension encounters repetitive errors or rapid state changes. * - * @param message what to print to the console + * @param {string} message - Message to print to the console. */ function log(message) { - console.log("[GG/plotter] " + message) + rateLimitedLog("[GG/plotter] " + message, 'log') } /** - * most complicated javascript debugging tool + * Verbose debug logger. Includes an ISO timestamp and prints Error + * objects with their stack trace once (the stack is printed directly + * to the console so developers can inspect it in DevTools). * - * @param message what to print to the console + * This is intentionally conservative: it uses the same rate-limited + * backend as `log` so high-volume messages are compacted. + * + * @param {string|Error} message - Debug message or Error object. */ function debug(message) { - //uncomment for debugging lol - console.debug("[GG/plotter] " + message) + try { + const ts = new Date().toISOString() + if (message instanceof Error) { + // show a short debug line and print the full stack once + rateLimitedLog("[GG/plotter] [DEBUG] " + ts + " - Error: " + (message && message.message), 'error') + console.error(message) + } else { + rateLimitedLog("[GG/plotter] [DEBUG] " + ts + " - " + message, 'debug') + } + } catch (e) { + // if anything in debug() throws, fall back to a best-effort log + try { rateLimitedLog("[GG/plotter] " + String(message), 'debug') } catch (ee) {} + } +} + +// Simple rate-limited logger to collapse repeated identical messages. +// This prevents runaway console output by tracking the last message and +// suppressing subsequent identical messages inside a short window. +const _rl = { + last: null, // last message string + lastTs: 0, // timestamp of last message + count: 0, // repetition count within window + windowMs: 2000 // time window to collapse repeats +} + +/** + * Rate-limited logging implementation. + * Behavior: + * - If the exact same message appears repeatedly within windowMs, we + * print the first 3 repeats and then a compact suppressed line. + * - When a different message arrives after a burst, we print a + * summary of how many times the previous message repeated. + * + * @param {string} msg - Message to log. + * @param {'log'|'debug'|'error'} level - Desired console method. + */ +function rateLimitedLog(msg, level) { + try { + const now = Date.now() + if (msg === _rl.last && (now - _rl.lastTs) < _rl.windowMs) { + _rl.count += 1 + // print first 3 repeats, then a compact suppressed message once + if (_rl.count <= 3) { + if (level === 'error') console.error(msg) + else if (level === 'debug') console.debug(msg) + else console.log(msg) + } else if (_rl.count === 4) { + console.log(`${msg} (repeated)`) + } else { + // suppressed - intentionally do nothing to avoid spam + } + _rl.lastTs = now + return + } + + // different message or outside of rate window + if (_rl.count > 3) { + // summarize previous burst so debugging still shows context + console.log(`[GG/plotter] previous message repeated ${_rl.count}x`) + } + _rl.last = msg + _rl.lastTs = now + _rl.count = 1 + + if (level === 'error') console.error(msg) + else if (level === 'debug') console.debug(msg) + else console.log(msg) + } catch (e) { + // best-effort fallback: don't throw from the logger + try { console.log(msg) } catch (ee) {} + } +} + +/** + * log an Error object and surface a small message in the plotter UI if present + * @param err {Error|any} + * @param context {string} + */ +/** + * Log an Error object and surface a short message in the plotter UI (if + * present). This function is used for non-fatal diagnostics and is + * intentionally resilient: it will swallow any exceptions while trying to + * report the original error. + * + * @param {Error|any} err - The error value being logged. + * @param {string} [context] - Short context string to help identify where + * the error happened. + */ +function errorLog(err, context) { + try { + const ts = new Date().toISOString() + console.error("[GG/plotter] [ERROR] " + ts + " " + (context || ""), err) + // Show a short message to the user inside the plotter UI when + // available. This helps non-technical users surface the basic + // problem without opening DevTools. + const reportNode = document.querySelector && document.querySelector("#gg-plotter-report") + if (reportNode) { + const msg = (err && err.message) ? err.message : String(err) + reportNode.textContent = `Error${context ? ' ('+context+')' : ''}: ${msg}` + } + } catch (e) { + // ensure logging never throws - best-effort fallback + try { console.error("[GG/plotter] errorLog failed", e) } catch (ee) {} + } +} + +// expose for other modules (global namespace used by extension) +try { + window.GGPlotter = window.GGPlotter || {} + window.GGPlotter.log = log + window.GGPlotter.debug = debug + window.GGPlotter.errorLog = errorLog +} catch (e) { + // ignore in case `window` is not writable +} + +// Lightweight failure suppression to avoid log storms when a persistent +// error condition exists (for example: missing term resulting in many +// repeated API 400 responses). We expose a small API on the global +// `window.GGPlotter` object so other modules can check whether work +// should proceed, and report failures so backoff is applied centrally. +try { + const gp = window.GGPlotter || {} + gp._failureCount = gp._failureCount || 0 + gp._suppressUntil = gp._suppressUntil || 0 + gp._updating = gp._updating || false + gp._failureCooldownBaseMs = gp._failureCooldownBaseMs || 2000 // 2s + + /** + * Return true when the plotter may proceed with work. This respects an + * in-progress guard (`_updating`) and a suppression window used to + * implement exponential backoff after repeated failures. + * @returns {boolean} + */ + gp.shouldRun = function () { + const now = Date.now() + if (gp._updating) return false + if (now < gp._suppressUntil) return false + return true + } + + /** + * Record a failure, increment the failure counter, schedule an + * exponential backoff, and emit a compact log entry. We print the full + * stack for the first few failures and then switch to compact messages + * to avoid spamming DevTools. + * + * @param {Error|any} err + * @param {string} [context] + */ + gp.recordFailure = function (err, context) { + try { + gp._failureCount = (gp._failureCount || 0) + 1 + const backoff = Math.min(60000, gp._failureCooldownBaseMs * Math.pow(2, gp._failureCount - 1)) + gp._suppressUntil = Date.now() + backoff + if (gp._failureCount <= 3) { + errorLog(err, context) + } else { + debug(`suppressed repeated error (${gp._failureCount}) - last: ${context}`) + } + } catch (e) { + console.error('[GG/plotter] recordFailure failed', e) + } + } + + gp.resetFailures = function () { + gp._failureCount = 0 + gp._suppressUntil = 0 + } + + window.GGPlotter = gp +} catch (e) { + // intentionally swallow errors installing the helper } log("loaded plotter/util.js") @@ -227,34 +405,70 @@ function pixelsToLatLong(sections) { * * @returns {?string} */ +/** + * Attempt to read the currently-displayed term from the page and convert + * it to a Schedule Builder `strm` string. The page breadcrumb typically + * contains readable strings such as "Fall 2025". If found, this function + * returns a string like "1259" (where year-1900 + semester code are + * concatenated). If no breadcrumb is available, fall back to computing + * the current term from the system date. + * + * @returns {?string} strm string (e.g. "1259") or null-ish value + */ function getTermStrm() { - //getting term from breadcrumbs (or so they're called) - let term = document.querySelector( - "#app-header > div > div.row.app-crumbs.hidden-print > div > ol > li:nth-child(2) > a") - if (term) { - term = term.textContent - term = strms[term] + // try to get term from breadcrumbs first + let termNode = document.querySelector( + "#app-header > div > div.row.app-crumbs.hidden-print > div > ol > li:nth-child(2) > a" + ) + if (termNode) { + const termText = termNode.textContent && termNode.textContent.trim() + // page breadcrumbs typically match keys in the `strms` map + const mapped = strms[termText] + if (mapped) return mapped + } + + // fallback: compute current term like the provided Python logic + const now = new Date() + const year = now.getFullYear() + const month = now.getMonth() + 1 + + let semester_code + if (1 <= month && month <= 5) { + semester_code = '3' // Spring + } else if (6 <= month && month <= 8) { + semester_code = '5' // Summer } else { - term = null + semester_code = '9' // Fall } - return term + const sterm = String(year - 1900) + semester_code + return sterm } - -const strms = function () { - let terms = {} - //future-proofing +/** + * Mapping of human-readable breadcrumb strings (e.g. "Fall 2025") to + * Schedule Builder `strm` numeric codes. Historically this mapping was + * generated ahead-of-time for a wide year range; keep a generated map so + * lookups are O(1) and brittle string parsing is avoided at runtime. + * + * If we ever need to shrink this code, `getTermStrm()` can be updated to + * parse breadcrumb text directly instead of consulting this table. + */ +const strms = (function () { + const terms = {} + // Generate entries for years 2020..2049 (matches legacy behaviour) for (let i = 20; i < 50; i++) { const year = 2000 + i - const strm = 1000 + 10 * i - terms["Spring " + year] = strm + 3 - terms["Summer " + year] = strm + 5 - terms["Fall " + year] = strm + 9 + const strmBase = 1000 + 10 * i + terms["Spring " + year] = strmBase + 3 + terms["Summer " + year] = strmBase + 5 + terms["Fall " + year] = strmBase + 9 } return terms -}() +})() -//todo succeed this garbage +// Locations list: authoritative mapping of building names to pixel +// coordinates on the embedded campus map. Keep this as data only; the +// mapping generation happens below. const locations = function () { const locs = [ {location: "Morrill Hall", x: 913, y: 614}, @@ -275,6 +489,7 @@ const locations = function () { {location: "Shepherd Labs", x: 1068, y: 612}, {location: "Rapson Hall", x: 1006, y: 583}, {location: "Pillsbury Hall", x: 916, y: 505}, + {location: "216 Pillsbury Drive", x: 850, y: 482}, // mental math for location unsure if it is 100% correct {location: "Nicholson Hall", x: 824, y: 477}, {location: "Williamson Hall", x: 887, y: 423}, {location: "Jones Hall", x: 850, y: 400}, diff --git a/chrome-extension/sidebar/sidebar.js b/chrome-extension/sidebar/sidebar.js index b245809..4d42d51 100644 --- a/chrome-extension/sidebar/sidebar.js +++ b/chrome-extension/sidebar/sidebar.js @@ -1,8 +1,25 @@ +/** + * Sidebar content script for embedding inline gopher-grades pages. + * + * This file contains a small set of helpers that: + * - inject an iframe into course pages and schedules + * - send a single postMessage containing the current user's email + * - debounce DOM-driven updates to avoid repeat work + * + */ + +/* Base URL used when embedding the static class pages */ const BASE_URL = "https://umn.lol"; +/* Small startup message helps quickly confirm the script loaded in DevTools */ console.log("sidebar grades is loaded :)"); -// listen for messages from iframes +/** + * Listen for messages from embedded iframes. + * + * Expected message shape (from the iframe): { url: } + * When a message with a url is received, open it in a new browser tab. + */ window.addEventListener("message", (event) => { console.log("[GG] received message from iframe", event); if (event.data?.url) { @@ -11,7 +28,16 @@ window.addEventListener("message", (event) => { } }); -// a debounce function to prevent the findCourses function from being called too many times + +/** + * Generic debounce utility used to avoid calling expensive DOM work + * repeatedly during rapid mutation sequences. + * + * @param {Function} func - Function to debounce. + * @param {number} [wait=20] - Milliseconds to wait before invoking. + * @param {boolean} [immediate=true] - If true, call on leading edge. + * @returns {Function} - Debounced wrapper. + */ const debounce = (func, wait = 20, immediate = true) => { let timeout; return function () { @@ -28,9 +54,15 @@ const debounce = (func, wait = 20, immediate = true) => { }; }; + +/* Cached internet id (email) so we avoid repeatedly querying DOM */ let internetId; -// get email so we can follow up on bug reports +/** + * Extract the user's internet id (email) from the page when available. + * This is used to send a small identifying message to embedded iframes + * so they can personalize content or surface follow-up contact details. + */ const getInternetId = () => { if (internetId) return internetId; const matches = document @@ -40,7 +72,14 @@ const getInternetId = () => { return internetId; }; -// code to turn template string into an actual html element + +/** + * Convert a HTML string into a DOM Element. Uses a template element so + * scripts are not executed and whitespace-only nodes are avoided. + * + * @param {string} html - HTML fragment + * @returns {Element} + */ const htmlToElement = (html) => { const template = document.createElement("template"); html = html.trim(); // Never return a text node of whitespace as the result @@ -48,6 +87,8 @@ const htmlToElement = (html) => { return template.content.firstChild; }; + +/* Templates used for iframe insertion */ const iframeTemplate = `
@@ -60,6 +101,17 @@ const iframePortalTemplate = (iframeId, courseName) => `
`; + +/** + * Ensure a portal container exists for a given course. If the portal is + * already present in the DOM we return it, otherwise we create and append + * it to the provided target node. + * + * @param {string} iframeId - id string used for the portal container + * @param {Element} target - DOM node to append the portal to + * @param {string} courseName - friendly label for the portal + * @returns {Element} the portal element + */ const appendPortal = (iframeId, target, courseName) => { const alreadyExists = document.querySelector(`#${iframeId}`); if (alreadyExists) return alreadyExists; @@ -68,7 +120,15 @@ const appendPortal = (iframeId, target, courseName) => { return portal; }; -// code to add the iframe to the page + +/** + * Insert an iframe into the page and post a single message containing the + * user's email. We attempt to send the message once shortly after + * insertion and once again when the iframe emits its `load` event. This + * avoids tight intervals and reduces the risk of infinite spam loops. + * + * Note: uses `direction` to support `prepend` (default) or `append`. + */ const prependFrame = (url, elem, direction = "prepend") => { if (elem.querySelector("iframe")) return; const frameContainer = htmlToElement(iframeTemplate); @@ -76,20 +136,37 @@ const prependFrame = (url, elem, direction = "prepend") => { const frame = frameContainer.querySelector("iframe"); frame.src = url.replace(/ /g, ""); elem[direction](frameContainer); - let interval = setInterval(() => { - console.log("[GG] sending message to iframe"); - if (!frame.contentWindow) { - clearInterval(interval); - return; + + // send a single postMessage when the iframe is ready. Also try once + // shortly after insertion in case the iframe is already available. + const sendMessage = () => { + try { + if (frame.contentWindow) { + console.log("[GG] posting message to iframe"); + frame.contentWindow.postMessage({ email: getInternetId() }, "*"); + return true; + } + } catch (e) { + console.log("[GG] postMessage error", e); } - frame.contentWindow.postMessage({ email: getInternetId() }, "*"); - }, 1000); + return false; + }; + + // try once immediately (after a small delay) and once on load event + setTimeout(sendMessage, 200); + frame.addEventListener('load', () => sendMessage()); }; -// if we need to go the other way, we can use this function (append instead of prepend) + +/* Append variant for readability */ const appendFrame = (url, elem) => prependFrame(url, elem, "append"); -// find courses in the course list + +/** + * Scan an instructor/course list and inject inline iframes for each course + * panel. This is debounced to avoid thrashing when the page mutates + * rapidly (search results updating, panel expansion, etc.). + */ const debouncedFindCourses = debounce((courseList) => { // list all ".panel" elements in the course list const coursePanels = courseList.querySelectorAll(".panel"); @@ -105,13 +182,21 @@ const debouncedFindCourses = debounce((courseList) => { }); }, 50); -// if we're on the course list page (search), load the courses + +/** + * Entry point when on the search/course list page. We debounce twice to + * guard against timing issues where the DOM is still stabilizing. + */ const loadCourses = (courseList) => { debouncedFindCourses(courseList); setTimeout(() => debouncedFindCourses(courseList), 200); }; -// if we're on the course info page, load the course with all the professors and sections + +/** + * When viewing a single course's info page, inject the main roadmap iframe + * and also inject per-instructor views for each professor panel. + */ const loadCourseInfo = (courseInfo) => { const courseTitle = courseInfo.querySelector("h2"); const courseId = courseTitle.innerText.split(":")[0]; @@ -121,7 +206,7 @@ const loadCourseInfo = (courseInfo) => { const url = `${BASE_URL}/class/${courseId}?static=all`; appendFrame(url, courseInfo); - // load all panels + // load all panels (one frame per professor panel) Array.from(document.querySelectorAll(".panel-body")).forEach((panel) => { const prof = panel ?.querySelector("table tbody tr td:nth-of-type(4)") @@ -135,7 +220,13 @@ const loadCourseInfo = (courseInfo) => { }); }; -// if we're on a built schedule, load the schedule + +/** + * If a built schedule is visible, find the courses listed there and inject + * portal iframes for each one. This method uses a lightweight debounce + * keyed on the current course list so we skip repeated processing when the + * same list is encountered repeatedly. + */ const loadCourseSchedule = (courseSchedule) => { const courses = Array.from( document.querySelectorAll("#schedule-courses tr:has(h4)") @@ -145,7 +236,28 @@ const loadCourseSchedule = (courseSchedule) => { tr, })); - console.log("[GG] scheduled courses", courses); + // lightweight debounce: if same course list was processed recently, skip + try { + const key = courses.map(c => c.courseId).join(",") + const now = Date.now() + if (key === _lastScheduledCoursesKey && (now - _lastScheduledCoursesTs) < _SCHEDULE_DEBOUNCE_MS) { + // use a minimal debug to indicate suppressed repeated update + if (window.GGPlotter && window.GGPlotter.debug) window.GGPlotter.debug('[GG] scheduled courses suppressed') + return + } + _lastScheduledCoursesKey = key + _lastScheduledCoursesTs = now + } catch (e) { + // fall back to always proceeding on error + } + + // log scheduled courses (compact) and inject iframes + try { + const dbg = (window.GGPlotter && window.GGPlotter.debug) ? window.GGPlotter.debug : console.log + dbg('[GG] scheduled courses (' + courses.length + ') ' + JSON.stringify(courses.map(c => c.courseId))) + } catch (e) { + console.log('[GG] scheduled courses', courses) + } for (let i = 0; i < courses.length; i++) { const { courseId, courseName } = courses[i]; const iframeTarget = document.querySelector("#app-main .col-xs-12"); @@ -155,6 +267,12 @@ const loadCourseSchedule = (courseSchedule) => { } }; + +/** + * Called on app-level changes (mutation observer). Detects which page we + * are on and delegates to the appropriate loader above. Honor the + * `sb:displayGraphsInline` setting so users can opt out of inline frames. + */ const onAppChange = async () => { const courseList = document.querySelector(".course-list-results"); const courseInfo = document.querySelector("#crse-info"); @@ -172,7 +290,15 @@ const onAppChange = async () => { else if (courseSchedule) loadCourseSchedule(courseSchedule); }; + +/* Initialization: observe the app container and call `onAppChange` when + * child nodes change. The lightweight MutationObserver avoids polling. + */ let loaded = false; +// track last scheduled courses to avoid repeated processing / logging +let _lastScheduledCoursesKey = null +let _lastScheduledCoursesTs = 0 +const _SCHEDULE_DEBOUNCE_MS = 2000 const onLoad = () => { if (loaded) return; loaded = true; From e42c2cf48f0787d68c9f507a256ba2bf9ab5b967 Mon Sep 17 00:00:00 2001 From: AssassinMagic Date: Sun, 2 Nov 2025 22:47:09 -0600 Subject: [PATCH 4/6] Comment about updating mapping --- chrome-extension/plotter/util.js | 213 ++++++++++++++++--------------- 1 file changed, 108 insertions(+), 105 deletions(-) diff --git a/chrome-extension/plotter/util.js b/chrome-extension/plotter/util.js index 786a22b..b3d0af4 100644 --- a/chrome-extension/plotter/util.js +++ b/chrome-extension/plotter/util.js @@ -1,3 +1,5 @@ +// Update Mapping Locations Every 2 Years + /** * Lightweight logger for informational messages. * Routes through a rate-limited logger to avoid console spam when the @@ -31,7 +33,7 @@ function debug(message) { } } catch (e) { // if anything in debug() throws, fall back to a best-effort log - try { rateLimitedLog("[GG/plotter] " + String(message), 'debug') } catch (ee) {} + try { rateLimitedLog("[GG/plotter] " + String(message), 'debug') } catch (ee) { } } } @@ -89,7 +91,7 @@ function rateLimitedLog(msg, level) { else console.log(msg) } catch (e) { // best-effort fallback: don't throw from the logger - try { console.log(msg) } catch (ee) {} + try { console.log(msg) } catch (ee) { } } } @@ -118,11 +120,11 @@ function errorLog(err, context) { const reportNode = document.querySelector && document.querySelector("#gg-plotter-report") if (reportNode) { const msg = (err && err.message) ? err.message : String(err) - reportNode.textContent = `Error${context ? ' ('+context+')' : ''}: ${msg}` + reportNode.textContent = `Error${context ? ' (' + context + ')' : ''}: ${msg}` } } catch (e) { // ensure logging never throws - best-effort fallback - try { console.error("[GG/plotter] errorLog failed", e) } catch (ee) {} + try { console.error("[GG/plotter] errorLog failed", e) } catch (ee) { } } } @@ -276,7 +278,7 @@ class Mapper { } doCircle(section) { - const {location, color} = section + const { location, color } = section this.ctx.beginPath() this.ctx.moveTo(location.x, location.y); this.ctx.arc(location.x, location.y, 16, 0, 2 * Math.PI); @@ -313,10 +315,10 @@ class Mapper { const { location } = sections[0] this.ctx.font = "60px Arial"; this.ctx.strokeStyle = "rgba(40, 40, 40, 0.7)"; - this.ctx.strokeText("Start", location.x-65, location.y+70); + this.ctx.strokeText("Start", location.x - 65, location.y + 70); this.ctx.strokeStyle = "black"; this.ctx.fillStyle = "rgb(128, 222, 160)"; - this.ctx.fillText("Start", location.x-65, location.y+70); + this.ctx.fillText("Start", location.x - 65, location.y + 70); } else if (sections.length === 0) { this.ctx.font = "360px Arial"; this.ctx.fillStyle = "rgba(40, 40, 40, 0.25)"; @@ -331,7 +333,7 @@ class Mapper { //a section associated with the highlighted // section will still have color section.color = "rgb(221, 221, 221)" - this.doCircle(section); + this.doCircle(section); }); const highlightedSection = sections.find(section => section.id === highlight) if (highlightedSection) { @@ -381,21 +383,21 @@ function pixelsToLatLong(sections) { } //degrees per pixel const scale = { - x: (a2.dg.x-anchor.dg.x)/(a2.px.x-anchor.px.x), - y: (a2.dg.y-anchor.dg.y)/(a2.px.y-anchor.px.y) + x: (a2.dg.x - anchor.dg.x) / (a2.px.x - anchor.px.x), + y: (a2.dg.y - anchor.dg.y) / (a2.px.y - anchor.px.y) } - + return sections.map(section => { //god why - let {x: y, y: x} = section.location + let { x: y, y: x } = section.location //offset st. paul campus if (y > 1500) { //yea - x = x*0.83-900 - y = y*0.83+3140 + x = x * 0.83 - 900 + y = y * 0.83 + 3140 } - const lat = anchor.dg.x + (x - anchor.px.x)*scale.x - const long = anchor.dg.y + (y - anchor.px.y)*scale.y + const lat = anchor.dg.x + (x - anchor.px.x) * scale.x + const long = anchor.dg.y + (y - anchor.px.y) * scale.y return [lat, long]; }) } @@ -469,97 +471,98 @@ const strms = (function () { // Locations list: authoritative mapping of building names to pixel // coordinates on the embedded campus map. Keep this as data only; the // mapping generation happens below. +// Mappings should be updated every 2 years const locations = function () { const locs = [ - {location: "Morrill Hall", x: 913, y: 614}, - {location: "Johnston Hall", x: 778, y: 613}, - {location: "John T. Tate Hall", x: 908, y: 672}, - {location: "Smith Hall", x: 778, y: 749}, - {location: "Vincent Hall", x: 891, y: 752}, - {location: "Murphy Hall", x: 926, y: 751}, - {location: "Ford Hall", x: 912, y: 811}, - {location: "Kolthoff Hall", x: 782, y: 812}, - {location: "Coffman Memorial Union", x: 847, y: 927}, - {location: "Amundson Hall", x: 1013, y: 815}, - {location: "Lind Hall", x: 980, y: 752}, - {location: "Mechanical Engineering", x: 1006, y: 670}, - {location: "Akerman Hall", x: 1072, y: 671}, - {location: "Kenneth H Keller Hall", x: 1048, y: 748}, - {location: "Physics & Nanotechnology Bldg", x: 1126, y: 661}, - {location: "Shepherd Labs", x: 1068, y: 612}, - {location: "Rapson Hall", x: 1006, y: 583}, - {location: "Pillsbury Hall", x: 916, y: 505}, - {location: "216 Pillsbury Drive", x: 850, y: 482}, // mental math for location unsure if it is 100% correct - {location: "Nicholson Hall", x: 824, y: 477}, - {location: "Williamson Hall", x: 887, y: 423}, - {location: "Jones Hall", x: 850, y: 400}, - {location: "Folwell Hall", x: 913, y: 365}, - {location: "Molecular Cellular Biology", x: 1047, y: 886}, - {location: "Jackson Hall", x: 981, y: 889}, - {location: "Hasselmo Hall", x: 929, y: 923}, - {location: "Bruininks Hall", x: 701, y: 797}, - {location: "Appleby Hall", x: 702, y: 715}, - {location: "Fraser Hall", x: 700, y: 642}, - {location: "Peik Hall", x: 720, y: 300}, - {location: "Cooke Hall", x: 1216, y: 597}, - {location: "Diehl Hall", x: 1143, y: 1035}, - {location: "Weaver-Densford Hall", x: 1176, y: 866}, - {location: "Scott Hall", x: 724, y: 516}, - {location: "Kaufert Laboratory", x: 1736, y: 319}, - {location: "Green Hall", x: 1741, y: 419}, - {location: "Skok Hall", x: 1727, y: 369}, - {location: "Hodson Hall", x: 1864, y: 320}, - {location: "Alderman Hall", x: 1879, y: 385}, - {location: "Borlaug Hall", x: 1913, y: 480}, - {location: "Gortner Lab", x: 1975, y: 585}, - {location: "McNeal Hall", x: 1893, y: 664}, - {location: "Biological Sciences Center", x: 1974, y: 718}, - {location: "Coffey Hall", x: 1729, y: 834}, - {location: "Ruttan Hall", x: 1798, y: 877}, - {location: "Magrath Library", x: 1867, y: 812}, - {location: "Biosystems/Agricultural Eng", x: 1727, y: 956}, - {location: "Haecker Hall", x: 1739, y: 1045}, - {location: "Andrew Boss Laboratory", x: 1772, y: 1104}, - {location: "Food Science/Nutrition", x: 1766, y: 1168}, - {location: "Stakman Hall", x: 1944, y: 470}, - {location: "Hayes Hall", x: 1959, y: 521}, - {location: "Christensen Lab", x: 1993, y: 483}, - {location: "Walter Library", x: 774, y: 673}, - {location: "Mondale Hall", x: 178, y: 887}, - {location: "Willey Hall", x: 232, y: 914}, - {location: "Andersen Library", x: 284, y: 886}, - {location: "Anderson Hall", x: 336, y: 1010}, - {location: "Social Sciences", x: 285, y: 1056}, - {location: "Heller Hall", x: 223, y: 1045}, - {location: "Blegen Hall", x: 257, y: 1045}, - {location: "Wilson Library", x: 234, y: 1118}, - {location: "Rarig Center", x: 314, y: 1171}, - {location: "Ferguson Hall", x: 364, y: 1109}, - {location: "Ted Mann Concert Hall", x: 400, y: 1152}, - {location: "Carlson School of Management", x: 146, y: 1174}, - {location: "Hanson Hall", x: 157, y: 1249}, - {location: "Hubert H Humphrey School", x: 160, y: 1064}, - {location: "Pattee Hall", x: 631, y: 318}, - {location: "Wilkins Hall", x: 601, y: 186}, - {location: "Learning & Environmental Sci.", x: 1990, y: 814}, - {location: "St. Paul Student Center", x: 1721, y: 706}, - {location: "Eng. & Fisheries Lab", x: 1775, y: 961}, - {location: "Animal Science/Veterinary Med", x: 1826, y: 1032}, - {location: "Veterinary Science", x: 1889, y: 1182}, - {location: "Peters Hall", x: 2103, y: 890}, - {location: "Continuing Education & Conference Center", x: 2152, y: 846}, - {location: "Chiller Bldg", x: 2202, y: 902}, - {location: "Armory Building", x: 1078, y: 478}, - {location: "Northrop", x: 850, y: 556}, - {location: "Wulling Hall", x: 695, y: 574}, - {location: "Elliott Hall", x: 666, y: 509}, - {location: "Burton Hall", x: 682, y: 422}, - {location: "Shelvin Hall", x: 651, y: 369}, - {location: "Eddy Hall", x: 754, y: 430}, - {location: "10 Church Street SE", x: 1000, y: 420}, - {location: "Health Sciences Education Cent", x: 1190, y: 1010}, - {location: "Civil Engineering Building", x: 1130, y: 582}, - {location: "Phillips-Wangensteen Building", x: 1138, y: 976}, + { location: "Morrill Hall", x: 913, y: 614 }, + { location: "Johnston Hall", x: 778, y: 613 }, + { location: "John T. Tate Hall", x: 908, y: 672 }, + { location: "Smith Hall", x: 778, y: 749 }, + { location: "Vincent Hall", x: 891, y: 752 }, + { location: "Murphy Hall", x: 926, y: 751 }, + { location: "Ford Hall", x: 912, y: 811 }, + { location: "Kolthoff Hall", x: 782, y: 812 }, + { location: "Coffman Memorial Union", x: 847, y: 927 }, + { location: "Amundson Hall", x: 1013, y: 815 }, + { location: "Lind Hall", x: 980, y: 752 }, + { location: "Mechanical Engineering", x: 1006, y: 670 }, + { location: "Akerman Hall", x: 1072, y: 671 }, + { location: "Kenneth H Keller Hall", x: 1048, y: 748 }, + { location: "Physics & Nanotechnology Bldg", x: 1126, y: 661 }, + { location: "Shepherd Labs", x: 1068, y: 612 }, + { location: "Rapson Hall", x: 1006, y: 583 }, + { location: "Pillsbury Hall", x: 916, y: 505 }, + { location: "216 Pillsbury Drive", x: 850, y: 482 }, // mental math for location unsure if it is 100% correct + { location: "Nicholson Hall", x: 824, y: 477 }, + { location: "Williamson Hall", x: 887, y: 423 }, + { location: "Jones Hall", x: 850, y: 400 }, + { location: "Folwell Hall", x: 913, y: 365 }, + { location: "Molecular Cellular Biology", x: 1047, y: 886 }, + { location: "Jackson Hall", x: 981, y: 889 }, + { location: "Hasselmo Hall", x: 929, y: 923 }, + { location: "Bruininks Hall", x: 701, y: 797 }, + { location: "Appleby Hall", x: 702, y: 715 }, + { location: "Fraser Hall", x: 700, y: 642 }, + { location: "Peik Hall", x: 720, y: 300 }, + { location: "Cooke Hall", x: 1216, y: 597 }, + { location: "Diehl Hall", x: 1143, y: 1035 }, + { location: "Weaver-Densford Hall", x: 1176, y: 866 }, + { location: "Scott Hall", x: 724, y: 516 }, + { location: "Kaufert Laboratory", x: 1736, y: 319 }, + { location: "Green Hall", x: 1741, y: 419 }, + { location: "Skok Hall", x: 1727, y: 369 }, + { location: "Hodson Hall", x: 1864, y: 320 }, + { location: "Alderman Hall", x: 1879, y: 385 }, + { location: "Borlaug Hall", x: 1913, y: 480 }, + { location: "Gortner Lab", x: 1975, y: 585 }, + { location: "McNeal Hall", x: 1893, y: 664 }, + { location: "Biological Sciences Center", x: 1974, y: 718 }, + { location: "Coffey Hall", x: 1729, y: 834 }, + { location: "Ruttan Hall", x: 1798, y: 877 }, + { location: "Magrath Library", x: 1867, y: 812 }, + { location: "Biosystems/Agricultural Eng", x: 1727, y: 956 }, + { location: "Haecker Hall", x: 1739, y: 1045 }, + { location: "Andrew Boss Laboratory", x: 1772, y: 1104 }, + { location: "Food Science/Nutrition", x: 1766, y: 1168 }, + { location: "Stakman Hall", x: 1944, y: 470 }, + { location: "Hayes Hall", x: 1959, y: 521 }, + { location: "Christensen Lab", x: 1993, y: 483 }, + { location: "Walter Library", x: 774, y: 673 }, + { location: "Mondale Hall", x: 178, y: 887 }, + { location: "Willey Hall", x: 232, y: 914 }, + { location: "Andersen Library", x: 284, y: 886 }, + { location: "Anderson Hall", x: 336, y: 1010 }, + { location: "Social Sciences", x: 285, y: 1056 }, + { location: "Heller Hall", x: 223, y: 1045 }, + { location: "Blegen Hall", x: 257, y: 1045 }, + { location: "Wilson Library", x: 234, y: 1118 }, + { location: "Rarig Center", x: 314, y: 1171 }, + { location: "Ferguson Hall", x: 364, y: 1109 }, + { location: "Ted Mann Concert Hall", x: 400, y: 1152 }, + { location: "Carlson School of Management", x: 146, y: 1174 }, + { location: "Hanson Hall", x: 157, y: 1249 }, + { location: "Hubert H Humphrey School", x: 160, y: 1064 }, + { location: "Pattee Hall", x: 631, y: 318 }, + { location: "Wilkins Hall", x: 601, y: 186 }, + { location: "Learning & Environmental Sci.", x: 1990, y: 814 }, + { location: "St. Paul Student Center", x: 1721, y: 706 }, + { location: "Eng. & Fisheries Lab", x: 1775, y: 961 }, + { location: "Animal Science/Veterinary Med", x: 1826, y: 1032 }, + { location: "Veterinary Science", x: 1889, y: 1182 }, + { location: "Peters Hall", x: 2103, y: 890 }, + { location: "Continuing Education & Conference Center", x: 2152, y: 846 }, + { location: "Chiller Bldg", x: 2202, y: 902 }, + { location: "Armory Building", x: 1078, y: 478 }, + { location: "Northrop", x: 850, y: 556 }, + { location: "Wulling Hall", x: 695, y: 574 }, + { location: "Elliott Hall", x: 666, y: 509 }, + { location: "Burton Hall", x: 682, y: 422 }, + { location: "Shelvin Hall", x: 651, y: 369 }, + { location: "Eddy Hall", x: 754, y: 430 }, + { location: "10 Church Street SE", x: 1000, y: 420 }, + { location: "Health Sciences Education Cent", x: 1190, y: 1010 }, + { location: "Civil Engineering Building", x: 1130, y: 582 }, + { location: "Phillips-Wangensteen Building", x: 1138, y: 976 }, //leave out for testing purposes (gracefully handle missing locations) // {location: "University Field House", x: 1166, y: 516} ]; From f8a683501fc7a02843c2cb1bfc0e56a0be6c3435 Mon Sep 17 00:00:00 2001 From: AssassinMagic Date: Sun, 2 Nov 2025 23:11:04 -0600 Subject: [PATCH 5/6] Revert "fixed semaphore issue in enhance" This reverts commit 54e16a1bb1e27097cc37ed3d4d0da74a7036ac1b. --- data-app/src/enhance/abstract.py | 4 ++-- data-app/src/enhance/courseInfo.py | 15 ++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/data-app/src/enhance/abstract.py b/data-app/src/enhance/abstract.py index b498130..5d6a5c0 100644 --- a/data-app/src/enhance/abstract.py +++ b/data-app/src/enhance/abstract.py @@ -15,7 +15,7 @@ def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: async def enhance(self, dept_dists: list[DepartmentDistribution]) -> None: """Enhance the data for a list of department distributions in a multiprocessing pool.""" - semaphore = asyncio.Semaphore(9) # Limit concurrent tasks to something under 5 due to rate limiting + semaphore = asyncio.Semaphore(10) # Limit concurrent tasks to 10 - tasks = [self.enhance_helper(dept, semaphore) for dept in dept_dists] + tasks = [self.enhance_helper(dept) for dept in dept_dists] await asyncio.gather(*tasks) diff --git a/data-app/src/enhance/courseInfo.py b/data-app/src/enhance/courseInfo.py index 450030b..a16cfa4 100644 --- a/data-app/src/enhance/courseInfo.py +++ b/data-app/src/enhance/courseInfo.py @@ -5,7 +5,6 @@ from db.Models import DepartmentDistribution, ClassDistribution, Libed, Session, and_ import httpx import datetime -import json from mapping.mappings import libed_mapping import asyncio @@ -67,9 +66,9 @@ def _process_course_data(self, courses: list[dict], dept: str, campus: str) -> N finally: session.close() - async def enhance_helper(self, dept_dist: DepartmentDistribution, semaphore: asyncio.Semaphore) -> None: + async def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: - async with semaphore: + async with asyncio.Semaphore(10): dept = dept_dist.dept_abbr campus = dept_dist.campus campus_str = str(campus) @@ -88,14 +87,8 @@ async def enhance_helper(self, dept_dist: DepartmentDistribution, semaphore: asy response.raise_for_status() req = response.json() courses = req.get("courses", []) - except httpx.HTTPStatusError as e: - print(f"[CI HTTP ERROR] Failed for {dept}. Status: {e.response.status_code}. URL: {e.request.url}") - return - except json.JSONDecodeError as e: - print(f"[CI PARSE ERROR] Failed to parse JSON for {dept}. URL: {link}. Error: {repr(e)}") - return - except httpx.RequestError as e: - print(f"[CI REQUEST ERROR] Failed to fetch data for {dept}. Error: {repr(e)}") + except (httpx.RequestError, ValueError) as e: + print(f"[CI ERROR] Failed to fetch or parse data for {dept} at {campus_str}: {e}") return if not courses: From bad3a80b2a7055d28786e8711851c054f857a089 Mon Sep 17 00:00:00 2001 From: AssassinMagic Date: Sun, 2 Nov 2025 23:11:17 -0600 Subject: [PATCH 6/6] Revert "updated enhance to use asyncio for multithreading" This reverts commit 62ab6521a208b551c911f03bc0992411e500b23d. --- data-app/main.py | 3 +- data-app/src/enhance/abstract.py | 12 ++- data-app/src/enhance/courseInfo.py | 117 ++++++++++++----------------- 3 files changed, 53 insertions(+), 79 deletions(-) diff --git a/data-app/main.py b/data-app/main.py index 68c696f..ba41497 100644 --- a/data-app/main.py +++ b/data-app/main.py @@ -1,7 +1,6 @@ import argparse import pandas as pd import numpy as np -import asyncio from db.Models import Session, Professor, DepartmentDistribution, TermDistribution from src.generation.process import Process @@ -88,7 +87,7 @@ session = Session() dept_dists = session.query(DepartmentDistribution).all() session.close() - asyncio.run(CourseInfoEnhance().enhance(dept_dists)) + CourseInfoEnhance().enhance(dept_dists) print("[MAIN] Finished CourseInfo Updating") if not args.DisableRMP: diff --git a/data-app/src/enhance/abstract.py b/data-app/src/enhance/abstract.py index 5d6a5c0..7c02b84 100644 --- a/data-app/src/enhance/abstract.py +++ b/data-app/src/enhance/abstract.py @@ -1,9 +1,10 @@ from abc import ABC, abstractmethod from db.Models import DepartmentDistribution -import asyncio +from multiprocessing import Pool class EnhanceBase(ABC): """Base class for data enhancement operations.""" + def __init__(self): pass @@ -12,10 +13,7 @@ def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: """Abstract method to be implemented by subclasses for enhancing data.""" pass - async def enhance(self, dept_dists: list[DepartmentDistribution]) -> None: + def enhance(self, dept_dists: list[DepartmentDistribution]) -> None: """Enhance the data for a list of department distributions in a multiprocessing pool.""" - - semaphore = asyncio.Semaphore(10) # Limit concurrent tasks to 10 - - tasks = [self.enhance_helper(dept) for dept in dept_dists] - await asyncio.gather(*tasks) + with Pool() as pool: + pool.map(self.enhance_helper, dept_dists) \ No newline at end of file diff --git a/data-app/src/enhance/courseInfo.py b/data-app/src/enhance/courseInfo.py index a16cfa4..279a161 100644 --- a/data-app/src/enhance/courseInfo.py +++ b/data-app/src/enhance/courseInfo.py @@ -3,10 +3,10 @@ from .abstract import EnhanceBase from db.Models import DepartmentDistribution, ClassDistribution, Libed, Session, and_ -import httpx +import requests import datetime from mapping.mappings import libed_mapping -import asyncio + class CourseInfoEnhance(EnhanceBase): @@ -29,76 +29,53 @@ def _calculate_current_term(self) -> str: sterm = f"{year - 1900}{semester_code}" return sterm - def _process_course_data(self, courses: list[dict], dept: str, campus: str) -> None: - session = Session() - try: - for course in courses: - course_nbr = course["catalog_number"] - class_dist = session.query(ClassDistribution).filter(and_(ClassDistribution.dept_abbr == dept, ClassDistribution.course_num == course_nbr, ClassDistribution.campus == campus)).first() - if class_dist: - class_dist.libeds.clear() - - for attribute in course.get("course_attributes", []): - family = attribute.get("family", "") - attr_id = attribute.get("attribute_id", "") - api_key = f"{family}_{attr_id}" - - if api_key in libed_mapping: - libed_name = libed_mapping[api_key] - elif attr_id in libed_mapping: - libed_name = libed_mapping[attr_id] - else: - continue - # Removed logging for libed not found due to many irrelevant unmapped attributes making it hard to monitor - # [CI Enhanced] Libed mapping not found for attribute: ONL_ONLINE / ONLINE - # [CI Enhanced] Libed mapping not found for attribute: DELM_08 / 08 - # There are more examples like this but they are not useful to log - - libed_dist = session.query(Libed).filter(Libed.name == libed_name).first() - if libed_dist and class_dist not in libed_dist.class_dists: - libed_dist.class_dists.append(class_dist) - - print(f"[CI Enhance] Updated [{class_dist.campus}] {class_dist.dept_abbr} {class_dist.course_num} : Libeds: ({class_dist.libeds})") - session.commit() - except Exception as e: - print(f"[CI ERROR] Error processing course data for {dept} at {campus}: {e}") - session.rollback() - finally: - session.close() - - async def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: - - async with asyncio.Semaphore(10): - dept = dept_dist.dept_abbr - campus = dept_dist.campus - campus_str = str(campus) - - # Only process UMNTC and UMNRO campuses - if campus_str not in ["UMNTC", "UMNRO"]: - return - - current_term = self._calculate_current_term() - link = f"https://courses.umn.edu/campuses/{campus_str.lower()}/terms/{current_term}/courses.json?q=subject_id={dept}" + def enhance_helper(self, dept_dist: DepartmentDistribution) -> None: + dept = dept_dist.dept_abbr + campus = dept_dist.campus + campus_str = str(campus) - courses = [] - async with httpx.AsyncClient() as client: - try: - response = await client.get(link) - response.raise_for_status() - req = response.json() - courses = req.get("courses", []) - except (httpx.RequestError, ValueError) as e: - print(f"[CI ERROR] Failed to fetch or parse data for {dept} at {campus_str}: {e}") - return + # Only process UMNTC and UMNRO campuses + if campus_str not in ["UMNTC", "UMNRO"]: + return - if not courses: - print(f"[CI INFO] No courses found for {dept} at {campus_str}") - return + current_term = self._calculate_current_term() + link = f"https://courses.umn.edu/campuses/{campus_str.lower()}/terms/{current_term}/courses.json?q=subject_id={dept}" + with requests.get(link) as url: try: - await asyncio.to_thread(self._process_course_data, courses, dept, campus) - except Exception as e: - print(f"[CI ERROR] Error in processing thread for {dept}: {e}") + req = url.json() + courses = req.get("courses", []) + except ValueError: + print("Json malformed, icky!") return - - \ No newline at end of file + + for course in courses: + course_nbr = course["catalog_number"] + session = Session() + class_dist = session.query(ClassDistribution).filter(and_(ClassDistribution.dept_abbr == dept, ClassDistribution.course_num == course_nbr, ClassDistribution.campus == campus)).first() + if class_dist: + class_dist.libeds.clear() + + for attribute in course.get("course_attributes", []): + family = attribute.get("family", "") + attr_id = attribute.get("attribute_id", "") + api_key = f"{family}_{attr_id}" + + if api_key in libed_mapping: + libed_name = libed_mapping[api_key] + elif attr_id in libed_mapping: + libed_name = libed_mapping[attr_id] + else: + continue + # Removed logging for libed not found due to many irrelevant unmapped attributes making it hard to monitor + # [CI Enhanced] Libed mapping not found for attribute: ONL_ONLINE / ONLINE + # [CI Enhanced] Libed mapping not found for attribute: DELM_08 / 08 + # There are more examples like this but they are not useful to log + + libed_dist = session.query(Libed).filter(Libed.name == libed_name).first() + if class_dist not in libed_dist.class_dists: + libed_dist.class_dists.append(class_dist) + + print(f"[CI Enhance] Updated [{class_dist.campus}] {class_dist.dept_abbr} {class_dist.course_num} : Libeds: ({class_dist.libeds})") + session.commit() + session.close()