New CivicsProject API. Fixing sorting and non-processed bills

src/app/BillExplorer.tsx

@@ -11,6 +11,7 @@ import {
 import { useIsMobile } from "@/components/ui/use-mobile";
 import type { JudgementValue } from "@/components/Judgement/judgement.component";
 import { TenetEvaluation } from "@/models/Bill";
+import { sortBillsByMostRecent } from "@/utils/stages-to-dates/stages-to-dates";
 
 interface BillExplorerProps {
   bills: BillSummary[];
@@ -30,28 +31,6 @@ function normalizeStatus(s?: string) {
   return (s || "unknown").toLowerCase().replace(/\s+/g, " ").trim();
 }
 
-// /** Pull a best-effort "updated at" date; fall back to introducedOn */
-// function getUpdatedAt(b: any): number {
-//   const candidates: Array<string | undefined> = [
-//     b.updatedAt,
-//     b.lastUpdated,
-//     b.lastActionDate,
-//     b.latestEventDate,
-//     b.lastEventAt,
-//     b.modifiedAt,
-//     b.introducedOn,
-//   ];
-//   for (const d of candidates) {
-//     const t = d ? new Date(d).getTime() : NaN;
-//     if (!Number.isNaN(t)) return t;
-//   }
-//   return 0;
-// }
-
-/**
- * Assign an advancement rank (lower = more advanced).
- * Covers common Canadian bill stages + generic fallbacks.
- */
 function statusRank(statusKey: string): number {
   // order buckets from most advanced to least
   const buckets: Array<[number, RegExp]> = [
@@ -217,50 +196,9 @@ function BillExplorer({ bills }: BillExplorerProps) {
       return true;
     });
 
-    console.log("Filtering results:", {
-      totalBills: bills.length,
-      filteredBills: filtered.length,
-      activeFilters: filters,
-    });
-
-    return filtered;
+    return filtered.sort(sortBillsByMostRecent);
   }, [bills, filters]);
 
-  // Grouping disabled for now
-  // const grouped: GroupedBills = useMemo(() => {
-  //   // collect originals per status for nicer headings
-  //   const originalsByKey = new Map<string, string[]>();
-  //   const groups = new Map<string, BillSummary[]>();
-  //
-  //   for (const b of filteredBills) {
-  //     const key = normalizeStatus(b.status);
-  //     if (!groups.has(key)) groups.set(key, []);
-  //     groups.get(key)!.push(b);
-  //
-  //     const origs = originalsByKey.get(key) || [];
-  //     origs.push(b.status || "Unknown");
-  //     originalsByKey.set(key, origs);
-  //   }
-  //
-  //   // build array with rank + sort items by updated desc
-  //   const arr: GroupedBills = [...groups.entries()].map(([key, items]) => {
-  //     const rank = statusRank(key);
-  //     items.sort((a, b) => getUpdatedAt(b) - getUpdatedAt(a));
-  //     const label = pickLabel(key, originalsByKey.get(key) || []);
-  //     return { statusLabel: label, key, rank, items };
-  //   });
-  //
-  //   // sort groups by advancement rank, then by most recent item in each group
-  //   arr.sort((g1, g2) => {
-  //     if (g1.rank !== g2.rank) return g1.rank - g2.rank;
-  //     const g1Latest = getUpdatedAt(g1.items[0]);
-  //     const g2Latest = getUpdatedAt(g2.items[0]);
-  //     return g2Latest - g1Latest;
-  //   });
-  //
-  //   return arr;
-  // }, [filteredBills]);
-
   // Sidebar filter options (normalize statuses for consistency)
   const filterOptions: FilterOptions = useMemo(() => {
     const statusKeyToLabel = new Map<string, string>();
@@ -295,7 +233,7 @@ function BillExplorer({ bills }: BillExplorerProps) {
     // present statuses sorted by advancement rank
     const statuses = [...statusKeyToLabel.entries()]
       .map(([key, label]) => ({ key, label, rank: statusRank(key) }))
-      .sort((a, b) => a.rank - b.rank || a.label.localeCompare(b.label))
+      // .sort((a, b) => a.rank - b.rank || a.label.localeCompare(b.label))
       .map((x) => x.label);
 
     const options = {
@@ -305,9 +243,6 @@ function BillExplorer({ bills }: BillExplorerProps) {
       categories: Array.from(categorySet).sort(),
     };
 
-    console.log("Filter options generated:", options);
-    console.log("Total bills:", bills.length);
-
     return options;
   }, [bills]);
 

src/app/page.tsx

@@ -14,6 +14,12 @@ import FAQModalTrigger from "./FAQModalTrigger";
 const CANADIAN_PARLIAMENT_NUMBER = 45;
 type HomeSearchParams = { cache?: string };
 
+function toIsoString(value?: Date | string): string | undefined {
+  if (!value) return undefined;
+  const parsed = value instanceof Date ? value : new Date(value);
+  return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString();
+}
+
 // Force runtime generation (avoid build-time pre-render) and cache in-memory.
 export const dynamic = "auto";
 // Next.js requires route segment configs to be literal values (not imported constants)
@@ -77,7 +83,7 @@ let mergedBillsCache: { data: BillSummary[]; expiresAt: number } | null = null;
 async function getApiBills(): Promise<BillSummary[]> {
   try {
     const response = await fetch(
-      `https://api.civicsproject.org/bills/region/canada/${CANADIAN_PARLIAMENT_NUMBER}`,
+      `${env.CIVICS_PROJECT_BASE_URL}/canada/bills/${CANADIAN_PARLIAMENT_NUMBER}`,
       {
         // Cache for 5 minutes in production, no cache in development
         ...(process.env.NODE_ENV === "production"
@@ -105,7 +111,6 @@ async function getApiBills(): Promise<BillSummary[]> {
 async function getMergedBills(): Promise<BillSummary[]> {
   const apiBills = await getApiBills();
   const uri = process.env.MONGO_URI || "";
-  console.log({ "THE MONGO URI IS": uri });
   const hasValidMongoUri =
     uri.startsWith("mongodb://") || uri.startsWith("mongodb+srv://");
   const dbBills = hasValidMongoUri ? await getAllBillsFromDB() : [];
@@ -161,9 +166,9 @@ async function getMergedBills(): Promise<BillSummary[]> {
           (dbBill.chamber as "House of Commons" | "Senate") ||
           "House of Commons",
         introducedOn:
-          dbBill.introducedOn?.toISOString() || new Date().toISOString(),
+          toIsoString(dbBill.introducedOn) || new Date().toISOString(),
         lastUpdatedOn:
-          dbBill.lastUpdatedOn?.toISOString() || new Date().toISOString(),
+          toIsoString(dbBill.lastUpdatedOn) || new Date().toISOString(),
         summary: dbBill.summary,
         isSocialIssue: dbBill.isSocialIssue,
         final_judgment: dbBill.final_judgment as BillSummary["final_judgment"],
@@ -178,9 +183,6 @@ async function getMergedBills(): Promise<BillSummary[]> {
     }
   }
 
-  console.log(
-    `Merged ${mergedBills.length} bills (${apiBills.length} from API, ${dbBills.length} from DB${hasValidMongoUri ? "" : " - Mongo disabled"})`,
-  );
   return mergedBills;
 }
 

src/app/types.ts

@@ -5,6 +5,13 @@ export type BillStatus =
   | "Failed"
   | "Paused";
 
+export type BillStage = {
+  stage: string;
+  state: string;
+  house: string;
+  date: string | Date;
+};
+
 export interface BillSummary {
   billID: string;
   title: string;
@@ -35,11 +42,8 @@ export interface BillSummary {
   } | null>;
   parliamentNumber?: number;
   sessionNumber?: number;
-  stages: {
-    stage: string;
-    state: string;
-    house: string;
-    date: Date;
-  }[];
+  billStages?: BillStage[];
+  /** Some sources use `stages`; keep both to smooth over schema differences. */
+  stages?: BillStage[];
   isSocialIssue?: boolean;
 }

src/components/BillCard.tsx

@@ -6,18 +6,16 @@ import { DynamicIcon } from "lucide-react/dynamic";
 import dayjs from "dayjs";
 
 import { getCategoryIcon } from "@/utils/bill-category-to-icon/bill-category-to-icon.util";
-import { getBillStageDates } from "@/utils/stages-to-dates/stages-to-dates";
+import { getBillMostRecentDate } from "@/utils/stages-to-dates/stages-to-dates";
 import { TenetEvaluation } from "@/models/Bill";
 
 interface BillCardProps {
   bill: BillSummary & { tenet_evaluations?: TenetEvaluation[] };
 }
 
 function BillCard({ bill }: BillCardProps) {
-  const { lastUpdated } = getBillStageDates(bill.stages);
-  const dateDisplay = lastUpdated
-    ? dayjs(lastUpdated).format("MMM D, YYYY")
-    : "N/A";
+  const bestDate = getBillMostRecentDate(bill);
+  const dateDisplay = bestDate ? dayjs(bestDate).format("MMM D, YYYY") : "N/A";
 
   const judgementValue: JudgementValue = bill.final_judgment || "abstain";
 

src/env.ts

@@ -14,6 +14,8 @@ function optional(
   return value && value.trim() !== "" ? value : undefined;
 }
 
+const ENDPOINT = "https://civics-project-kiyv.vercel.app";
+
 export const env = {
   NODE_ENV: process.env.NODE_ENV || "development",
   NEXTAUTH_URL: optional("NEXTAUTH_URL", process.env.NEXTAUTH_URL),
@@ -29,6 +31,7 @@ export const env = {
     "CIVICS_PROJECT_API_KEY",
     process.env.CIVICS_PROJECT_API_KEY,
   ),
+  CIVICS_PROJECT_BASE_URL: optional("CIVICS_PROJECT_BASE_URL", ENDPOINT),
   MONGO_URI: optional(
     "MONGO_URI",
     (process.env.MONGO_URI || process.env.MONGODB_URI)?.trim(),

src/server/get-all-bills-from-civics-project.ts

@@ -1,8 +1,9 @@
 import { BillSummary } from "@/app/types";
+import { env } from "@/env";
 
 export async function getBillsFromCivicsProject(): Promise<BillSummary[]> {
   const response = await fetch(
-    "https://api.civicsproject.org/bills/region/canada/45",
+    `${env.CIVICS_PROJECT_BASE_URL}/canada/bills/45`,
     {
       cache: "no-store",
       headers: {

src/server/get-all-bills-from-db.ts

@@ -18,10 +18,8 @@ export const getAllBillsFromDB = async (): Promise<BillDocument[]> => {
     const bills = await Bill.find({}).lean().exec();
     console.log(`Fetched ${bills.length} bills from MongoDB`);
 
-    // Ensure proper serialization by converting to plain objects
-    return bills.map((bill) =>
-      JSON.parse(JSON.stringify(bill)),
-    ) as BillDocument[];
+    // Return lean results directly so date fields stay as Date instances
+    return bills as unknown as BillDocument[];
   } catch (error) {
     console.error("Error fetching bills from MongoDB:", error);
     return [];

src/services/billApi.ts

@@ -2,6 +2,7 @@ import { xmlToMarkdown } from "@/utils/xml-to-md/xml-to-md.util";
 import { SUMMARY_AND_VOTE_PROMPT } from "@/prompt/summary-and-vote-prompt";
 import OpenAI from "openai";
 import { BILL_API_REVALIDATE_INTERVAL } from "@/consts/general";
+import { env } from "@/env";
 
 export type ApiStage = {
   stage: string;
@@ -49,7 +50,7 @@ export interface BillAnalysis {
     explanation: string;
   }>;
   final_judgment: "yes" | "no" | "abstain";
-  rationale: string;
+  rationale?: string;
   needs_more_info: boolean;
   missing_details: string[];
   steel_man: string;
@@ -59,7 +60,7 @@ export interface BillAnalysis {
 export async function getBillFromCivicsProjectApi(
   billId: string,
 ): Promise<ApiBillDetail | null> {
-  const URL = `https://api.civicsproject.org/bills/canada/${billId.toLowerCase()}/${CANADIAN_PARLIAMENT_NUMBER}`;
+  const URL = `${env.CIVICS_PROJECT_BASE_URL}/canada/bills/${CANADIAN_PARLIAMENT_NUMBER}/${billId}`;
   const response = await fetch(URL, {
     // Cache individual bills.
     ...(process.env.NODE_ENV === "production"
@@ -152,7 +153,7 @@ export async function summarizeBillText(input: string): Promise<BillAnalysis> {
         },
       ],
       final_judgment: "abstain",
-      rationale: "Analysis requires AI capabilities",
+      rationale: undefined,
       needs_more_info: true,
       missing_details: ["AI analysis capabilities required"],
       steel_man:
@@ -190,7 +191,7 @@ export async function summarizeBillText(input: string): Promise<BillAnalysis> {
         short_title: parsed.short_title ?? undefined,
         tenet_evaluations: parsed.tenet_evaluations ?? [],
         final_judgment: normalizedFj,
-        rationale: parsed.rationale ?? "",
+        rationale: parsed.rationale ?? undefined,
         needs_more_info: parsed.needs_more_info ?? false,
         missing_details: parsed.missing_details ?? [],
         steel_man: parsed.steel_man ?? "",
@@ -271,7 +272,7 @@ export async function summarizeBillText(input: string): Promise<BillAnalysis> {
           },
         ],
         final_judgment: "abstain",
-        rationale: "Analysis parsing failed",
+        rationale: undefined,
         needs_more_info: true,
         missing_details: ["Valid AI response format"],
         steel_man: "Analysis parsing failed",
@@ -476,11 +477,6 @@ export async function onBillNotInDatabase(params: {
         ? params.bill.stages[params.bill.stages.length - 1].date
         : (params.bill.updatedAt ?? params.bill.date);
 
-    const house =
-      params.bill.stages && params.bill.stages.length > 0
-        ? params.bill.stages[params.bill.stages.length - 1].house
-        : undefined;
-
     const classifiedIsSocialIssue = params.isSocialIssue;
 
     const billData = {
@@ -498,7 +494,9 @@ export async function onBillNotInDatabase(params: {
       steel_man: params.analysis.steel_man,
       status: params.bill.status,
       sponsorParty: params.bill.sponsorParty,
-      chamber: house as "House of Commons" | "Senate" | undefined,
+      chamber: params.bill.billID.startsWith("S")
+        ? "Senate"
+        : "House of Commons",
       genres: params.bill.genres,
       supportedRegion: params.bill.supportedRegion,
       introducedOn: new Date(params.bill.date),

src/utils/billConverters.ts

@@ -120,12 +120,15 @@ export async function fromCivicsProjectApiBill(
       : undefined;
 
   let billMarkdown: string | null = null;
-  if (bill.source) {
-    billMarkdown = await fetchBillMarkdown(bill.source);
+
+  const latestBillSource =
+    bill.source || (bill.billTexts?.[0] as { url?: string })?.url;
+
+  if (latestBillSource) {
+    billMarkdown = await fetchBillMarkdown(latestBillSource);
   }
 
   // Check if we need to regenerate summary based on source changes from Civics Project API
-  const currentSource = bill.source || null;
   let analysis: BillAnalysis = {
     summary: bill.header || "",
     tenet_evaluations: [
@@ -181,12 +184,11 @@ export async function fromCivicsProjectApiBill(
       },
     ],
     final_judgment: "no",
-    rationale: "Not analyzed",
+    rationale: undefined,
     needs_more_info: false,
     missing_details: [],
     steel_man: "Not analyzed",
   };
-  let shouldRegenerateSummary = true;
 
   // Check existing bill in database to see if source changed
   try {
@@ -199,7 +201,7 @@ export async function fromCivicsProjectApiBill(
         .lean()
         .exec()) as BillDocument | null;
 
-      if (existingBill && existingBill.source === currentSource) {
+      if (existingBill && !analysis.rationale) {
         // Source hasn't changed, use existing analysis
         analysis = {
           summary: existingBill.summary,
@@ -221,7 +223,6 @@ export async function fromCivicsProjectApiBill(
             existingBill.missing_details || analysis.missing_details,
           steel_man: existingBill.steel_man || analysis.steel_man,
         };
-        shouldRegenerateSummary = false;
         console.log(
           `Using existing analysis for ${bill.billID} (source unchanged)`,
         );
@@ -232,7 +233,7 @@ export async function fromCivicsProjectApiBill(
     // Continue with regeneration if DB check fails
   }
 
-  if (shouldRegenerateSummary && billMarkdown) {
+  if (!analysis?.rationale && billMarkdown) {
     console.log(`Regenerating analysis for ${bill.billID} (source changed)`);
     analysis = await summarizeBillText(billMarkdown);
   }