From b508c661f66a6d79526426854eff7481562482c9 Mon Sep 17 00:00:00 2001 From: Quan Pham Date: Wed, 28 Jan 2026 12:10:33 -0500 Subject: [PATCH] Fix compatibility issues with pandas 3 While not entirely clear, it seems the recent Pandas relase (3.0.0) changed `read_csv()` cast to pyarrow datatypes, causing an error. Specifying the `pyarrow` engine seems to fix the issue Pinned pandas version to >=3.0, <4.0 --- requirements.txt | 2 +- setup.cfg | 2 +- .../management/commands/fetch_daily_billable_usage.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7c960af4..b1e6c65b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ boto3>=1.42.6,<2.0 kubernetes openshift coldfront >= 1.1.0 -pandas +pandas>=3.0, <4.0 pyarrow pydantic python-cinderclient # TODO: Set version for OpenStack Clients diff --git a/setup.cfg b/setup.cfg index d88cf8bf..c95cf4d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ install_requires = python-swiftclient requests < 3.0 simplejson < 4.0 - pandas + pandas >=3.0, <4.0 pyarrow pytz diff --git a/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py b/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py index 18d4c9ac..3ae273e4 100644 --- a/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py +++ b/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py @@ -192,6 +192,7 @@ def s3_client(self): def load_csv(location) -> DataFrameGroupBy: df = pandas.read_csv( location, + engine="pyarrow", dtype={INVOICE_COLUMN_COST: pandas.ArrowDtype(pyarrow.decimal128(12, 2))}, ) return df.groupby(INVOICE_COLUMN_ALLOCATION_ID)