From 8cf42b92af044c5cccf6bb93834ef7ff1c53ab6d Mon Sep 17 00:00:00 2001 From: Bradley Gauthier <2234748+bradleygauthier@users.noreply.github.com> Date: Tue, 17 Mar 2026 00:45:21 -0500 Subject: [PATCH] fix: widen signed_at and signed_by storage columns to prevent PostgreSQL overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Seal.seal()` stamps `signed_at` with `datetime.now(UTC)`, whose `.isoformat()` produces 32-character strings (e.g. `2026-03-17T05:24:42.485699+00:00`). Both `CapsuleModel` and `CapsuleModelPG` declared this column as `String(30)`, causing every PostgreSQL write to raise `StorageError: value too long for type character varying(30)`. SQLite was unaffected (it ignores VARCHAR length). Changes: - Widen `signed_at` from String(30) to String(40) in both models - Widen `signed_by` from String(16) to String(32) for headroom (legacy hex fingerprint is exactly 16 chars — zero margin for format changes) - Bump version to 1.5.1 - Add 24 regression tests covering schema introspection, isoformat length validation, and store-retrieve roundtrips for both storage backends - Document storage schema with full column reference in API docs - Add migration SQL for existing PostgreSQL deployments --- CHANGELOG.md | 24 ++ reference/python/docs/api.md | 46 ++- reference/python/pyproject.toml | 2 +- reference/python/src/qp_capsule/__init__.py | 2 +- reference/python/src/qp_capsule/storage.py | 4 +- reference/python/src/qp_capsule/storage_pg.py | 4 +- .../python/tests/test_chain_concurrency.py | 2 +- reference/python/tests/test_cli.py | 2 +- reference/python/tests/test_column_width.py | 344 ++++++++++++++++++ 9 files changed, 419 insertions(+), 11 deletions(-) create mode 100644 reference/python/tests/test_column_width.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa3a50..95ead0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,28 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- +## [1.5.1] - 2026-03-17 + +Storage column width fix. Prevents PostgreSQL `StorageError` on every capsule write. + +### Fixed + +- **`signed_at` column overflow on PostgreSQL** — `String(30)` was too narrow for `datetime.isoformat()` output from timezone-aware datetimes (32 characters for UTC `+00:00` suffix). Widened to `String(40)` in both `CapsuleModel` (SQLite) and `CapsuleModelPG` (PostgreSQL). SQLite was unaffected (it doesn't enforce `VARCHAR` length), but every `seal_and_store()` against PostgreSQL raised `StorageError: value too long for type character varying(30)`. +- **`signed_by` column zero headroom** — `String(16)` exactly matched the legacy 16-character hex fingerprint with no margin. Widened to `String(32)` in both models. The keyring `qp_key_XXXX` format (11 chars) was safe, but any future fingerprint format change would have caused the same overflow. + +### Migration + +- **New installations**: columns are created with the correct width by `create_all()`. +- **Existing PostgreSQL databases**: + ```sql + ALTER TABLE quantumpipes_capsules + ALTER COLUMN signed_at TYPE VARCHAR(40), + ALTER COLUMN signed_by TYPE VARCHAR(32); + ``` +- **Existing SQLite databases**: no action required (SQLite does not enforce `VARCHAR` length). + +--- + ## [1.5.0] - 2026-03-15 Hash chain concurrency protection. Prevents race conditions where concurrent writes could fork the chain. @@ -178,6 +200,8 @@ Initial public release of the Capsule Protocol Specification (CPS) v1.0 referenc --- +[1.5.1]: https://github.com/quantumpipes/capsule/releases/tag/v1.5.1 +[1.5.0]: https://github.com/quantumpipes/capsule/releases/tag/v1.5.0 [1.4.0]: https://github.com/quantumpipes/capsule/releases/tag/v1.4.0 [1.3.0]: https://github.com/quantumpipes/capsule/releases/tag/v1.3.0 [1.2.0]: https://github.com/quantumpipes/capsule/releases/tag/v1.2.0 diff --git a/reference/python/docs/api.md b/reference/python/docs/api.md index ac50d9b..57cd3c2 100644 --- a/reference/python/docs/api.md +++ b/reference/python/docs/api.md @@ -1,13 +1,14 @@ --- title: "API Reference" description: "Complete API reference for Capsule: every class, method, parameter, and type." -date_modified: "2026-03-09" +date_modified: "2026-03-17" ai_context: | - Complete Python API reference for the qp-capsule package v1.3.0. Covers Capsule model + Complete Python API reference for the qp-capsule package v1.5.0. Covers Capsule model (6 sections, 8 CapsuleTypes), Seal (seal, verify, verify_with_key, compute_hash, keyring integration), Keyring (epoch-based key rotation, NIST SP 800-57), CapsuleChain (add, verify, seal_and_store), CapsuleStorageProtocol (7 methods), - CapsuleStorage (SQLite), PostgresCapsuleStorage (multi-tenant), exception hierarchy, + CapsuleStorage (SQLite), PostgresCapsuleStorage (multi-tenant), storage schema with + column constraints (signed_at String(40), signed_by String(32)), exception hierarchy, CLI (verify, inspect, keys, hash), and the high-level API: Capsules class, @audit() decorator, current() context variable, and mount_capsules() FastAPI integration. --- @@ -507,6 +508,45 @@ The `list()` and `count()` methods accept extra parameters beyond the Protocol: --- +## Storage Schema + +Both storage backends persist seal metadata in dedicated columns. The column widths are sized to accommodate all values produced by `Seal.seal()`. + + + + +| Column | Type | Description | +|---|---|---| +| `id` | `String(36)` | UUIDv4 primary key | +| `type` | `String(20)` | `CapsuleType` value | +| `sequence` | `Integer` | Chain position (indexed, unique per tenant) | +| `previous_hash` | `String(64)` | SHA3-256 hash of previous Capsule (nullable) | +| `data` | `Text` | Full Capsule as JSON | +| `hash` | `String(64)` | SHA3-256 content hash (indexed) | +| `signature` | `Text` | Ed25519 signature (hex) | +| `signature_pq` | `Text` | ML-DSA-65 signature (hex, empty if PQ disabled) | +| `signed_at` | `String(40)` | `datetime.isoformat()` — UTC-aware produces 32 chars (e.g., `2026-03-17T05:24:42.485699+00:00`); 40 provides headroom | +| `signed_by` | `String(32)` | Key fingerprint — legacy hex prefix is 16 chars, keyring `qp_key_XXXX` is 11 chars; 32 provides headroom | +| `session_id` | `String(36)` | Conversation session UUID (nullable, indexed) | +| `domain` | `String(50)` | Capsule domain (PostgreSQL only, indexed) | +| `tenant_id` | `String(36)` | Tenant isolation (PostgreSQL only, nullable, indexed) | + +### Migration from pre-1.5.1 + +Versions prior to 1.5.1 used `String(30)` for `signed_at` and `String(16)` for `signed_by`. The `signed_at` column was 2 characters too narrow for UTC-aware `datetime.isoformat()` output, causing `StorageError` on every PostgreSQL write. + +For existing PostgreSQL deployments: + +```sql +ALTER TABLE quantumpipes_capsules + ALTER COLUMN signed_at TYPE VARCHAR(40), + ALTER COLUMN signed_by TYPE VARCHAR(32); +``` + +SQLite does not enforce `VARCHAR` length, so no migration is needed for SQLite databases. + +--- + ## Exceptions diff --git a/reference/python/pyproject.toml b/reference/python/pyproject.toml index 134fa75..c83e93d 100644 --- a/reference/python/pyproject.toml +++ b/reference/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "qp-capsule" -version = "1.5.0" +version = "1.5.1" description = "Capsule Protocol Specification (CPS) — tamper-evident audit records for AI operations. Create, seal, verify, and chain Capsules in Python." readme = "README.md" license = "Apache-2.0" diff --git a/reference/python/src/qp_capsule/__init__.py b/reference/python/src/qp_capsule/__init__.py index 9e2b4ba..452311f 100644 --- a/reference/python/src/qp_capsule/__init__.py +++ b/reference/python/src/qp_capsule/__init__.py @@ -36,7 +36,7 @@ Spec: https://github.com/quantumpipes/capsule """ -__version__ = "1.5.0" +__version__ = "1.5.1" __author__ = "Quantum Pipes Technologies, LLC" __license__ = "Apache-2.0" diff --git a/reference/python/src/qp_capsule/storage.py b/reference/python/src/qp_capsule/storage.py index 06d4a5e..cae627f 100644 --- a/reference/python/src/qp_capsule/storage.py +++ b/reference/python/src/qp_capsule/storage.py @@ -65,8 +65,8 @@ class CapsuleModel(Base): hash: Mapped[str] = mapped_column(String(64), index=True) signature: Mapped[str] = mapped_column(Text) # Ed25519 (classical) signature_pq: Mapped[str] = mapped_column(Text, default="") # ML-DSA-65 (post-quantum) - signed_at: Mapped[str | None] = mapped_column(String(30), nullable=True) - signed_by: Mapped[str | None] = mapped_column(String(16), nullable=True) + signed_at: Mapped[str | None] = mapped_column(String(40), nullable=True) + signed_by: Mapped[str | None] = mapped_column(String(32), nullable=True) # v1.0.0: Session tracking for conversation queries session_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) diff --git a/reference/python/src/qp_capsule/storage_pg.py b/reference/python/src/qp_capsule/storage_pg.py index cb60585..073e66d 100644 --- a/reference/python/src/qp_capsule/storage_pg.py +++ b/reference/python/src/qp_capsule/storage_pg.py @@ -61,8 +61,8 @@ class CapsuleModelPG(PGBase): hash: Mapped[str] = mapped_column(String(64), index=True) signature: Mapped[str] = mapped_column(Text) # Ed25519 (classical) signature_pq: Mapped[str] = mapped_column(Text, default="") # ML-DSA-65 (post-quantum) - signed_at: Mapped[str | None] = mapped_column(String(30), nullable=True) - signed_by: Mapped[str | None] = mapped_column(String(16), nullable=True) + signed_at: Mapped[str | None] = mapped_column(String(40), nullable=True) + signed_by: Mapped[str | None] = mapped_column(String(32), nullable=True) session_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) domain: Mapped[str] = mapped_column(String(50), default="agents", index=True) tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) diff --git a/reference/python/tests/test_chain_concurrency.py b/reference/python/tests/test_chain_concurrency.py index d423c9e..0f005cb 100644 --- a/reference/python/tests/test_chain_concurrency.py +++ b/reference/python/tests/test_chain_concurrency.py @@ -484,7 +484,7 @@ def test_returns_version_string(self): from qp_capsule.cli import _get_version version = _get_version() - assert version == "1.5.0" + assert version == "1.5.1" def test_matches_package_version(self): import qp_capsule diff --git a/reference/python/tests/test_cli.py b/reference/python/tests/test_cli.py index cbf834d..b9f5b6f 100644 --- a/reference/python/tests/test_cli.py +++ b/reference/python/tests/test_cli.py @@ -632,7 +632,7 @@ def test_version_flag(self, capsys, monkeypatch): main(["--version"]) out = capsys.readouterr().out assert "capsule" in out - assert "1.5.0" in out + assert "1.5.1" in out def test_verify_via_main(self, seal, temp_dir, monkeypatch): monkeypatch.setenv("NO_COLOR", "1") diff --git a/reference/python/tests/test_column_width.py b/reference/python/tests/test_column_width.py new file mode 100644 index 0000000..f67fa4a --- /dev/null +++ b/reference/python/tests/test_column_width.py @@ -0,0 +1,344 @@ +""" +Tests for signed_at / signed_by column width fix. + +Validates that timezone-aware datetime.isoformat() values (32 chars) +fit within the widened String(40) columns, and that signed_by handles +both legacy hex prefix (16 chars) and keyring qp_key_XXXX formats. + +Regression tests for: String(30) overflow on PostgreSQL when storing +capsule.signed_at.isoformat() for UTC-aware datetimes. +""" + +from datetime import UTC, datetime, timedelta, timezone + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from qp_capsule.capsule import Capsule +from qp_capsule.keyring import Keyring +from qp_capsule.seal import Seal +from qp_capsule.storage import CapsuleModel, CapsuleStorage +from qp_capsule.storage_pg import CapsuleModelPG, PostgresCapsuleStorage + +# ========================================================================= +# Schema — Column width declarations +# ========================================================================= + + +class TestSignedAtSchemaWidth: + """Verify both models declare signed_at as String(40).""" + + def test_sqlite_model_column_width(self): + col = CapsuleModel.__table__.columns["signed_at"] + assert col.type.length == 40 + + def test_pg_model_column_width(self): + col = CapsuleModelPG.__table__.columns["signed_at"] + assert col.type.length == 40 + + +class TestSignedBySchemaWidth: + """Verify both models declare signed_by as String(32).""" + + def test_sqlite_model_column_width(self): + col = CapsuleModel.__table__.columns["signed_by"] + assert col.type.length == 32 + + def test_pg_model_column_width(self): + col = CapsuleModelPG.__table__.columns["signed_by"] + assert col.type.length == 32 + + +# ========================================================================= +# Isoformat length — prove the old limit was too narrow +# ========================================================================= + + +class TestIsoformatLengths: + """Demonstrate that timezone-aware isoformat exceeds the old String(30).""" + + def test_utc_isoformat_is_32_chars(self): + dt = datetime(2026, 3, 17, 5, 24, 42, 485699, tzinfo=timezone.utc) + iso = dt.isoformat() + assert len(iso) == 32 + assert iso.endswith("+00:00") + + def test_positive_offset_isoformat_is_32_chars(self): + tz = timezone(timedelta(hours=5, minutes=30)) + dt = datetime(2026, 3, 17, 10, 54, 42, 485450, tzinfo=tz) + assert len(dt.isoformat()) == 32 + + def test_negative_offset_isoformat_is_32_chars(self): + tz = timezone(timedelta(hours=-7)) + dt = datetime(2026, 3, 17, 22, 24, 42, 123456, tzinfo=tz) + assert len(dt.isoformat()) == 32 + + def test_naive_isoformat_is_26_chars(self): + dt = datetime(2026, 3, 17, 5, 24, 42, 485699) + assert len(dt.isoformat()) == 26 + + def test_seal_produces_utc_aware_datetime(self, tmp_path): + """Seal.seal() stamps signed_at with UTC — the 32-char variant.""" + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + + assert capsule.signed_at is not None + assert capsule.signed_at.tzinfo is not None + assert len(capsule.signed_at.isoformat()) == 32 + + def test_all_variants_fit_within_40(self): + """Every plausible isoformat output fits String(40).""" + variants = [ + datetime(2026, 3, 17, 5, 24, 42, 485699, tzinfo=timezone.utc), + datetime(2026, 3, 17, 10, 54, 42, 0, tzinfo=timezone(timedelta(hours=5, minutes=30))), + datetime(2026, 3, 17, 22, 24, 42, 123456, tzinfo=timezone(timedelta(hours=-12))), + datetime(2026, 3, 17, 5, 24, 42, 485699), + datetime(9999, 12, 31, 23, 59, 59, 999999, tzinfo=timezone.utc), + ] + for dt in variants: + iso = dt.isoformat() + assert len(iso) <= 40, f"{iso!r} is {len(iso)} chars" + + def test_old_limit_would_truncate(self): + """Confirm String(30) would have truncated UTC isoformat.""" + dt = datetime.now(UTC) + iso = dt.isoformat() + assert len(iso) > 30, "UTC isoformat must exceed old String(30) limit" + + +# ========================================================================= +# signed_by length — both fingerprint formats +# ========================================================================= + + +class TestSignedByLengths: + """Validate signed_by values fit within String(32).""" + + def test_legacy_hex_prefix_is_16_chars(self, tmp_path): + """Without keyring, signed_by is the 16-char hex prefix.""" + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + + assert len(capsule.signed_by) == 16 + assert len(capsule.signed_by) <= 32 + + def test_keyring_fingerprint_is_11_chars(self, tmp_path): + """With keyring, signed_by is qp_key_XXXX (11 chars).""" + key_dir = tmp_path / "keys" + seal = Seal( + key_path=key_dir / "key", + keyring=Keyring( + keyring_path=key_dir / "keyring.json", + key_path=key_dir / "key", + ), + ) + capsule = Capsule() + seal.seal(capsule) + + assert capsule.signed_by.startswith("qp_key_") + assert len(capsule.signed_by) == 11 + assert len(capsule.signed_by) <= 32 + + +# ========================================================================= +# SQLite roundtrip — store and retrieve +# ========================================================================= + + +class TestSQLiteSignedAtRoundtrip: + """signed_at/signed_by survive store → get via CapsuleStorage.""" + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_get(self, tmp_path): + storage = CapsuleStorage(db_path=tmp_path / "test.db") + seal = Seal(key_path=tmp_path / "key") + + capsule = Capsule() + seal.seal(capsule) + original_at = capsule.signed_at + + await storage.store(capsule) + retrieved = await storage.get(str(capsule.id)) + + assert retrieved is not None + assert retrieved.signed_at == original_at + assert retrieved.signed_at.isoformat() == original_at.isoformat() + await storage.close() + + @pytest.mark.asyncio + async def test_signed_by_preserved_on_get(self, tmp_path): + storage = CapsuleStorage(db_path=tmp_path / "test.db") + seal = Seal(key_path=tmp_path / "key") + + capsule = Capsule() + seal.seal(capsule) + original_by = capsule.signed_by + + await storage.store(capsule) + retrieved = await storage.get(str(capsule.id)) + + assert retrieved is not None + assert retrieved.signed_by == original_by + await storage.close() + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_list(self, tmp_path): + storage = CapsuleStorage(db_path=tmp_path / "test.db") + seal = Seal(key_path=tmp_path / "key") + + capsule = Capsule() + seal.seal(capsule) + original_at = capsule.signed_at + + await storage.store(capsule) + results = await storage.list() + + assert len(results) == 1 + assert results[0].signed_at == original_at + await storage.close() + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_get_latest(self, tmp_path): + storage = CapsuleStorage(db_path=tmp_path / "test.db") + seal = Seal(key_path=tmp_path / "key") + + capsule = Capsule() + seal.seal(capsule) + original_at = capsule.signed_at + + await storage.store(capsule) + latest = await storage.get_latest() + + assert latest is not None + assert latest.signed_at == original_at + await storage.close() + + +# ========================================================================= +# PostgresCapsuleStorage roundtrip (SQLite-backed for tests) +# ========================================================================= + + +@pytest.fixture +async def pg_storage(tmp_path): + """PostgresCapsuleStorage backed by SQLite for testing.""" + db_url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}" + s = PostgresCapsuleStorage.__new__(PostgresCapsuleStorage) + s.database_url = db_url + s._engine = create_async_engine(db_url, echo=False) + s._session_factory = async_sessionmaker( + s._engine, class_=AsyncSession, expire_on_commit=False + ) + s._initialized = False + yield s + await s.close() + + +class TestPGSignedAtRoundtrip: + """signed_at/signed_by survive store → get via PostgresCapsuleStorage.""" + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_get(self, tmp_path, pg_storage): + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + original_at = capsule.signed_at + + await pg_storage.store(capsule) + retrieved = await pg_storage.get(str(capsule.id)) + + assert retrieved is not None + assert retrieved.signed_at == original_at + assert retrieved.signed_at.isoformat() == original_at.isoformat() + + @pytest.mark.asyncio + async def test_signed_by_legacy_hex_preserved(self, tmp_path, pg_storage): + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + + assert len(capsule.signed_by) == 16 + await pg_storage.store(capsule) + retrieved = await pg_storage.get(str(capsule.id)) + + assert retrieved is not None + assert retrieved.signed_by == capsule.signed_by + + @pytest.mark.asyncio + async def test_signed_by_keyring_format_preserved(self, tmp_path, pg_storage): + key_dir = tmp_path / "keys" + seal = Seal( + key_path=key_dir / "key", + keyring=Keyring( + keyring_path=key_dir / "keyring.json", + key_path=key_dir / "key", + ), + ) + capsule = Capsule() + seal.seal(capsule) + + assert capsule.signed_by.startswith("qp_key_") + await pg_storage.store(capsule) + retrieved = await pg_storage.get(str(capsule.id)) + + assert retrieved is not None + assert retrieved.signed_by == capsule.signed_by + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_list(self, tmp_path, pg_storage): + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + original_at = capsule.signed_at + + await pg_storage.store(capsule) + results = await pg_storage.list() + + assert len(results) == 1 + assert results[0].signed_at == original_at + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_get_latest(self, tmp_path, pg_storage): + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + original_at = capsule.signed_at + + await pg_storage.store(capsule) + latest = await pg_storage.get_latest() + + assert latest is not None + assert latest.signed_at == original_at + + @pytest.mark.asyncio + async def test_signed_at_preserved_on_get_all_ordered(self, tmp_path, pg_storage): + seal = Seal(key_path=tmp_path / "key") + + originals = [] + for seq in range(3): + c = Capsule() + c.sequence = seq + seal.seal(c) + originals.append(c) + await pg_storage.store(c) + + ordered = await pg_storage.get_all_ordered() + assert len(ordered) == 3 + for original, restored in zip(originals, ordered): + assert restored.signed_at == original.signed_at + assert restored.signed_by == original.signed_by + + @pytest.mark.asyncio + async def test_verify_after_roundtrip(self, tmp_path, pg_storage): + """Capsule still verifies after store+retrieve (seal integrity).""" + seal = Seal(key_path=tmp_path / "key") + capsule = Capsule() + seal.seal(capsule) + + await pg_storage.store(capsule) + retrieved = await pg_storage.get(str(capsule.id)) + + assert retrieved is not None + assert seal.verify(retrieved) is True