From 2d81a8a91c111909d029584f3a8b27b92837f704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 7 Oct 2025 11:49:49 +0200 Subject: [PATCH 01/36] Fix capitalization in dockerfile --- ooniapi/services/ooniprobe/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/Dockerfile b/ooniapi/services/ooniprobe/Dockerfile index 42fae6d0b..1c2305356 100644 --- a/ooniapi/services/ooniprobe/Dockerfile +++ b/ooniapi/services/ooniprobe/Dockerfile @@ -1,5 +1,5 @@ # Python builder -FROM python:3.11-bookworm as builder +FROM python:3.11-bookworm AS builder ARG BUILD_LABEL=docker WORKDIR /build @@ -14,10 +14,10 @@ RUN find /build -type f -name '._*' -delete RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL -RUN hatch build +RUN make build ### Actual image running on the host -FROM python:3.11-bookworm as runner +FROM python:3.11-bookworm AS runner WORKDIR /app From e6c5fdce77d73091cc42f2be50e62e3de7c0fc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 7 Oct 2025 11:50:07 +0200 Subject: [PATCH 02/36] Clone userauth repo into include dir --- ooniapi/services/ooniprobe/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ooniapi/services/ooniprobe/Makefile b/ooniapi/services/ooniprobe/Makefile index afa989499..7b1475f38 100644 --- a/ooniapi/services/ooniprobe/Makefile +++ b/ooniapi/services/ooniprobe/Makefile @@ -50,6 +50,14 @@ test-cov: hatch run test-cov build: + mkdir -p include + # Git pull if already present or git clone if not + cd include && \ + if [ -d "userauth/.git" ]; then \ + cd userauth && git pull && cd ..; \ + else \ + git clone -b str-serialization https://github.com/ooni/userauth.git; \ + fi hatch build clean: From 294af3217098e88c085e282477317e37286f7bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 7 Oct 2025 16:04:08 +0200 Subject: [PATCH 03/36] Add ooniauth-py as dependency --- ooniapi/services/ooniprobe/Makefile | 8 -------- ooniapi/services/ooniprobe/pyproject.toml | 10 +++++++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ooniapi/services/ooniprobe/Makefile b/ooniapi/services/ooniprobe/Makefile index 7b1475f38..afa989499 100644 --- a/ooniapi/services/ooniprobe/Makefile +++ b/ooniapi/services/ooniprobe/Makefile @@ -50,14 +50,6 @@ test-cov: hatch run test-cov build: - mkdir -p include - # Git pull if already present or git clone if not - cd include && \ - if [ -d "userauth/.git" ]; then \ - cd userauth && git pull && cd ..; \ - else \ - git clone -b str-serialization https://github.com/ooni/userauth.git; \ - fi hatch build clean: diff --git a/ooniapi/services/ooniprobe/pyproject.toml b/ooniapi/services/ooniprobe/pyproject.toml index b8c6fb32b..c206b23c1 100644 --- a/ooniapi/services/ooniprobe/pyproject.toml +++ b/ooniapi/services/ooniprobe/pyproject.toml @@ -28,7 +28,8 @@ dependencies = [ "fastapi-utils[all] ~= 0.8.0", "zstd ~= 1.5.7.2", "boto3 ~= 1.39.3", - "boto3-stubs[s3] ~= 1.39.3" + "boto3-stubs[s3] ~= 1.39.3", + "ooniauth-py @ https://github.com/ooni/userauth/raw/refs/heads/str-serialization/ooniauth-py/wheels/ooniauth_py-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl" ] readme = "README.md" @@ -82,6 +83,13 @@ test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --s cov-report = ["coverage report"] cov = ["test-cov", "cov-report"] +# Allows specifyng dependencies as a direct reference like an URL +# or a file path. +# Required by ooniauth-py +# See: https://hatch.pypa.io/1.13/config/metadata/#allowing-direct-references +[tool.hatch.metadata] +allow-direct-references = true + [tool.pytest.ini_options] addopts = ["--import-mode=importlib"] From 6754df8a520b38d789f784fe69733468661bb889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 10:15:41 +0200 Subject: [PATCH 04/36] Remove unused imports --- ooniapi/services/ooniprobe/src/ooniprobe/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 1cfc2e104..794887906 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -1,8 +1,7 @@ from datetime import datetime -from typing import Dict from .common.models import UtcDateTime from .common.postgresql import Base -from sqlalchemy import ForeignKey, Sequence, String, Integer +from sqlalchemy import ForeignKey, Sequence, String from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column, relationship From 42b5ec5db85c267120727f2f3d310681d9e1a729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 10:55:02 +0200 Subject: [PATCH 05/36] Add server state model for anonymous credentials --- ...f_add_server_state_table_for_anonymous_.py | 45 +++++++++++++++++++ .../ooniprobe/src/ooniprobe/models.py | 19 ++++++++ 2 files changed, 64 insertions(+) create mode 100644 ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py diff --git a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py new file mode 100644 index 000000000..725ef7225 --- /dev/null +++ b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py @@ -0,0 +1,45 @@ +"""Add server state table for anonymous credentials + +Revision ID: 7e28b5d17a7f +Revises: 8e7ecea5c2f5 +Create Date: 2025-10-08 10:36:55.796144 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlalchemy.schema as sc + + +# revision identifiers, used by Alembic. +revision: str = '7e28b5d17a7f' +down_revision: Union[str, None] = '8e7ecea5c2f5' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# --- +ooniprobe_server_state_id_seq = sc.Sequence("ooniprobe_server_state_id_seq", start=1) +def upgrade() -> None: + op.execute(sc.CreateSequence(ooniprobe_server_state_id_seq)) + + op.create_table( + "ooniprobe_server_state", + sa.Column( + "id", + sa.String(), + nullable=False, + server_default=ooniprobe_server_state_id_seq.next_value(), + primary_key=True + ), + sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), + sa.Column("secret_key", sa.String(), nullable=False), + sa.Column("public_parameters", sa.String(), nullable=False) + ) + pass + + +def downgrade() -> None: + op.drop_table("ooniprobe_server_state") + op.execute(sc.DropSequence(ooniprobe_server_state_id_seq)) + pass diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 794887906..5965211d9 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -48,3 +48,22 @@ class OONIProbeVPNProviderEndpoint(Base): provider_id = mapped_column(ForeignKey("ooniprobe_vpn_provider.id")) provider = relationship("OONIProbeVPNProvider", back_populates="endpoints") + + +class OONIProbeServerState(Base): + """ + Server state used for the anonymous credentials protocol. + Stores public parameters and secret key used for credential + generation + """ + __tablename__ = "ooniprobe_server_state" + + id: Mapped[str] = mapped_column( + String, + Sequence("ooniprobe_server_state_id_seq", start=1), + primary_key=True, + nullable=False + ) + date_created: Mapped[datetime] = mapped_column(UtcDateTime()) + secret_key: Mapped[str] = mapped_column() + public_parameters: Mapped[str] = mapped_column() \ No newline at end of file From 7c118748abd36ecdcfd472cd8e20405a290b9e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 10:55:50 +0200 Subject: [PATCH 06/36] remove unnecessary pass --- .../7e28b5d17a7f_add_server_state_table_for_anonymous_.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py index 725ef7225..5fcbbd929 100644 --- a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py +++ b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py @@ -36,10 +36,8 @@ def upgrade() -> None: sa.Column("secret_key", sa.String(), nullable=False), sa.Column("public_parameters", sa.String(), nullable=False) ) - pass def downgrade() -> None: op.drop_table("ooniprobe_server_state") op.execute(sc.DropSequence(ooniprobe_server_state_id_seq)) - pass From 0970cb5905ce282b4c18855e704744213549f7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 11:56:49 +0200 Subject: [PATCH 07/36] Add utility functions to work with protocol objects --- .../ooniprobe/src/ooniprobe/models.py | 46 +++++++++++++++++-- .../ooniprobe/routers/v1/probe_services.py | 2 +- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 5965211d9..482f68ceb 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -1,9 +1,12 @@ from datetime import datetime +from typing import Self from .common.models import UtcDateTime from .common.postgresql import Base from sqlalchemy import ForeignKey, Sequence, String from sqlalchemy.orm import Mapped -from sqlalchemy.orm import mapped_column, relationship +from sqlalchemy.orm import mapped_column, relationship, Session +from sqlalchemy import desc +from ooniauth_py import ServerState class OONIProbeVPNProvider(Base): @@ -56,14 +59,51 @@ class OONIProbeServerState(Base): Stores public parameters and secret key used for credential generation """ + __tablename__ = "ooniprobe_server_state" id: Mapped[str] = mapped_column( String, Sequence("ooniprobe_server_state_id_seq", start=1), primary_key=True, - nullable=False + nullable=False, ) date_created: Mapped[datetime] = mapped_column(UtcDateTime()) secret_key: Mapped[str] = mapped_column() - public_parameters: Mapped[str] = mapped_column() \ No newline at end of file + public_parameters: Mapped[str] = mapped_column() + + @classmethod + def get_latest(cls, session: Session) -> Self | None: + return ( + session + .query(cls) + .order_by(desc(cls.date_created)) + .limit(1) + .one_or_none() + ) + + @classmethod + def make_new_state(cls, session: Session, state : ServerState | None = None) -> Self: + """ + Creates a new state, saving it to db + If no state value is provided, create a default one with random values + """ + if state is None: + state = ServerState() + + new = cls( + secret_key=state.get_secret_key(), + public_parameters = state.get_public_parameters() + ) + session.add(new) + session.commit() + session.refresh(new) + + return new + + def to_protocol(self) -> ServerState: + """ + Converts this instance into a protocol object + """ + + return ServerState.from_creds(self.public_parameters, self.secret_key) \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 8d4bc2e65..93dfd2d4f 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timezone, timedelta import time -from typing import List, Optional, Any, Dict, Tuple, Optional +from typing import List, Any, Dict, Tuple, Optional import random import geoip2 From c97f3ba695ee23ab7c24a3241ff77d8d758a4ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 12:58:17 +0200 Subject: [PATCH 08/36] Add initialization of server state table --- .../ooniprobe/src/ooniprobe/dependencies.py | 4 ++-- .../services/ooniprobe/src/ooniprobe/main.py | 17 ++++++++++++++ .../ooniprobe/src/ooniprobe/models.py | 22 +++++++++++++++---- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py index 9f1e13279..4ad38ecc1 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -6,7 +6,7 @@ import geoip2.database from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import sessionmaker, Session from clickhouse_driver import Client as Clickhouse @@ -23,13 +23,13 @@ def get_postgresql_session(settings: SettingsDep): engine = create_engine(settings.postgresql_url) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - db = SessionLocal() try: yield db finally: db.close() +PostgresSessionDep = Annotated[Session, get_postgresql_session] def get_cc_reader(settings: SettingsDep): db_path = Path(settings.geoip_db_dir, "cc.mmdb") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py index ca5909bef..fc6d71157 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/main.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -8,6 +8,8 @@ from fastapi_utils.tasks import repeat_every +from sqlalchemy.orm import Session + from pydantic import BaseModel from prometheus_fastapi_instrumentator import Instrumentator @@ -48,6 +50,11 @@ async def lifespan( if repeating_tasks_active: await setup_repeating_tasks(settings) + db = get_postgresql_session(settings) + session = next(db) + models.OONIProbeServerState.init_table(session) + next(db, None) # closes the connection + yield @@ -133,6 +140,16 @@ async def health( if settings.prometheus_metrics_password == "CHANGEME": errors.append("bad_prometheus_password") + # check that we have at least one server state object for credentials validation + try: + state = models.OONIProbeServerState.get_latest(db) + if state is None: + errors.append("no_server_state_entry") + except Exception as exc: + log.error("Error trying to retrieve server state") + log.error(exc) + pass # Database error already reported above + status = "ok" if len(errors) > 0: status = "fail" diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 482f68ceb..743f3e64d 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -2,11 +2,13 @@ from typing import Self from .common.models import UtcDateTime from .common.postgresql import Base -from sqlalchemy import ForeignKey, Sequence, String +from sqlalchemy import ForeignKey, Sequence, String, func from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column, relationship, Session from sqlalchemy import desc from ooniauth_py import ServerState +import logging +log = logging.getLogger(__name__) class OONIProbeVPNProvider(Base): @@ -57,7 +59,7 @@ class OONIProbeServerState(Base): """ Server state used for the anonymous credentials protocol. Stores public parameters and secret key used for credential - generation + generation and validation """ __tablename__ = "ooniprobe_server_state" @@ -68,7 +70,7 @@ class OONIProbeServerState(Base): primary_key=True, nullable=False, ) - date_created: Mapped[datetime] = mapped_column(UtcDateTime()) + date_created: Mapped[datetime] = mapped_column(UtcDateTime(), default=func.now()) secret_key: Mapped[str] = mapped_column() public_parameters: Mapped[str] = mapped_column() @@ -106,4 +108,16 @@ def to_protocol(self) -> ServerState: Converts this instance into a protocol object """ - return ServerState.from_creds(self.public_parameters, self.secret_key) \ No newline at end of file + return ServerState.from_creds(self.public_parameters, self.secret_key) + + @classmethod + def init_table(cls, session: Session): + """ + Creates a new entry if none exists. + """ + latest = cls.get_latest(session) + if latest is None: + log.info("No OONIProbeServerState entry found. Creating a new one...") + cls.make_new_state(session) + else: + log.info("OONIProbeServerState already initialized!") From 950cbe3cf21277d6fc334bac003397d1afb820db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 13:23:49 +0200 Subject: [PATCH 09/36] Add manifest endpoint --- .../ooniprobe/src/ooniprobe/dependencies.py | 10 +++++++++ .../ooniprobe/routers/v1/probe_services.py | 21 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py index 4ad38ecc1..68f364667 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -15,6 +15,7 @@ from .common.config import Settings from .common.dependencies import get_settings +from .models import OONIProbeServerState SettingsDep: TypeAlias = Annotated[Settings, Depends(get_settings)] @@ -64,3 +65,12 @@ def get_s3_client() -> S3Client: S3ClientDep = Annotated[S3Client, Depends(get_s3_client)] + + +def get_latest_state(session: PostgresSessionDep): + state = OONIProbeServerState.get_latest(session) + assert state is not None, "Uninitialized `OONIProbeServerState` table" + + return state + +LatestStateDep = Annotated[OONIProbeServerState, Depends(get_latest_state)] \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 93dfd2d4f..6822ee4e4 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -16,7 +16,7 @@ lookup_probe_cc, lookup_probe_network, ) -from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep +from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestStateDep from ...common.dependencies import get_settings from ...common.routers import BaseModel from ...common.auth import create_jwt, decode_jwt, jwt @@ -590,3 +590,22 @@ def random_web_test_helpers(th_list: List[str]) -> List[Dict]: for th_addr in th_list: out.append({"address": th_addr, "type": "https"}) return out + +# -- ------------------------------------ + +class ManifestResponse(BaseModel): + nym_scope: str + public_parameters: str + submission_policy: Dict[str, Any] + # TODO: Is the manifest version different from the server state? For now we assume it's the same + # and use the `date_created` as version + date_created: str + +@router.get("/manifest") +def manifest(state : LatestStateDep): + return ManifestResponse( + nym_scope="ooni.org/{probe_cc}/{probe_asn}", + public_parameters=state.public_parameters, + submission_policy={}, + date_created=state.date_created.isoformat() + ) \ No newline at end of file From a9990ed64a83f9d20fb2c02ae3b32848034eb2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 14:15:59 +0200 Subject: [PATCH 10/36] Fix error with postgres dep --- ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py | 5 ++--- ooniapi/services/ooniprobe/src/ooniprobe/main.py | 2 -- .../ooniprobe/src/ooniprobe/routers/v1/probe_services.py | 9 +++++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py index 68f364667..f6eedb2b8 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -30,7 +30,7 @@ def get_postgresql_session(settings: SettingsDep): finally: db.close() -PostgresSessionDep = Annotated[Session, get_postgresql_session] +PostgresSessionDep = Annotated[Session, Depends(get_postgresql_session)] def get_cc_reader(settings: SettingsDep): db_path = Path(settings.geoip_db_dir, "cc.mmdb") @@ -67,10 +67,9 @@ def get_s3_client() -> S3Client: S3ClientDep = Annotated[S3Client, Depends(get_s3_client)] -def get_latest_state(session: PostgresSessionDep): +def get_latest_state(session : PostgresSessionDep) -> OONIProbeServerState: state = OONIProbeServerState.get_latest(session) assert state is not None, "Uninitialized `OONIProbeServerState` table" - return state LatestStateDep = Annotated[OONIProbeServerState, Depends(get_latest_state)] \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py index fc6d71157..baae4b6af 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/main.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -8,8 +8,6 @@ from fastapi_utils.tasks import repeat_every -from sqlalchemy.orm import Session - from pydantic import BaseModel from prometheus_fastapi_instrumentator import Instrumentator diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 6822ee4e4..147fe87c4 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -599,13 +599,14 @@ class ManifestResponse(BaseModel): submission_policy: Dict[str, Any] # TODO: Is the manifest version different from the server state? For now we assume it's the same # and use the `date_created` as version - date_created: str + date_created: datetime -@router.get("/manifest") -def manifest(state : LatestStateDep): +@router.get("/manifest", tags=["anonymous_credentials"]) +def manifest(response: Response, state : LatestStateDep): + setnocacheresponse(response) return ManifestResponse( nym_scope="ooni.org/{probe_cc}/{probe_asn}", public_parameters=state.public_parameters, submission_policy={}, - date_created=state.date_created.isoformat() + date_created=state.date_created ) \ No newline at end of file From 94a79364e94e1122ec11ba4b612297ac501236a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 14:20:28 +0200 Subject: [PATCH 11/36] Init server state dep in tests --- ooniapi/services/ooniprobe/tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index f15438931..aa63f7de0 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -16,6 +16,8 @@ from ooniprobe.dependencies import get_s3_client from ooniprobe.main import app from ooniprobe.download_geoip import try_update +from ooniprobe.dependencies import get_postgresql_session +from ooniprobe.models import OONIProbeServerState def make_override_get_settings(**kw): @@ -100,6 +102,13 @@ def geoip_db_dir(fixture_path): def client(clickhouse_server, test_settings, geoip_db_dir): app.dependency_overrides[get_settings] = test_settings app.dependency_overrides[get_s3_client] = get_s3_client_mock + + # Initialize server state + db = get_postgresql_session(test_settings()) + session = next(db) + OONIProbeServerState.init_table(session) + next(db, None) + # lifespan won't run so do this here to have the DB try_update(geoip_db_dir) client = TestClient(app) From bcffac5188497afbd5c6d9fcb51afa39e4c76c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 15:59:31 +0200 Subject: [PATCH 12/36] fix bare except --- ooniapi/services/ooniprobe/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index aa63f7de0..76c32eaff 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -182,5 +182,5 @@ def is_fastpath_running(url: str) -> bool: try: resp = urlopen(url) return resp.status == 200 - except: + except Exception: return False \ No newline at end of file From 7b602c29c4d01c5b7b8b6a28f72501d851447c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 16:01:23 +0200 Subject: [PATCH 13/36] remove unused import --- ooniapi/services/ooniprobe/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index 76c32eaff..895b4edf6 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -1,4 +1,3 @@ -from tempfile import tempdir import pathlib from pathlib import Path import pytest From 50e0c478bb59a5c80505e1fa405318bdb659cfa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 16:15:51 +0200 Subject: [PATCH 14/36] Add test file for anonymous credentials --- .../services/ooniprobe/tests/test_anoncred.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 ooniapi/services/ooniprobe/tests/test_anoncred.py diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py new file mode 100644 index 000000000..bbc211c21 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -0,0 +1,19 @@ +from datetime import datetime +from typing import Any, Dict +from httpx import Client +from fastapi import status +from ooniprobe.models import OONIProbeServerState +from ooniprobe.common.routers import ISO_FORMAT_DATETIME + +def getj(client : Client, url: str, params: Dict[str, Any] = {}) -> Dict[str, Any]: + resp = client.get(url) + assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" + return resp.json() + +def test_manifest_basic(client, db): + latest = OONIProbeServerState.get_latest(db) + assert latest is not None, "Server state not initialized" + + m = getj(client, "/api/v1/manifest") + assert latest.public_parameters == m['public_parameters'] + assert datetime.strftime(latest.date_created, ISO_FORMAT_DATETIME) == m['date_created'] From ad1498787e29e59d8d1e0e43a0a1fa4ed2e199a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 16:43:15 +0200 Subject: [PATCH 15/36] Add register endpoint for anonymous credentials --- .../ooniprobe/src/ooniprobe/models.py | 9 ++++ .../ooniprobe/routers/v1/probe_services.py | 52 +++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 743f3e64d..56ba643aa 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -74,6 +74,15 @@ class OONIProbeServerState(Base): secret_key: Mapped[str] = mapped_column() public_parameters: Mapped[str] = mapped_column() + @classmethod + def get_by_datetime(cls, session: Session, dt : datetime) -> Self | None: + return ( + session + .query(cls) + .where(cls.date_created == dt) + .one_or_none() + ) + @classmethod def get_latest(cls, session: Session) -> Self | None: return ( diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 147fe87c4..8ab9e7b08 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -6,9 +6,10 @@ import geoip2 import geoip2.errors -from fastapi import APIRouter, Depends, HTTPException, Response, Request +from fastapi import APIRouter, Depends, HTTPException, Response, Request, status from prometheus_client import Counter, Info, Gauge from pydantic import Field +from ooniauth_py import ProtocolError, CredentialError, DeserializationFailed from ...utils import ( generate_report_id, @@ -16,12 +17,13 @@ lookup_probe_cc, lookup_probe_network, ) -from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestStateDep +from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestStateDep, PostgresSessionDep from ...common.dependencies import get_settings from ...common.routers import BaseModel from ...common.auth import create_jwt, decode_jwt, jwt from ...common.config import Settings from ...common.utils import setnocacheresponse +from ...models import OONIProbeServerState from ...prio import generate_test_list router = APIRouter(prefix="/v1") @@ -602,11 +604,51 @@ class ManifestResponse(BaseModel): date_created: datetime @router.get("/manifest", tags=["anonymous_credentials"]) -def manifest(response: Response, state : LatestStateDep): - setnocacheresponse(response) +def manifest(Response, state : LatestStateDep): return ManifestResponse( nym_scope="ooni.org/{probe_cc}/{probe_asn}", public_parameters=state.public_parameters, submission_policy={}, date_created=state.date_created - ) \ No newline at end of file + ) + +class RegisterRequest(BaseModel): + manifest_date_created: datetime + credential_sign_request: str + +class RegisterResponse(BaseModel): + credential_sign_response: str + emission_day: int + +@router.get("/register", tags=["anonymous_credentials"]) +def register(register_request: RegisterRequest, session : PostgresSessionDep): + + state = OONIProbeServerState.get_by_datetime(session, register_request.manifest_date_created) + if state is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + {"error" : f"No manifest with creation date '{register_request.manifest_date_created.isoformat()}' was found"} + ) + + protocol_state = state.to_protocol() + + try: + resp = protocol_state.handle_registration_request(register_request.credential_sign_request) + except (ProtocolError, CredentialError, DeserializationFailed) as e: + raise to_http_exception(e) + + return RegisterResponse( + credential_sign_response=resp, + emission_day=protocol_state.today() + ) + + +def to_http_exception(error: ProtocolError | CredentialError | DeserializationFailed): + if isinstance(error, (ProtocolError, DeserializationFailed)): + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail={"error": str(error)} + ) + if isinstance(error, CredentialError): + return HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail={"error": str(error)} + ) From d286c76f4d84a4670662181be349d73a75c36c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 17:01:31 +0200 Subject: [PATCH 16/36] Add test for credential sign request --- .../ooniprobe/routers/v1/probe_services.py | 6 +++--- .../services/ooniprobe/tests/test_anoncred.py | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 8ab9e7b08..ed5c6528a 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -604,7 +604,7 @@ class ManifestResponse(BaseModel): date_created: datetime @router.get("/manifest", tags=["anonymous_credentials"]) -def manifest(Response, state : LatestStateDep): +def manifest(state : LatestStateDep) -> ManifestResponse: return ManifestResponse( nym_scope="ooni.org/{probe_cc}/{probe_asn}", public_parameters=state.public_parameters, @@ -620,8 +620,8 @@ class RegisterResponse(BaseModel): credential_sign_response: str emission_day: int -@router.get("/register", tags=["anonymous_credentials"]) -def register(register_request: RegisterRequest, session : PostgresSessionDep): +@router.post("/sign_credential", tags=["anonymous_credentials"]) +def sign_credential(register_request: RegisterRequest, session : PostgresSessionDep): state = OONIProbeServerState.get_by_datetime(session, register_request.manifest_date_created) if state is None: diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index bbc211c21..a80bf363a 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -4,12 +4,18 @@ from fastapi import status from ooniprobe.models import OONIProbeServerState from ooniprobe.common.routers import ISO_FORMAT_DATETIME +from ooniauth_py import UserState def getj(client : Client, url: str, params: Dict[str, Any] = {}) -> Dict[str, Any]: resp = client.get(url) assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" return resp.json() +def postj(client : Client, url: str, json: Dict[str, Any] | None = None) -> Dict[str, Any]: + resp = client.post(url, json=json) + assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" + return resp.json() + def test_manifest_basic(client, db): latest = OONIProbeServerState.get_latest(db) assert latest is not None, "Server state not initialized" @@ -17,3 +23,18 @@ def test_manifest_basic(client, db): m = getj(client, "/api/v1/manifest") assert latest.public_parameters == m['public_parameters'] assert datetime.strftime(latest.date_created, ISO_FORMAT_DATETIME) == m['date_created'] + +def test_registration_basic(client): + + manifest = getj(client, "/api/v1/manifest") + + user_state = UserState(manifest['public_parameters']) + sign_req = user_state.make_registration_request() + resp = postj( + client, + "/api/v1/sign_credential", + { + "credential_sign_request" : sign_req, + "manifest_date_created" : manifest['date_created'] + } + ) From bfb5600fb7f0aa5865e88cfb70144c29b5fde174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 8 Oct 2025 17:04:30 +0200 Subject: [PATCH 17/36] Add line to check if user is able to verify credential --- ooniapi/services/ooniprobe/tests/test_anoncred.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index a80bf363a..2a0e97a47 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -38,3 +38,5 @@ def test_registration_basic(client): "manifest_date_created" : manifest['date_created'] } ) + # should be able to verify this credential + user_state.handle_registration_response(resp['credential_sign_response']) # should not crash From f582c8f5c1fd06ac8953c67abdd2b38143b448e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 9 Oct 2025 10:27:55 +0200 Subject: [PATCH 18/36] Improve error reporting --- .../ooniprobe/routers/v1/probe_services.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index ed5c6528a..49c205c60 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -627,7 +627,9 @@ def sign_credential(register_request: RegisterRequest, session : PostgresSession if state is None: raise HTTPException( status.HTTP_404_NOT_FOUND, - {"error" : f"No manifest with creation date '{register_request.manifest_date_created.isoformat()}' was found"} + { + "error" : "manifest_not_found", + "detail" : f"No manifest with creation date '{register_request.manifest_date_created.isoformat()}' was found"} ) protocol_state = state.to_protocol() @@ -644,11 +646,22 @@ def sign_credential(register_request: RegisterRequest, session : PostgresSession def to_http_exception(error: ProtocolError | CredentialError | DeserializationFailed): + + error_to_string = { + ProtocolError : "protocol_error", + DeserializationFailed : "deserialization_failed", + CredentialError : "credential_error" + } + + error_str = error_to_string[type(error)] + if isinstance(error, (ProtocolError, DeserializationFailed)): return HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail={"error": str(error)} + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": error_str, "detail": str(error)} ) if isinstance(error, CredentialError): return HTTPException( - status_code=status.HTTP_403_FORBIDDEN, detail={"error": str(error)} + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": error_str, "detail": str(error)} ) From db5763ae954cb2894a7d87060eb931a079cc9e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 9 Oct 2025 11:08:52 +0200 Subject: [PATCH 19/36] Add test for not found error --- .../src/ooniprobe/routers/v1/probe_services.py | 4 ++-- .../services/ooniprobe/tests/test_anoncred.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 49c205c60..40854458a 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -629,7 +629,7 @@ def sign_credential(register_request: RegisterRequest, session : PostgresSession status.HTTP_404_NOT_FOUND, { "error" : "manifest_not_found", - "detail" : f"No manifest with creation date '{register_request.manifest_date_created.isoformat()}' was found"} + "message" : f"No manifest with creation date '{register_request.manifest_date_created.isoformat()}' was found"} ) protocol_state = state.to_protocol() @@ -663,5 +663,5 @@ def to_http_exception(error: ProtocolError | CredentialError | DeserializationFa if isinstance(error, CredentialError): return HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail={"error": error_str, "detail": str(error)} + detail={"error": error_str, "message": str(error)} ) diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 2a0e97a47..038bef6d5 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -40,3 +40,21 @@ def test_registration_basic(client): ) # should be able to verify this credential user_state.handle_registration_response(resp['credential_sign_response']) # should not crash + +def test_registration_errors(client): + + manifest = getj(client, "/api/v1/manifest") + user_state = UserState(manifest['public_parameters']) + sign_req = user_state.make_registration_request() + bad_date = datetime.strftime(datetime(2012, 12, 21), ISO_FORMAT_DATETIME) + resp = client.post("/api/v1/sign_credential", + json={ + "credential_sign_request" : sign_req, + "manifest_date_created" : bad_date + } + ) + # Bad date for credential should raise 404 + assert resp.status_code == 404, resp.content + j = resp.json() + assert 'error' in j['detail'] and 'message' in j['detail'], j + assert j['detail']['error'] == "manifest_not_found" From f06ce429948d349e14c28e279c523c5f12193188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 9 Oct 2025 11:21:38 +0200 Subject: [PATCH 20/36] Add test for protocol error --- .../ooniprobe/routers/v1/probe_services.py | 5 +++-- .../services/ooniprobe/tests/test_anoncred.py | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 40854458a..be4ada422 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -620,6 +620,7 @@ class RegisterResponse(BaseModel): credential_sign_response: str emission_day: int +# TODO: choose a better name for this endpoint @router.post("/sign_credential", tags=["anonymous_credentials"]) def sign_credential(register_request: RegisterRequest, session : PostgresSessionDep): @@ -655,12 +656,12 @@ def to_http_exception(error: ProtocolError | CredentialError | DeserializationFa error_str = error_to_string[type(error)] - if isinstance(error, (ProtocolError, DeserializationFailed)): + if isinstance(error, DeserializationFailed): return HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail={"error": error_str, "detail": str(error)} ) - if isinstance(error, CredentialError): + if isinstance(error, (CredentialError, ProtocolError)): # return HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail={"error": error_str, "message": str(error)} diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 038bef6d5..df28db603 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -4,7 +4,7 @@ from fastapi import status from ooniprobe.models import OONIProbeServerState from ooniprobe.common.routers import ISO_FORMAT_DATETIME -from ooniauth_py import UserState +from ooniauth_py import UserState, ServerState def getj(client : Client, url: str, params: Dict[str, Any] = {}) -> Dict[str, Any]: resp = client.get(url) @@ -43,18 +43,28 @@ def test_registration_basic(client): def test_registration_errors(client): - manifest = getj(client, "/api/v1/manifest") - user_state = UserState(manifest['public_parameters']) - sign_req = user_state.make_registration_request() bad_date = datetime.strftime(datetime(2012, 12, 21), ISO_FORMAT_DATETIME) resp = client.post("/api/v1/sign_credential", json={ - "credential_sign_request" : sign_req, + "credential_sign_request" : "doesntmatter", "manifest_date_created" : bad_date } ) - # Bad date for credential should raise 404 + # Bad manifest date should raise 404 assert resp.status_code == 404, resp.content j = resp.json() assert 'error' in j['detail'] and 'message' in j['detail'], j assert j['detail']['error'] == "manifest_not_found" + + # Not using the right public params should not verify + manifest = getj(client, "/api/v1/manifest") + bad_server = ServerState() + user = UserState(bad_server.get_public_parameters()) + resp = client.post("/api/v1/sign_credential", json={ + "credential_sign_request" : user.make_registration_request(), + "manifest_date_created" : manifest['date_created'] + }) + + assert resp.status_code == status.HTTP_403_FORBIDDEN, resp.content + j = resp.json() + assert j['detail']['error'] == 'protocol_error' From 08413d7831ef68f1c1c2684754f72e69cd55254f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 9 Oct 2025 11:35:36 +0200 Subject: [PATCH 21/36] Add test for bad serialization --- ooniapi/services/ooniprobe/tests/test_anoncred.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index df28db603..dcbead732 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -68,3 +68,18 @@ def test_registration_errors(client): assert resp.status_code == status.HTTP_403_FORBIDDEN, resp.content j = resp.json() assert j['detail']['error'] == 'protocol_error' + + # Changing random characters should mess with the serialization + user = UserState(manifest['public_parameters']) + sign_req = user.make_registration_request() + bad = "bad" + assert len(sign_req) >= len(bad), sign_req + sign_req = bad + sign_req[len(bad):] + resp = client.post("/api/v1/sign_credential", json={ + "credential_sign_request" : sign_req, + "manifest_date_created" : manifest['date_created'] + }) + + assert resp.status_code == status.HTTP_400_BAD_REQUEST, resp.content + j = resp.json() + assert j['detail']['error'] == 'deserialization_failed', j \ No newline at end of file From 75eead1da42df6ffb349792f001ee33b079887ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 9 Oct 2025 13:23:00 +0200 Subject: [PATCH 22/36] Refactor to share metrics to the entire module --- .../ooniprobe/src/ooniprobe/metrics.py | 78 +++++++++ .../src/ooniprobe/routers/reports.py | 94 +---------- .../ooniprobe/routers/v1/probe_services.py | 157 +++++++++++++----- .../services/ooniprobe/src/ooniprobe/utils.py | 34 +++- 4 files changed, 240 insertions(+), 123 deletions(-) create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/metrics.py diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py b/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py new file mode 100644 index 000000000..568e5bf0e --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py @@ -0,0 +1,78 @@ +from prometheus_client import Counter, Info, Gauge + +class Metrics: + # -- < Measurement submission > ------------------------------------ + MSMNT_DISCARD_ASN0 = Counter( + "receive_measurement_discard_asn_0", + "How many measurements were discarded due to probe_asn == ASN0", + ) + + MSMNT_DISCARD_CC_ZZ = Counter( + "receive_measurement_discard_cc_zz", + "How many measurements were discarded due to probe_cc == ZZ", + ) + + MSMNT_RECEIVED_CNT = Counter( + "receive_measurement_count", + "Count of incomming measurements", + ) + + PROBE_CC_ASN_MATCH = Counter( + "probe_cc_asn_match", + "How many matches between reported and observed probe_cc and asn", + ) + + PROBE_CC_ASN_NO_MATCH = Counter( + "probe_cc_asn_nomatch", + "How many mismatches between reported and observed probe_cc and asn", + labelnames=["mismatch"], + ) + + MISSED_MSMNTS = Counter( + "missed_msmnts", "Measurements that failed to be sent to the fast path." + ) + + SEND_FASTPATH_FAILURE = Counter( + "measurement_fastpath_send_failure_count", + "How many times ooniprobe failed to send a measurement to fastpath", + ) + + SEND_S3_FAILURE = Counter( + "measurement_s3_upload_failure_count", + "How many times ooniprobe failed to send a measurement to s3. " + "Measurements are sent to s3 when they can't be sent to the fastpath", + ) + + # -- < Probe services > ---------------------------------------------------- + PROBE_LOGIN = Counter( + "probe_login_requests", + "Requests made to the probe login endpoint", + labelnames=["state", "detail", "login"], + ) + + PROBE_UPDATE_INFO = Info( + "probe_update_info", + "Information reported in the probe update endpoint", + ) + + CHECK_IN_TEST_LIST_COUNT = Gauge( + "check_in_test_list_count", "Amount of test lists present in each experiment" + ) + + GEOIP_ADDR_FOUND = Counter( + "geoip_ipaddr_found", + "If the ip address was found by geoip", + labelnames=["probe_cc", "asn"], + ) + + GEOIP_ADDR_NOT_FOUND = Counter( + "geoip_ipaddr_not_found", "We couldn't look up the IP address in the database" + ) + + GEOIP_CC_DIFFERS = Counter( + "geoip_cc_differs", "There's a mismatch between reported CC and observed CC" + ) + + GEOIP_ASN_DIFFERS = Counter( + "geoip_asn_differs", "There's a mismatch between reported ASN and observed ASN" + ) \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py index f8c3e931c..d575167a5 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py @@ -1,25 +1,22 @@ -from typing import List, Annotated, Dict, Any +from typing import List, Dict, Any import asyncio -from pathlib import Path import logging from hashlib import sha512 -from urllib.request import urlopen from datetime import datetime, timezone import io import random -from fastapi import Request, Response, APIRouter, HTTPException, Header, Body +from fastapi import Request, Response, APIRouter, Header import httpx from pydantic import Field -from prometheus_client import Counter import zstd from ..utils import ( generate_report_id, - extract_probe_ipaddr, - lookup_probe_cc, - lookup_probe_network, + error, + compare_probe_msmt_cc_asn ) +from ..metrics import Metrics from ..dependencies import SettingsDep, ASNReaderDep, CCReaderDep, S3ClientDep from ..common.routers import BaseModel from ..common.utils import setnocacheresponse @@ -30,49 +27,6 @@ log = logging.getLogger(__name__) -class Metrics: - MSMNT_DISCARD_ASN0 = Counter( - "receive_measurement_discard_asn_0", - "How many measurements were discarded due to probe_asn == ASN0", - ) - - MSMNT_DISCARD_CC_ZZ = Counter( - "receive_measurement_discard_cc_zz", - "How many measurements were discarded due to probe_cc == ZZ", - ) - - MSMNT_RECEIVED_CNT = Counter( - "receive_measurement_count", - "Count of incomming measurements", - ) - - PROBE_CC_ASN_MATCH = Counter( - "probe_cc_asn_match", - "How many matches between reported and observed probe_cc and asn", - ) - - PROBE_CC_ASN_NO_MATCH = Counter( - "probe_cc_asn_nomatch", - "How many mismatches between reported and observed probe_cc and asn", - labelnames=["mismatch"], - ) - - MISSED_MSMNTS = Counter( - "missed_msmnts", "Measurements that failed to be sent to the fast path." - ) - - SEND_FASTPATH_FAILURE = Counter( - "measurement_fastpath_send_failure_count", - "How many times ooniprobe failed to send a measurement to fastpath", - ) - - SEND_S3_FAILURE = Counter( - "measurement_s3_upload_failure_count", - "How many times ooniprobe failed to send a measurement to s3. " - "Measurements are sent to s3 when they can't be sent to the fastpath", - ) - - class OpenReportRequest(BaseModel): """ Open report @@ -187,10 +141,11 @@ async def receive_measurement( data = await request.body() if content_encoding == "zstd": try: + compressed_len = len(data) data = zstd.decompress(data) - ratio = len(data) / len(data) + ratio = compressed_len / len(data) log.debug(f"Zstd compression ratio {ratio}") - except Exception as e: + except Exception: log.info("Failed zstd decompression") error("Incorrect format") @@ -213,7 +168,7 @@ async def receive_measurement( async with httpx.AsyncClient() as client: resp = await client.post(url, content=data, timeout=59) - + assert resp.status_code == 200, resp.content return ReceiveMeasurementResponse(measurement_uid=msmt_uid) @@ -247,34 +202,3 @@ def close_report(report_id): Close a report """ return {} - - -def error(msg: str, status_code: int = 400): - raise HTTPException(status_code=status_code, detail=msg) - - -def compare_probe_msmt_cc_asn( - cc: str, - asn: str, - request: Request, - cc_reader: CCReaderDep, - asn_reader: ASNReaderDep, -): - """Compares CC/ASN from measurement with CC/ASN from HTTPS connection ipaddr - Generates a metric. - """ - try: - cc = cc.upper() - ipaddr = extract_probe_ipaddr(request) - db_probe_cc = lookup_probe_cc(ipaddr, cc_reader) - db_asn, _ = lookup_probe_network(ipaddr, asn_reader) - if db_asn.startswith("AS"): - db_asn = db_asn[2:] - if db_probe_cc == cc and db_asn == asn: - Metrics.PROBE_CC_ASN_MATCH.inc() - elif db_probe_cc != cc: - Metrics.PROBE_CC_ASN_NO_MATCH.labels(mismatch="cc").inc() - elif db_asn != asn: - Metrics.PROBE_CC_ASN_NO_MATCH.labels(mismatch="asn").inc() - except Exception: - pass diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index be4ada422..edb0e0c71 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -3,21 +3,30 @@ import time from typing import List, Any, Dict, Tuple, Optional import random +import zstd +from hashlib import sha512 +import asyncio +import io import geoip2 import geoip2.errors -from fastapi import APIRouter, Depends, HTTPException, Response, Request, status +from fastapi import APIRouter, Depends, HTTPException, Response, Request, status, Header from prometheus_client import Counter, Info, Gauge from pydantic import Field from ooniauth_py import ProtocolError, CredentialError, DeserializationFailed +import httpx from ...utils import ( generate_report_id, extract_probe_ipaddr, lookup_probe_cc, lookup_probe_network, + error, + compare_probe_msmt_cc_asn ) -from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestStateDep, PostgresSessionDep + +from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestStateDep, PostgresSessionDep, S3ClientDep +from ..reports import Metrics from ...common.dependencies import get_settings from ...common.routers import BaseModel from ...common.auth import create_jwt, decode_jwt, jwt @@ -31,41 +40,6 @@ log = logging.getLogger(__name__) -class Metrics: - PROBE_LOGIN = Counter( - "probe_login_requests", - "Requests made to the probe login endpoint", - labelnames=["state", "detail", "login"], - ) - - PROBE_UPDATE_INFO = Info( - "probe_update_info", - "Information reported in the probe update endpoint", - ) - - CHECK_IN_TEST_LIST_COUNT = Gauge( - "check_in_test_list_count", "Amount of test lists present in each experiment" - ) - - GEOIP_ADDR_FOUND = Counter( - "geoip_ipaddr_found", - "If the ip address was found by geoip", - labelnames=["probe_cc", "asn"], - ) - - GEOIP_ADDR_NOT_FOUND = Counter( - "geoip_ipaddr_not_found", "We couldn't look up the IP address in the database" - ) - - GEOIP_CC_DIFFERS = Counter( - "geoip_cc_differs", "There's a mismatch between reported CC and observed CC" - ) - - GEOIP_ASN_DIFFERS = Counter( - "geoip_asn_differs", "There's a mismatch between reported ASN and observed ASN" - ) - - class ProbeLogin(BaseModel): # Allow None username and password # to deliver informational 401 error when they're missing @@ -666,3 +640,112 @@ def to_http_exception(error: ProtocolError | CredentialError | DeserializationFa status_code=status.HTTP_403_FORBIDDEN, detail={"error": error_str, "message": str(error)} ) + +class SubmitMeasurementResponse(BaseModel): + """ + Acknowledge + """ + + measurement_uid: str | None = Field( + examples=["20210208220710.181572_MA_ndt_7888edc7748936bf"], default=None + ) + +@router.post("/submit_measurement/{report_id}") +async def submit_measurement( + report_id: str, + request: Request, + response: Response, + cc_reader: CCReaderDep, + asn_reader: ASNReaderDep, + settings: SettingsDep, + s3_client: S3ClientDep, + content_encoding: str = Header(default=None), +) -> SubmitMeasurementResponse | Dict[str, Any]: + """ + Submit measurement + """ + setnocacheresponse(response) + empty_measurement = {} + try: + rid_timestamp, test_name, cc, asn, format_cid, rand = report_id.split("_") + except Exception: + log.info("Unexpected report_id %r", report_id[:200]) + raise error("Incorrect format") + + # TODO validate the timestamp? + good = len(cc) == 2 and test_name.isalnum() and 1 < len(test_name) < 30 + if not good: + log.info("Unexpected report_id %r", report_id[:200]) + error("Incorrect format") + + try: + asn_i = int(asn) + except ValueError: + log.info("ASN value not parsable %r", asn) + error("Incorrect format") + + if asn_i == 0: + log.info("Discarding ASN == 0") + Metrics.MSMNT_DISCARD_ASN0.inc() + return empty_measurement + + if cc.upper() == "ZZ": + log.info("Discarding CC == ZZ") + Metrics.MSMNT_DISCARD_CC_ZZ.inc() + return empty_measurement + + data = await request.body() + if content_encoding == "zstd": + try: + compressed_len = len(data) + data = zstd.decompress(data) + ratio = compressed_len / len(data) + log.debug(f"Zstd compression ratio {ratio}") + except Exception as e: + log.info("Failed zstd decompression") + error("Incorrect format") + + # Write the whole body of the measurement in a directory based on a 1-hour + # time window + now = datetime.now(timezone.utc) + h = sha512(data).hexdigest()[:16] + ts = now.strftime("%Y%m%d%H%M%S.%f") + + # msmt_uid is a unique id based on upload time, cc, testname and hash + msmt_uid = f"{ts}_{cc}_{test_name}_{h}" + Metrics.MSMNT_RECEIVED_CNT.inc() + + compare_probe_msmt_cc_asn(cc, asn, request, cc_reader, asn_reader) + # Use exponential back off with jitter between retries + N_RETRIES = 3 + for t in range(N_RETRIES): + try: + url = f"{settings.fastpath_url}/{msmt_uid}" + + async with httpx.AsyncClient() as client: + resp = await client.post(url, content=data, timeout=59) + + assert resp.status_code == 200, resp.content + return SubmitMeasurementResponse(measurement_uid=msmt_uid) + + except Exception as exc: + log.error( + f"[Try {t+1}/{N_RETRIES}] Error trying to send measurement to the fastpath. Error: {exc}" + ) + sleep_time = random.uniform(0, min(3, 0.3 * 2 ** t)) + await asyncio.sleep(sleep_time) + + Metrics.SEND_FASTPATH_FAILURE.inc() + + # wasn't possible to send msmnt to fastpath, try to send it to s3 + try: + s3_client.upload_fileobj( + io.BytesIO(data), Bucket=settings.failed_reports_bucket, Key=report_id + ) + except Exception as exc: + log.error(f"Unable to upload measurement to s3. Error: {exc}") + Metrics.SEND_S3_FAILURE.inc() + + log.error(f"Unable to send report to fastpath. report_id: {report_id}") + Metrics.MISSED_MSMNTS.inc() + return empty_measurement diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py index 311c075bf..dd85ff089 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py @@ -11,11 +11,12 @@ import logging from typing import Dict, List, Mapping, TypedDict, Tuple -from fastapi import Request +from fastapi import Request, HTTPException from sqlalchemy.orm import Session import pem import httpx +from .metrics import Metrics from .common.config import Settings from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint from .dependencies import CCReaderDep, ASNReaderDep @@ -144,3 +145,34 @@ def lookup_probe_network(ipaddr: str, asn_reader: ASNReaderDep) -> Tuple[str, st "AS{}".format(resp.autonomous_system_number), resp.autonomous_system_organization or "0", ) + + +def error(msg: str, status_code: int = 400): + raise HTTPException(status_code=status_code, detail=msg) + + +def compare_probe_msmt_cc_asn( + cc: str, + asn: str, + request: Request, + cc_reader: CCReaderDep, + asn_reader: ASNReaderDep, +): + """Compares CC/ASN from measurement with CC/ASN from HTTPS connection ipaddr + Generates a metric. + """ + try: + cc = cc.upper() + ipaddr = extract_probe_ipaddr(request) + db_probe_cc = lookup_probe_cc(ipaddr, cc_reader) + db_asn, _ = lookup_probe_network(ipaddr, asn_reader) + if db_asn.startswith("AS"): + db_asn = db_asn[2:] + if db_probe_cc == cc and db_asn == asn: + Metrics.PROBE_CC_ASN_MATCH.inc() + elif db_probe_cc != cc: + Metrics.PROBE_CC_ASN_NO_MATCH.labels(mismatch="cc").inc() + elif db_asn != asn: + Metrics.PROBE_CC_ASN_NO_MATCH.labels(mismatch="asn").inc() + except Exception: + pass \ No newline at end of file From ddf860a4da18ea78eab2f4f302484bdffa6ee524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 11:07:53 +0200 Subject: [PATCH 23/36] Creating manifest table --- ...f_add_server_state_table_for_anonymous_.py | 19 ++++++ .../ooniprobe/src/ooniprobe/models.py | 66 ++++++++++++------- .../ooniprobe/routers/v1/probe_services.py | 32 +++++---- .../services/ooniprobe/tests/test_anoncred.py | 30 ++++++++- 4 files changed, 109 insertions(+), 38 deletions(-) diff --git a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py index 5fcbbd929..05b64c437 100644 --- a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py +++ b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py @@ -20,6 +20,7 @@ # --- ooniprobe_server_state_id_seq = sc.Sequence("ooniprobe_server_state_id_seq", start=1) +ooniprobe_manifest_id_seq = sc.Sequence("ooniprobe_manifest_id_seq", start=1) def upgrade() -> None: op.execute(sc.CreateSequence(ooniprobe_server_state_id_seq)) @@ -37,7 +38,25 @@ def upgrade() -> None: sa.Column("public_parameters", sa.String(), nullable=False) ) + op.execute(sc.CreateSequence(ooniprobe_manifest_id_seq)) + op.create_table( + "ooniprobe_manifest", + sa.Column( + "version", + sa.String(), + nullable=False, + server_default=ooniprobe_manifest_id_seq.next_value(), + primary_key=True + ), + sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), + sa.Column("nym_scope", sa.JSON(), nullable=False), + sa.Column("submission_policy", sa.JSON(), nullable=False) + + ) + def downgrade() -> None: op.drop_table("ooniprobe_server_state") + op.drop_table("ooniprobe_manifest") op.execute(sc.DropSequence(ooniprobe_server_state_id_seq)) + op.execute(sc.DropSequence(ooniprobe_manifest_id_seq)) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 56ba643aa..d4fec1f98 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Self +from typing import Self, Dict, Any from .common.models import UtcDateTime from .common.postgresql import Base from sqlalchemy import ForeignKey, Sequence, String, func @@ -74,25 +74,6 @@ class OONIProbeServerState(Base): secret_key: Mapped[str] = mapped_column() public_parameters: Mapped[str] = mapped_column() - @classmethod - def get_by_datetime(cls, session: Session, dt : datetime) -> Self | None: - return ( - session - .query(cls) - .where(cls.date_created == dt) - .one_or_none() - ) - - @classmethod - def get_latest(cls, session: Session) -> Self | None: - return ( - session - .query(cls) - .order_by(desc(cls.date_created)) - .limit(1) - .one_or_none() - ) - @classmethod def make_new_state(cls, session: Session, state : ServerState | None = None) -> Self: """ @@ -124,9 +105,50 @@ def init_table(cls, session: Session): """ Creates a new entry if none exists. """ - latest = cls.get_latest(session) - if latest is None: + entry = session.query(cls).limit(1).one_or_none() + if entry is None: log.info("No OONIProbeServerState entry found. Creating a new one...") cls.make_new_state(session) else: log.info("OONIProbeServerState already initialized!") + + +class OONIProbeManifest(Base): + """ + Manifest used to share with clients the information they need to provide + registration and validation of measurement submissions + """ + + __tablename__ = "ooniprobe_manifest" + + version: Mapped[str] = mapped_column( + String, + Sequence("ooniprobe_manifest_id_seq", start=1), + primary_key=True, + nullable=False, + ) + date_created: Mapped[datetime] = mapped_column(UtcDateTime(), default=func.now()) + nym_scope: Mapped[str] = mapped_column(default="ooni.org/{probe_cc}/{probe_asn}") + submission_policy: Mapped[Dict[str, Any]] = mapped_column(default={}) + + server_state_id = mapped_column(ForeignKey("ooniprobe_server_state.id")) + server_state = relationship("ServerState") + + @classmethod + def get_latest(cls, session: Session) -> Self | None: + return ( + session + .query(cls) + .order_by(desc(cls.date_created)) + .limit(1) + .one_or_none() + ) + + @classmethod + def get_by_version(cls, session: Session, version: str) -> Self | None: + return ( + session + .query(cls) + .where(cls.version == version) + .one_or_none() + ) \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index edb0e0c71..5106ebc02 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -11,7 +11,6 @@ import geoip2 import geoip2.errors from fastapi import APIRouter, Depends, HTTPException, Response, Request, status, Header -from prometheus_client import Counter, Info, Gauge from pydantic import Field from ooniauth_py import ProtocolError, CredentialError, DeserializationFailed import httpx @@ -569,13 +568,14 @@ def random_web_test_helpers(th_list: List[str]) -> List[Dict]: # -- ------------------------------------ +# make manifest table class ManifestResponse(BaseModel): nym_scope: str public_parameters: str submission_policy: Dict[str, Any] # TODO: Is the manifest version different from the server state? For now we assume it's the same # and use the `date_created` as version - date_created: datetime + date_created: datetime # change to version @router.get("/manifest", tags=["anonymous_credentials"]) def manifest(state : LatestStateDep) -> ManifestResponse: @@ -650,10 +650,20 @@ class SubmitMeasurementResponse(BaseModel): examples=["20210208220710.181572_MA_ndt_7888edc7748936bf"], default=None ) +class SubmitMeasurementRequest(BaseModel): + + format: str + content: Dict[str, Any] + # not post quantum, in the future we might want to use a hashed key for storage + nym: str + zkp_request: str + age_range: Tuple[int,int] + msm_range: Tuple[int,int] + @router.post("/submit_measurement/{report_id}") async def submit_measurement( report_id: str, - request: Request, + submit_request: SubmitMeasurementRequest, response: Response, cc_reader: CCReaderDep, asn_reader: ASNReaderDep, @@ -694,16 +704,12 @@ async def submit_measurement( Metrics.MSMNT_DISCARD_CC_ZZ.inc() return empty_measurement - data = await request.body() - if content_encoding == "zstd": - try: - compressed_len = len(data) - data = zstd.decompress(data) - ratio = compressed_len / len(data) - log.debug(f"Zstd compression ratio {ratio}") - except Exception as e: - log.info("Failed zstd decompression") - error("Incorrect format") + + # TODO + # Parse data into a json + # verify with anonymous credentials parameters + # add additional information to the json + # convert to data again # Write the whole body of the measurement in a directory based on a 1-hour # time window diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index dcbead732..128268a32 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -11,11 +11,16 @@ def getj(client : Client, url: str, params: Dict[str, Any] = {}) -> Dict[str, An assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" return resp.json() -def postj(client : Client, url: str, json: Dict[str, Any] | None = None) -> Dict[str, Any]: - resp = client.post(url, json=json) +def postj(client : Client, url: str, json: Dict[str, Any] | None = None, headers: Dict[str, Any] | None = None) -> Dict[str, Any]: + resp = client.post(url, json=json, headers=headers) assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" return resp.json() +def post(client, url, data, headers=None): + response = client.post(url, data=data, headers=headers) + assert response.status_code == 200 + return response.json() + def test_manifest_basic(client, db): latest = OONIProbeServerState.get_latest(db) assert latest is not None, "Server state not initialized" @@ -82,4 +87,23 @@ def test_registration_errors(client): assert resp.status_code == status.HTTP_400_BAD_REQUEST, resp.content j = resp.json() - assert j['detail']['error'] == 'deserialization_failed', j \ No newline at end of file + assert j['detail']['error'] == 'deserialization_failed', j + +def test_submission_basic(client): + # open report + j = { + "data_format_version": "0.2.0", + "format": "json", + "probe_asn": "AS34245", + "probe_cc": "IE", + "software_name": "miniooni", + "software_version": "0.17.0-beta", + "test_name": "web_connectivity", + "test_start_time": "2020-09-09 14:11:11", + "test_version": "0.1.0", + } + c = postj(client, "/report", json=j) + rid = c.pop("report_id") + msmt = dict(test_keys={}, probe_cc = "VE", asn = "AS1234", test_name = "web_connectivity") + c = postj(client, f"/api/v1/submit_measurement/{rid}", {"format":"json", "content":msmt}) + assert c == {} From 2baff8473f907452594b164c417ad593e7fd1595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 11:29:36 +0200 Subject: [PATCH 24/36] Added foreign key to server state to fetch keys --- ...f_add_server_state_table_for_anonymous_.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py index 05b64c437..364c9d7d6 100644 --- a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py +++ b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py @@ -5,6 +5,7 @@ Create Date: 2025-10-08 10:36:55.796144 """ + from typing import Sequence, Union from alembic import op @@ -13,14 +14,16 @@ # revision identifiers, used by Alembic. -revision: str = '7e28b5d17a7f' -down_revision: Union[str, None] = '8e7ecea5c2f5' +revision: str = "7e28b5d17a7f" +down_revision: Union[str, None] = "8e7ecea5c2f5" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None # --- ooniprobe_server_state_id_seq = sc.Sequence("ooniprobe_server_state_id_seq", start=1) ooniprobe_manifest_id_seq = sc.Sequence("ooniprobe_manifest_id_seq", start=1) + + def upgrade() -> None: op.execute(sc.CreateSequence(ooniprobe_server_state_id_seq)) @@ -31,11 +34,11 @@ def upgrade() -> None: sa.String(), nullable=False, server_default=ooniprobe_server_state_id_seq.next_value(), - primary_key=True + primary_key=True, ), sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), sa.Column("secret_key", sa.String(), nullable=False), - sa.Column("public_parameters", sa.String(), nullable=False) + sa.Column("public_parameters", sa.String(), nullable=False), ) op.execute(sc.CreateSequence(ooniprobe_manifest_id_seq)) @@ -46,12 +49,18 @@ def upgrade() -> None: sa.String(), nullable=False, server_default=ooniprobe_manifest_id_seq.next_value(), - primary_key=True + primary_key=True, ), sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), - sa.Column("nym_scope", sa.JSON(), nullable=False), - sa.Column("submission_policy", sa.JSON(), nullable=False) + sa.Column("nym_scope", sa.String(), nullable=False), + sa.Column("submission_policy", sa.JSON(), nullable=False), + sa.Column( + "ooniprobe_server_state_id", + sa.String(), + sa.ForeignKey("ooniprobe_server_state.id"), + nullable=False, + ), ) From 4dd14befd3ca16558122995337d5623796ddb963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 11:50:53 +0200 Subject: [PATCH 25/36] Change register endpoint to use manifest version --- ooniapi/services/ooniprobe/src/ooniprobe/models.py | 5 +++-- .../src/ooniprobe/routers/v1/probe_services.py | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index d4fec1f98..0f4f0a36e 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -4,7 +4,7 @@ from .common.postgresql import Base from sqlalchemy import ForeignKey, Sequence, String, func from sqlalchemy.orm import Mapped -from sqlalchemy.orm import mapped_column, relationship, Session +from sqlalchemy.orm import mapped_column, relationship, Session, joinedload from sqlalchemy import desc from ooniauth_py import ServerState import logging @@ -132,7 +132,7 @@ class OONIProbeManifest(Base): submission_policy: Mapped[Dict[str, Any]] = mapped_column(default={}) server_state_id = mapped_column(ForeignKey("ooniprobe_server_state.id")) - server_state = relationship("ServerState") + server_state = relationship("OONIProbeServerState") @classmethod def get_latest(cls, session: Session) -> Self | None: @@ -149,6 +149,7 @@ def get_by_version(cls, session: Session, version: str) -> Self | None: return ( session .query(cls) + .options(joinedload(cls.server_state)) .where(cls.version == version) .one_or_none() ) \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 5106ebc02..11dfd9948 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -31,7 +31,7 @@ from ...common.auth import create_jwt, decode_jwt, jwt from ...common.config import Settings from ...common.utils import setnocacheresponse -from ...models import OONIProbeServerState +from ...models import OONIProbeManifest, OONIProbeServerState from ...prio import generate_test_list router = APIRouter(prefix="/v1") @@ -587,7 +587,7 @@ def manifest(state : LatestStateDep) -> ManifestResponse: ) class RegisterRequest(BaseModel): - manifest_date_created: datetime + manifest_version: str credential_sign_request: str class RegisterResponse(BaseModel): @@ -598,15 +598,15 @@ class RegisterResponse(BaseModel): @router.post("/sign_credential", tags=["anonymous_credentials"]) def sign_credential(register_request: RegisterRequest, session : PostgresSessionDep): - state = OONIProbeServerState.get_by_datetime(session, register_request.manifest_date_created) - if state is None: + manifest = OONIProbeManifest.get_by_version(session, register_request.manifest_version) + if manifest is None: raise HTTPException( status.HTTP_404_NOT_FOUND, { "error" : "manifest_not_found", - "message" : f"No manifest with creation date '{register_request.manifest_date_created.isoformat()}' was found"} + "message" : f"No manifest with version '{register_request.manifest_version}' was found"} ) - + state: OONIProbeServerState = manifest.server_state protocol_state = state.to_protocol() try: From 9810800f0288499a2c4ff3094ad80aecebefbe20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 13:01:23 +0200 Subject: [PATCH 26/36] Use new manifest style in endpoints --- ...f_add_server_state_table_for_anonymous_.py | 2 +- .../ooniprobe/src/ooniprobe/dependencies.py | 12 +++++----- .../services/ooniprobe/src/ooniprobe/main.py | 7 ++++-- .../ooniprobe/src/ooniprobe/models.py | 23 +++++++++++++++++-- .../ooniprobe/routers/v1/probe_services.py | 13 ++++------- ooniapi/services/ooniprobe/tests/conftest.py | 10 ++++---- .../services/ooniprobe/tests/test_anoncred.py | 19 ++++++++------- 7 files changed, 54 insertions(+), 32 deletions(-) diff --git a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py index 364c9d7d6..bb0ba2745 100644 --- a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py +++ b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py @@ -56,7 +56,7 @@ def upgrade() -> None: sa.Column("submission_policy", sa.JSON(), nullable=False), sa.Column( - "ooniprobe_server_state_id", + "server_state_id", sa.String(), sa.ForeignKey("ooniprobe_server_state.id"), nullable=False, diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py index f6eedb2b8..4ab815065 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -15,7 +15,7 @@ from .common.config import Settings from .common.dependencies import get_settings -from .models import OONIProbeServerState +from .models import OONIProbeServerState, OONIProbeManifest SettingsDep: TypeAlias = Annotated[Settings, Depends(get_settings)] @@ -67,9 +67,9 @@ def get_s3_client() -> S3Client: S3ClientDep = Annotated[S3Client, Depends(get_s3_client)] -def get_latest_state(session : PostgresSessionDep) -> OONIProbeServerState: - state = OONIProbeServerState.get_latest(session) - assert state is not None, "Uninitialized `OONIProbeServerState` table" - return state +def get_latest_manifest(session : PostgresSessionDep) -> OONIProbeManifest: + manifest = OONIProbeManifest.get_latest(session) + assert manifest is not None, "Uninitialized `OONIProbeServerState` table" + return manifest -LatestStateDep = Annotated[OONIProbeServerState, Depends(get_latest_state)] \ No newline at end of file +LatestManifestDep = Annotated[OONIProbeManifest, Depends(get_latest_manifest)] \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py index baae4b6af..ecf3285a7 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/main.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -50,7 +50,7 @@ async def lifespan( db = get_postgresql_session(settings) session = next(db) - models.OONIProbeServerState.init_table(session) + models.OONIProbeManifest.init_table(session) next(db, None) # closes the connection yield @@ -140,9 +140,12 @@ async def health( # check that we have at least one server state object for credentials validation try: - state = models.OONIProbeServerState.get_latest(db) + state = db.query(models.OONIProbeManifest).limit(1).one_or_none() if state is None: errors.append("no_server_state_entry") + manifest = models.OONIProbeManifest.get_latest(db) + if manifest == None: + errors.append("no_manifest_entry") except Exception as exc: log.error("Error trying to retrieve server state") log.error(exc) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 0f4f0a36e..17457c5ce 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -139,7 +139,8 @@ def get_latest(cls, session: Session) -> Self | None: return ( session .query(cls) - .order_by(desc(cls.date_created)) + .options(joinedload(cls.server_state)) + .order_by(desc(cls.version)) .limit(1) .one_or_none() ) @@ -152,4 +153,22 @@ def get_by_version(cls, session: Session, version: str) -> Self | None: .options(joinedload(cls.server_state)) .where(cls.version == version) .one_or_none() - ) \ No newline at end of file + ) + + @classmethod + def init_table(cls, session: Session): + """ + Creates a new manifest entry if none exists + """ + entry = session.query(cls).limit(1).one_or_none() + if entry is None: + log.info("No OONIProbeManifest entry found. Creating a new one...") + + # Make sure there's a server state + OONIProbeServerState.init_table(session) + state = session.query(OONIProbeServerState).order_by(desc(OONIProbeServerState.date_created)).limit(1).one() + entry = cls(server_state_id=state.id) + session.add(entry) + session.commit() + else: + log.info("OONIProbeServerState already initialized!") \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 11dfd9948..ca88bc5da 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -24,7 +24,7 @@ compare_probe_msmt_cc_asn ) -from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestStateDep, PostgresSessionDep, S3ClientDep +from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep, LatestManifestDep, PostgresSessionDep, S3ClientDep from ..reports import Metrics from ...common.dependencies import get_settings from ...common.routers import BaseModel @@ -568,22 +568,19 @@ def random_web_test_helpers(th_list: List[str]) -> List[Dict]: # -- ------------------------------------ -# make manifest table class ManifestResponse(BaseModel): nym_scope: str public_parameters: str submission_policy: Dict[str, Any] - # TODO: Is the manifest version different from the server state? For now we assume it's the same - # and use the `date_created` as version - date_created: datetime # change to version + version: str @router.get("/manifest", tags=["anonymous_credentials"]) -def manifest(state : LatestStateDep) -> ManifestResponse: +def manifest(manifest : LatestManifestDep) -> ManifestResponse: return ManifestResponse( nym_scope="ooni.org/{probe_cc}/{probe_asn}", - public_parameters=state.public_parameters, + public_parameters=manifest.server_state.public_parameters, submission_policy={}, - date_created=state.date_created + version=manifest.version ) class RegisterRequest(BaseModel): diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index 895b4edf6..1b0c3804a 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -16,7 +16,7 @@ from ooniprobe.main import app from ooniprobe.download_geoip import try_update from ooniprobe.dependencies import get_postgresql_session -from ooniprobe.models import OONIProbeServerState +from ooniprobe.models import OONIProbeManifest def make_override_get_settings(**kw): @@ -79,7 +79,7 @@ def client_with_bad_settings(): @pytest.fixture(scope="session") def fixture_path(): """ - Directory for this fixtures used to store temporary data, will be + Directory for this fixtures used to store temporary data, will be deleted after the tests are finished """ FIXTURE_PATH = Path(os.path.dirname(os.path.realpath(__file__))) / "data" @@ -105,7 +105,7 @@ def client(clickhouse_server, test_settings, geoip_db_dir): # Initialize server state db = get_postgresql_session(test_settings()) session = next(db) - OONIProbeServerState.init_table(session) + OONIProbeManifest.init_table(session) next(db, None) # lifespan won't run so do this here to have the DB @@ -177,8 +177,8 @@ def fastpath_server(docker_ip, docker_services): ) yield url -def is_fastpath_running(url: str) -> bool: - try: +def is_fastpath_running(url: str) -> bool: + try: resp = urlopen(url) return resp.status == 200 except Exception: diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 128268a32..20ab1677f 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -2,7 +2,7 @@ from typing import Any, Dict from httpx import Client from fastapi import status -from ooniprobe.models import OONIProbeServerState +from ooniprobe.models import OONIProbeServerState, OONIProbeManifest from ooniprobe.common.routers import ISO_FORMAT_DATETIME from ooniauth_py import UserState, ServerState @@ -22,12 +22,15 @@ def post(client, url, data, headers=None): return response.json() def test_manifest_basic(client, db): - latest = OONIProbeServerState.get_latest(db) + latest = db.query(OONIProbeServerState).limit(1).one_or_none() + manifest = OONIProbeManifest.get_latest(db) assert latest is not None, "Server state not initialized" + assert manifest is not None, "Manifest not initialized" m = getj(client, "/api/v1/manifest") + assert latest.public_parameters == m['public_parameters'] - assert datetime.strftime(latest.date_created, ISO_FORMAT_DATETIME) == m['date_created'] + assert manifest.version == m['version'] def test_registration_basic(client): @@ -40,7 +43,7 @@ def test_registration_basic(client): "/api/v1/sign_credential", { "credential_sign_request" : sign_req, - "manifest_date_created" : manifest['date_created'] + "manifest_version" : manifest['version'] } ) # should be able to verify this credential @@ -48,11 +51,11 @@ def test_registration_basic(client): def test_registration_errors(client): - bad_date = datetime.strftime(datetime(2012, 12, 21), ISO_FORMAT_DATETIME) + bad_version = "999" resp = client.post("/api/v1/sign_credential", json={ "credential_sign_request" : "doesntmatter", - "manifest_date_created" : bad_date + "manifest_version" : bad_version } ) # Bad manifest date should raise 404 @@ -67,7 +70,7 @@ def test_registration_errors(client): user = UserState(bad_server.get_public_parameters()) resp = client.post("/api/v1/sign_credential", json={ "credential_sign_request" : user.make_registration_request(), - "manifest_date_created" : manifest['date_created'] + "manifest_version" : manifest['version'] }) assert resp.status_code == status.HTTP_403_FORBIDDEN, resp.content @@ -82,7 +85,7 @@ def test_registration_errors(client): sign_req = bad + sign_req[len(bad):] resp = client.post("/api/v1/sign_credential", json={ "credential_sign_request" : sign_req, - "manifest_date_created" : manifest['date_created'] + "manifest_version" : manifest['version'] }) assert resp.status_code == status.HTTP_400_BAD_REQUEST, resp.content From 355e0596cc8c2df13e1aaf9247b8afeebe18b30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 13:42:08 +0200 Subject: [PATCH 27/36] Use extra fields in measurement submission --- .../ooniprobe/routers/v1/probe_services.py | 50 +++++++++++++++++-- .../services/ooniprobe/src/ooniprobe/utils.py | 4 +- .../services/ooniprobe/tests/test_anoncred.py | 18 ++++--- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index ca88bc5da..99f8be826 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -3,7 +3,7 @@ import time from typing import List, Any, Dict, Tuple, Optional import random -import zstd +import ujson from hashlib import sha512 import asyncio import io @@ -651,21 +651,25 @@ class SubmitMeasurementRequest(BaseModel): format: str content: Dict[str, Any] + # -- < Anonymous Credentials > ---------------------- # not post quantum, in the future we might want to use a hashed key for storage nym: str zkp_request: str age_range: Tuple[int,int] msm_range: Tuple[int,int] + manifest_version: str @router.post("/submit_measurement/{report_id}") async def submit_measurement( report_id: str, + request: Request, submit_request: SubmitMeasurementRequest, response: Response, cc_reader: CCReaderDep, asn_reader: ASNReaderDep, settings: SettingsDep, s3_client: S3ClientDep, + session: PostgresSessionDep, content_encoding: str = Header(default=None), ) -> SubmitMeasurementResponse | Dict[str, Any]: """ @@ -708,10 +712,48 @@ async def submit_measurement( # add additional information to the json # convert to data again + # Retrieve manifest known by the client + manifest = OONIProbeManifest.get_by_version(session, submit_request.manifest_version) + if manifest is None: + error({ + "detail" : f"No manifest with version '{submit_request.manifest_version}' was found", + "error" : "manifest_not_found" + }) + + # Run verification + assert manifest + assert "probe_cc" in submit_request.content and isinstance(submit_request.content['probe_cc'], str) + assert "probe_asn" in submit_request.content and isinstance(submit_request.content['probe_asn'], str) + + state: OONIProbeServerState = manifest.server_state + protocol_state = state.to_protocol() + + try: + protocol_state.handle_submit_request( + submit_request.nym, + submit_request.zkp_request, + submit_request.content["probe_cc"], + submit_request.content["probe_asn"], + list(submit_request.age_range), + list(submit_request.msm_range), + ) + is_verified = True + except (DeserializationFailed, ProtocolError, CredentialError): + # proof failed + # TODO might be a good idea to add a "why not verified" field to the measurement + is_verified = False + + data = submit_request.model_dump() + data['is_verified'] = is_verified + data_buff = io.BytesIO() + stream = io.TextIOWrapper(data_buff, "utf-8") + ujson.dump(data, stream) + data_bin = data_buff.getvalue() + # Write the whole body of the measurement in a directory based on a 1-hour # time window now = datetime.now(timezone.utc) - h = sha512(data).hexdigest()[:16] + h = sha512(data_bin).hexdigest()[:16] ts = now.strftime("%Y%m%d%H%M%S.%f") # msmt_uid is a unique id based on upload time, cc, testname and hash @@ -726,7 +768,7 @@ async def submit_measurement( url = f"{settings.fastpath_url}/{msmt_uid}" async with httpx.AsyncClient() as client: - resp = await client.post(url, content=data, timeout=59) + resp = await client.post(url, content=data_bin, timeout=59) assert resp.status_code == 200, resp.content return SubmitMeasurementResponse(measurement_uid=msmt_uid) @@ -743,7 +785,7 @@ async def submit_measurement( # wasn't possible to send msmnt to fastpath, try to send it to s3 try: s3_client.upload_fileobj( - io.BytesIO(data), Bucket=settings.failed_reports_bucket, Key=report_id + data_buff, Bucket=settings.failed_reports_bucket, Key=report_id ) except Exception as exc: log.error(f"Unable to upload measurement to s3. Error: {exc}") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py index dd85ff089..a5b8dcbb5 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py @@ -9,7 +9,7 @@ from datetime import datetime, timezone import itertools import logging -from typing import Dict, List, Mapping, TypedDict, Tuple +from typing import Dict, List, TypedDict, Tuple, Any from fastapi import Request, HTTPException from sqlalchemy.orm import Session @@ -147,7 +147,7 @@ def lookup_probe_network(ipaddr: str, asn_reader: ASNReaderDep) -> Tuple[str, st ) -def error(msg: str, status_code: int = 400): +def error(msg: str | Dict[str, Any], status_code: int = 400): raise HTTPException(status_code=status_code, detail=msg) diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 20ab1677f..2e61fd663 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -95,15 +95,17 @@ def test_registration_errors(client): def test_submission_basic(client): # open report j = { - "data_format_version": "0.2.0", "format": "json", - "probe_asn": "AS34245", - "probe_cc": "IE", - "software_name": "miniooni", - "software_version": "0.17.0-beta", - "test_name": "web_connectivity", - "test_start_time": "2020-09-09 14:11:11", - "test_version": "0.1.0", + "content": { + "data_format_version": "0.2.0", + "probe_asn": "AS34245", + "probe_cc": "IE", + "software_name": "miniooni", + "software_version": "0.17.0-beta", + "test_name": "web_connectivity", + "test_start_time": "2020-09-09 14:11:11", + "test_version": "0.1.0", + }, } c = postj(client, "/report", json=j) rid = c.pop("report_id") From b9b925014f77affdce3af727044bec04873cd6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 16:29:17 +0200 Subject: [PATCH 28/36] Testing measurement submission --- .../services/ooniprobe/tests/test_anoncred.py | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 2e61fd663..9370e1805 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Dict +from typing import Any, Dict, Tuple from httpx import Client from fastapi import status from ooniprobe.models import OONIProbeServerState, OONIProbeManifest @@ -8,12 +8,12 @@ def getj(client : Client, url: str, params: Dict[str, Any] = {}) -> Dict[str, Any]: resp = client.get(url) - assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" + assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code} - {url}. {resp.content}" return resp.json() def postj(client : Client, url: str, json: Dict[str, Any] | None = None, headers: Dict[str, Any] | None = None) -> Dict[str, Any]: resp = client.post(url, json=json, headers=headers) - assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code}. {resp.content}" + assert resp.status_code == status.HTTP_200_OK, f"Unexpected status code: {resp.status_code} - {url}. {resp.content}" return resp.json() def post(client, url, data, headers=None): @@ -95,20 +95,48 @@ def test_registration_errors(client): def test_submission_basic(client): # open report j = { + "data_format_version": "0.2.0", + "format": "json", + "probe_asn": "AS34245", + "probe_cc": "IE", + "software_name": "miniooni", + "software_version": "0.17.0-beta", + "test_name": "web_connectivity", + "test_start_time": "2020-09-09 14:11:11", + "test_version": "0.1.0", + } + resp = postj(client, "/report", json=j) + rid = resp.pop("report_id") + + # Create user + user, manifest_version, emission_day = setup_user(client) + + submit_request = user.make_submit_request("IE", "AS34245", emission_day) + msm = { "format": "json", "content": { - "data_format_version": "0.2.0", + "test_name": "web_connectivity", "probe_asn": "AS34245", "probe_cc": "IE", - "software_name": "miniooni", - "software_version": "0.17.0-beta", - "test_name": "web_connectivity", "test_start_time": "2020-09-09 14:11:11", - "test_version": "0.1.0", }, + "nym": submit_request.nym, + "zkp_request": submit_request.request, + "age_range": [emission_day - 30, emission_day + 1], + "msm_range": [0, 10], + "manifest_version": manifest_version } - c = postj(client, "/report", json=j) - rid = c.pop("report_id") - msmt = dict(test_keys={}, probe_cc = "VE", asn = "AS1234", test_name = "web_connectivity") - c = postj(client, f"/api/v1/submit_measurement/{rid}", {"format":"json", "content":msmt}) + c = postj(client, f"/api/v1/submit_measurement/{rid}", msm) assert c == {} + +def setup_user(client) -> Tuple[UserState, str, int]: # user, manifest version + manifest = getj(client, "/api/v1/manifest") + user = UserState(manifest['public_parameters']) + req = user.make_registration_request() + resp = postj(client, "/api/v1/sign_credential", json = { + "credential_sign_request" : req, + "manifest_version" : manifest['version'] + }) + user.handle_registration_response(resp['credential_sign_response']) + + return (user, manifest['version'], resp['emission_day']) \ No newline at end of file From cff7f32b9259a06bda6703cdf020344e8e16eaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Oct 2025 17:01:22 +0200 Subject: [PATCH 29/36] Improve testing of measurement submission --- .../ooniprobe/routers/v1/probe_services.py | 27 +++++++++++-------- .../services/ooniprobe/tests/fakepath/main.py | 5 +++- .../services/ooniprobe/tests/test_anoncred.py | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 99f8be826..8684e5cc2 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -638,14 +638,6 @@ def to_http_exception(error: ProtocolError | CredentialError | DeserializationFa detail={"error": error_str, "message": str(error)} ) -class SubmitMeasurementResponse(BaseModel): - """ - Acknowledge - """ - - measurement_uid: str | None = Field( - examples=["20210208220710.181572_MA_ndt_7888edc7748936bf"], default=None - ) class SubmitMeasurementRequest(BaseModel): @@ -659,6 +651,18 @@ class SubmitMeasurementRequest(BaseModel): msm_range: Tuple[int,int] manifest_version: str +class SubmitMeasurementResponse(BaseModel): + """ + Acknowledge + """ + + measurement_uid: str | None = Field( + examples=["20210208220710.181572_MA_ndt_7888edc7748936bf"], default=None + ) + is_verified: bool = Field( + description="if the ZKP was able to verify this request" + ) + @router.post("/submit_measurement/{report_id}") async def submit_measurement( report_id: str, @@ -738,9 +742,10 @@ async def submit_measurement( list(submit_request.msm_range), ) is_verified = True - except (DeserializationFailed, ProtocolError, CredentialError): + except (DeserializationFailed, ProtocolError, CredentialError) as e: # proof failed # TODO might be a good idea to add a "why not verified" field to the measurement + log.error(f"ZKP Failed: {e}") is_verified = False data = submit_request.model_dump() @@ -771,11 +776,11 @@ async def submit_measurement( resp = await client.post(url, content=data_bin, timeout=59) assert resp.status_code == 200, resp.content - return SubmitMeasurementResponse(measurement_uid=msmt_uid) + return SubmitMeasurementResponse(measurement_uid=msmt_uid, is_verified=is_verified) except Exception as exc: log.error( - f"[Try {t+1}/{N_RETRIES}] Error trying to send measurement to the fastpath. Error: {exc}" + f"[Try {t+1}/{N_RETRIES}] Error trying to send measurement to the fastpath ({settings.fastpath_url}). Error: {exc}" ) sleep_time = random.uniform(0, min(3, 0.3 * 2 ** t)) await asyncio.sleep(sleep_time) diff --git a/ooniapi/services/ooniprobe/tests/fakepath/main.py b/ooniapi/services/ooniprobe/tests/fakepath/main.py index 6ffe633ef..8d7505c5b 100644 --- a/ooniapi/services/ooniprobe/tests/fakepath/main.py +++ b/ooniapi/services/ooniprobe/tests/fakepath/main.py @@ -11,5 +11,8 @@ @app.get("/") def health(): - return + return +@app.post("/{msmt_uid}") +def receive_msm(): + return {} diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 9370e1805..833b28475 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -127,7 +127,7 @@ def test_submission_basic(client): "manifest_version": manifest_version } c = postj(client, f"/api/v1/submit_measurement/{rid}", msm) - assert c == {} + assert c == {'is_verified' : True, "measurement_uid" : rid} def setup_user(client) -> Tuple[UserState, str, int]: # user, manifest version manifest = getj(client, "/api/v1/manifest") From ec638953f3747c8455d9a103bc80a023ca758472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 14 Oct 2025 11:26:24 +0200 Subject: [PATCH 30/36] Fix broken test --- .../src/ooniprobe/routers/v1/probe_services.py | 17 +++++++---------- .../services/ooniprobe/tests/test_anoncred.py | 11 ++++++----- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 8684e5cc2..7d6ae6c5f 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -662,6 +662,9 @@ class SubmitMeasurementResponse(BaseModel): is_verified: bool = Field( description="if the ZKP was able to verify this request" ) + submit_response: str | None = Field( + description="Anonymous credential verification response. Null if verification failed" + ) @router.post("/submit_measurement/{report_id}") async def submit_measurement( @@ -709,13 +712,6 @@ async def submit_measurement( Metrics.MSMNT_DISCARD_CC_ZZ.inc() return empty_measurement - - # TODO - # Parse data into a json - # verify with anonymous credentials parameters - # add additional information to the json - # convert to data again - # Retrieve manifest known by the client manifest = OONIProbeManifest.get_by_version(session, submit_request.manifest_version) if manifest is None: @@ -733,7 +729,7 @@ async def submit_measurement( protocol_state = state.to_protocol() try: - protocol_state.handle_submit_request( + submit_response = protocol_state.handle_submit_request( submit_request.nym, submit_request.zkp_request, submit_request.content["probe_cc"], @@ -744,9 +740,10 @@ async def submit_measurement( is_verified = True except (DeserializationFailed, ProtocolError, CredentialError) as e: # proof failed - # TODO might be a good idea to add a "why not verified" field to the measurement + # TODO Q: should we add a "why not verified" field to the measurement? log.error(f"ZKP Failed: {e}") is_verified = False + submit_response = None data = submit_request.model_dump() data['is_verified'] = is_verified @@ -776,7 +773,7 @@ async def submit_measurement( resp = await client.post(url, content=data_bin, timeout=59) assert resp.status_code == 200, resp.content - return SubmitMeasurementResponse(measurement_uid=msmt_uid, is_verified=is_verified) + return SubmitMeasurementResponse(measurement_uid=msmt_uid, is_verified=is_verified, submit_response=submit_response) except Exception as exc: log.error( diff --git a/ooniapi/services/ooniprobe/tests/test_anoncred.py b/ooniapi/services/ooniprobe/tests/test_anoncred.py index 833b28475..6247ab26e 100644 --- a/ooniapi/services/ooniprobe/tests/test_anoncred.py +++ b/ooniapi/services/ooniprobe/tests/test_anoncred.py @@ -1,9 +1,7 @@ -from datetime import datetime from typing import Any, Dict, Tuple from httpx import Client from fastapi import status from ooniprobe.models import OONIProbeServerState, OONIProbeManifest -from ooniprobe.common.routers import ISO_FORMAT_DATETIME from ooniauth_py import UserState, ServerState def getj(client : Client, url: str, params: Dict[str, Any] = {}) -> Dict[str, Any]: @@ -123,13 +121,16 @@ def test_submission_basic(client): "nym": submit_request.nym, "zkp_request": submit_request.request, "age_range": [emission_day - 30, emission_day + 1], - "msm_range": [0, 10], + "msm_range": [0, 100], "manifest_version": manifest_version } c = postj(client, f"/api/v1/submit_measurement/{rid}", msm) - assert c == {'is_verified' : True, "measurement_uid" : rid} + assert c['is_verified'] == True # noqa: E712 -def setup_user(client) -> Tuple[UserState, str, int]: # user, manifest version + assert c['submit_response'], "Submit response should not be null if the proof was verified" + user.handle_submit_response(c['submit_response']) + +def setup_user(client) -> Tuple[UserState, str, int]: # user, manifest version, emission day manifest = getj(client, "/api/v1/manifest") user = UserState(manifest['public_parameters']) req = user.make_registration_request() From 6c667f041ca191df372a848a0baeb499bd458858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 14 Oct 2025 11:53:05 +0200 Subject: [PATCH 31/36] Fix broken tests --- ooniapi/services/ooniprobe/tests/integ/test_reports.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_reports.py b/ooniapi/services/ooniprobe/tests/integ/test_reports.py index 4abf3d092..01f3bd19b 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_reports.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_reports.py @@ -60,10 +60,10 @@ def test_collector_upload_msmt_valid(client): msmt = dict(test_keys={}) c = postj(client, f"/report/{rid}", {"format":"json", "content":msmt}) - assert c == {} + assert c['measurement_uid'].endswith("_IE_webconnectivity_e7889aeba0b36729"), c c = postj(client, f"/report/{rid}/close", json={}) - assert c == {} + assert c == {}, c def test_collector_upload_msmt_valid_zstd(client): rid = "20230101T000000Z_integtest_IT_1_n1_integtest0000000" @@ -71,4 +71,5 @@ def test_collector_upload_msmt_valid_zstd(client): zmsmt = zstd.compress(msmt) headers = [("Content-Encoding", "zstd")] c = post(client, f"/report/{rid}", zmsmt, headers=headers) - assert c == {} + assert len(c) == 1 + assert c['measurement_uid'].endswith("_IT_integtest_50be3cd5406bca65"), c From 985e89adfba0d8933cf2fdfb5dda2af05eb9ecf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 14 Oct 2025 11:57:10 +0200 Subject: [PATCH 32/36] Fix bad fiture downgrade --- .../7e28b5d17a7f_add_server_state_table_for_anonymous_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py index bb0ba2745..99e35c2a1 100644 --- a/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py +++ b/ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py @@ -65,7 +65,7 @@ def upgrade() -> None: def downgrade() -> None: - op.drop_table("ooniprobe_server_state") op.drop_table("ooniprobe_manifest") + op.drop_table("ooniprobe_server_state") op.execute(sc.DropSequence(ooniprobe_server_state_id_seq)) op.execute(sc.DropSequence(ooniprobe_manifest_id_seq)) From 4328d278f425c20b4de9c18b4631ec23ab62851e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 14 Oct 2025 14:51:15 +0200 Subject: [PATCH 33/36] Add anonymous credentials related data to fastpath --- fastpath/clickhouse_init.sql | 8 ++++++-- fastpath/fastpath/core.py | 18 +++++++++++++++--- fastpath/fastpath/db.py | 10 ++++++++++ fastpath/makefile | 18 +++++++++--------- .../src/ooniprobe/routers/v1/probe_services.py | 2 ++ 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/fastpath/clickhouse_init.sql b/fastpath/clickhouse_init.sql index da49958ce..2cde490b4 100644 --- a/fastpath/clickhouse_init.sql +++ b/fastpath/clickhouse_init.sql @@ -38,7 +38,12 @@ CREATE TABLE IF NOT EXISTS default.fastpath `blocking_type` String, `test_helper_address` LowCardinality(String), `test_helper_type` LowCardinality(String), - `ooni_run_link_id` Nullable(UInt64) + `ooni_run_link_id` Nullable(UInt64), + `is_verified` Int8, + `nym` Nullable(String), + `zkp_request` Nullable(String), + `age_range` Nullable(String), + `msm_range` Nullable(String), ) ENGINE = ReplacingMergeTree ORDER BY (measurement_start_time, report_id, input) @@ -202,4 +207,3 @@ CREATE TABLE IF NOT EXISTS default.fingerprints_http ) ENGINE = EmbeddedRocksDB PRIMARY KEY name; - diff --git a/fastpath/fastpath/core.py b/fastpath/fastpath/core.py index a06ca72ce..161b030f7 100644 --- a/fastpath/fastpath/core.py +++ b/fastpath/fastpath/core.py @@ -1616,7 +1616,7 @@ def flag_measurements_with_wrong_date(msm: dict, msmt_uid: str, scores: dict) -> scores["msg"] = "Measurement start time too old" def write_measurement_to_disk(msm_tup) -> None: - """Write this measurement to disk so that it can be + """Write this measurement to disk so that it can be processed by the measurement uploader Args: @@ -1633,7 +1633,7 @@ def write_measurement_to_disk(msm_tup) -> None: msmtdir = spooldir / "incoming" / dirname msmtdir.mkdir(parents=True, exist_ok=True) - try: + try: msmt_f_tmp = msmtdir / f"{msmt_uid}.post.tmp" msmt_f_tmp.write_bytes(data) msmt_f = msmtdir / f"{msmt_uid}.post" @@ -1654,7 +1654,14 @@ def process_measurement(msm_tup, buffer_writes=False) -> None: assert msmt_uid if measurement is None: measurement = ujson.loads(msm_jstr) - if sorted(measurement.keys()) == ["content", "format"]: + + is_verified = g(measurement, 'is_verified', False) + nym = g(measurement, 'nym') + zkp_request = g(measurement, 'zkp_request') + age_range = g(measurement, 'age_range') + msm_range = g(measurement, 'msm_range') + + if "content" in measurement and "format" in measurement: measurement = unwrap_msmt(measurement) rid = measurement.get("report_id") inp = measurement.get("input") @@ -1742,6 +1749,11 @@ def process_measurement(msm_tup, buffer_writes=False) -> None: test_helper_address, test_helper_type, ooni_run_link_id, + is_verified, + nym, + zkp_request, + age_range, + msm_range, buffer_writes=buffer_writes, ) diff --git a/fastpath/fastpath/db.py b/fastpath/fastpath/db.py index e8decb6ce..8de7f5c6d 100644 --- a/fastpath/fastpath/db.py +++ b/fastpath/fastpath/db.py @@ -209,6 +209,11 @@ def clickhouse_upsert_summary( test_helper_address: str, test_helper_type: str, ooni_run_link_id: Optional[int], + is_verified: bool, + zkp_request: Optional[str], + nym: Optional[str], + age_range: Optional[Tuple[int, int]], + msm_range: Optional[Tuple[int, int]], buffer_writes=False, ) -> None: """Insert a row in the fastpath table. Overwrite an existing one.""" @@ -257,6 +262,11 @@ def tf(v: bool) -> str: test_helper_address=test_helper_address, test_helper_type=test_helper_type, ooni_run_link_id=ooni_run_link_id, + is_verified=tf(is_verified), + nym=nym, + zkp_request=zkp_request, + age_range=ujson.dumps(age_range), + msm_range=ujson.dumps(msm_range) ) if buffer_writes: diff --git a/fastpath/makefile b/fastpath/makefile index 10f18430c..b0c87355c 100644 --- a/fastpath/makefile +++ b/fastpath/makefile @@ -80,18 +80,18 @@ docker: # Runs docker in foreground, useful for checking errors in the image before it runs docker-fg: - docker compose --profile default up --build + docker compose --profile default up --build -# Runs both fastpath and the testing clickhous. +# Runs both fastpath and the testing clickhous. # Mind the fastpath configuration in fastpath.conf docker-all: docker-clickhouse echo "Waiting for clickhouse..." - sleep 4 - docker compose --profile default up --build -d + sleep 4 + docker compose --profile default up --build -d # Turns off every service docker-down: - docker compose --profile all down + docker compose --profile all down # If you need to test the fastpath locally you can use this rule to spawn the clickhouse database # locally and then use `make docker` or `make docker-fg` to start the fastpath container. Ex: @@ -109,15 +109,15 @@ docker-login: # Get logs from the fastpath docker service docker-logs: - docker compose logs fastpath -f + docker compose logs fastpath -f -# Get logs for a specified service. Example: +# Get logs for a specified service. Example: # `make docker-logs-for args="clickhouse-server"` docker-logs-for: - docker compose logs $(args) -f + docker compose logs $(args) -f # Used for actually building the fastpath docker image -docker-build: +docker-build: # We need to use tar -czh to resolve the common dir symlink tar -czh . | docker build \ --build-arg BUILD_LABEL=${BUILD_LABEL} \ diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 7d6ae6c5f..9f3ec791d 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -746,6 +746,8 @@ async def submit_measurement( submit_response = None data = submit_request.model_dump() + + # Add verification-related data. data['is_verified'] = is_verified data_buff = io.BytesIO() stream = io.TextIOWrapper(data_buff, "utf-8") From 28cb8e0c8dd8ac7088708694f43139a1d585e5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 15 Oct 2025 14:08:11 +0200 Subject: [PATCH 34/36] fix broken tests --- fastpath/fastpath/db.py | 10 +++++++--- fastpath/fastpath/tests/test_functional_nodb.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/fastpath/fastpath/db.py b/fastpath/fastpath/db.py index 8de7f5c6d..5ed0d9f63 100644 --- a/fastpath/fastpath/db.py +++ b/fastpath/fastpath/db.py @@ -10,7 +10,7 @@ from datetime import datetime from textwrap import dedent from urllib.parse import urlparse -from typing import List, Tuple, Dict, Optional +from typing import List, Tuple, Dict, Optional, Any import logging try: @@ -229,6 +229,10 @@ def nn(features: dict, k: str) -> str: def tf(v: bool) -> str: return "t" if v else "f" + def s_or_n(x : Optional[Any]) -> Optional[str]: + """Serialize to string if not None, return None otherwise""" + return ujson.dumps(x) if x is not None else None + test_name = msm.get("test_name", None) or "" input_, domain = extract_input_domain(msm, test_name) asn = int(msm["probe_asn"][2:]) # AS123 @@ -265,8 +269,8 @@ def tf(v: bool) -> str: is_verified=tf(is_verified), nym=nym, zkp_request=zkp_request, - age_range=ujson.dumps(age_range), - msm_range=ujson.dumps(msm_range) + age_range=s_or_n(age_range), + msm_range=s_or_n(msm_range) ) if buffer_writes: diff --git a/fastpath/fastpath/tests/test_functional_nodb.py b/fastpath/fastpath/tests/test_functional_nodb.py index 80b658c7f..5af5b33b0 100644 --- a/fastpath/fastpath/tests/test_functional_nodb.py +++ b/fastpath/fastpath/tests/test_functional_nodb.py @@ -154,6 +154,11 @@ def test_score_web_connectivity_bug_610_2(fprints): "test_helper_address": "https://0.th.ooni.org", "test_helper_type": "https", "ooni_run_link_id": None, + "is_verified" : "f", + "nym" : None, + "zkp_request" : None, + "age_range" : None, + "msm_range" : None, } ] @@ -201,6 +206,11 @@ def test_score_browser_web(fprints): "test_runtime": 0.35740000000037253, "test_start_time": datetime.datetime(2023, 3, 20, 18, 26, 35), "test_version": "0.1.0", + "is_verified" : "f", + "nym" : None, + "zkp_request" : None, + "age_range" : None, + "msm_range" : None, }, ] @@ -252,6 +262,11 @@ def test_score_openvpn(): "test_helper_address": "", "test_helper_type": "", "ooni_run_link_id": None, + "is_verified" : "f", + "nym" : None, + "zkp_request" : None, + "age_range" : None, + "msm_range" : None, } ] From 3b3bfb2cfe1d77a69c2e78e1f6a0067d945d429f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 15 Oct 2025 16:48:29 +0200 Subject: [PATCH 35/36] Add fastpath migration to oonimeasurements tests --- .../tests/migrations/0_clickhouse_init_tables.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql b/ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql index 7245f4aee..2ee79c98e 100644 --- a/ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql +++ b/ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql @@ -36,13 +36,18 @@ CREATE TABLE IF NOT EXISTS default.fastpath `blocking_type` String, `test_helper_address` LowCardinality(String), `test_helper_type` LowCardinality(String), - `ooni_run_link_id` Nullable(UInt64) + `ooni_run_link_id` Nullable(UInt64), + `is_verified` Int8, + `nym` Nullable(String), + `zkp_request` Nullable(String), + `age_range` Nullable(String), + `msm_range` Nullable(String) ) ENGINE = ReplacingMergeTree ORDER BY (measurement_start_time, report_id, input) SETTINGS index_granularity = 8192; -CREATE TABLE IF NOT EXISTS default.citizenlab +CREATE TABLE IF NOT EXISTS default.citizenlab ( `domain` String, `url` String, From e0f443209dc7b50e8a80ab8373b6546066be4362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Oct 2025 10:53:42 +0200 Subject: [PATCH 36/36] refactor to_http_exception --- .../ooniprobe/routers/v1/probe_services.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 9f3ec791d..740b805d1 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -619,25 +619,23 @@ def sign_credential(register_request: RegisterRequest, session : PostgresSession def to_http_exception(error: ProtocolError | CredentialError | DeserializationFailed): - error_to_string = { + type_to_str = { ProtocolError : "protocol_error", DeserializationFailed : "deserialization_failed", CredentialError : "credential_error" } + type_str = type_to_str[type(error)] - error_str = error_to_string[type(error)] - - if isinstance(error, DeserializationFailed): - return HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": error_str, "detail": str(error)} - ) - if isinstance(error, (CredentialError, ProtocolError)): # - return HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail={"error": error_str, "message": str(error)} - ) + assert isinstance(error, (ProtocolError, CredentialError, DeserializationFailed)) + status_code = status.HTTP_400_BAD_REQUEST if isinstance(error, DeserializationFailed) else status.HTTP_403_FORBIDDEN + return HTTPException( + status_code=status_code, + detail={ + "error" : type_str, + "message" : str(error) + } + ) class SubmitMeasurementRequest(BaseModel): @@ -747,7 +745,7 @@ async def submit_measurement( data = submit_request.model_dump() - # Add verification-related data. + # Add verification-related data. data['is_verified'] = is_verified data_buff = io.BytesIO() stream = io.TextIOWrapper(data_buff, "utf-8")