Merge pull request #20 from openaleph/develop

ftm-datalake

Author: simonwoerpel

.bumpversion.cfg

@@ -10,4 +10,4 @@ message = 🔖 Bump version: {current_version} → {new_version}
 search = version = "{current_version}"
 replace = version = "{new_version}"
 
-[bumpversion:file:leakrfc/__init__.py]
+[bumpversion:file:ftm-datalake/__init__.py]

.github/dependabot.yml

@@ -5,14 +5,11 @@ updates:
     open-pull-requests-limit: 99
     schedule:
       interval: "daily"
-    target-branch: "develop"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "daily"
-    target-branch: "develop"
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "weekly"
-    target-branch: "develop"

.github/workflows/docker.yml

@@ -19,12 +19,12 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ghcr.io/investigativedata/leakrfc
+          images: ghcr.io/openaleph/ftm-datalake
           tags: |
             type=ref,event=branch
             type=semver,pattern={{version}}
             type=sha
-            type=raw,value=latest
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags') }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:

.github/workflows/python.yml

@@ -43,7 +43,7 @@ jobs:
         path: ~/.cache/pre-commit
         key: pre-commit-${{ runner.os }}-${{ env.PY }}-${{ hashFiles('.pre-commit-config.yaml') }}
     - name: Install dependencies
-      run: poetry install --with dev
+      run: poetry install --with dev --all-extras
     - name: Run pre-commit hooks
       run: poetry run pre-commit run
     - name: Lint with flake8

.gitignore

@@ -1,7 +1,7 @@
 _wip
 archive/*
 .anystore/*
-tests/fixtures/**/.leakrfc/*
+tests/fixtures/**/.ftm-datalake/*
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.pre-commit-config.yaml

@@ -32,13 +32,13 @@ repos:
     - id: absolufy-imports
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
       - id: isort
         args: ["--profile", "black"]
 
   - repo: https://github.com/psf/black
-    rev: 24.10.0
+    rev: 25.1.0
     hooks:
       - id: black
 
@@ -51,7 +51,7 @@ repos:
         exclude: (test_[\w]+\.py|\.csv|\.json|\.lock)$
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         exclude: (test_[\w]+\.py|\.csv|\.i?json|\.lock)$
@@ -69,7 +69,7 @@ repos:
       - id: rst-inline-touching-normal
 
   - repo: https://github.com/python-poetry/poetry
-    rev: 2.0.1
+    rev: 2.1.3
     hooks:
       - id: poetry-check
       - id: poetry-lock

Dockerfile

@@ -1,23 +1,15 @@
-FROM python:3.13-bookworm
+FROM ghcr.io/dataresearchcenter/ftmq:latest
 
-RUN apt-get -qq update && apt-get -qq -y upgrade
-RUN apt-get install -qq -y pkg-config libicu-dev
-RUN apt-get -qq -y autoremove && apt-get clean
 
-RUN pip install --no-cache-dir -q -U pip setuptools
-
-COPY leakrfc /src/leakrfc
+COPY ftm_datalake /src/ftm_datalake
 COPY setup.py /src/setup.py
-# COPY requirements.txt /src/requirements.txt
 COPY README.md /src/README.md
 COPY pyproject.toml /src/pyproject.toml
 COPY VERSION /src/VERSION
 COPY LICENSE /src/LICENSE
 COPY NOTICE /src/NOTICE
 
 WORKDIR /src
-# RUN pip install -r requirements.txt
 RUN pip install --no-cache-dir -q "."
-RUN pip install --no-cache-dir -q -U redis sqlalchemy psycopg2-binary
 
-ENTRYPOINT ["leakrfc"]
+ENTRYPOINT ["ftm-datalake"]

Makefile

@@ -1,24 +1,24 @@
 all: clean install test
 
 api:
-	LEAKRFC_ARCHIVE__URI=./tests/fixtures/archive DEBUG=1 uvicorn leakrfc.api:app --reload --port 5000
+	LEAKRFC_ARCHIVE__URI=./tests/fixtures/archive DEBUG=1 uvicorn ftm_datalake.api:app --reload --port 5000
 
 install:
-	poetry install --with dev
+	poetry install --with dev --all-extras
 
 lint:
-	poetry run flake8 leakrfc --count --select=E9,F63,F7,F82 --show-source --statistics
-	poetry run flake8 leakrfc --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+	poetry run flake8 ftm_datalake --count --select=E9,F63,F7,F82 --show-source --statistics
+	poetry run flake8 ftm_datalake --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
 pre-commit:
 	poetry run pre-commit install
 	poetry run pre-commit run -a
 
 typecheck:
-	poetry run mypy --strict leakrfc
+	poetry run mypy --strict ftm_datalake
 
 test:
-	poetry run pytest -v --capture=sys --cov=leakrfc --cov-report lcov
+	poetry run pytest -v --capture=sys --cov=ftm_datalake --cov-report lcov
 
 build:
 	poetry run build
@@ -36,4 +36,4 @@ clean:
 
 documentation:
 	mkdocs build
-	aws --endpoint-url https://s3.investigativedata.org s3 sync ./site s3://docs.investigraph.dev/lib/leakrfc
+	aws --profile nbg1 --endpoint-url https://s3.investigativedata.org s3 sync ./site s3://openaleph.org/docs/lib/ftm-datalake

NOTICE

@@ -1,8 +1,8 @@
 LEAKRFC, (C) 2024 investigativedata.io
-LEAKRFC, (C) 2025 investigativedata.io
+LEAKRFC, (C) 2025 Data and Research Center – DARC
 
-This product includes software developed at investigativedata.io
-(https://investigativedata.io)
+This product includes software developed at the Data and Research Center
+(https://dataresearchcenter.org)
 
 LEAKRFC contains unmodified subcomponents too with separate copyright notices
 and license terms. Your use of the source code for these subcomponents is

README.md

@@ -1,24 +1,26 @@
-# leakrfc
+[![ftm-datalake on pypi](https://img.shields.io/pypi/v/ftm-datalake)](https://pypi.org/project/ftm-datalake/)
+[![Python test and package](https://github.com/dataresearchcenter/ftm-datalake/actions/workflows/python.yml/badge.svg)](https://github.com/dataresearchcenter/ftm-datalake/actions/workflows/python.yml)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Coverage Status](https://coveralls.io/repos/github/dataresearchcenter/ftm-datalake/badge.svg?branch=main)](https://coveralls.io/github/dataresearchcenter/ftm-datalake?branch=main)
+[![AGPLv3+ License](https://img.shields.io/pypi/l/ftm-datalake)](./LICENSE)
 
-"_A RFC for leaks_"
+# ftm-datalake
 
-[leak-rfc.org](https://leak-rfc.org)
+`ftm-datalake` provides a _data standard_ and _archive storage_ for structured [FollowTheMoney](https://followthemoney.tech) data, leaked data, private and public document collections. The concepts and implementations are originally inspired by [mmmeta](https://github.com/simonwoerpel/mmmeta) and [Aleph's servicelayer archive](https://github.com/alephdata/servicelayer) and are [discussed here](https://aleph.discourse.group/t/rfc-followthemoney-data-lake-specification/276/3)
 
-`leakrfc` provides a _data standard_ and _archive storage_ for leaked data, private and public document collections. The concepts and implementations are originally inspired by [mmmeta](https://github.com/simonwoerpel/mmmeta) and [Aleph's servicelayer archive](https://github.com/alephdata/servicelayer).
-
-`leakrfc` acts as a multi-tenant storage and retrieval mechanism for documents and their metadata. It provides a high-level interface for generating and sharing document collections and importing them into various search and analysis platforms, such as [_ICIJ Datashare_](https://datashare.icij.org/), [_Liquid Investigations_](https://github.com/liquidinvestigations/), and [_Aleph_](https://docs.aleph.occrp.org/).
+`ftm-datalake` acts as a multi-tenant storage and retrieval mechanism for structured entity data, documents and their metadata. It provides a high-level interface for generating and sharing document collections and importing them into various search and analysis platforms, such as [_ICIJ Datashare_](https://datashare.icij.org/), [_Liquid Investigations_](https://github.com/liquidinvestigations/), and [_OpenAleph_](https://openaleph.org/).
 
 ## Installation
 
 Requires python 3.11 or later.
 
 ```bash
-pip install leakrfc
+pip install ftm-datalake
 ```
 
 ## Documentation
 
-[docs.investigraph.dev/lib/leakrfc](https://docs.investigraph.dev/lib/leakrfc)
+[openaleph.org/lib/ftm-datalake](https://openaleph.org/lib/ftm-datalake)
 
 ## Development
 
@@ -38,15 +40,16 @@ Before creating a commit, this checks for correct code formatting (isort, black)
 
 ### testing
 
-`leakrfc` uses [pytest](https://docs.pytest.org/en/stable/) as the testing framework.
+`ftm-datalake` uses [pytest](https://docs.pytest.org/en/stable/) as the testing framework.
 
     make test
 
 ## License and Copyright
 
-`leakrfc`, (C) 2024 investigativedata.io
-`leakrfc`, (C) 2025 investigativedata.io
+`ftm-datalake`, (C) 2024 investigativedata.io
+
+`ftm-datalake`, (C) 2025 [Data and Resear Center – DARC](https://dataresearchcenter.org)
 
-`leakrfc` is licensed under the AGPLv3 or later license.
+`ftm-datalake` is licensed under the AGPLv3 or later license.
 
 see [NOTICE](./NOTICE) and [LICENSE](./LICENSE)