Klondike v0.3.0

.github/workflows/pr_checks.yml

@@ -0,0 +1,94 @@
+name: PR Checks
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  ruff-lint:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run ruff linting
+        run: uv run ruff check .
+
+  ruff-format:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run ruff formatting check
+        run: uv run ruff format --check .
+
+  mypy:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run mypy type checking
+        run: uv run mypy klondike/
+
+  pytest:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run pytest
+        run: uv run pytest -rf tests/

.github/workflows/publish.yml

@@ -0,0 +1,36 @@
+name: Publish to PyPI
+
+on:
+  push:
+    branches:
+      - main  # This triggers when a PR is merged into main
+
+jobs:
+  build-n-publish:
+    name: Build and publish Klondike to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      url: https://pypi.org/p/klondike
+    permissions:
+      id-token: write  # MANDATORY for Trusted Publishers
+      contents: read   # Required to check out the code
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: uv run python -m pip install build
+
+      - name: Build distributions
+        run: uv run python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -8,6 +8,7 @@ venv/
 .vscode/
 dev/
 service_accounts/
+development/
 
 build/
 dist/

.pre-commit-config.yaml

@@ -1,8 +1,7 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.9.2
+    rev: v0.11.2
     hooks:
-      # Run the linter.
       - id: ruff
+      - id: ruff-format
 

Makefile

@@ -5,6 +5,20 @@ push:
 setup:
 	@bash run_setup.sh
 
-clean:
-	@ruff check --fix .
-	@pytest -rf tests/*
+
+### CI Checks (these run locally in the CI pipeline, but can be run locally with `make all-checks`) ###
+
+ruff:
+	@uv run ruff check --fix .
+	@uv run ruff format .
+
+pytest:
+	@uv run pytest -rf tests
+
+mypy:
+	@uv run mypy klondike
+
+all-checks:
+	@make ruff
+	@make pytest
+	@make mypy

README.md

@@ -1,6 +1,8 @@
 # Klondike
 
-Klondike offers a lightweight API to read and write data to Google BigQuery using Polars DataFrames.
+![Klondike Logo](https://raw.githubusercontent.com/IanRFerguson/klondike/main/assets/logo.png)
+
+Klondike offers a lightweight, unified API to read and write data to and from various database and data warehouse types using Polars DataFrames.
 
 ## Installation
 

klondike/__init__.py

@@ -1,35 +1,19 @@
 import importlib
-import logging
-import os
 
-# Define project logger
-logger = logging.getLogger(__name__)
+from klondike.utilities.logger import logger
 
-_handler = logging.StreamHandler()
-_formatter = logging.Formatter("%(module)s %(levelname)s %(message)s")
-_handler.setFormatter(_formatter)
-
-logger.addHandler(_handler)
-
-# Define logging behavior
-if any([os.environ.get("TESTING"), os.environ.get("TEST")]):
-    logger.setLevel("WARNING")
-elif os.environ.get("DEBUG"):
-    logger.setLevel("DEBUG")
-    logger.debug("Debugger active...")
-else:
-    logger.setLevel("INFO")
-
-#####
-
-__all__ = []
+__all__: list[str] = []
 POLAR_OBJECTS = [
     ("klondike.gcp.bigquery", "BigQueryConnector"),
+    ("klondike.gcp.cloud_storage", "CloudStorageConnector"),
     ("klondike.snowflake.snowflake", "SnowflakeConnector"),
+    ("klondike.scripts.stream_csv_to_database", "stream_csv_to_database"),
 ]
 
 for module_, object_ in POLAR_OBJECTS:
     try:
         globals()[object_] = getattr(importlib.import_module(module_), object_)
     except ImportError:
-        logger.debug(f"Could not import {module_}.{object_} ... skipping")
+        logger.warning(
+            f"Could not import {object_} from {module_}. It will not be available."
+        )

klondike/aws/redshift.py

@@ -1,8 +1,6 @@
-from klondike.base.abc_klondike import KlondikeBaseDBConnector
-
-#####
+from klondike.base.abc_klondike import KlondikeBaseDatabaseConnector
 
 
 # TODO
-class RedshiftConnector(KlondikeBaseDBConnector):
+class RedshiftConnector(KlondikeBaseDatabaseConnector):
     pass

klondike/aws/s3.py

@@ -1,8 +1,6 @@
-from klondike.base.abc_klondike import KlondikeBaseFlatFileConnector
-
-#####
+from klondike.base.abc_klondike import KlondikeBaseStorageConnector
 
 
 # TODO
-class S3Connector(KlondikeBaseFlatFileConnector):
+class S3Connector(KlondikeBaseStorageConnector):
     pass