feat: Switch to devservices instead of sentry devservices (#88)

Use direnv + devservices to have a smaller dependency footprint. Sentry
will also be moving to devservices in the new year and already has
opt-in config for using devservices now.

Fixes #85

Co-authored-by: Ian Woodard <17186604+IanWoodard@users.noreply.github.com>

Author: markstory

.envrc

@@ -0,0 +1,14 @@
+#!/bin/bash
+# shellcheck disable=SC1091
+if [[ -f .env ]]; then
+    dotenv .env
+fi
+
+if [ ! -d .venv ]; then
+    echo "warning: creating virtualenv for the first time"
+    make setup
+    source python/.venv/bin/activate
+else
+    source python/.venv/bin/activate
+    unset PS1
+fi

.github/workflows/ci.yml

@@ -26,10 +26,10 @@ jobs:
       - uses: swatinem/rust-cache@v2
 
       - name: Run Rustfmt
-        run: cargo fmt --all -- --check
+        run: make style
 
       - name: Run Clippy
-        run: cargo clippy --workspace --all-features --tests -- -D clippy::all
+        run: make lint
 
   test:
     name: Tests (ubuntu)
@@ -38,27 +38,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Checkout sentry
-        uses: actions/checkout@v4
-        with:
-          repository: getsentry/sentry
-          path: sentry
+      - name: Install Python Dependencies
+        run: |
+          make install-py-dev
 
-      - name: Setup steps
+      - name: Start services
         id: setup
         run: |
-          # We cannot execute actions that are not placed under .github of the main repo
-          mkdir -p .github/actions
-          cp -r sentry/.github/actions/* .github/actions
-
-      - name: Setup Sentry
-        id: setup-sentry
-        uses: ./.github/actions/setup-sentry
-        with:
-          workdir: sentry
-          snuba: false
-          kafka: true
-          clickhouse: false
+          devservices up
 
       - name: Install protoc
         uses: arduino/setup-protoc@v3
@@ -79,12 +66,6 @@ jobs:
           command: test
           args: --all
 
-      - name: Install Python Dependencies
-        run: |
-          python -m venv python/.venv
-          . python/.venv/bin/activate
-          pip install -r python/requirements-dev.txt
-
       - name: Run Python Integration Tests
         run: |
           export PYTEST_ADDOPTS=""

Makefile

@@ -1,5 +1,5 @@
 # Development
-setup: setup-git ## run setup tasks to create and configure a development environment
+setup: setup-git install-py-dev ## run setup tasks to create and configure a development environment
 .PHONY: setup
 
 setup-git: .git/hooks/pre-commit ## Setup git-hooks
@@ -27,7 +27,7 @@ style: ## Run style checking tools (cargo-fmt)
 
 lint: ## Run linting tools (cargo-clippy)
 	@rustup component add clippy 2> /dev/null
-	cargo clippy --workspace --all-targets --all-features --no-deps --allow-dirty -- -D warnings
+	cargo clippy --workspace --all-targets --all-features --no-deps -- -D warnings
 .PHONY: format
 
 format: ## Run autofix mode for formatting and lint

config/config-sentry-dev.yaml

@@ -1,2 +1,2 @@
 upkeep_task_interval_ms: 2000
-kafka_cluster: sentry_kafka:9093
+kafka_cluster: 127.0.0.1:9092

devservices/config.yml

@@ -0,0 +1,19 @@
+# Ignored by docker compose, used by devservices
+x-sentry-service-config:
+  version: 0.1
+  service_name: taskbroker
+  dependencies:
+    kafka:
+      description: Shared instance of kafka used by sentry services
+      remote:
+        repo_name: sentry-shared-kafka
+        branch: main
+        repo_link: https://github.com/getsentry/sentry-shared-kafka
+  modes:
+    default: [kafka]
+    containerized: [kafka]
+
+networks:
+  devservices:
+    name: devservices
+    external: true

python/integration_tests/helpers.py

@@ -18,7 +18,7 @@ def check_topic_exists(topic_name: str) -> bool:
         check_topic_cmd = [
             "docker",
             "exec",
-            "sentry_kafka",
+            "kafka-kafka-1",
             "kafka-topics",
             "--bootstrap-server",
             "localhost:9092",
@@ -37,7 +37,7 @@ def create_topic(topic_name: str, num_partitions: int) -> None:
         create_topic_cmd = [
             "docker",
             "exec",
-            "sentry_kafka",
+            "kafka-kafka-1",
             "kafka-topics",
             "--bootstrap-server",
             "localhost:9092",
@@ -59,7 +59,7 @@ def update_topic_partitions(topic_name: str, num_partitions: int) -> None:
         create_topic_cmd = [
             "docker",
             "exec",
-            "sentry_kafka",
+            "kafka-kafka-1",
             "kafka-topics",
             "--bootstrap-server",
             "localhost:9092",

python/requirements-dev.txt

@@ -1,4 +1,5 @@
 confluent_kafka>=2.3.0
+devservices==1.0.5
 orjson>=3.10.10
 protobuf>=5.28.3
 pytest>=8.3.3