Merge pull request #2 from workflowai/guillaume/e2e-tests

Feat add first e2e test

Author: guillaq

.env.sample

@@ -1,2 +1,6 @@
 WORKFLOWAI_API_URL=
 WORKFLOWAI_API_KEY=
+
+# Used when running e2e tests
+WORKFLOWAI_TEST_API_URL=
+WORKFLOWAI_TEST_API_KEY=

.github/workflows/e2e.yml

@@ -0,0 +1,101 @@
+name: E2E tests
+on:
+  push:
+    branches:
+      - main
+  schedule:
+    # Every night at midnight
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+env:
+  WORKFLOWAI_TEST_API_URL: https://api.workflowai.dev
+
+jobs:
+  e2e-tests:
+    # TODO: we should run on multiple environments
+    environment: staging
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Poetry
+        run: pipx install poetry==1.8.0
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --all-extras
+
+      - name: Run tests
+        run: poetry run pytest tests/e2e
+        env:
+          WORKFLOWAI_TEST_API_KEY: ${{ secrets.WORKFLOWAI_TEST_API_KEY }}
+
+      - name: Send Slack Notification
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        run: |
+          curl -X POST -H 'Content-type: application/json' --data '{
+            "blocks": [
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": "*:white_check_mark: Python E2E Tests Completed*"
+                    }
+                }
+            ]
+          }' $SLACK_WEBHOOK_URL
+
+      - name: Notify Slack on Failure
+        if: failure()
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          GITHUB_RUN_URL: ${{ github.server_url }}/{{ github.repository }}/actions/runs/{{ github.run_id }}
+        run: |
+          curl -X POST -H 'Content-type: application/json' --data '{
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": ":warning: *Python E2E Tests Failed!* \n <!channel>"
+                }
+              },
+              {
+                "type": "section",
+                "fields": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Job:* ${{ github.job }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Run Number:* ${{ github.run_number }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Repository:* ${{ github.repository }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Branch:* ${{ github.ref }}"
+                  }
+                ]
+              },
+              {
+                "type": "actions",
+                "elements": [
+                  {
+                    "type": "button",
+                    "text": {
+                      "type": "plain_text",
+                      "text": "View Action"
+                    },
+                    "url": "${{ env.GITHUB_RUN_URL }}"
+                  }
+                ]
+              }
+            ]
+          }' $SLACK_WEBHOOK_URL

.github/workflows/quality.yml

@@ -47,4 +47,4 @@ jobs:
           poetry install --all-extras
 
       - name: Run tests
-        run: poetry run pytest
+        run: poetry run pytest --ignore=tests/e2e

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
       - id: poetry-lock
         name: poetry lock
-        entry: poetry check --lock 
+        entry: poetry check --lock
         language: system
         pass_filenames: false
         always_run: true
@@ -22,6 +22,6 @@ repos:
         types: [file, python]
       - id: pytest
         name: testing (pytest)
-        entry: pytest .
+        entry: pytest . --ignore tests/e2e
         language: system
         pass_filenames: false

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "workflowai"
-version = "0.1.2"
+version = "0.1.3"
 description = ""
 authors = ["Guillaume Aquilina <guillaume@workflowai.com>"]
 readme = "README.md"
@@ -22,6 +22,7 @@ ruff = "^0.4.9"
 freezegun = "^1.5.1"
 pre-commit = "^3.7.1"
 pytest-httpx = "^0.30.0"
+python-dotenv = "^1.0.1"
 
 [tool.poetry.extras]
 cli = ["typer", "rich"]

pyproject.toml

@@ -0,0 +1,16 @@
+import os
+
+import pytest
+from dotenv import load_dotenv
+
+from workflowai import Client, start
+
+load_dotenv()
+
+
+@pytest.fixture(scope="session")
+def wai() -> Client:
+    return start(
+        url=os.environ["WORKFLOWAI_TEST_API_URL"],
+        api_key=os.environ["WORKFLOWAI_TEST_API_KEY"],
+    )

tests/e2e/conftest.py

@@ -0,0 +1,41 @@
+import uuid
+
+import workflowai
+from examples.city_to_capital_task import CityToCapitalTask, CityToCapitalTaskInput
+from workflowai.core.domain.task_version_reference import TaskVersionReference
+
+
+async def test_deploy_task(wai: workflowai.Client):
+    task = CityToCapitalTask(
+        id=f"citytocapital-{uuid.uuid4()}",
+        schema_id=0,
+        version=TaskVersionReference.with_properties(model=""),
+    )
+
+    await wai.register(task)
+
+    assert task.schema_id == 1
+
+    # Run task with input
+    task_run = await wai.run(task, task_input=CityToCapitalTaskInput(city="Osaka"))
+    assert task_run.task_output.capital == "Tokyo"
+    assert task_run.version.iteration == 1
+
+    # Deploy group to dev environnment
+    version = await wai.deploy_version(task, iteration=1, environment="dev")
+    assert version.iteration == 1
+    assert version.aliases == {"environment=dev"}
+
+    # Run using the environment and the same input
+    task_run2 = await wai.run(
+        task, task_input=CityToCapitalTaskInput(city="Osaka"), environment="dev"
+    )
+    # IDs will match since we are using cache
+    assert task_run.id == task_run2.id
+
+    # Run using the environment and a different input
+    task_run3 = await wai.run(
+        task, task_input=CityToCapitalTaskInput(city="Toulouse"), environment="dev"
+    )
+    assert task_run3.task_output.capital == "Paris"
+    assert task_run3.id != task_run2.id

tests/e2e/deploy_test.py

@@ -0,0 +1,26 @@
+{
+    "id": "0a35fd2e55def802918880d094f634f9",
+    "iteration": 1,
+    "properties": {
+        "model": "gpt-4o-2024-05-13",
+        "provider": "openai",
+        "temperature": 0,
+        "instructions": "To complete the task 'AddSixToNumber', follow these steps: 1. Take an input object that contains a 'number' field. 2. Add 6 to the value of the 'number' field. 3. Return an output object that contains a 'result' field with the computed value. Ensure the input and output objects conform to their respective schemas.",
+        "max_tokens": null,
+        "runner_name": "WorkflowAI",
+        "runner_version": "v0.1.0",
+        "few_shot": null,
+        "template_name": "v1",
+        "task_variant_id": "216123bda0159257f03ed2723f59731f"
+    },
+    "tags": [
+        "model=gpt-4o-2024-05-13",
+        "provider=openai",
+        "temperature=0",
+        "template_name=v1"
+    ],
+    "aliases": [
+        "environment=dev"
+    ],
+    "is_external": null
+}

tests/fixtures/task_version.json

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from workflowai.core import client
+from workflowai.core.client import Client as Client
 from workflowai.core.domain.cache_usage import CacheUsage as CacheUsage
 from workflowai.core.domain.errors import NotFoundError as NotFoundError
 from workflowai.core.domain.llm_completion import LLMCompletion as LLMCompletion
@@ -14,7 +14,7 @@
 )
 
 
-def start(url: Optional[str] = None, api_key: Optional[str] = None) -> "client.Client":
+def start(url: Optional[str] = None, api_key: Optional[str] = None) -> Client:
     """Create a new workflowai client
 
     Args: