Async Behaviour Tree implementation for the Redis Release process

.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh

@@ -45,4 +45,4 @@ fi
 # Clean up temporary file
 rm -f "$TEMP_ARCHIVE"
 
-echo "Redis archive validation completed successfully"
+echo "Redis archive validation completed successfully"

.github/workflows/run-tests.yml

@@ -0,0 +1,22 @@
+on:
+  push:
+  pull_request:
+
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install packages
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          pip install -e .[dev]
+
+      - name: Run tests
+        run: |
+          . venv/bin/activate
+          pytest

.gitignore

@@ -236,8 +236,6 @@ $RECYCLE.BIN/
 aws-credentials.json
 
 # Local test files
-test_*.py
-*_test.py
 temp_*.py
 
 # Local configuration

README.md

@@ -4,14 +4,18 @@ A command-line tool for automating Redis OSS releases across multiple package re
 
 ## Installation
 
-### From Source
+### Using uv
 
 ```bash
 git clone https://github.com/redis/redis-oss-release-automation.git
 cd redis-oss-release-automation
-pip install -e .
+uv sync
 ```
 
+After `uv sync`, you can run the tool in two ways:
+- **With `uv run`**: `uv run redis-release <command>`
+- **Activate virtual environment**: `. .venv/bin/activate` then `redis-release <command>`
+
 ## Prerequisites
 
 1. **GitHub Token**: Personal access token with workflow permissions
@@ -24,20 +28,38 @@ pip install -e .
 export GITHUB_TOKEN="ghp_xxxxxxxxxxxx"
 export AWS_ACCESS_KEY_ID="your-access-key-id"
 export AWS_SECRET_ACCESS_KEY="your-secret-access-key"
-export AWS_SESSION_TOKEN="your-session-token"  
+export AWS_SESSION_TOKEN="your-session-token"
 export REDIS_RELEASE_STATE_BUCKET="redis-release-state"
 ```
 
+### AWS SSO Login
+
+In AWS, you can also use `aws sso login` prior to running the tool to authenticate.
+
 ## Usage
 
 ### Basic Release
 
+By default, `config.yaml` is used. You can specify a different config file with `--config`:
+
 ```bash
-# Start a new release
+# Start a new release (uses config.yaml by default)
 redis-release release 8.2.0
 
-# Force rebuild packages
-redis-release release 8.2.0 --force-rebuild
+# Use custom config file
+redis-release release 8.2.0 --config custom-config.yaml
+
+# Force rebuild all packages (WARNING: This will delete all existing state!)
+redis-release release 8.2.0 --force-rebuild all
+
+# Force rebuild specific package
+redis-release release 8.2.0 --force-rebuild package-name
+
+# Release only specific packages (can be used multiple times)
+redis-release release 8.2.0 --only-packages package1 --only-packages package2
+
+# Force release type (changes release-type even for existing state)
+redis-release release 8.2.0 --force-release-type rc
 ```
 
 ### Check Status
@@ -47,9 +69,34 @@ redis-release release 8.2.0 --force-rebuild
 redis-release status 8.2.0
 ```
 
-### Advanced Options
+## Troubleshooting
+
+### Dangling Release Locks
+
+If you encounter a dangling lock file, you can delete it from the S3 bucket:
 
 ```bash
-# Dry run mode (simulate without changes)
-redis-release release 8.2.0 --dry-run
+aws s3 rm s3://redis-release-state/release-locks/TAG.lock
 ```
+
+Replace `TAG` with the release tag (e.g., `8.2.0`).
+
+## Diagrams
+
+Generate release workflow diagrams using:
+
+```bash
+# Generate full release diagram
+redis-release release-print
+
+# Generate diagram with custom name (list available with --help)
+redis-release release-print --name NAME
+```
+
+**Note**: Graphviz is required to generate diagrams.
+
+## Configuration
+
+The tool uses a YAML configuration file to define release packages and their settings. By default, `config.yaml` is used.
+
+See `config.yaml` for an example configuration file.

config.yaml

@@ -0,0 +1,34 @@
+version: 1
+packages:
+  docker:
+    # available types: docker, debian
+    package_type: docker
+    # repo where the workflow is started
+    repo: redis/docker-library-redis
+    # ref to use for workflow runs, if not specified it
+    # will be determined based on the tag and existing branches
+    # Should be empty for real use cases, is to be used for testing only
+    # ref: branch_name
+    # build workflow name
+    build_workflow: release_build_and_test.yml
+    # build workflow timeout in minutes, optional
+    build_timeout_minutes: 45
+    # static workflow inputs for build workflow
+    build_inputs: {}
+    # whether to run publish workflow for internal releases
+    publish_internal_release: no
+    # publish workflow name
+    publish_workflow: release_publish.yml
+    # publish workflow timeout in minutes, optional
+    publish_timeout_minutes: 10
+    # static workflow inputs for publish workflow
+    publish_inputs: {}
+  debian:
+    package_type: debian
+    repo: redis/redis-debian
+    build_workflow: release_build_and_test.yml
+    build_inputs: {}
+    publish_internal_release: yes
+    publish_workflow: release_publish.yml
+    publish_timeout_minutes: 10
+    publish_inputs: {}

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling>=1.13"]
 build-backend = "hatchling.build"
 
 [project]
@@ -15,23 +15,27 @@ classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
     "typer[all]>=0.9.0",
     "requests>=2.28.0",
+    "aiohttp>=3.8.0",
     "boto3>=1.26.0",
     "rich>=13.0.0",
     "pydantic>=2.0.0",
+    "py_trees>=2.2,<3.0",
+    "pyyaml>=6.0.3",
 ]
 
 [project.optional-dependencies]
 dev = [
     "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
     "pytest-cov>=4.0.0",
     "black>=23.0.0",
     "isort>=5.12.0",
@@ -71,3 +75,11 @@ python_version = "3.8"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+log_cli = true
+log_cli_level = "DEBUG"
+log_cli_format = "%(asctime)s [%(levelname)8s] %(name)s - %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"

src/redis_release/bht/args.py

@@ -0,0 +1 @@
+"""Arguments for release automation."""

src/redis_release/bht/backchain.py

@@ -0,0 +1,145 @@
+"""Tools for creating behavior trees using backchaining.
+
+See Michele Colledanchise and Petter Ögren
+Behavior Trees in Robotics and AI
+3.5 Creating Deliberative BTs using Backchaining
+"""
+
+import logging
+from typing import Optional, Union
+
+from py_trees.behaviour import Behaviour
+from py_trees.composites import Selector, Sequence
+
+logger = logging.getLogger(__name__)
+
+
+def find_chain_anchor_point(
+    root: Behaviour,
+) -> Sequence:
+    """Find the anchor point (Sequence) to which we can latch the next chain.
+
+    We assume that the anchor point is the leftmost non empty Sequence in the tree.
+    """
+    for child in root.children:
+        if (isinstance(child, Sequence) or isinstance(child, Selector)) and len(
+            child.children
+        ) > 0:
+            return find_chain_anchor_point(child)
+    if isinstance(root, Sequence):
+        return root
+    else:
+        raise Exception("No chain anchor_point found")
+
+
+def latch_chains(*chains: Union[Selector, Sequence]) -> None:
+    assert len(chains) >= 2
+    first = chains[0]
+    for chain in chains[1:]:
+        latch_chain_to_chain(first, chain)
+
+
+def latch_chain_to_chain(
+    first: Behaviour,
+    next: Union[Selector, Sequence],
+) -> None:
+    """Latch two chains together. Both are expected to be formed using PPAs.
+
+    If both precondition in the anchor point and postcondition of the next
+    chain exist, and they are the same type then the precondition in the
+    anchor point is replaced by the next chain.
+    Otherwise the next chain is added as a leftmost child to the anchor point.
+
+    If the next chain is a sequence, its children are merged into the anchor point.
+
+    Args:
+        ppa: PPA composite to latch to
+        link: Link composite to latch
+    """
+    anchor_point = find_chain_anchor_point(first)
+    next_postcondition: Optional[Behaviour] = None
+    anchor_precondition: Optional[Behaviour] = None
+
+    logger.debug(f'Latching "{next.name}" to "{anchor_point.name}"')
+
+    # Trying to guess from the structure which node may be a postcondition
+    # Later we compare it with the anchor point precondition and when they match
+    # we assume it is the postcondition that could be removed as a part of backchaining
+    if type(next) == Selector and len(next.children) > 0:
+        next_postcondition = next.children[0]
+    if type(next) == Sequence:
+        if len(next.children) == 1:
+            # This is a PPA with only one action which may be interpreted as a postcondition
+            # Like Sequence --> IsWorkflowSuccessful?
+            next_postcondition = next.children[0]
+        elif (
+            len(next.children) > 1
+            and type(next.children[0]) == Selector
+            and len(next.children[-1].children) == 0
+        ):
+            # The same as above but when another chain is already latched to this PPA
+            # and therefore it now has leftmost Selector children and rightmost action
+            next_postcondition = next.children[-1]
+
+    assert len(anchor_point.children) > 0
+    anchor_precondition = anchor_point.children[0]
+
+    logger.debug(
+        f"Anchor precondition: {anchor_precondition.name}, Next postcondition: {next_postcondition.name if next_postcondition else 'None'}"
+    )
+
+    # If anchor point has both precondition and action, remove anchor_precondition if it matches the next_postcondition
+    # very weak check that the anchor_precondition is the same as the next_postcondition:
+    if (
+        len(anchor_point.children) == 2
+        and next_postcondition is not None
+        and type(next_postcondition) == type(anchor_precondition)
+        and next_postcondition.name == anchor_precondition.name
+    ):
+        anchor_point.children.pop(0)
+        logger.debug(f"Removed precondition from PPA {anchor_precondition.name}")
+
+    if type(next) == Sequence:
+        # If next is a sequence, merge next's children into achor_point sequence to the left
+        for child in reversed(next.children):
+            child.parent = anchor_point
+            anchor_point.children.insert(0, child)
+            logger.debug(
+                f"Merged child {child.name} to anchor point {anchor_point.name}"
+            )
+    else:
+        next.parent = anchor_point
+        anchor_point.children.insert(0, next)
+        logger.debug(
+            f"Added chain {next.name} directly to anchor point {anchor_point.name}"
+        )
+
+
+def create_PPA(
+    name: str,
+    action: Behaviour,
+    postcondition: Optional[Behaviour] = None,
+    precondition: Optional[Behaviour] = None,
+) -> Union[Sequence, Selector]:
+    """Create a PPA (Precondition-Postcondition-Action) composite."""
+
+    sequence = Sequence(
+        name=f"{name}",
+        memory=False,
+        children=[],
+    )
+    if precondition is not None:
+        sequence.add_child(precondition)
+    sequence.add_child(action)
+
+    if postcondition is not None:
+        selector = Selector(
+            name=f"{name} Goal",
+            memory=False,
+            children=[],
+        )
+        selector.add_child(postcondition)
+        selector.add_child(sequence)
+        return selector
+    else:
+        return sequence