From bb248683c563f4355cdf3eb5292d1ce9e66785e6 Mon Sep 17 00:00:00 2001 From: BIN Zhang Date: Mon, 16 Mar 2026 01:08:22 +0800 Subject: [PATCH 1/2] feat: add file-based plugin discovery config --- .gitignore | 1 + README.md | 11 +++ demos/discovery.py | 140 ++++++++++++++++++++++++++++++--- demos/persona_workflow_demo.py | 3 +- demos/task_registry.py | 30 +++++-- plugin_config.example.json | 10 +++ 6 files changed, 175 insertions(+), 20 deletions(-) create mode 100644 plugin_config.example.json diff --git a/.gitignore b/.gitignore index f06f792..069d966 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build/ dist/ .pytest_cache/ demos/generated/ +plugin_config.json diff --git a/README.md b/README.md index 86c83df..1a74335 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,17 @@ Start here: This dry-run demo shows 3 portable persona objects projected into 3 CrewAI-style runtime roles. +## Plugin Discovery + +The demo workflow can load registry extensions from builtin modules and +external plugin packages. By default, the runtime looks for a local +`plugin_config.json` file at the repository root if present. An example +template is available at [`plugin_config.example.json`](plugin_config.example.json). + +For temporary overrides, you can also point discovery at external plugin +packages with `POP_PLUGIN_CONFIG_FILE`, `POP_PLUGIN_PACKAGES`, and +`POP_PLUGIN_PACKAGE_PATHS`. + ## Persona Layer Diagram ```mermaid diff --git a/demos/discovery.py b/demos/discovery.py index 40622e1..6d95946 100644 --- a/demos/discovery.py +++ b/demos/discovery.py @@ -1,14 +1,28 @@ from __future__ import annotations import importlib +import importlib.util +import json import os import pkgutil import sys +from dataclasses import dataclass from pathlib import Path from types import ModuleType +from typing import Any +PROJECT_ROOT = Path(__file__).resolve().parents[1] PLUGIN_PACKAGES_ENV = "POP_PLUGIN_PACKAGES" PLUGIN_PACKAGE_PATHS_ENV = "POP_PLUGIN_PACKAGE_PATHS" +PLUGIN_CONFIG_FILE_ENV = "POP_PLUGIN_CONFIG_FILE" +DEFAULT_PLUGIN_CONFIG_PATH = PROJECT_ROOT / "plugin_config.json" + + +@dataclass(frozen=True) +class PluginDiscoveryConfig: + config_path: Path | None + package_names: tuple[str, ...] + package_paths: tuple[Path, ...] def split_package_names(value: str) -> tuple[str, ...]: @@ -23,6 +37,108 @@ def split_search_paths(value: str) -> tuple[Path, ...]: ) +def unique_strings(values: tuple[str, ...]) -> tuple[str, ...]: + ordered: list[str] = [] + seen: set[str] = set() + for value in values: + if value and value not in seen: + ordered.append(value) + seen.add(value) + return tuple(ordered) + + +def unique_paths(values: tuple[Path, ...]) -> tuple[Path, ...]: + ordered: list[Path] = [] + seen: set[Path] = set() + for value in values: + if value not in seen: + ordered.append(value) + seen.add(value) + return tuple(ordered) + + +def resolve_plugin_config_path(config_file: str | Path | None = None) -> Path | None: + if config_file is not None: + candidate = Path(config_file).expanduser().resolve() + if not candidate.exists(): + raise FileNotFoundError(f"Plugin config file does not exist: {candidate}") + return candidate + + env_value = os.environ.get(PLUGIN_CONFIG_FILE_ENV, "").strip() + if env_value: + candidate = Path(env_value).expanduser().resolve() + if not candidate.exists(): + raise FileNotFoundError(f"Plugin config file does not exist: {candidate}") + return candidate + + if DEFAULT_PLUGIN_CONFIG_PATH.exists(): + return DEFAULT_PLUGIN_CONFIG_PATH + + return None + + +def resolve_relative_path(value: str, base_dir: Path) -> Path: + path = Path(value).expanduser() + if path.is_absolute(): + return path.resolve() + return (base_dir / path).resolve() + + +def normalize_plugin_paths( + payload: dict[str, Any], + base_dir: Path, +) -> tuple[Path, ...]: + paths: list[Path] = [] + for raw_path in payload.get("plugin_package_paths", ()): + if str(raw_path).strip(): + paths.append(resolve_relative_path(str(raw_path), base_dir)) + + for entry in payload.get("plugin_paths", ()): + if isinstance(entry, str): + if entry.strip(): + paths.append(resolve_relative_path(entry, base_dir)) + continue + if isinstance(entry, dict): + raw_path = str(entry.get("path", "")).strip() + if raw_path: + paths.append(resolve_relative_path(raw_path, base_dir)) + + return unique_paths(tuple(paths)) + + +def normalize_plugin_packages(payload: dict[str, Any]) -> tuple[str, ...]: + package_names: list[str] = [] + for package_name in payload.get("plugin_packages", ()): + normalized = str(package_name).strip() + if normalized: + package_names.append(normalized) + + for entry in payload.get("plugin_paths", ()): + if isinstance(entry, dict): + package_name = str(entry.get("package_name", "")).strip() + if package_name: + package_names.append(package_name) + + return unique_strings(tuple(package_names)) + + +def load_plugin_config( + config_file: str | Path | None = None, +) -> PluginDiscoveryConfig: + config_path = resolve_plugin_config_path(config_file) + if config_path is None: + return PluginDiscoveryConfig(None, (), ()) + + payload = json.loads(config_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("Plugin config must be a JSON object.") + + base_dir = config_path.parent + package_paths = normalize_plugin_paths(payload, base_dir) + package_names = normalize_plugin_packages(payload) + return PluginDiscoveryConfig(config_path, package_names, package_paths) + + def configure_plugin_search_paths(paths: tuple[Path, ...]) -> tuple[Path, ...]: configured: list[Path] = [] for path in paths: @@ -35,21 +151,21 @@ def configure_plugin_search_paths(paths: tuple[Path, ...]) -> tuple[Path, ...]: return tuple(configured) -def configured_package_names( +def configured_plugin_sources( default_packages: tuple[str, ...] = ("demos",), extra_packages: tuple[str, ...] = (), -) -> tuple[str, ...]: - plugin_paths = split_search_paths(os.environ.get(PLUGIN_PACKAGE_PATHS_ENV, "")) - configure_plugin_search_paths(plugin_paths) + config_file: str | Path | None = None, +) -> PluginDiscoveryConfig: + file_config = load_plugin_config(config_file) + env_paths = split_search_paths(os.environ.get(PLUGIN_PACKAGE_PATHS_ENV, "")) + configured_paths = configure_plugin_search_paths( + unique_paths((*file_config.package_paths, *env_paths)) + ) env_packages = split_package_names(os.environ.get(PLUGIN_PACKAGES_ENV, "")) - - ordered_packages: list[str] = [] - seen: set[str] = set() - for package_name in (*default_packages, *extra_packages, *env_packages): - if package_name and package_name not in seen: - ordered_packages.append(package_name) - seen.add(package_name) - return tuple(ordered_packages) + package_names = unique_strings( + (*default_packages, *extra_packages, *file_config.package_names, *env_packages) + ) + return PluginDiscoveryConfig(file_config.config_path, package_names, configured_paths) def import_matching_modules( diff --git a/demos/persona_workflow_demo.py b/demos/persona_workflow_demo.py index 267ed1d..c6dc51f 100644 --- a/demos/persona_workflow_demo.py +++ b/demos/persona_workflow_demo.py @@ -274,7 +274,8 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Run the persona workflow demo with dynamic task input. " - "External plugin packages can be added with " + "External plugin packages can be added with plugin_config.json or " + "POP_PLUGIN_CONFIG_FILE, and overridden with " "POP_PLUGIN_PACKAGES and POP_PLUGIN_PACKAGE_PATHS." ) ) diff --git a/demos/task_registry.py b/demos/task_registry.py index 2813570..08ee679 100644 --- a/demos/task_registry.py +++ b/demos/task_registry.py @@ -42,6 +42,8 @@ class TaskTypeDefinition: STAGE_HANDLER_REGISTRY: dict[str, StageHandlerDefinition] = {} TASK_REGISTRY: dict[str, TaskTypeDefinition] = {} LOADED_REGISTRY_PACKAGES: tuple[str, ...] = () +LOADED_PLUGIN_SEARCH_PATHS: tuple[Path, ...] = () +LOADED_PLUGIN_CONFIG_PATH: Path | None = None def register_persona(definition: PersonaDefinition) -> None: @@ -110,6 +112,14 @@ def loaded_registry_packages() -> tuple[str, ...]: return LOADED_REGISTRY_PACKAGES +def loaded_plugin_search_paths() -> tuple[Path, ...]: + return LOADED_PLUGIN_SEARCH_PATHS + + +def loaded_plugin_config_path() -> Path | None: + return LOADED_PLUGIN_CONFIG_PATH + + def register_task_type(definition: TaskTypeDefinition) -> None: TASK_REGISTRY[definition.task_type] = definition @@ -196,30 +206,32 @@ def build_deliverable(task_type: str, context: Any) -> dict[str, Any]: def load_registry( extra_packages: tuple[str, ...] = (), reset: bool = True, + config_file: str | Path | None = None, ) -> tuple[str, ...]: - from demos.discovery import collect_module_exports, configured_package_names + from demos.discovery import collect_module_exports, configured_plugin_sources - package_names = configured_package_names( + plugin_sources = configured_plugin_sources( default_packages=("demos",), extra_packages=extra_packages, + config_file=config_file, ) if reset: reset_registry() handlers = collect_module_exports( - package_names, + plugin_sources.package_names, "STAGE_HANDLER_DEFINITIONS", exact_names=("stage_handlers",), prefixes=("stage_handlers_",), ) personas = collect_module_exports( - package_names, + plugin_sources.package_names, "PERSONA_DEFINITIONS", exact_names=("persona_definitions",), prefixes=("persona_definitions_",), ) task_types = collect_module_exports( - package_names, + plugin_sources.package_names, "TASK_TYPE_DEFINITIONS", exact_names=("task_types",), prefixes=("task_types_",), @@ -232,8 +244,12 @@ def load_registry( for task_type in task_types: register_task_type(task_type) global LOADED_REGISTRY_PACKAGES - LOADED_REGISTRY_PACKAGES = package_names - return package_names + global LOADED_PLUGIN_SEARCH_PATHS + global LOADED_PLUGIN_CONFIG_PATH + LOADED_REGISTRY_PACKAGES = plugin_sources.package_names + LOADED_PLUGIN_SEARCH_PATHS = plugin_sources.package_paths + LOADED_PLUGIN_CONFIG_PATH = plugin_sources.config_path + return plugin_sources.package_names def load_builtin_registry() -> tuple[str, ...]: diff --git a/plugin_config.example.json b/plugin_config.example.json new file mode 100644 index 0000000..b851a76 --- /dev/null +++ b/plugin_config.example.json @@ -0,0 +1,10 @@ +{ + "plugin_paths": [ + { + "path": "/absolute/path/to/pop_plugin_package_root", + "package_name": "pop_demo_plugins" + } + ], + "plugin_packages": [], + "plugin_package_paths": [] +} From 715a9145a284029dd7e34c832670f067b68df9da Mon Sep 17 00:00:00 2001 From: Bin Zhang <138868899+joy7758@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:26:32 +0800 Subject: [PATCH 2/2] feat: validate plugin config schema (#22) * feat: validate plugin config schema * feat: validate plugin config in ci (#23) --- .github/workflows/validate.yml | 4 ++ README.md | 12 +++- demos/discovery.py | 28 ++++++++++ plugin_config.example.json | 1 + plugin_config.schema.json | 55 ++++++++++++++++++ validation/scripts/validate_plugin_config.py | 59 ++++++++++++++++++++ 6 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 plugin_config.schema.json create mode 100644 validation/scripts/validate_plugin_config.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 39022e8..7f03da7 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -23,6 +23,10 @@ jobs: python -m pip install --upgrade pip pip install -e ".[schema]" + - name: Validate plugin config example + run: | + python validation/scripts/validate_plugin_config.py + - name: Validate all example POP objects run: | for f in examples/*.json; do diff --git a/README.md b/README.md index 1a74335..8246efa 100644 --- a/README.md +++ b/README.md @@ -47,12 +47,22 @@ CrewAI-style runtime roles. The demo workflow can load registry extensions from builtin modules and external plugin packages. By default, the runtime looks for a local `plugin_config.json` file at the repository root if present. An example -template is available at [`plugin_config.example.json`](plugin_config.example.json). +template is available at [`plugin_config.example.json`](plugin_config.example.json), +and the config shape is defined by +[`plugin_config.schema.json`](plugin_config.schema.json). +The example config is also validated in CI so schema drift is caught during PR review. For temporary overrides, you can also point discovery at external plugin packages with `POP_PLUGIN_CONFIG_FILE`, `POP_PLUGIN_PACKAGES`, and `POP_PLUGIN_PACKAGE_PATHS`. +If you use file-based plugin config validation outside this repository, +install the optional schema dependency: + +```bash +pip install "pop-persona[schema]" +``` + ## Persona Layer Diagram ```mermaid diff --git a/demos/discovery.py b/demos/discovery.py index 6d95946..b1ab134 100644 --- a/demos/discovery.py +++ b/demos/discovery.py @@ -16,6 +16,7 @@ PLUGIN_PACKAGE_PATHS_ENV = "POP_PLUGIN_PACKAGE_PATHS" PLUGIN_CONFIG_FILE_ENV = "POP_PLUGIN_CONFIG_FILE" DEFAULT_PLUGIN_CONFIG_PATH = PROJECT_ROOT / "plugin_config.json" +PLUGIN_CONFIG_SCHEMA_PATH = PROJECT_ROOT / "plugin_config.schema.json" @dataclass(frozen=True) @@ -132,6 +133,7 @@ def load_plugin_config( payload = json.loads(config_path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError("Plugin config must be a JSON object.") + validate_plugin_config_payload(payload) base_dir = config_path.parent package_paths = normalize_plugin_paths(payload, base_dir) @@ -139,6 +141,32 @@ def load_plugin_config( return PluginDiscoveryConfig(config_path, package_names, package_paths) +def load_plugin_config_schema() -> dict[str, Any]: + return json.loads(PLUGIN_CONFIG_SCHEMA_PATH.read_text(encoding="utf-8")) + + +def validation_error_path(error: Any) -> str: + parts = [str(part) for part in getattr(error, "absolute_path", ())] + return ".".join(parts) if parts else "" + + +def validate_plugin_config_payload(payload: dict[str, Any]) -> None: + try: + import jsonschema + except ModuleNotFoundError as exc: + raise RuntimeError( + "Plugin config validation requires `jsonschema`. " + "Install `pop-persona[schema]` or add `jsonschema` to your environment." + ) from exc + + schema = load_plugin_config_schema() + try: + jsonschema.validate(instance=payload, schema=schema) + except jsonschema.ValidationError as exc: + path = validation_error_path(exc) + raise ValueError(f"Invalid plugin config at {path}: {exc.message}") from exc + + def configure_plugin_search_paths(paths: tuple[Path, ...]) -> tuple[Path, ...]: configured: list[Path] = [] for path in paths: diff --git a/plugin_config.example.json b/plugin_config.example.json index b851a76..3c3e3e5 100644 --- a/plugin_config.example.json +++ b/plugin_config.example.json @@ -1,4 +1,5 @@ { + "$schema": "./plugin_config.schema.json", "plugin_paths": [ { "path": "/absolute/path/to/pop_plugin_package_root", diff --git a/plugin_config.schema.json b/plugin_config.schema.json new file mode 100644 index 0000000..049573f --- /dev/null +++ b/plugin_config.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/joy7758/persona-object-protocol/plugin_config.schema.json", + "title": "POP Plugin Discovery Config", + "type": "object", + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "plugin_paths": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_name": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "path" + ] + } + ] + } + }, + "plugin_packages": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "plugin_package_paths": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } +} diff --git a/validation/scripts/validate_plugin_config.py b/validation/scripts/validate_plugin_config.py new file mode 100644 index 0000000..a7677ef --- /dev/null +++ b/validation/scripts/validate_plugin_config.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator + + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG_PATH = ROOT / "plugin_config.example.json" +DEFAULT_SCHEMA_PATH = ROOT / "plugin_config.schema.json" + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Validate a plugin config file against the repository schema." + ) + parser.add_argument( + "config_path", + nargs="?", + type=Path, + default=DEFAULT_CONFIG_PATH, + help="Path to the plugin config file to validate.", + ) + parser.add_argument( + "--schema", + type=Path, + default=DEFAULT_SCHEMA_PATH, + help="Path to the plugin config schema file.", + ) + args = parser.parse_args() + + config_path = args.config_path.resolve() + schema_path = args.schema.resolve() + schema = load_json(schema_path) + payload = load_json(config_path) + + validator = Draft202012Validator(schema) + errors = sorted( + validator.iter_errors(payload), + key=lambda error: [str(part) for part in error.absolute_path], + ) + if errors: + first_error = errors[0] + path = ".".join(str(part) for part in first_error.absolute_path) or "" + raise ValueError(f"{config_path}: invalid at {path}: {first_error.message}") + + print(f"Validated {config_path.relative_to(ROOT)} against {schema_path.relative_to(ROOT)}") + + +if __name__ == "__main__": + main()