From c8fa46893f53699e8c33f48d5885f98bc92aae57 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Thu, 11 Dec 2025 11:09:57 +0100 Subject: [PATCH 1/8] Add migration to backfill `is_task=true` on core tasks --- ...b9185b334_add_generic_workflows_to_core.py | 1 + ...3c8b9185c221_add_validate_products_task.py | 1 + ...reate_linker_table_workflow_apscheduler.py | 2 +- ...e3eba_set_is_task_true_on_certain_tasks.py | 40 +++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 orchestrator/migrations/versions/schema/2025-12-10_9736496e3eba_set_is_task_true_on_certain_tasks.py diff --git a/orchestrator/migrations/versions/schema/2020-10-19_a76b9185b334_add_generic_workflows_to_core.py b/orchestrator/migrations/versions/schema/2020-10-19_a76b9185b334_add_generic_workflows_to_core.py index 6a2db72d2..7d4c4a964 100644 --- a/orchestrator/migrations/versions/schema/2020-10-19_a76b9185b334_add_generic_workflows_to_core.py +++ b/orchestrator/migrations/versions/schema/2020-10-19_a76b9185b334_add_generic_workflows_to_core.py @@ -17,6 +17,7 @@ branch_labels = None depends_on = None +# NOTE: this migration forgot to insert these workflows with is_task=true. Make sure to correct that if you copy this. workflows = [ {"name": "modify_note", "description": "Modify Note", "workflow_id": uuid4(), "target": "MODIFY"}, {"name": "task_clean_up_tasks", "description": "Clean up old tasks", "workflow_id": uuid4(), "target": "SYSTEM"}, diff --git a/orchestrator/migrations/versions/schema/2021-04-06_3c8b9185c221_add_validate_products_task.py b/orchestrator/migrations/versions/schema/2021-04-06_3c8b9185c221_add_validate_products_task.py index 8630af6a1..f8f7d0678 100644 --- a/orchestrator/migrations/versions/schema/2021-04-06_3c8b9185c221_add_validate_products_task.py +++ b/orchestrator/migrations/versions/schema/2021-04-06_3c8b9185c221_add_validate_products_task.py @@ -17,6 +17,7 @@ branch_labels = None depends_on = None +# NOTE: this migration forgot to insert these workflows with is_task=true. Make sure to correct that if you copy this. workflows = [ {"name": "task_validate_products", "description": "Validate products", "workflow_id": uuid4(), "target": "SYSTEM"}, ] diff --git a/orchestrator/migrations/versions/schema/2025-11-18_961eddbd4c13_create_linker_table_workflow_apscheduler.py b/orchestrator/migrations/versions/schema/2025-11-18_961eddbd4c13_create_linker_table_workflow_apscheduler.py index a1de23146..131610373 100644 --- a/orchestrator/migrations/versions/schema/2025-11-18_961eddbd4c13_create_linker_table_workflow_apscheduler.py +++ b/orchestrator/migrations/versions/schema/2025-11-18_961eddbd4c13_create_linker_table_workflow_apscheduler.py @@ -17,7 +17,7 @@ branch_labels = None depends_on = None - +# NOTE: this migration forgot to insert these workflows with is_task=true. Make sure to correct that if you copy this. workflows = [ { "name": "task_validate_subscriptions", diff --git a/orchestrator/migrations/versions/schema/2025-12-10_9736496e3eba_set_is_task_true_on_certain_tasks.py b/orchestrator/migrations/versions/schema/2025-12-10_9736496e3eba_set_is_task_true_on_certain_tasks.py new file mode 100644 index 000000000..cabb2b9b4 --- /dev/null +++ b/orchestrator/migrations/versions/schema/2025-12-10_9736496e3eba_set_is_task_true_on_certain_tasks.py @@ -0,0 +1,40 @@ +"""Set is_task=true on certain tasks. + +This is required to make them appear in the completed tasks in the UI, and for the cleanup task to be able to +remove them. + +Revision ID: 9736496e3eba +Revises: 961eddbd4c13 +Create Date: 2025-12-10 16:42:29.060382 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "9736496e3eba" +down_revision = "961eddbd4c13" +branch_labels = None +depends_on = None + +task_names = [ + # Added in a76b9185b334 + "task_clean_up_tasks", + "task_resume_workflows", + # Added in 3c8b9185c221 + "task_validate_products", + # Added in 961eddbd4c13 + "task_validate_subscriptions", +] + + +def upgrade() -> None: + conn = op.get_bind() + query = sa.text("UPDATE workflows SET is_task=true WHERE name = :task_name and is_task=false") + for task_name in task_names: + conn.execute(query, parameters={"task_name": task_name}) + + +def downgrade() -> None: + pass # Does not make sense to downgrade back to a 'bad' state. From 459ce84fe340d67b45ebf749f10f9f298420ff06 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Thu, 11 Dec 2025 12:00:00 +0100 Subject: [PATCH 2/8] Change show-schedule command to table, add schedule name and derive source --- orchestrator/cli/scheduler.py | 26 +++++++++- test/unit_tests/schedules/test_scheduling.py | 53 ++++++++++++++++---- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/orchestrator/cli/scheduler.py b/orchestrator/cli/scheduler.py index 4eb8c0af0..4f1d6d873 100644 --- a/orchestrator/cli/scheduler.py +++ b/orchestrator/cli/scheduler.py @@ -67,8 +67,30 @@ def show_schedule() -> None: in cli underscore is replaced by a dash `show-schedule` """ - for task in get_all_scheduler_tasks(): - typer.echo(f"[{task.id}] Next run: {task.next_run_time} | Trigger: {task.trigger}") + from rich.console import Console + from rich.table import Table + + from orchestrator.schedules.service import get_linker_entries_by_schedule_ids + + console = Console() + + table = Table(title="Scheduled Tasks") + table.add_column("id", no_wrap=True) + table.add_column("name") + table.add_column("source") + table.add_column("next run time") + table.add_column("trigger") + + scheduled_tasks = get_all_scheduler_tasks() + _schedule_ids = [task.id for task in scheduled_tasks] + api_managed = {str(i.schedule_id) for i in get_linker_entries_by_schedule_ids(_schedule_ids)} + + for task in scheduled_tasks: + source = "API" if task.id in api_managed else "decorator" + run_time = str(task.next_run_time.replace(microsecond=0)) + table.add_row(task.id, task.name, source, str(run_time), str(task.trigger)) + + console.print(table) @app.command() diff --git a/test/unit_tests/schedules/test_scheduling.py b/test/unit_tests/schedules/test_scheduling.py index 0d002fc03..0082e60cf 100644 --- a/test/unit_tests/schedules/test_scheduling.py +++ b/test/unit_tests/schedules/test_scheduling.py @@ -1,8 +1,11 @@ +import re +from datetime import datetime from unittest import mock from typer.testing import CliRunner from orchestrator.cli.scheduler import app +from orchestrator.db.models import WorkflowApschedulerJob runner = CliRunner() @@ -15,22 +18,50 @@ def test_run_scheduler(mock_scheduler): assert result.exit_code == 130 -@mock.patch("orchestrator.schedules.scheduler.get_scheduler_store") -def test_show_schedule_command(mock_get_scheduler_store): +def make_mock_job(id_, name, next_run_time_str, trigger): mock_job = mock.MagicMock() - mock_job.id = "job1" - mock_job.next_run_time = "2025-08-05 12:00:00" - mock_job.trigger = "trigger_info" + mock_job.id = id_ + mock_job.name = name + mock_job.next_run_time = datetime.fromisoformat(next_run_time_str) + mock_job.trigger = trigger + return mock_job + + +def to_ascii_line(line: str): + # Remove unicode from a rich.table outputted line + return line.encode("ascii", "ignore").decode("ascii").strip() + + +def to_regex(mock_job, *, source): + # Create regex to match mock job in show-schedule output + return re.compile(rf"{mock_job.id}\s+{mock_job.name}\s+{source}\s+.*", flags=re.MULTILINE) + + +@mock.patch("orchestrator.schedules.service.get_linker_entries_by_schedule_ids") +@mock.patch("orchestrator.schedules.scheduler.get_scheduler_store") +def test_show_schedule_command(mock_get_scheduler_store, mock_get_linker_entries): + # given + mock_job1 = make_mock_job("job1", "My Job 1", "2025-08-05T12:00:00", "trigger_info") + mock_job2 = make_mock_job("6faf2c63-44de-48bc-853d-bb3f57225055", "My Job 2", "2025-08-05T14:00:00", "trigger_info") mock_scheduler = mock.MagicMock() - mock_scheduler.get_all_jobs.return_value = [mock_job] + mock_scheduler.get_all_jobs.return_value = [mock_job1, mock_job2] mock_get_scheduler_store.return_value.__enter__.return_value = mock_scheduler - result = runner.invoke(app, ["show-schedule"]) - assert result.exit_code == 0 - assert "[job1]" in result.output - assert "Next run: 2025-08-05 12:00:00" in result.output - assert "trigger_info" in result.output + mock_linker_entry_job2 = mock.MagicMock(spec=WorkflowApschedulerJob) + mock_linker_entry_job2.schedule_id = mock_job2.id + mock_get_linker_entries.return_value = [mock_linker_entry_job2] # only job 2 is defined in API + + regex1 = to_regex(mock_job1, source="decorator") + regex2 = to_regex(mock_job2, source="API") + + # when + result = runner.invoke(app, ["show-schedule"], env={"COLUMNS": "300", "LINES": "200"}) + output_stripped = "\n".join([to_ascii_line(line) for line in result.output.splitlines()]) + + # then + assert regex1.findall(output_stripped) != [], f"Regex {regex1} did not match output {output_stripped}" + assert regex2.findall(output_stripped) != [], f"Regex {regex2} did not match output {output_stripped}" @mock.patch("orchestrator.schedules.scheduler.get_scheduler_store") From f4a194d4f2e56a054fd6ef26e4644236f366e130 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Thu, 11 Dec 2025 13:49:52 +0100 Subject: [PATCH 3/8] Pin click<=8.2.1 for docs dependencies, to make mkdocs serve reload on changes --- pyproject.toml | 1 + uv.lock | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d6907f3e8..cc9b3137f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ agent = [ # Local dependencies for development [dependency-groups] docs = [ + "click<=8.2.1", # hotreloading is broken https://github.com/squidfunk/mkdocs-material/issues/8478 "mkdocs>=1.6.1", "mkdocs-embed-external-markdown>=3.0.2", "mkdocs-include-markdown-plugin>=7.1.6", diff --git a/uv.lock b/uv.lock index f797511d9..6a4ba1cac 100644 --- a/uv.lock +++ b/uv.lock @@ -565,14 +565,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, ] [[package]] @@ -2438,6 +2438,7 @@ dev = [ { name = "watchdog" }, ] docs = [ + { name = "click" }, { name = "mkdocs" }, { name = "mkdocs-embed-external-markdown" }, { name = "mkdocs-include-markdown-plugin" }, @@ -2537,6 +2538,7 @@ dev = [ { name = "watchdog", specifier = ">=6.0.0" }, ] docs = [ + { name = "click", specifier = "<=8.2.1" }, { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-embed-external-markdown", specifier = ">=3.0.2" }, { name = "mkdocs-include-markdown-plugin", specifier = ">=7.1.6" }, From 4f1ec46e7862a47b733be17f52e3f6e736de9362 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Fri, 12 Dec 2025 12:46:50 +0100 Subject: [PATCH 4/8] Add upgrade guide for 4.7, update docs considering 5.0, link to development tickets --- docs/guides/tasks.md | 34 ++++-- docs/guides/upgrading/4.0.md | 23 ---- docs/guides/upgrading/4.7.md | 163 +++++++++++++++++++++++++++ mkdocs.yml | 7 +- orchestrator/schedules/__init__.py | 4 +- orchestrator/schedules/scheduling.py | 6 +- 6 files changed, 198 insertions(+), 39 deletions(-) create mode 100644 docs/guides/upgrading/4.7.md diff --git a/docs/guides/tasks.md b/docs/guides/tasks.md index 10dec5879..530f7c302 100644 --- a/docs/guides/tasks.md +++ b/docs/guides/tasks.md @@ -112,11 +112,11 @@ Even if the task does not have any form input, an entry will still need to be ma } ``` -## The schedule file (DEPRECATED) -> This section is deprecated and will be removed in version 5.0.0, please refer to the [new scheduling system](#the-schedule-api) -> below. +## The schedule file {: .deprecated } -> from `4.3.0` we switched from [schedule] package to [apscheduler] to allow schedules to be stored in the DB and schedule tasks from the API. +!!! Warning + As of [v4.7.0] this is deprecated, and it will be removed in v5.0.0. + Please use the [new scheduling system](#the-schedule-api) below. The schedule file is essentially the crontab associated with the task. Continuing with our previous example: @@ -157,11 +157,17 @@ To keep things organized and consistent (similar to how workflows are handled), ## The schedule API -> from `4.3.0` we switched from [schedule] package to [apscheduler] to allow schedules to be stored in the DB and schedule tasks from the API. +!!! Info + In [v4.4.0] we switched from [schedule] package to [apscheduler] to allow schedules to be stored in the DB and retrieve schedule tasks from the API. + The apscheduler library has its own decorator to schedule tasks: `@scheduler.scheduled_job()` (from `orchestrator.schedules.scheduler`). + We therefore deprecated the old `@schedule` decorator (from `orchestrator.schedules.scheduling`) and made it forwards compatible. + + In [v4.7.0] we deprecated `@scheduler.scheduled_job()` provided by [apscheduler] in favor of a more dynamic API based system described below. + Although we no longer support the `@scheduler.scheduled_job()` decorator, it is still available because it is part of [apscheduler]. + Therefore, we do NOT recommend using it for new schedules. Because you will miss a Linker Table join between schedules and workflows/tasks. + + Consult the [v4.7 upgrade guide] for more details. -> from `4.7.0` we deprecated `@scheduler.scheduled_job()` provided by [apscheduler] in favor of a more dynamic API based system. -> Although we do no longer support the `@scheduler.scheduled_job()` decorator, it is still available because it is part of [apscheduler]. -> Therefore, we do NOT recommend using it for new schedules. Because you will miss a Linker Table join between schedules and workflows/tasks. Schedules can be created, updated, and deleted via the REST API, and retrieved via the already existing GraphQL API. It @@ -226,9 +232,10 @@ For detailed configuration options, see the [APScheduler scheduling docs]. The scheduler automatically loads any schedules that are imported before the scheduler starts. -> In previous versions, schedules needed to be explicitly listed in an ALL_SCHEDULERS variable. -> This is no longer required; ALL_SCHEDULERS is deprecated as of orchestrator-core 4.7.0 and will be removed in 5.0.0. -> Follow-up ticket to remove deprecated code: [#1276](https://github.com/workfloworchestrator/orchestrator-core/issues/1276) +!!! Info + In previous versions, schedules needed to be explicitly added in the `ALL_SCHEDULERS` variable. + This is no longer required; `ALL_SCHEDULERS` is deprecated as of orchestrator-core v4.7.0 and will be removed in v5.0.0. + Follow-up ticket to remove deprecated code: [#1276](https://github.com/workfloworchestrator/orchestrator-core/issues/1276) ## The scheduler @@ -236,6 +243,7 @@ The scheduler is invoked via `python main.py scheduler`. Try `--help` or review the [CLI docs][cli-docs] to learn more. ### Initial schedules + From version orchestrator-core >= `4.7.0`, the scheduler uses the database to store schedules instead of hard-coded schedule files. Previous versions (orchestrator-core < `4.7.0` had hard-coded schedules. These can be ported to the new system by creating them via the API or CLI. Run the following CLI command to import previously existing orchestrator-core schedules and change them if needed via the API. @@ -300,6 +308,7 @@ if [ $status -ne 0 ]; then fi ``` + [schedule]: https://pypi.org/project/schedule/ [apscheduler]: https://pypi.org/project/APScheduler/ @@ -313,3 +322,6 @@ fi [trigger docs]: https://apscheduler.readthedocs.io/en/master/api.html#triggers [registering-workflows]: ../../../getting-started/workflows#register-workflows [cli-docs]: ../../../reference-docs/cli/#orchestrator.cli.scheduler.show_schedule +[v4.4.0]: https://github.com/workfloworchestrator/orchestrator-core/releases/tag/4.4.0 +[v4.7.0]: https://github.com/workfloworchestrator/orchestrator-core/releases/tag/4.7.0 +[v4.7 upgrade guide]: ../guides/upgrading/4.7.md diff --git a/docs/guides/upgrading/4.0.md b/docs/guides/upgrading/4.0.md index 02ad7c06e..9889ee463 100644 --- a/docs/guides/upgrading/4.0.md +++ b/docs/guides/upgrading/4.0.md @@ -71,26 +71,3 @@ This will update the `target` for all workflows that are `SYSTEM` or `VALIDATE` This is a breaking change, so you will need to test your workflows after making this change to ensure that they are working as expected. - -## Scheduling flows -In 4.4.0 we have introduced a new way to schedule workflows via the decorator [@scheduler.scheduled_job(...)](../tasks.md#the-schedule-file-deprecated). -In 4.7.0 we introduce a new way to schedule workflows via the [Scheduler API](../tasks.md#the-schedule-api). - -The 4.7.0 migration will create the `apscheduler_jobs` table if missing, so upgrades from older versions still succeed, but the table will initially be empty. - -### Who needs to take action? - -**Users upgrading from 4.4–4.6:** - - Nothing special required; your scheduler data already exists. - -**Users upgrading from <4.4 who used the old decorator:** - - Your schedules will not automatically reappear. Restore them by running: - - - ``` - python main.py scheduler load-initial-schedule - ``` - (More information: [Scheduler API](../tasks.md#the-schedule-api).) - Make sure your schedule definitions are imported so the scheduler can discover them. - -**Users who never used the decorator:** -- No action required; the migration creates the table and you can add schedules normally. diff --git a/docs/guides/upgrading/4.7.md b/docs/guides/upgrading/4.7.md new file mode 100644 index 000000000..8928c1259 --- /dev/null +++ b/docs/guides/upgrading/4.7.md @@ -0,0 +1,163 @@ +from orchestrator import begin + +# 4.7 Upgrade Guide + +In this document we describe important changes to be aware of when upgrading from `orchestrator-core` v4.x to v4.7. + +If you are not yet using v4.x, please consult the 4.0 upgrade guide. + +## About 4.7 + +This release introduces a deprecation and a breaking change to a beta feature: + +1. Scheduling tasks via decorator is **deprecated** in favor of using the API +2. [Workflow authorization callbacks] (Beta) **change** from `def` to `async def` + +### Scheduling tasks via decorator is deprecated in favor of using the API + +In [v4.4.0] we introduced the [apscheduler] library with a persistent jobstore. +The decorator [@scheduler.scheduled_job(...)](../tasks.md#the-schedule-file) allowed to schedule tasks from code. + +In [v4.7.0] this has evolved to scheduling tasks via the [Scheduler API]. +Since tasks scheduled from code cannot be related to their task definition in the DB, they cannot be managed through the API. +Therefore, using the decorator to schedule tasks is deprecated. +This release contains a DB migration which ensures you can update from any v4.x version to v4.7.0. + +**Please check which of the 3 scenarios applies for you and which action is required.** + +#### 1. I use the scheduler with only the default schedules + +**Short answer** + +If you want to keep using the default schedules, run `python main.py scheduler load-initial-schedule` from the CLI where you normally run your scheduler. + +Otherwise, no action is required. + +**Long answer** + +As of v4.6.5, the default core schedules that are scheduled with the decorator were: + +- [clean-tasks]: Runs the task `task_clean_up_tasks` every 6 hours +- [resume-workflows]: Runs the task `task_resume_workflows` task every hour +- [validate-products]: Runs the task `task_validate_products` every night (only if no previous run is incomplete) +- [subscriptions-validator]: Runs a validate workflow for each active subscription + +In v4.7.0 the only remaining schedule is `validate-products`. (this will be removed in [#1250]) + +The other 3 schedules have been removed. +When you upgrade from anywhere between v4.4.0 - v4.6.x then the DB will still contain the scheduled jobs. +You can remove them from the `apscheduler_jobs` table manually, or simply wait for the scheduler to clean them up, which will produce these 3 messages: + +``` +[error ] Unable to restore job "resume-workflows" -- removing it [apscheduler.jobstores.default] +[error ] Unable to restore job "clean-tasks" -- removing it [apscheduler.jobstores.default] +[error ] Unable to restore job "subscriptions-validator" -- removing it [apscheduler.jobstores.default] +``` + +If you upgrade from an older v4.x version, this does not apply. + +If you want to recreate these 3 schedules by using the API, we have provided a command to do so: + +``` +python main.py scheduler load-initial-schedule +``` + +You can also choose to only schedule a specific task, or change when they run. +For now this can only be done through the API; in the future this will be possible from the UI. ([orchestrator-ui-library#2215]) + +For more information: [Scheduler API]. + + +#### 2. I use the scheduler with both the default schedules AND my own schedules + +**Short answer** + +Check the actions for scenario 1. + +No further action is required as your own schedules will continue to work. + +**Long answer** + +Please check scenario 1 regarding the default schedules. + +For your own schedules, you may remove their definitions in code and schedule them using the [Schedule API]. +However, this is not yet a requirement as the decorator-scheduled jobs can be used alongside API-scheduled jobs. +This will only be a requirement for orchestrator-core v5.0.0. + +You can determine how jobs have been scheduled by running the [show-schedule] CLI command and inspecting the `source` column. +For example, an excerpt of the SURF schedule: + +``` +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓ +┃ id ┃ name ┃ source ┃ next run time ┃ trigger ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩ +│ 6db5c270-8239-455c-97e4-14a6a865c68d │ Task Resume Workflows │ API │ 2025-12-11 11:07:42+00:00 │ interval[1:00:00] │ +│ import-crm-to-ims │ Import CRM data to IMS │ decorator │ 2025-12-11 12:00:00+00:00 │ cron │ +│ 70c0b5ce-88f5-48fb-ba5f-d19422628bef │ Task Clean Up Tasks │ API │ 2025-12-11 16:07:42+00:00 │ interval[6:00:00] │ +│ import-customers-from-crm │ Import customers from CRM │ decorator │ 2025-12-11 19:00:00+00:00 │ cron │ +``` + +#### 3. I do not use the scheduler + +No action required! + +### Workflow authorization callbacks change from `def` to `async def` + +In [v4.7.0] the workflow authorization callbacks change from synchronous to asynchronous. +This was done for consistency with API authorization callback functions which are also async. +Additionally, it is always possible to run a sync function in an async context, but the other way around is much harder. + +While this is a breaking change, we decided to add it in this minor release since workflow authorization is currently still a beta feature. + +#### Example + +Given a synchronous callback, like this: + +```python +from oauth2_lib.fastapi import OIDCUserModel +from orchestrator.workflow import workflow, StepList, begin + +def authorize_workflow_access(user: OIDCUserModel) -> bool: + ... + +@workflow("My workflow", authorize_callback=authorize_workflow_access) +def my_workflow() -> StepList: + return begin >> ... +``` + +The required change is to make the callback look like this: + +```python +async def authorize_workflow_access(user: OIDCUserModel) -> bool: + ... +``` + +!!! Info + **Important**: check your callback for synchronous I/O calls. + These block the eventloop, which has a negative impact on performance of the API. + + For example, if your callback performs a `requests.get()` call or a sqlalchemy query, this will block the eventloop. + You can resolve this either using an asynchronous alternative that can be `await`ed, or by using [to_thread] to run the synchronous call in a thread. + Running in a thread is not as optimal as truly async code, but it's still a lot better than blocking the loop. + +For more information see [Workflow authorization callbacks]. + + + +[v4.4.0]: https://github.com/workfloworchestrator/orchestrator-core/releases/tag/4.4.0 +[v4.7.0]: https://github.com/workfloworchestrator/orchestrator-core/releases/tag/4.7.0 +[Workflow authorization callbacks]: ../../reference-docs/auth-backend-and-frontend.md#authorization-and-workflows +[Scheduler API]: ../tasks.md#the-schedule-api +[apscheduler]: https://pypi.org/project/APScheduler/ +[show-schedule]: ../../reference-docs/cli.md#scheduler +[load-initial-schedule]: ../../reference-docs/cli.md#scheduler + +[clean-tasks]: https://github.com/workfloworchestrator/orchestrator-core/blob/4.6.5/orchestrator/schedules/task_vacuum.py +[resume-workflows]: https://github.com/workfloworchestrator/rchestrator-core/blob/4.6.5/orchestrator/schedules/resume_workflows.py +[validate-products]: https://github.com/workfloworchestrator/orchestrator-core/blob/4.6.5/orchestrator/schedules/validate_products.py +[subscriptions-validator]: https://github.com/workfloworchestrator/orchestrator-core/blob/4.6.5/orchestrator/schedules/validate_subscriptions.py + +[#1250]: https://github.com/workfloworchestrator/orchestrator-core/issues/1250 +[orchestrator-ui-library#2215]: https://github.com/workfloworchestrator/orchestrator-ui-library/issues/2215 + +[to_thread]: https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread diff --git a/mkdocs.yml b/mkdocs.yml index 90ad810ae..ba84c965c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -184,9 +184,10 @@ nav: # - Workflows # - Tasks - Upgrade Guides: - - 2.x: guides/upgrading/2.0.md - - 3.x: guides/upgrading/3.0.md - - 4.x: guides/upgrading/4.0.md + - v2.x: guides/upgrading/2.0.md + - v3.x: guides/upgrading/3.0.md + - v4.x: guides/upgrading/4.0.md + - v4.7: guides/upgrading/4.7.md - Reference Documentation: - TL;DR: reference-docs/tldr.md - API docs: diff --git a/orchestrator/schedules/__init__.py b/orchestrator/schedules/__init__.py index 5a38248f9..4ad83da46 100644 --- a/orchestrator/schedules/__init__.py +++ b/orchestrator/schedules/__init__.py @@ -15,7 +15,9 @@ from orchestrator.schedules.validate_products import validate_products warnings.warn( - "ALL_SCHEDULERS is deprecated; scheduling is now handled entirely through the scheduler API.", + "ALL_SCHEDULERS is deprecated and will be removed in 5.0.0. " + "Scheduling tasks can now be handled entirely through the API. " + "For more details, please consult https://workfloworchestrator.org/orchestrator-core/guides/upgrading/4.7/", DeprecationWarning, stacklevel=2, ) diff --git a/orchestrator/schedules/scheduling.py b/orchestrator/schedules/scheduling.py index 4be41a4d0..011c03580 100644 --- a/orchestrator/schedules/scheduling.py +++ b/orchestrator/schedules/scheduling.py @@ -23,7 +23,11 @@ @deprecated( - reason="We changed from scheduler to apscheduler which has its own decoractor, use `@scheduler.scheduled_job()` from `from orchestrator.scheduling.scheduler import scheduler`" + reason=( + "Scheduling tasks with a decorator is deprecated in favor of using the API. " + "This decorator will be removed in 5.0.0. " + "For more details, please consult https://workfloworchestrator.org/orchestrator-core/guides/upgrading/4.7/" + ) ) def scheduler( name: str, From 3471ec6a0f7be9ba09e3d80576668b4c3a6968b9 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Fri, 12 Dec 2025 12:49:02 +0100 Subject: [PATCH 5/8] Add beta marker to Authorization and Workflows section and include markers.css in mkdocs.yml --- docs/css/markers.css | 54 +++++++++++++++++++ .../auth-backend-and-frontend.md | 2 +- mkdocs.yml | 1 + 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 docs/css/markers.css diff --git a/docs/css/markers.css b/docs/css/markers.css new file mode 100644 index 000000000..af8651977 --- /dev/null +++ b/docs/css/markers.css @@ -0,0 +1,54 @@ +/** + * Styles for adding markers to headings that do not change the URL anchor. + * This works thanks to the mkdocs-material plugin. + * + * Example usage: + * + * ## Feature X {: .beta } + * + * ## Feature Y {: .deprecated } + * +*/ + +.experimental::after { + content: "EXPERIMENTAL"; + background-color: #e91e63; + color: white; + padding: 2px 6px; + border-radius: 4px; + font-size: 0.7em; + margin-left: 10px; + font-weight: bold; +} + +.beta::after { + content: "BETA"; + background-color: #ffc107; + color: black; + padding: 2px 6px; + border-radius: 4px; + font-size: 0.8em; + margin-left: 10px; +} + +.new::after { + content: "NEW"; + background-color: #2196f3; + color: white; + padding: 2px 6px; + border-radius: 4px; + font-size: 0.7em; + margin-left: 10px; + font-weight: bold; +} + +.deprecated::after { + content: "DEPRECATED"; + background-color: #757575; + color: white; + padding: 2px 6px; + border-radius: 4px; + font-size: 0.7em; + margin-left: 10px; + font-weight: bold; +} diff --git a/docs/reference-docs/auth-backend-and-frontend.md b/docs/reference-docs/auth-backend-and-frontend.md index 575a1f18b..4d791bf84 100644 --- a/docs/reference-docs/auth-backend-and-frontend.md +++ b/docs/reference-docs/auth-backend-and-frontend.md @@ -269,7 +269,7 @@ app.register_authorization(authorization_instance) app.register_graphql_authorization(graphql_authorization_instance) ``` -## Authorization and Workflows +## Authorization and Workflows {: .beta } !!! Warning Role-based access control for workflows is currently in beta. diff --git a/mkdocs.yml b/mkdocs.yml index ba84c965c..b1a88cf77 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -118,6 +118,7 @@ extra_css: - "css/termynal.css" - "css/custom.css" - "css/style.css" + - "css/markers.css" extra_javascript: - "js/termynal.js" - "js/custom.js" From 8a4a63b1ef71ac636bb8213e1f045a5837596444 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Fri, 12 Dec 2025 12:50:00 +0100 Subject: [PATCH 6/8] Update documentation for CLI commands and scheduler subcommands --- docs/reference-docs/cli.md | 61 +++++++++++++++++++++++++++-------- orchestrator/cli/scheduler.py | 37 +++++++++++++++++---- 2 files changed, 77 insertions(+), 21 deletions(-) diff --git a/docs/reference-docs/cli.md b/docs/reference-docs/cli.md index 63bd8ae30..d69341415 100644 --- a/docs/reference-docs/cli.md +++ b/docs/reference-docs/cli.md @@ -1,5 +1,30 @@ # Command Line Interface Commands +This page documents CLI commands available in orchestrator-core. + +The syntax of a CLI command is: + +``` +python main.py +``` + +Where: + +- `` is one of the top-level headings in this page +- `` is one of the secondary headings in this page + +Some examples: + +``` +python main.py db migrate_tasks + +python main.py generate workflows + +python main.py scheduler show-schedule +``` + +Each command can also be run with `--help` to get information directly in the CLI. + Top level options: `--install-completion [bash|zsh|fish|powershell|pwsh]` @@ -17,14 +42,15 @@ Interact with the application database. By default, does nothing, specify `main. ::: orchestrator.cli.database options: - show_signature: false - show_root_heading: false docstring_style: google + separate_signature: false show_docstring_parameters: false show_docstring_returns: false + show_root_heading: false show_root_toc_entry: false - show_symbol_type_toc: false + show_signature: false show_symbol_type_heading: false + show_symbol_type_toc: false members: - downgrade - heads @@ -432,6 +458,8 @@ the code for the blocks used in other blocks should be generated first. ### config file + + An example of a simple product configuration: ```yaml @@ -742,17 +770,22 @@ None] ## scheduler -Access all the scheduler functions. +Commands to interact with the scheduler and scheduled jobs. ::: orchestrator.cli.scheduler options: - heading_level: 3 - members: - - run - - force - -### show-schedule - -::: orchestrator.cli.scheduler.show_schedule - options: - heading_level: 4 + docstring_style: google + separate_signature: false + show_docstring_parameters: false + show_docstring_returns: false + show_root_heading: false + show_root_toc_entry: false + show_signature: false + show_symbol_type_heading: false + show_symbol_type_toc: false + heading_level: 3 + members: + - "run" + - "force" + - "show_schedule" + - "load_initial_schedule" diff --git a/orchestrator/cli/scheduler.py b/orchestrator/cli/scheduler.py index 4f1d6d873..210d2f6ac 100644 --- a/orchestrator/cli/scheduler.py +++ b/orchestrator/cli/scheduler.py @@ -36,7 +36,13 @@ @app.command() def run() -> None: - """Start scheduler and loop eternally to keep thread alive.""" + """Starts the scheduler in the foreground. + + While running, this process will: + + * Periodically wake up when the next schedule is due for execution, and run it + * Process schedule changes made through the schedule API + """ def _get_scheduled_task_item_from_queue(redis_conn: Redis) -> tuple[str, bytes] | None: """Get an item from the Redis Queue for scheduler tasks.""" @@ -63,10 +69,7 @@ def _get_scheduled_task_item_from_queue(redis_conn: Redis) -> tuple[str, bytes] @app.command() def show_schedule() -> None: - """Show the currently configured schedule. - - in cli underscore is replaced by a dash `show-schedule` - """ + """The `show-schedule` command shows an overview of the scheduled jobs.""" from rich.console import Console from rich.table import Table @@ -95,7 +98,16 @@ def show_schedule() -> None: @app.command() def force(task_id: str) -> None: - """Force the execution of (a) scheduler(s) based on a task_id.""" + """Force the execution of (a) scheduler(s) based on a schedule ID. + + Use the `show-schedule` command to determine the ID of the schedule to execute. + + CLI Arguments: + ```sh + Arguments: + SCHEDULE_ID ID of the schedule to execute + ``` + """ task = get_scheduler_task(task_id) if not task: @@ -113,7 +125,18 @@ def force(task_id: str) -> None: @app.command() def load_initial_schedule() -> None: - """Load the initial schedule into the scheduler.""" + """The `load-initial-schedule` command loads the initial schedule using the scheduler API. + + The initial schedules are: + - Task Resume Workflows + - Task Clean Up Tasks + - Task Validate Subscriptions + + !!! Warning + This command is not idempotent. + + Please run `show-schedule` first to determine if the schedules already exist. + """ initial_schedules = [ { "name": "Task Resume Workflows", From 4b63349e91e6e7268c1a8247d2fb26cd4d1d7643 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Fri, 12 Dec 2025 12:50:30 +0100 Subject: [PATCH 7/8] Fix workflow auth callback names for `@workflow` and `@inputstep` decorators --- .../auth-backend-and-frontend.md | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/docs/reference-docs/auth-backend-and-frontend.md b/docs/reference-docs/auth-backend-and-frontend.md index 4d791bf84..af46d5c48 100644 --- a/docs/reference-docs/auth-backend-and-frontend.md +++ b/docs/reference-docs/auth-backend-and-frontend.md @@ -281,43 +281,45 @@ In other words, authorization callbacks are async, take a nullable OIDCUserModel A table (below) is available for comparing possible configuration states with the policy that will be enforced. ### `@workflow` -The `@workflow` decorator accepts the optional parameters `auth: Authorizer` and `retry_auth: Authorizer`. -`auth` will be used to determine the authorization of a user to start the workflow. -If `auth` is omitted, the workflow is authorized for any logged in user. +The `@workflow` decorator accepts the optional parameters `authorize_callback: Authorizer` and `retry_auth_callback: Authorizer`. -`retry_auth` will be used to determine the authorization of a user to start, resume, or retry the workflow from a failed step. -If `retry_auth` is omitted, then `auth` is used to authorize. +`authorize_callback` will be used to determine the authorization of a user to start the workflow. +If `authorize_callback` is omitted, the workflow is authorized for any logged in user. -(This does not percolate past an `@inputstep` that specifies `resume_auth` or `retry_auth`.) +`retry_auth_callback` will be used to determine the authorization of a user to start, resume, or retry the workflow from a failed step. +If `retry_auth_callback` is omitted, then `authorize_callback` is used to authorize. + +(This does not percolate past an `@inputstep` that specifies `resume_auth_callback` or `retry_auth_callback`.) Examples: -* `auth=None, retry_auth=None`: any user may run the workflow. -* `auth=A, retry_auth=B`: users authorized by A may start the workflow. Users authorized by B may retry on failure. +* `authorize_callback=None, retry_auth_callback=None`: any user may run the workflow. +* `authorize_callback=A, retry_auth_callback=B`: users authorized by A may start the workflow. Users authorized by B may retry on failure. * Example: starting the workflow is a decision that must be made by a product owner. Retrying can be made by an on-call member of the operations team. -* `auth=None, retry_auth=B`: any user can start the workflow, but only users authorized by B may retry on failure. +* `authorize_callback=None, retry_auth_callback=B`: any user can start the workflow, but only users authorized by B may retry on failure. ### `@inputstep` -The `@inputstep` decorator accepts the optional parameters `resume_auth: Authorizer` and `retry_auth: Authorizer`. +The `@inputstep` decorator accepts the optional parameters `resume_auth_callback: Authorizer` and `retry_auth_callback: Authorizer`. -`resume_auth` will be used to determine the authorization of a user to resume the workflow when suspended at this inputstep. -If `resume_auth` is omitted, then the workflow's `auth` will be used. +`resume_auth_callback` will be used to determine the authorization of a user to resume the workflow when suspended at this inputstep. +If `resume_auth_callback` is omitted, then the workflow's `authorize_callback` will be used. -`retry_auth` will be used to determine the authorization of a user to retry the workflow from a failed step following the inputstep. -If `retry_auth` is omitted, then `resume_auth` is used to authorize retries. -If `resume_auth` is also omitted, then the workflow’s `retry_auth` is checked, and then the workflow’s `auth`. +`retry_auth_callback` will be used to determine the authorization of a user to retry the workflow from a failed step following the inputstep. +If `retry_auth_callback` is omitted, then `resume_auth_callback` is used to authorize retries. +If `resume_auth_callback` is also omitted, then the workflow’s `retry_auth_callback` is checked, and then the workflow’s `authorize_callback`. In summary: -* A workflow establishes `auth` for starting, resuming, or retrying. -* The workflow can also establish `retry_auth`, which will override `auth` for retries. - * An inputstep can override the existing `auth` with `resume_auth` and the existing `retry_auth` with its own `retry_auth`. +* A workflow establishes `authorize_callback` for starting, resuming, or retrying. +* The workflow can also establish `retry_auth_callback`, which will override `authorize_callback` for retries. + * An inputstep can override the existing `authorize_callback` with `resume_auth_callback` and the existing `retry_auth_callback` with its own `retry_auth_callback`. * Subsequent inputsteps can do the same, but any None will not overwrite a previous not-None. ### Policy resolutions Below is an exhaustive table of how policies (implemented as callbacks `A`, `B`, `C`, and `D`) are prioritized in different workflow and inputstep configurations. +For brevity, the `_callback` parameter suffix has been ommitted. @@ -334,7 +336,7 @@ are prioritized in different workflow and inputstep configurations. - + @@ -551,11 +553,11 @@ We can now construct a variety of authorization policies. Suppose we have a workflow W that needs to pause on inputstep `approval` for approval from finance. Ops (and only ops) should be able to start the workflow and retry any failed steps. Finance (and only finance) should be able to resume at the input step. ```python - @workflow("An expensive workflow", auth=allow_roles("ops")) + @workflow("An expensive workflow", authorize_callback=allow_roles("ops")) def W(...): return begin >> A >> ... >> notify_finance >> approval >> ... >> Z - @inputstep("Approval", resume_auth=allow_roles("finance"), retry_auth=allow_roles("ops")) + @inputstep("Approval", resume_auth_callback=allow_roles("finance"), retry_auth_callback=allow_roles("ops")) def approval(...): ... ``` @@ -568,27 +570,27 @@ We can now construct a variety of authorization policies. Dev can start the workflow and retry steps prior to S. Once step S is reached, Platform (and only Platform) can resume the workflow and retry later failed steps. ```python - @workflow("An expensive workflow", auth=allow_roles("dev")) + @workflow("An expensive workflow", authorize_callback=allow_roles("dev")) def W(...): return begin >> A >> ... >> notify_platform >> handoff >> ... >> Z - @inputstep("Hand-off", resume_auth=allow_roles("platform")) + @inputstep("Hand-off", resume_auth_callback=allow_roles("platform")) def handoff(...): ... ``` - Notice that default behaviors let us ignore `retry_auth` arguments in both decorators. + Notice that default behaviors let us ignore `retry_auth_callback` arguments in both decorators. #### Restricted Retries Model !!!example Suppose we have a workflow that anyone can run, but with steps that should only be retried by users with certain backend access. ```python - @workflow("A workflow for any user", retry_auth=allow_roles("admin")) + @workflow("A workflow for any user", retry_auth_callback=allow_roles("admin")) def W(...): return begin >> A >> ... >> S >> ... >> Z ``` - Note that we could specify `auth=allow_roles("user")` if helpful, or we can omit `auth` to fail open to any logged in user. + Note that we could specify `authorize_callback=allow_roles("user")` if helpful, or we can omit `authorize_callback` to fail over to any logged in user. [1]: https://github.com/workfloworchestrator/example-orchestrator-ui [2]: https://github.com/workfloworchestrator/example-orchestrator From 438cba366555dfcf881d624b170afcdf7b126373 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Fri, 12 Dec 2025 12:51:33 +0100 Subject: [PATCH 8/8] Bump version to 4.7.0rc2 --- .bumpversion.cfg | 2 +- orchestrator/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e7d18694f..7624245f7 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.7.0rc1 +current_version = 4.7.0rc2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(rc(?P\d+))? diff --git a/orchestrator/__init__.py b/orchestrator/__init__.py index 488f990e4..4a4fefcbf 100644 --- a/orchestrator/__init__.py +++ b/orchestrator/__init__.py @@ -13,7 +13,7 @@ """This is the orchestrator workflow engine.""" -__version__ = "4.7.0rc1" +__version__ = "4.7.0rc2" from structlog import get_logger
authauthorize retry_auth resume_auth retry_auth