diff --git a/api-reference/python/tilebox.workflows/Client.configure_logging.mdx b/api-reference/python/tilebox.workflows/Client.configure_logging.mdx new file mode 100644 index 0000000..8f392fb --- /dev/null +++ b/api-reference/python/tilebox.workflows/Client.configure_logging.mdx @@ -0,0 +1,38 @@ +--- +title: Client.configure_logging +icon: laptop-code +--- + +```python +def Client.configure_logging( + level: int, + runner_level: int | None = None, +) -> None +``` + +Configure the log level for logs exported by this workflow client. + +## Parameters + + + Logging level for task logs emitted with `context.logger`, for example `logging.INFO` or `logging.DEBUG`. + + + + Logging level for internal task runner logs. If omitted, the value of `level` is used. + + +## Returns + +`None` + + +```python Python +import logging + +from tilebox.workflows import Client + +client = Client(name="sentinel-2-runner") +client.configure_logging(level=logging.DEBUG, runner_level=logging.INFO) +``` + diff --git a/api-reference/python/tilebox.workflows/Client.mdx b/api-reference/python/tilebox.workflows/Client.mdx index 2d21185..ddf9971 100644 --- a/api-reference/python/tilebox.workflows/Client.mdx +++ b/api-reference/python/tilebox.workflows/Client.mdx @@ -4,7 +4,12 @@ icon: code --- ```python -class Client(url: str, token: str) +class Client( + *, + url: str = "https://api.tilebox.com", + token: str | None = None, + name: str | None = None, +) ``` Create a Tilebox workflows client. @@ -19,6 +24,10 @@ Create a Tilebox workflows client. The API Key to authenticate with. If not set the `TILEBOX_API_KEY` environment variable will be used. + + Optional service name for workflow telemetry. If not set, the default service name is used. + + ## Sub clients The workflows client exposes sub clients for interacting with different parts of the Tilebox workflows API. @@ -41,6 +50,14 @@ def Client.automations() -> AutomationClient A client for scheduling automations. +## Logging + +```python +def Client.configure_logging(level: int, runner_level: int | None = None) -> None +``` + +Configure which task and runner logs this client exports. See [`Client.configure_logging`](/api-reference/python/tilebox.workflows/Client.configure_logging). + ## Task runners ```python @@ -59,7 +76,8 @@ client = Client() # or provide connection details directly client = Client( url="https://api.tilebox.com", - token="YOUR_TILEBOX_API_KEY" + token="YOUR_TILEBOX_API_KEY", + name="sentinel-2-runner", ) # access sub clients diff --git a/api-reference/python/tilebox.workflows/ExecutionContext.logger.mdx b/api-reference/python/tilebox.workflows/ExecutionContext.logger.mdx new file mode 100644 index 0000000..7edf762 --- /dev/null +++ b/api-reference/python/tilebox.workflows/ExecutionContext.logger.mdx @@ -0,0 +1,39 @@ +--- +title: Context.logger +icon: folder-gear +--- + +```python +ExecutionContext.logger: StructuredLogger +``` + +Structured logger for logs emitted by the currently executing task. + +## Methods + +```python +context.logger.debug(message, /, *args, **attributes) -> None +context.logger.info(message, /, *args, **attributes) -> None +context.logger.warning(message, /, *args, **attributes) -> None +context.logger.error(message, /, *args, **attributes) -> None +context.logger.exception(message, /, *args, **attributes) -> None +context.logger.critical(message, /, *args, **attributes) -> None +context.logger.bind(**attributes) -> StructuredLogger +``` + +Structured attributes are attached to the log record and become searchable telemetry attributes. + + +```python Python +from tilebox.workflows import ExecutionContext, Task + +class ProcessScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + context.logger.info("Started", scene_id=self.scene_id) + + log = context.logger.bind(component="download") + log.debug("Fetching input") +``` + diff --git a/api-reference/python/tilebox.workflows/ExecutionContext.tracer.mdx b/api-reference/python/tilebox.workflows/ExecutionContext.tracer.mdx new file mode 100644 index 0000000..2609a11 --- /dev/null +++ b/api-reference/python/tilebox.workflows/ExecutionContext.tracer.mdx @@ -0,0 +1,32 @@ +--- +title: Context.tracer +icon: folder-gear +--- + +```python +ExecutionContext.tracer: WorkflowTracer +``` + +Tracer for creating custom spans inside the currently executing task. + +## Methods + +```python +context.tracer.span(name: str, *args, **kwargs) -> ContextManager[Span] +context.tracer.current_span() -> Span +``` + +Custom spans are nested under the current task span and are exported to Tilebox with the job trace. + + +```python Python +from tilebox.workflows import ExecutionContext, Task + +class ProcessScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + with context.tracer.span("download-scene") as span: + span.set_attribute("scene_id", self.scene_id) +``` + diff --git a/api-reference/python/tilebox.workflows/JobClient.query_logs.mdx b/api-reference/python/tilebox.workflows/JobClient.query_logs.mdx new file mode 100644 index 0000000..5c56fc1 --- /dev/null +++ b/api-reference/python/tilebox.workflows/JobClient.query_logs.mdx @@ -0,0 +1,33 @@ +--- +title: JobClient.query_logs +icon: rectangle-terminal +--- + +```python +def JobClient.query_logs(job_id: Job | UUID | str) -> LogRecords +``` + +Query logs emitted while running a job. Pagination is handled automatically. + +## Parameters + + + The job, job ID, or job ID string to query logs for. + + +## Returns + +A `LogRecords` list. Each `LogRecord` contains `time`, `severity_number`, `severity_text`, `body`, `trace_id`, `span_id`, `attributes`, and `runner_attributes`. + +`LogRecords.to_pandas()` converts the records to a pandas DataFrame. + + +```python Python +logs = client.jobs().query_logs(job) + +for record in logs: + print(record.time, record.severity_text, record.body) + +df = logs.to_pandas() +``` + diff --git a/api-reference/python/tilebox.workflows/JobClient.query_spans.mdx b/api-reference/python/tilebox.workflows/JobClient.query_spans.mdx new file mode 100644 index 0000000..9f19f05 --- /dev/null +++ b/api-reference/python/tilebox.workflows/JobClient.query_spans.mdx @@ -0,0 +1,33 @@ +--- +title: JobClient.query_spans +icon: chart-gantt +--- + +```python +def JobClient.query_spans(job_id: Job | UUID | str) -> Spans +``` + +Query spans emitted while running a job. Pagination is handled automatically. + +## Parameters + + + The job, job ID, or job ID string to query spans for. + + +## Returns + +A `Spans` list. Each `Span` contains `start_time`, `end_time`, `duration`, `trace_id`, `span_id`, `parent_span_id`, `name`, `status_code`, `status_message`, `attributes`, `runner_attributes`, and `events`. + +`Spans.to_pandas()` converts the spans to a pandas DataFrame and includes a computed `duration` column. + + +```python Python +spans = client.jobs().query_spans(job.id) + +for span in spans: + print(span.name, span.status_code, span.duration) + +df = spans.to_pandas() +``` + diff --git a/assets/changelog/2026-05-observability-dark.png b/assets/changelog/2026-05-observability-dark.png new file mode 100644 index 0000000..b08e348 Binary files /dev/null and b/assets/changelog/2026-05-observability-dark.png differ diff --git a/assets/changelog/2026-05-observability-light.png b/assets/changelog/2026-05-observability-light.png new file mode 100644 index 0000000..a6790b8 Binary files /dev/null and b/assets/changelog/2026-05-observability-light.png differ diff --git a/assets/console/api-keys-dark.png b/assets/console/api-keys-dark.png index 6ab286a..d9179bb 100644 Binary files a/assets/console/api-keys-dark.png and b/assets/console/api-keys-dark.png differ diff --git a/assets/console/api-keys-light.png b/assets/console/api-keys-light.png index 14475eb..da1e40e 100644 Binary files a/assets/console/api-keys-light.png and b/assets/console/api-keys-light.png differ diff --git a/assets/console/automation-dark.png b/assets/console/automation-dark.png index 9cb643e..50ace4b 100644 Binary files a/assets/console/automation-dark.png and b/assets/console/automation-dark.png differ diff --git a/assets/console/automation-edit-dark.png b/assets/console/automation-edit-dark.png index c6d33ec..2b44bf0 100644 Binary files a/assets/console/automation-edit-dark.png and b/assets/console/automation-edit-dark.png differ diff --git a/assets/console/automation-edit-light.png b/assets/console/automation-edit-light.png index 4cbbffc..f224525 100644 Binary files a/assets/console/automation-edit-light.png and b/assets/console/automation-edit-light.png differ diff --git a/assets/console/automation-light.png b/assets/console/automation-light.png index 89776b0..934e021 100644 Binary files a/assets/console/automation-light.png and b/assets/console/automation-light.png differ diff --git a/assets/console/automations-dark.png b/assets/console/automations-dark.png index 883d81a..b0ed2a4 100644 Binary files a/assets/console/automations-dark.png and b/assets/console/automations-dark.png differ diff --git a/assets/console/automations-light.png b/assets/console/automations-light.png index cc0c8a6..e0fa40a 100644 Binary files a/assets/console/automations-light.png and b/assets/console/automations-light.png differ diff --git a/assets/console/cluster-dark.png b/assets/console/cluster-dark.png index a148263..c031c37 100644 Binary files a/assets/console/cluster-dark.png and b/assets/console/cluster-dark.png differ diff --git a/assets/console/cluster-light.png b/assets/console/cluster-light.png index 9c8102b..90baebd 100644 Binary files a/assets/console/cluster-light.png and b/assets/console/cluster-light.png differ diff --git a/assets/console/datapoint-dark.png b/assets/console/datapoint-dark.png index a71e284..4235dde 100644 Binary files a/assets/console/datapoint-dark.png and b/assets/console/datapoint-dark.png differ diff --git a/assets/console/datapoint-light.png b/assets/console/datapoint-light.png index 262b826..d2d881c 100644 Binary files a/assets/console/datapoint-light.png and b/assets/console/datapoint-light.png differ diff --git a/assets/console/datasets-create-dark.png b/assets/console/datasets-create-dark.png index c674fb3..a2874fd 100644 Binary files a/assets/console/datasets-create-dark.png and b/assets/console/datasets-create-dark.png differ diff --git a/assets/console/datasets-create-light.png b/assets/console/datasets-create-light.png index 8bce8f9..a39eab6 100644 Binary files a/assets/console/datasets-create-light.png and b/assets/console/datasets-create-light.png differ diff --git a/assets/console/datasets-create-temporal-dark.png b/assets/console/datasets-create-temporal-dark.png index 75c84ee..543118d 100644 Binary files a/assets/console/datasets-create-temporal-dark.png and b/assets/console/datasets-create-temporal-dark.png differ diff --git a/assets/console/datasets-create-temporal-light.png b/assets/console/datasets-create-temporal-light.png index 6a3ebbf..e70f686 100644 Binary files a/assets/console/datasets-create-temporal-light.png and b/assets/console/datasets-create-temporal-light.png differ diff --git a/assets/console/datasets-documentation-dark.png b/assets/console/datasets-documentation-dark.png index c71bb4c..8b7f492 100644 Binary files a/assets/console/datasets-documentation-dark.png and b/assets/console/datasets-documentation-dark.png differ diff --git a/assets/console/datasets-documentation-light.png b/assets/console/datasets-documentation-light.png index 1f48150..92a9be8 100644 Binary files a/assets/console/datasets-documentation-light.png and b/assets/console/datasets-documentation-light.png differ diff --git a/assets/console/datasets-explorer-dark.png b/assets/console/datasets-explorer-dark.png index 62b235b..a7cffc8 100644 Binary files a/assets/console/datasets-explorer-dark.png and b/assets/console/datasets-explorer-dark.png differ diff --git a/assets/console/datasets-explorer-light.png b/assets/console/datasets-explorer-light.png index 57e99fb..5f3c1a9 100644 Binary files a/assets/console/datasets-explorer-light.png and b/assets/console/datasets-explorer-light.png differ diff --git a/assets/console/datasets-overview-dark.png b/assets/console/datasets-overview-dark.png index b2a9604..9bcfe7e 100644 Binary files a/assets/console/datasets-overview-dark.png and b/assets/console/datasets-overview-dark.png differ diff --git a/assets/console/datasets-overview-light.png b/assets/console/datasets-overview-light.png index 094c024..e416dcb 100644 Binary files a/assets/console/datasets-overview-light.png and b/assets/console/datasets-overview-light.png differ diff --git a/assets/console/job-execution-dark.png b/assets/console/job-execution-dark.png new file mode 100644 index 0000000..d289a52 Binary files /dev/null and b/assets/console/job-execution-dark.png differ diff --git a/assets/console/job-execution-light.png b/assets/console/job-execution-light.png new file mode 100644 index 0000000..567a05a Binary files /dev/null and b/assets/console/job-execution-light.png differ diff --git a/assets/console/job-logs-dark.png b/assets/console/job-logs-dark.png new file mode 100644 index 0000000..97a4327 Binary files /dev/null and b/assets/console/job-logs-dark.png differ diff --git a/assets/console/job-logs-light.png b/assets/console/job-logs-light.png new file mode 100644 index 0000000..c7e3206 Binary files /dev/null and b/assets/console/job-logs-light.png differ diff --git a/assets/console/storage-event-dark.png b/assets/console/storage-event-dark.png index 95e6063..8fe0568 100644 Binary files a/assets/console/storage-event-dark.png and b/assets/console/storage-event-dark.png differ diff --git a/assets/console/storage-event-light.png b/assets/console/storage-event-light.png index fd0adc3..deb328e 100644 Binary files a/assets/console/storage-event-light.png and b/assets/console/storage-event-light.png differ diff --git a/assets/console/usage-dark.png b/assets/console/usage-dark.png index 4fb1185..a5d9455 100644 Binary files a/assets/console/usage-dark.png and b/assets/console/usage-dark.png differ diff --git a/assets/console/usage-light.png b/assets/console/usage-light.png index 1066f35..223ba75 100644 Binary files a/assets/console/usage-light.png and b/assets/console/usage-light.png differ diff --git a/assets/workflows/observability/job-logs-pandas-dark.png b/assets/workflows/observability/job-logs-pandas-dark.png new file mode 100644 index 0000000..1ec22dd Binary files /dev/null and b/assets/workflows/observability/job-logs-pandas-dark.png differ diff --git a/assets/workflows/observability/job-logs-pandas-light.png b/assets/workflows/observability/job-logs-pandas-light.png new file mode 100644 index 0000000..ac632b1 Binary files /dev/null and b/assets/workflows/observability/job-logs-pandas-light.png differ diff --git a/assets/workflows/observability/job-traces-pandas-dark.png b/assets/workflows/observability/job-traces-pandas-dark.png new file mode 100644 index 0000000..c9de948 Binary files /dev/null and b/assets/workflows/observability/job-traces-pandas-dark.png differ diff --git a/assets/workflows/observability/job-traces-pandas-light.png b/assets/workflows/observability/job-traces-pandas-light.png new file mode 100644 index 0000000..918379c Binary files /dev/null and b/assets/workflows/observability/job-traces-pandas-light.png differ diff --git a/authentication.mdx b/authentication.mdx index 37c59be..9dc06f9 100644 --- a/authentication.mdx +++ b/authentication.mdx @@ -6,7 +6,7 @@ icon: key ## Creating an API key -To create an API key, log into the [Tilebox Console](https://console.tilebox.com). Navigate to [Account -> API Keys](https://console.tilebox.com/account/api-keys) and click the "Create API Key" button. +To create an API key, log into the [Tilebox Console](https://console.tilebox.com). Navigate to [Settings -> API Keys](https://console.tilebox.com/settings/api-keys) and click the "Create API Key" button. Keep your API keys secure. Deactivate any keys if you suspect they have been compromised. diff --git a/changelog.mdx b/changelog.mdx index d402cac..9a58db0 100644 --- a/changelog.mdx +++ b/changelog.mdx @@ -4,6 +4,21 @@ description: A chronological record of new features, improvements, and other not icon: rss --- + + + Job Execution Trace View + Job Execution Trace View + + + ## Workflow Observability + + Tilebox Workflows now includes built-in observability for jobs and task runners. Tilebox captures workflow logs, traces, task status, and runner context, then correlates them with jobs and tasks. + + The Console includes a built-in explorer for workflow observability, so you can inspect task logs, trace timing, failures, and runner behavior from the job view. + + See the [Workflow observability documentation](/workflows/observability/introduction) for examples and integration options. + + ![Optional Tasks](/assets/changelog/2026-02-optional-tasks.png) diff --git a/docs.json b/docs.json index 63d56ca..b1cb0fb 100644 --- a/docs.json +++ b/docs.json @@ -88,9 +88,18 @@ "group": "Observability", "icon": "eye", "pages": [ - "workflows/observability/open-telemetry", + "workflows/observability/introduction", "workflows/observability/tracing", - "workflows/observability/logging" + "workflows/observability/logging", + "workflows/observability/query", + { + "group": "Integrations", + "icon": "plug", + "pages": [ + "workflows/observability/integrations/open-telemetry", + "workflows/observability/integrations/axiom" + ] + } ] }, { @@ -187,8 +196,11 @@ "api-reference/python/tilebox.workflows/Client", "api-reference/python/tilebox.workflows/Task", "api-reference/python/tilebox.workflows/Client.runner", + "api-reference/python/tilebox.workflows/Client.configure_logging", "api-reference/python/tilebox.workflows/TaskRunner.run_all", "api-reference/python/tilebox.workflows/TaskRunner.run_forever", + "api-reference/python/tilebox.workflows/ExecutionContext.logger", + "api-reference/python/tilebox.workflows/ExecutionContext.tracer", "api-reference/python/tilebox.workflows/ExecutionContext.submit_subtask", "api-reference/python/tilebox.workflows/ExecutionContext.submit_subtasks", "api-reference/python/tilebox.workflows/ExecutionContext.job_cache", @@ -203,6 +215,8 @@ "api-reference/python/tilebox.workflows/JobClient.retry", "api-reference/python/tilebox.workflows/JobClient.cancel", "api-reference/python/tilebox.workflows/JobClient.visualize", + "api-reference/python/tilebox.workflows/JobClient.query_logs", + "api-reference/python/tilebox.workflows/JobClient.query_spans", "api-reference/python/tilebox.workflows/JobClient.query" ] } diff --git a/guides/datasets/access-sentinel2-data.mdx b/guides/datasets/access-sentinel2-data.mdx index 72b2a8b..2e60586 100644 --- a/guides/datasets/access-sentinel2-data.mdx +++ b/guides/datasets/access-sentinel2-data.mdx @@ -4,7 +4,7 @@ description: Query Sentinel-2 satellite imagery metadata from a Tilebox dataset, icon: database --- -This guide assume you already [signed up](https://console.tilebox.com/sign-up) for a Tilebox account (free) and [created an API key](https://console.tilebox.com/account/api-keys). +This guide assume you already [signed up](https://console.tilebox.com/sign-up) for a Tilebox account (free) and [created an API key](https://console.tilebox.com/settings/api-keys). ## Install Tilebox package diff --git a/quickstart.mdx b/quickstart.mdx index 6711a63..8c14dbc 100644 --- a/quickstart.mdx +++ b/quickstart.mdx @@ -40,7 +40,7 @@ If you prefer to work locally, follow these steps to get started. - Create an API key by logging into the [Tilebox Console](https://console.tilebox.com), navigating to [Account -> API Keys](https://console.tilebox.com/account/api-keys), and clicking the "Create API Key" button. + Create an API key by logging into the [Tilebox Console](https://console.tilebox.com), navigating to [Settings -> API Keys](https://console.tilebox.com/settings/api-keys), and clicking the "Create API Key" button. Tilebox Console @@ -138,7 +138,7 @@ If you prefer to work locally, follow these steps to get started. ``` - Create an API key by logging into the [Tilebox Console](https://console.tilebox.com), navigating to [Account -> API Keys](https://console.tilebox.com/account/api-keys), and clicking the "Create API Key" button. + Create an API key by logging into the [Tilebox Console](https://console.tilebox.com), navigating to [Settings -> API Keys](https://console.tilebox.com/settings/api-keys), and clicking the "Create API Key" button. Tilebox Console diff --git a/workflows/concepts/task-runners.mdx b/workflows/concepts/task-runners.mdx index 35b4853..fe9a6af 100644 --- a/workflows/concepts/task-runners.mdx +++ b/workflows/concepts/task-runners.mdx @@ -366,4 +366,4 @@ This mechanism ensures that scenarios such as power outages, hardware failures, ## Observability -Task runners are continuously running processes, making it essential to monitor their health and performance. You can achieve observability by collecting and analyzing logs, metrics, and traces from task runners. Tilebox Workflows provides tools to enable this data collection and analysis. To learn how to configure task runners for observability, head over to the [Observability](/workflows/observability) section. +Task runners are continuously running processes, making it essential to monitor their health and performance. Tilebox Workflows collects logs and traces from task runners automatically. To learn how to inspect and customize workflow observability, see [Observability](/workflows/observability/introduction). diff --git a/workflows/observability/integrations/axiom.mdx b/workflows/observability/integrations/axiom.mdx new file mode 100644 index 0000000..667d5fd --- /dev/null +++ b/workflows/observability/integrations/axiom.mdx @@ -0,0 +1,67 @@ +--- +title: Axiom +description: Export Tilebox workflow logs and traces to Axiom datasets in addition to Tilebox Console. +icon: chart-line +--- + +Tilebox can export workflow telemetry to [Axiom](https://axiom.co/) through OTLP. Use this when your team already analyzes logs and traces in Axiom or wants long-term telemetry in Axiom datasets. + +Built-in Tilebox Console observability does not require Axiom. Axiom export is optional and additive. + + + Tilebox is pre-integrated with Axiom + + +## Configure Axiom export + +Create Axiom datasets for logs and traces and an API key with ingest permissions. Then configure export when the runner process starts. + +```python Python +from tilebox.workflows import Client +from tilebox.workflows.observability.logging import configure_otel_logging_axiom +from tilebox.workflows.observability.tracing import configure_otel_tracing_axiom + +from my_workflow import ProcessScene + +configure_otel_tracing_axiom( + service="sentinel-2-runner", + dataset="workflow-traces", + api_key="", +) +configure_otel_logging_axiom( + service="sentinel-2-runner", + dataset="workflow-logs", + api_key="", +) + +client = Client(name="sentinel-2-runner") +runner = client.runner(tasks=[ProcessScene]) +runner.run_forever() +``` + +## Environment variables + +You can omit credentials from code by setting environment variables: + +| Variable | Used by | +| --- | --- | +| `AXIOM_API_KEY` | log and trace export | +| `AXIOM_LOGS_DATASET` | `configure_otel_logging_axiom()` | +| `AXIOM_TRACES_DATASET` | `configure_otel_tracing_axiom()` | + +```python Python +configure_otel_tracing_axiom(service="sentinel-2-runner") +configure_otel_logging_axiom(service="sentinel-2-runner") +``` + +## Existing Axiom screenshots + + + Tilebox workflow logs in Axiom + Tilebox workflow logs in Axiom + + + + Tilebox workflow traces in Axiom + Tilebox workflow traces in Axiom + diff --git a/workflows/observability/integrations/open-telemetry.mdx b/workflows/observability/integrations/open-telemetry.mdx new file mode 100644 index 0000000..4b50da9 --- /dev/null +++ b/workflows/observability/integrations/open-telemetry.mdx @@ -0,0 +1,61 @@ +--- +title: OpenTelemetry +description: Export Tilebox workflow logs and traces to any OTLP-compatible backend in addition to Tilebox Console. +icon: telescope +--- + +Tilebox uses OpenTelemetry data models for workflow telemetry. Built-in Tilebox observability works without any OpenTelemetry setup, but you can add OTLP export when you need the same logs and traces in another backend. + +## Configure OTLP export + +Call the configuration functions when the runner process starts, before creating the client or runner. + +```python Python +from tilebox.workflows import Client +from tilebox.workflows.observability.logging import configure_otel_logging +from tilebox.workflows.observability.tracing import configure_otel_tracing + +from my_workflow import ProcessScene + +configure_otel_tracing( + service="sentinel-2-runner", + endpoint="http://localhost:4318/v1/traces", + headers={"Authorization": "Bearer "}, +) +configure_otel_logging( + service="sentinel-2-runner", + endpoint="http://localhost:4318/v1/logs", + headers={"Authorization": "Bearer "}, +) + +client = Client(name="sentinel-2-runner") +runner = client.runner(tasks=[ProcessScene]) +runner.run_forever() +``` + +If the endpoint does not include `/v1/traces` or `/v1/logs`, the Python SDK adds the correct path automatically. + +## Environment variables + +You can omit endpoint and interval arguments by setting environment variables: + +| Variable | Used by | +| --- | --- | +| `OTEL_TRACES_ENDPOINT` | `configure_otel_tracing()` | +| `OTEL_LOGS_ENDPOINT` | `configure_otel_logging()` | +| `OTEL_EXPORT_INTERVAL` | tracing and logging exporters | + +`OTEL_EXPORT_INTERVAL` accepts durations such as `5s`, `30s`, or `2m`. + +## Local collector example + +For local trace testing, run Jaeger with OTLP HTTP enabled: + +```bash +docker run --rm --name jaeger \ + -p 16686:16686 \ + -p 4318:4318 \ + jaegertracing/jaeger:2.9.0 +``` + +Then configure tracing with `endpoint="http://localhost:4318"` and open the Jaeger UI at [http://localhost:16686](http://localhost:16686). diff --git a/workflows/observability/introduction.mdx b/workflows/observability/introduction.mdx new file mode 100644 index 0000000..dbe5b3c --- /dev/null +++ b/workflows/observability/introduction.mdx @@ -0,0 +1,141 @@ +--- +title: Workflow observability +sidebarTitle: Overview +description: Inspect workflow logs, traces, task status, and runner behavior. +icon: lightbulb +--- + +Tilebox Workflows gives each job a live observability record. As task runners execute work, Tilebox captures logs, traces, task status, and runner context. You can follow a job from the root task through its subtasks, inspect failures, and compare slow steps across distributed runners. + + + Job Execution Trace View + Job Execution Trace View + + +Use the built-in view for day-to-day debugging and operations. Add structured log fields and custom spans when you need more detail inside your task code. + + + + + + + + +## How Tilebox observes workflows + +A submitted job starts a trace. Each task run creates a span, and custom spans sit under the task that creates them. Log records emitted from task code attach to active spans, which connects messages to timing data. + +Tilebox adds job, task, runner, and service metadata to telemetry records. This data helps you filter by job, inspect a single task run, or compare work across task runners. + +## Observability example + + +```python Python +from tilebox.workflows import Client, ExecutionContext, Task + +class ProcessScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + context.logger.info("Processing scene", scene_id=self.scene_id) + + with context.tracer.span("plan-subtasks"): + thumbnail = context.submit_subtask(BuildThumbnail(scene_id=self.scene_id)) + context.submit_subtask(PublishScene(scene_id=self.scene_id), depends_on=[thumbnail]) + +class BuildThumbnail(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + context.logger.info("Building thumbnail", scene_id=self.scene_id) + +class PublishScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + context.logger.info("Publishing scene", scene_id=self.scene_id) + +client = Client(name="sentinel-2-runner") +runner = client.runner(tasks=[ProcessScene, BuildThumbnail, PublishScene]) +runner.run_forever() +``` +```go Go +package tasks + +import ( + "context" + "fmt" + "log/slog" + + "github.com/tilebox/tilebox-go" + "github.com/tilebox/tilebox-go/workflows/v1" + "github.com/tilebox/tilebox-go/workflows/v1/subtask" +) + +type ProcessScene struct { + SceneID string +} + +func (t *ProcessScene) Execute(ctx context.Context) error { + slog.InfoContext(ctx, "processing scene", slog.String("scene_id", t.SceneID)) + + return tilebox.WithSpan(ctx, "plan-subtasks", func(ctx context.Context) error { + thumbnail, err := workflows.SubmitSubtask(ctx, &BuildThumbnail{SceneID: t.SceneID}) + if err != nil { + return fmt.Errorf("failed to submit thumbnail subtask: %w", err) + } + + _, err = workflows.SubmitSubtask(ctx, &PublishScene{SceneID: t.SceneID}, subtask.WithDependencies(thumbnail)) + if err != nil { + return fmt.Errorf("failed to submit publish subtask: %w", err) + } + + return nil + }) +} + +type BuildThumbnail struct { + SceneID string +} + +func (t *BuildThumbnail) Execute(ctx context.Context) error { + slog.InfoContext(ctx, "building thumbnail", slog.String("scene_id", t.SceneID)) + return nil +} + +type PublishScene struct { + SceneID string +} + +func (t *PublishScene) Execute(ctx context.Context) error { + slog.InfoContext(ctx, "publishing scene", slog.String("scene_id", t.SceneID)) + return nil +} +``` + + +The parent task, spawned subtasks, task logs, and spans share the same job trace. This keeps orchestration and task-level work connected. + +## Integrate with external observability platforms + +If your team uses another observability platform, configure [OpenTelemetry](/workflows/observability/integrations/open-telemetry) or [Axiom](/workflows/observability/integrations/axiom) export in the runner process. Tilebox keeps the workflow state, while your platform receives the same logs and traces for alerting, long-term storage, or analysis. diff --git a/workflows/observability/logging.mdx b/workflows/observability/logging.mdx index e9c8598..89fdf1a 100644 --- a/workflows/observability/logging.mdx +++ b/workflows/observability/logging.mdx @@ -1,328 +1,123 @@ --- title: Logging -description: Collect logs from distributed task runners into a centralized observability backend so you can search and correlate them across your workflow cluster. +description: Emit structured task logs and tune how workflow clients export log records. icon: rectangle-terminal --- -## Overview - -Tilebox workflows are designed for distributed execution, making it essential to set up logging to a centralized system. Tilebox supports OpenTelemetry logging, which simplifies sending log messages from your tasks to a chosen backend. Collecting and visualizing logs from a distributed cluster of task runners in a tool like [Axiom](https://axiom.co/) can look like this: +Tilebox collects workflow logs automatically. When a task runner is created from a `Client`, logs emitted through the task execution context are exported to Tilebox and correlated with the active job, task, and trace. - Tilebox Workflows logging in Axiom - Tilebox Workflows logging in Axiom + Job Logs View + Job Logs View -## Configure logging - -The Tilebox workflow SDKs include support for exporting OpenTelemetry logs. To enable logging, call the appropriate configuration functions during the startup of your[task runner](/workflows/concepts/task-runners). Then, use the provided `logger` to send log messages from your tasks. - - - - To configure logging with Axiom, you first need to create a [Axiom Dataset](https://axiom.co/docs/reference/datasets) to export your workflow logs to. You will also need an [Axiom API key](https://axiom.co/docs/reference/tokens) with the necessary write permissions for your Axiom dataset. - - - ```python Python - from tilebox.workflows import Client, Task, ExecutionContext - from tilebox.workflows.observability.logging import configure_otel_logging_axiom - - # your own workflow: - from my_workflow import MyTask - - def main(): - configure_otel_logging_axiom( - # specify an Axiom dataset to export logs to - dataset="my-axiom-logs-dataset", - # along with an Axiom API key with ingest permissions for that dataset - api_key="my-axiom-api-key", - ) - - # the task runner will export logs from - # the executed tasks to the specified dataset - client = Client() - runner = client.runner(tasks=[MyTask]) - runner.run_forever() - - if __name__ == "__main__": - main() - ``` -```go Go -package main +## Write task logs -import ( - "context" - "log/slog" +Use `context.logger` inside `Task.execute`. It supports standard log levels and structured attributes. - "github.com/tilebox/tilebox-go/examples/workflows/axiom" - "github.com/tilebox/tilebox-go/observability" - "github.com/tilebox/tilebox-go/observability/logger" - "github.com/tilebox/tilebox-go/workflows/v1" -) + +```python Python +from tilebox.workflows import ExecutionContext, Task -// specify a service name and version to identify the instrumenting application in traces and logs -var service = &observability.Service{Name: "task-runner", Version: "dev"} - -func main() { - ctx := context.Background() - - // Setup OpenTelemetry logging and slog - // It uses AXIOM_API_KEY and AXIOM_LOGS_DATASET from the environment - axiomHandler, shutdownLogger, err := logger.NewAxiomHandlerFromEnv(ctx, service, - logger.WithLevel(slog.LevelInfo), // export logs at info level and above as OTEL logs - ) - defer shutdownLogger(ctx) - if err != nil { - slog.Error("failed to set up axiom log handler", slog.Any("error", err)) - return - } - tileboxLogger := logger.New( // initialize a slog.Logger - axiomHandler, // export logs to the Axiom handler - logger.NewConsoleHandler(logger.WithLevel(slog.LevelWarn)), // and additionally, export WARN and ERROR logs to stdout - ) - slog.SetDefault(tileboxLogger) // all future slog calls will be forwarded to the tilebox logger - - client := workflows.NewClient() - - taskRunner, err := client.NewTaskRunner(ctx) - if err != nil { - slog.Error("failed to create task runner", slog.Any("error", err)) - return - } - - err = taskRunner.RegisterTasks(&MyTask{}) - if err != nil { - slog.Error("failed to register tasks", slog.Any("error", err)) - return - } - - taskRunner.RunForever(ctx) -} +class ProcessScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + context.logger.info("Started scene processing", scene_id=self.scene_id) + + log = context.logger.bind(component="atmospheric-correction") + log.debug("Fetching auxiliary data") + + try: + # process the scene + log.info("Scene processed", output_format="cog") + except Exception: + context.logger.exception("Scene processing failed", scene_id=self.scene_id) + raise ``` - - - - Setting the environment variables `AXIOM_API_KEY` and `AXIOM_LOGS_DATASET` allows you to omit these arguments in the `configure_otel_logging_axiom` function. - - - - - If you are using another OpenTelemetry-compatible backend besides Axiom, such as OpenTelemetry Collector or Jaeger, you can configure logging by specifying the URL endpoint to export log messages to. - - - ```python Python - from tilebox.workflows import Client - from tilebox.workflows.observability.logging import configure_otel_logging - - # your own workflow: - from my_workflow import MyTask - - def main(): - configure_otel_logging( - # specify an endpoint to export logs to, such as a - # locally running instance of OpenTelemetry Collector - endpoint="http://localhost:4318/v1/logs", - # optional headers for each request - headers={"Authorization": "Bearer some-api-key"}, - ) - - # the task runner will export logs from - # the executed tasks to the specified endpoint - client = Client() - runner = client.runner(tasks=[MyTask]) - runner.run_forever() - - if __name__ == "__main__": - main() - ``` ```go Go -package main +package tasks import ( "context" "log/slog" - - "github.com/tilebox/tilebox-go/examples/workflows/opentelemetry" - "github.com/tilebox/tilebox-go/observability" - "github.com/tilebox/tilebox-go/observability/logger" - "github.com/tilebox/tilebox-go/workflows/v1" ) -// specify a service name and version to identify the instrumenting application in traces and logs -var service = &observability.Service{Name: "task-runner", Version: "dev"} - -func main() { - ctx := context.Background() - - endpoint := "http://localhost:4318" - headers := map[string]string{ - "Authorization": "Bearer ", - } - - // Setup an OpenTelemetry log handler, exporting logs to an OTEL compatible log endpoint - otelHandler, shutdownLogger, err := logger.NewOtelHandler(ctx, service, - logger.WithEndpointURL(endpoint), - logger.WithHeaders(headers), - logger.WithLevel(slog.LevelInfo), // export logs at info level and above as OTEL logs - ) - defer shutdownLogger(ctx) - if err != nil { - slog.Error("failed to set up otel log handler", slog.Any("error", err)) - return - } - tileboxLogger := logger.New( // initialize a slog.Logger - otelHandler, // export logs to the OTEL handler - logger.NewConsoleHandler(logger.WithLevel(slog.LevelWarn)), // and additionally, export WARN and ERROR logs to stdout - ) - slog.SetDefault(tileboxLogger) // all future slog calls will be forwarded to the tilebox logger - - client := workflows.NewClient() - - taskRunner, err := client.NewTaskRunner(ctx) - if err != nil { - slog.Error("failed to create task runner", slog.Any("error", err)) - return - } - - err = taskRunner.RegisterTasks(&MyTask{}) - if err != nil { - slog.Error("failed to register tasks", slog.Any("error", err)) - return - } - - taskRunner.RunForever(ctx) +type ProcessScene struct{} + +func (t *ProcessScene) Execute(ctx context.Context) error { + slog.InfoContext(ctx, "started scene processing", slog.String("scene_id", "S2A_001")) + slog.DebugContext(ctx, "fetching auxiliary data", slog.String("component", "atmospheric-correction")) + return nil } ``` - - - - If you set the environment variable `OTEL_LOGS_ENDPOINT`, you can omit that argument in the `configure_otel_logging` function. - - - - To log messages to the standard console output, use the `configure_console_logging` function. - - - ```python Python - from tilebox.workflows import Client - from tilebox.workflows.observability.logging import configure_console_logging - - # your own workflow: - from my_workflow import MyTask - - def main(): - configure_console_logging() - - # the task runner will print log messages from - # the executed tasks to the console - client = Client() - runner = client.runner(tasks=[MyTask]) - runner.run_forever() - - if __name__ == "__main__": - main() - ``` -```go Go -package main + -import ( - "context" - "log/slog" +Structured attributes become searchable log attributes. Bind shared attributes to a logger when records need the same context. - "github.com/tilebox/tilebox-go/examples/workflows/opentelemetry" - "github.com/tilebox/tilebox-go/observability/logger" - "github.com/tilebox/tilebox-go/workflows/v1" -) +## What Tilebox adds automatically -func main() { - ctx := context.Background() +Logs emitted from a task include workflow metadata without extra code. Tilebox attaches job ID, job name, task ID, task display name, parent task data, task identifier name and version, runner service data, SDK version, host, OS, and process ID. When a log is emitted inside an active span, Tilebox also attaches `trace_id` and `span_id`. - tileboxLogger := logger.New(logger.NewConsoleHandler(logger.WithLevel(slog.LevelWarn))) - slog.SetDefault(tileboxLogger) // all future slog calls will be forwarded to the tilebox logger +Logs are also added as events on the active trace span, so a trace view can show important log messages inline with task execution. - client := workflows.NewClient() +## Configure local console logging - taskRunner, err := client.NewTaskRunner(ctx) - if err != nil { - slog.Error("failed to create task runner", slog.Any("error", err)) - return - } +Built-in Tilebox export does not require configuration. For local development, add a console handler to print Tilebox workflow logs to standard output. - err = taskRunner.RegisterTasks(&MyTask{}) - if err != nil { - slog.Error("failed to register tasks", slog.Any("error", err)) - return - } +```python Python +import logging - taskRunner.RunForever(ctx) -} -``` - +from tilebox.workflows import Client +from tilebox.workflows.observability.logging import configure_console_logging - - The console logging backend is not recommended for production use. Log messages will be emitted to the standard output of each task runner rather than a centralized logging system. It is intended for local development and testing of workflows. - - +from my_workflow import ProcessScene - +configure_console_logging(level=logging.DEBUG) -## Emitting log messages +client = Client() +runner = client.runner(tasks=[ProcessScene]) +runner.run_forever() +``` -Use the logger provided by the Tilebox SDK to emit log messages from your tasks. You can then use it to send log messages to the [configured logging backend](#configure-logging). -Log messages emitted within a task's `execute` method are also automatically recorded as span events for the current [job trace](/workflows/observability/tracing). +`configure_console_logging()` is process-wide for Tilebox workflow loggers. Use it for local runs and debugging distributed runners. - -```python Python -import logging -from tilebox.workflows import Task, ExecutionContext -from tilebox.workflows.observability.logging import get_logger +## Configure the client log level -logger = get_logger() +Use `Client.configure_logging()` to choose which task and runner logs a client exports to Tilebox. -class MyTask(Task): - def execute(self, context: ExecutionContext) -> None: - # emit a log message to the configured OpenTelemetry backend - logger.info("Hello world from configured logger!") -``` -```go Go -package tasks +```python Python +import logging -import ( - "context" - "log/slog" -) +from tilebox.workflows import Client -type MyTask struct{} +client = Client(name="sentinel-2-runner") -func (t *MyTask) Execute(context.Context) error { - // emit a log message to the configured OpenTelemetry backend - slog.Info("Hello world from configured logger!") - return nil -} +# Export task logs at DEBUG and internal runner logs at INFO. +client.configure_logging(level=logging.DEBUG, runner_level=logging.INFO) ``` - -## Logging task runner internals +The `level` argument applies to logs emitted with `context.logger`. The optional `runner_level` argument applies to internal task runner logs. If `runner_level` is omitted, it uses the same value as `level`. + +## Query logs -In python, Tilebox task runners also internally use a logger. By default, it's set to the WARNING level, but you can change it by explicitly configuring a logger for the workflows client when constructing the task runner. +You can retrieve logs for a job through the jobs client and convert the result to a pandas DataFrame. ```python Python from tilebox.workflows import Client -from tilebox.workflows.observability.logging import configure_otel_logging_axiom -from tilebox.workflows.observability.logging import get_logger - -# configure Axiom or another logging backend -configure_otel_logging_axiom( - dataset="my-axiom-logs-dataset", - api_key="my-axiom-api-key", -) - -# configure a logger for the Tilebox client at the INFO level client = Client() -client.configure_logger(get_logger(level=logging.INFO)) +job = client.jobs().submit("process-scene", ProcessScene(scene_id="S2A_001")) -# now the task runner inherits this logger and uses -# it to emit its own internal log messages as well -runner = client.runner(tasks=[MyTask]) -runner.run_forever() +logs = client.jobs().query_logs(job) +for record in logs: + print(record.time, record.severity_text, record.body, record.attributes) + +df = logs.to_pandas() ``` + +See [Query telemetry](/workflows/observability/query) for the log and span query APIs. + +## Export to another backend + +Tilebox stores logs by default. To export logs to your own observability backend as well, configure an [OpenTelemetry](/workflows/observability/integrations/open-telemetry) or [Axiom](/workflows/observability/integrations/axiom) integration when the runner process starts. diff --git a/workflows/observability/open-telemetry.mdx b/workflows/observability/open-telemetry.mdx deleted file mode 100644 index 19e66d5..0000000 --- a/workflows/observability/open-telemetry.mdx +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: OpenTelemetry Integration -sidebarTitle: OpenTelemetry -description: Gain full visibility into your workflow execution by integrating OpenTelemetry for distributed tracing and structured logging across your task runners. -icon: telescope ---- - -## Observability - -Effective observability is essential for building reliable workflows. Understanding and monitoring the execution of workflows and their tasks helps ensure correctness and efficiency. This section describes methods to gain insights into your workflow's execution. - - - - - - -## OpenTelemetry - -Tilebox Workflows is designed with [OpenTelemetry](https://opentelemetry.io/) in mind, which provides a set of APIs and libraries for instrumenting, generating, collecting, and exporting telemetry data (metrics, logs, and traces) in distributed systems. - - -Tilebox Workflows currently supports OpenTelemetry for tracing and logging, with plans to include metrics in the future. - - -## Integrations - -Tilebox exports telemetry data using the [OpenTelemetry Protocol](https://opentelemetry.io/docs/specs/otlp/). This allows you to send telemetry data to any OpenTelemetry-compatible backend, such as Axiom or Jaeger. - -### Axiom - -Tilebox is pre-integrated with Axiom - -Tilebox Workflows has built-in support for Axiom, a cloud-based observability and telemetry platform. The examples and screenshots in this section come from this integration. - -To get started, [sign up for a free Axiom account](https://axiom.co/), create an axiom dataset for traces and logs, and generate an API key with ingest permissions for those datasets. - -You can then configure Tilebox to export traces and logs to Axiom using [configure_otel_tracing_axiom](/workflows/observability/tracing#axiom) and [configure_otel_logging_axiom](/workflows/observability/logging#axiom). - -### Jaeger - -Tilebox works well with Jaeger - -Another popular option is [Jaeger](https://jaegertracing.io/), a popular distributed tracing system. You can use the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) to collect telemetry data from Tilebox Workflows and export it to Jaeger. - -An all-in-one Jaeger environment can be spun up using Docker: - -```bash -docker run --rm --name jaeger \ - -p 5778:5778 \ - -p 16686:16686 \ - -p 4318:4318 \ - jaegertracing/jaeger:2.9.0 -``` - -You can then configure Tilebox to export traces to Jaeger using `configure_otel_tracing`. - - -```python Python -from tilebox.workflows import Client -from tilebox.workflows.observability.tracing import configure_otel_tracing - -# your own workflow: -from my_workflow import MyTask - -def main(): - configure_otel_tracing( - # export traces to the local Jaeger instance - endpoint="http://localhost:4318", - ) - - # the following task runner generates traces for executed tasks and - # exports trace and span data to the specified endpoint - client = Client() - runner = client.runner(tasks=[MyTask]) - runner.run_forever() - -if __name__ == "__main__": - main() -``` -```go Go -package main - -import ( - "context" - "log/slog" - - "github.com/tilebox/tilebox-go/examples/workflows/opentelemetry" - "github.com/tilebox/tilebox-go/observability" - "github.com/tilebox/tilebox-go/observability/tracer" - "github.com/tilebox/tilebox-go/workflows/v1" - "go.opentelemetry.io/otel" -) - -// specify a service name and version to identify the instrumenting application in traces and logs -var service = &observability.Service{Name: "task-runner", Version: "dev"} - -func main() { - ctx := context.Background() - - endpoint := "http://localhost:4318" - - // Setup an OpenTelemetry trace span processor, exporting traces and spans to an OTEL compatible trace endpoint - tileboxTracerProvider, shutdown, err := tracer.NewOtelProvider(ctx, service, tracer.WithEndpointURL(endpoint)) - defer shutdown(ctx) - if err != nil { - slog.Error("failed to set up otel span processor", slog.Any("error", err)) - return - } - otel.SetTracerProvider(tileboxTracerProvider) // set the tilebox tracer provider as the global OTEL tracer provider - - client := workflows.NewClient() - - taskRunner, err := client.NewTaskRunner(ctx) - if err != nil { - slog.Error("failed to create task runner", slog.Any("error", err)) - return - } - - err = taskRunner.RegisterTasks(&MyTask{}) - if err != nil { - slog.Error("failed to register tasks", slog.Any("error", err)) - return - } - - taskRunner.RunForever(ctx) -} -``` - - -The generated workflow traces can then be viewed in the Jaeger UI at [http://localhost:16686](http://localhost:16686). diff --git a/workflows/observability/query.mdx b/workflows/observability/query.mdx new file mode 100644 index 0000000..4a7a337 --- /dev/null +++ b/workflows/observability/query.mdx @@ -0,0 +1,88 @@ +--- +title: Query telemetry +description: Query workflow logs and spans for a job from Python and convert the results to pandas DataFrames. +icon: magnifying-glass-chart +--- + +Tilebox stores logs and spans for each workflow job. Use the jobs client to query that telemetry from notebooks, scripts, or automated diagnostics. + +## Query job logs + +`query_logs()` returns a `LogRecords` list. Pagination is handled automatically. + +```python Python +from tilebox.workflows import Client + +client = Client() +job = client.jobs().find("019e07b1-916b-0630-f3ba-f1c33235d174") + +logs = client.jobs().query_logs(job) + +for record in logs: + print(record.time, record.severity_text, record.body) + print(record.attributes) +``` + +Each log record includes: + +- `time` +- `severity_number` and `severity_text` +- `body` +- `trace_id` and `span_id` +- `attributes` +- `runner_attributes` + +### As pandas DataFrame + +Use `to_pandas()` to convert log records to a pandas DataFrame. + +```python Python +logs_df = client.jobs().query_logs(job).to_pandas() +logs_df[["time", "severity_text", "body"]] +``` + + + Job Logs as Pandas DataFrame + Job Logs as Pandas DataFrame + + +## Query job spans + +`query_spans()` returns a `Spans` list. Pagination is handled automatically. + +```python Python +spans = client.jobs().query_spans(job.id) + +for span in spans: + print(span.name, span.status_code, span.duration) + print(span.attributes) +``` + +Each span includes: + +- `start_time` and `end_time` +- `duration` +- `trace_id`, `span_id`, and `parent_span_id` +- `name` +- `status_code` and `status_message` +- `attributes` +- `runner_attributes` +- `events` + +### As pandas DataFrame + +Use `to_pandas()` to convert spans to a pandas DataFrame. + +```python Python +spans_df = client.jobs().query_spans(job).to_pandas() + +slow_spans = spans_df.sort_values("duration", ascending=False).head(10) +slow_spans[["name", "duration", "start_time"]] +``` + + + Job Traces as Pandas DataFrame + Job Traces as Pandas DataFrame + + +Nested `attributes`, `runner_attributes`, and `events` stay as Python objects in DataFrame columns. Span DataFrames include a computed `duration` column. diff --git a/workflows/observability/tracing.mdx b/workflows/observability/tracing.mdx index 4e29f35..539ed68 100644 --- a/workflows/observability/tracing.mdx +++ b/workflows/observability/tracing.mdx @@ -1,206 +1,99 @@ --- title: Tracing -description: Record and visualize workflow job execution as structured traces to understand task ordering, parallelism, and duration across distributed task runners. +description: Use built-in workflow traces and custom spans to inspect job execution, task duration, and bottlenecks. icon: chart-gantt --- -## Overview - -Applying [OpenTelemetry traces](https://opentelemetry.io/docs/concepts/signals/traces/) to the concept of workflows allows you to monitor the execution of your jobs and their individual tasks. Visualizing the trace for a job in a tool like [Axiom](https://axiom.co/) may look like this: +Tilebox traces workflow jobs automatically. Job submission creates a root trace, task runners continue that trace across machines, and every task execution creates a span. - Tilebox Workflows tracing in Axiom - Tilebox Workflows tracing in Axiom + Job Execution Trace View + Job Execution Trace View -Tracing your workflows enables you to easily observe: - -- The order of task execution -- Which tasks run in parallel -- The [task runner](/workflows/concepts/task-runners) handling each task -- The duration of each task -- The outcome of each task (success or failure) - -This information helps identify bottlenecks and performance issues, ensuring that your workflows execute correctly. - -## Configure tracing - -The Tilebox workflow SDKs have built-in support for exporting OpenTelemetry traces. To enable tracing, call the appropriate configuration functions during the startup of your [task runner](/workflows/concepts/task-runners). - - - - To configure tracing with Axiom, you first need to create a [Axiom Dataset](https://axiom.co/docs/reference/datasets) to export your workflow traces to. You will also need an [Axiom API key](https://axiom.co/docs/reference/tokens) with the necessary write permissions for your Axiom dataset. - - - ```python Python - from tilebox.workflows import Client - from tilebox.workflows.observability.tracing import configure_otel_tracing_axiom - - # your own workflow: - from my_workflow import MyTask - - def main(): - configure_otel_tracing_axiom( - # specify an Axiom dataset to export traces to - dataset="my-axiom-traces-dataset", - # along with an Axiom API key for ingest permissions - api_key="my-axiom-api-key", - ) - - # the following task runner generates traces for executed tasks and - # exports trace and span data to the specified Axiom dataset - client = Client() - runner = client.runner(tasks=[MyTask]) - runner.run_forever() - - if __name__ == "__main__": - main() - ``` -```go Go -package main +Built-in traces connect task order, dependencies, parallel execution, task duration, task status, runner identity, service identity, and logs emitted while a span was active. -import ( - "context" - "log/slog" +## Add custom spans - "github.com/tilebox/tilebox-go/examples/workflows/axiom" - "github.com/tilebox/tilebox-go/observability" - "github.com/tilebox/tilebox-go/observability/tracer" - "github.com/tilebox/tilebox-go/workflows/v1" - "go.opentelemetry.io/otel" -) +Use `context.tracer` inside a task to add spans around meaningful parts of your own code. -// specify a service name and version to identify the instrumenting application in traces and logs -var service = &observability.Service{Name: "task-runner", Version: "dev"} - -func main() { - ctx := context.Background() - - // Setup an OpenTelemetry trace span processor, exporting traces and spans to Axiom - // It uses AXIOM_API_KEY and AXIOM_TRACES_DATASET from the environment - tileboxTracerProvider, shutdown, err := tracer.NewAxiomProviderFromEnv(ctx, service) - defer shutdown(ctx) - if err != nil { - slog.Error("failed to set up axiom tracer provider", slog.Any("error", err)) - return - } - otel.SetTracerProvider(tileboxTracerProvider) // set the tilebox tracer provider as the global OTEL tracer provider - - client := workflows.NewClient() - - taskRunner, err := client.NewTaskRunner(ctx) - if err != nil { - slog.Error("failed to create task runner", slog.Any("error", err)) - return - } - - err = taskRunner.RegisterTasks(&MyTask{}) - if err != nil { - slog.Error("failed to register tasks", slog.Any("error", err)) - return - } - - taskRunner.RunForever(ctx) -} + +```python Python +from tilebox.workflows import ExecutionContext, Task + +class ProcessScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + with context.tracer.span("download-scene") as span: + span.set_attribute("scene_id", self.scene_id) + # download input data + + with context.tracer.span("compute-index"): + # perform expensive computation + pass ``` - - - - Set the environment variables `AXIOM_API_KEY` and `AXIOM_TRACES_DATASET` to omit those arguments - in the `configure_otel_tracing_axiom` function. - - - - - If you are using another OpenTelemetry-compatible backend besides Axiom, like OpenTelemetry Collector or Jaeger, you can configure tracing by specifying the URL endpoint to export traces to. - - - ```python Python - from tilebox.workflows import Client - from tilebox.workflows.observability.tracing import configure_otel_tracing - - # your own workflow: - from my_workflow import MyTask - - def main(): - configure_otel_tracing( - # specify an endpoint for trace ingestion, such as a - # locally running instance of OpenTelemetry Collector - endpoint="http://localhost:4318/v1/traces", - # optional headers for each request - headers={"Authorization": "Bearer some-api-key"}, - ) - - # the following task runner generates traces for executed tasks and - # exports trace and span data to the specified endpoint - client = Client() - runner = client.runner(tasks=[MyTask]) - runner.run_forever() - - if __name__ == "__main__": - main() - ``` ```go Go -package main +package tasks import ( "context" - "log/slog" - "github.com/tilebox/tilebox-go/examples/workflows/opentelemetry" - "github.com/tilebox/tilebox-go/observability" - "github.com/tilebox/tilebox-go/observability/tracer" - "github.com/tilebox/tilebox-go/workflows/v1" - "go.opentelemetry.io/otel" + "github.com/tilebox/tilebox-go" ) -// specify a service name and version to identify the instrumenting application in traces and logs -var service = &observability.Service{Name: "task-runner", Version: "dev"} - -func main() { - ctx := context.Background() - - endpoint := "http://localhost:4318" - headers := map[string]string{ - "Authorization": "Bearer ", - } - - // Setup an OpenTelemetry trace span processor, exporting traces and spans to an OTEL compatible trace endpoint - tileboxTracerProvider, shutdown, err := tracer.NewOtelProvider(ctx, service, - tracer.WithEndpointURL(endpoint), - tracer.WithHeaders(headers), - ) - defer shutdown(ctx) - if err != nil { - slog.Error("failed to set up otel span processor", slog.Any("error", err)) - return - } - otel.SetTracerProvider(tileboxTracerProvider) // set the tilebox tracer provider as the global OTEL tracer provider - - client := workflows.NewClient() - - taskRunner, err := client.NewTaskRunner(ctx) - if err != nil { - slog.Error("failed to create task runner", slog.Any("error", err)) - return - } - - err = taskRunner.RegisterTasks(&MyTask{}) - if err != nil { - slog.Error("failed to register tasks", slog.Any("error", err)) - return - } - - taskRunner.RunForever(ctx) +type ProcessScene struct{} + +func (t *ProcessScene) Execute(ctx context.Context) error { + return tilebox.WithSpan(ctx, "compute-index", func(ctx context.Context) error { + // perform expensive computation + return nil + }) } ``` - + + +Custom spans are nested under the current task span. Logs emitted inside the span are correlated with its `trace_id` and `span_id`. + +## Span status and exceptions + +If a task raises an exception, Tilebox records the exception on the task span and marks the span as failed before the task is retried or marked failed. + +For finer-grained error reporting, record errors on your custom spans before re-raising them. + +```python Python +class ProcessScene(Task): + scene_id: str + + def execute(self, context: ExecutionContext) -> None: + with context.tracer.span("publish-output") as span: + try: + # publish output + pass + except Exception as error: + span.record_exception(error) + raise +``` + +## Query spans + +You can retrieve spans for a job through the jobs client and convert the result to a pandas DataFrame. + +```python Python +from tilebox.workflows import Client + +client = Client() +job = client.jobs().submit("process-scene", ProcessScene(scene_id="S2A_001")) + +spans = client.jobs().query_spans(job) +for span in spans: + print(span.name, span.status_code, span.duration) + +df = spans.to_pandas() +``` + +See [Query telemetry](/workflows/observability/query) for the log and span query APIs. - - Set the environment variable `OTEL_TRACES_ENDPOINT` to omit that argument - in the `configure_otel_tracing` function. - - - +## Export to another backend -Once the runner picks up tasks and executes them, corresponding traces and spans are automatically generated and exported to the configured backend. +Tilebox stores traces by default. To export spans to your own observability backend as well, configure an [OpenTelemetry](/workflows/observability/integrations/open-telemetry) or [Axiom](/workflows/observability/integrations/axiom) integration when the runner process starts.