diff --git a/.github/workflows/enable_log_forwarding_action_tests.yaml b/.github/workflows/enable_log_forwarding_action_tests.yaml new file mode 100644 index 00000000..73acd7c6 --- /dev/null +++ b/.github/workflows/enable_log_forwarding_action_tests.yaml @@ -0,0 +1,73 @@ +name: Enable Log Forwarding Action Tests + +on: + pull_request: + workflow_call: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + action: ${{ steps.filter.outputs.action }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v4 + id: filter + with: + filters: | + action: + - 'actions/enable_log_forwarding/**' + - '.github/workflows/enable_log_forwarding_action_tests.yaml' + + test-action: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.action == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Install tox + run: uv tool install tox --with tox-uv + + - name: Run lint, static checks, and unit tests + run: tox -e actions-lint,actions-static,actions-unit + + smoke-test-self-hosted: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.action == 'true' }} + runs-on: [self-hosted-linux-amd64-noble-edge] + env: + TEST_CONFIG_FILE: 98-enable-log-forwarding-smoke-${{ github.run_id }}-${{ github.run_attempt }}.yaml + steps: + - uses: actions/checkout@v6 + + - name: Run enable log forwarding action + uses: ./actions/enable_log_forwarding + with: + files: | + /var/log/syslog + config-file-name: ${{ env.TEST_CONFIG_FILE }} + otlp-endpoint: 127.0.0.1:4317 + + - name: Verify generated config file exists + run: | + sudo test -f /etc/otelcol/config.d/${TEST_CONFIG_FILE} + + - name: Verify generated config contains expected receiver + run: | + sudo grep -q '"filelog/github_runner_optin"' /etc/otelcol/config.d/${TEST_CONFIG_FILE} diff --git a/actions/enable_log_forwarding/action.yaml b/actions/enable_log_forwarding/action.yaml new file mode 100644 index 00000000..7337ed21 --- /dev/null +++ b/actions/enable_log_forwarding/action.yaml @@ -0,0 +1,33 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +name: Enable log forwarding +description: Opt in to forward selected log files from a self-hosted GitHub runner to Loki. + +inputs: + files: + description: | + Newline or comma separated list of file paths or glob patterns to forward. + Example: /var/log/chrony/*.log + required: true + otlp-endpoint: + description: | + Optional OTLP/gRPC endpoint for upstream OpenTelemetry Collector logs export. + When not set, the action falls back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT. + Example: otel-gateway.internal:4317 + required: false + default: "" + config-file-name: + description: File name for the generated collector fragment. + required: false + default: 90-github-runner-log-forwarding.yaml + +runs: + using: composite + steps: + - name: Configure collector for opt-in file log forwarding + shell: bash + env: + INPUT_FILES: ${{ inputs.files }} + INPUT_OTLP_ENDPOINT: ${{ inputs.otlp-endpoint }} + INPUT_CONFIG_FILE_NAME: ${{ inputs.config-file-name }} + run: python3 "${{ github.action_path }}/enable_log_forwarding.py" diff --git a/actions/enable_log_forwarding/collector_config.j2 b/actions/enable_log_forwarding/collector_config.j2 new file mode 100644 index 00000000..b48fe5ae --- /dev/null +++ b/actions/enable_log_forwarding/collector_config.j2 @@ -0,0 +1,23 @@ +{ + "receivers": { + "filelog/github_runner_optin": { + "include": {{ include_files }}, + "start_at": "end" + } + }, +{{ exporters_section }} + "processors": { + "resource/github_runner_optin": { + "attributes": {{ resource_attributes }} + } + }, + "service": { + "pipelines": { + "logs/github_runner_optin": { + "receivers": ["filelog/github_runner_optin"], + "processors": ["resource/github_runner_optin", "batch"], + "exporters": [{{ exporter_name }}] + } + } + } +} diff --git a/actions/enable_log_forwarding/enable_log_forwarding.py b/actions/enable_log_forwarding/enable_log_forwarding.py new file mode 100644 index 00000000..7e7aed3f --- /dev/null +++ b/actions/enable_log_forwarding/enable_log_forwarding.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +"""Configure OpenTelemetry Collector log forwarding for selected runner log files.""" + +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path +from typing import Sequence + +CONFIG_DIR = "/etc/otelcol/config.d" +EXPORTER_NAME = "otlp_grpc" +CONFIG_TEMPLATE_PATH = Path(__file__).with_name("collector_config.j2") +SNAP_CMD = shutil.which("snap") +SUDO_CMD = shutil.which("sudo") + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def run_as_root(*args: str) -> subprocess.CompletedProcess[bytes]: + """Run a command directly as root or through sudo when available.""" + if os.geteuid() == 0: # if running as root + return subprocess.run(args, capture_output=True, check=False) + if SUDO_CMD: # if sudo is available + return subprocess.run([SUDO_CMD, *args], capture_output=True, check=False) + logger.error("This action requires root privileges to update collector config.") + sys.exit(1) + + +def parse_files_into_list(files_input: str) -> list[str]: + """Parse comma/newline-separated file patterns into a normalized list.""" + entries = [] + # Split on commas or newlines, and strip whitespace. Ignore empty entries. + for item in re.split(r"[,\n]", files_input): + stripped = item.strip() + if stripped: + entries.append(stripped) + return entries + + +def resolve_endpoint() -> str: + """Resolve OTLP endpoint from explicit input, then workflow fallback variable.""" + # If INPUT_OTLP_ENDPOINT is not set, fall back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT + for env_var in ( + "INPUT_OTLP_ENDPOINT", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", + ): + val = os.getenv(env_var, "").strip() + if val: + return val + return "" + + +def check_exporter_exists() -> bool: + """Return whether the configured exporter is already defined in collector config files.""" + # Check for " otlp_grpc:" exporter definition + pattern = f"^ {re.escape(EXPORTER_NAME)}:[ \t]*$" + config_dir = Path(CONFIG_DIR) + if not config_dir.is_dir(): + return False + + matcher = re.compile(pattern) + for path in config_dir.rglob("*"): + if not path.is_file(): + continue + try: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if matcher.match(line): + return True + except OSError: + continue + return False + + +def render_template(template_path: Path, context: dict[str, str]) -> str: + """Render a minimal Jinja-style template with {{ var }} placeholders.""" + template = template_path.read_text(encoding="utf-8") + pattern = re.compile(r"{{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*}}") + + def replacer(match: re.Match[str]) -> str: + key = match.group(1) + if key not in context: + raise KeyError(f"Missing template variable: {key}") + return context[key] + + return pattern.sub(replacer, template) + + +def build_resource_attributes() -> list[dict[str, str]]: + """Build static GitHub resource attributes attached to forwarded logs.""" + attrs = [ + ("github.repository", os.getenv("GITHUB_REPOSITORY", "unknown")), + ("github.runner.name", os.getenv("RUNNER_NAME", "unknown")), + ("github.workflow", os.getenv("GITHUB_WORKFLOW", "unknown")), + ("github.job.id", os.getenv("GITHUB_JOB", "unknown")), + ("github.run.id", os.getenv("GITHUB_RUN_ID", "unknown")), + ("github.run.attempt", os.getenv("GITHUB_RUN_ATTEMPT", "unknown")), + ] + return [{"key": key, "value": value, "action": "upsert"} for key, value in attrs] + + +def build_exporters_section( + resolved_endpoint: str, exporter_already_exists: bool +) -> str: + """Build the optional exporters JSON fragment for the template.""" + if exporter_already_exists: + return "" + + exporters_block = { + "exporters": { + EXPORTER_NAME: { + "endpoint": resolved_endpoint, + } + } + } + block = json.dumps(exporters_block, indent=2) + inner = block.strip()[1:-1].strip() + return textwrap.indent(inner, " ") + ",\n" + + +def build_config( + files: Sequence[str], resolved_endpoint: str, exporter_already_exists: bool +) -> str: + """Build a collector pipeline config fragment for opt-in log forwarding.""" + context = { + "include_files": json.dumps(list(files)), + "resource_attributes": json.dumps(build_resource_attributes()), + "exporters_section": build_exporters_section( + resolved_endpoint, exporter_already_exists + ), + "exporter_name": json.dumps(EXPORTER_NAME), + } + return render_template(CONFIG_TEMPLATE_PATH, context) + + +def read_files_input() -> str: + """Read and validate the required files input.""" + files_input = os.getenv("INPUT_FILES", "").strip() + if not files_input: + logger.error("Input 'files' cannot be empty.") + sys.exit(1) + return files_input + + +def resolve_config_path() -> str: + """Resolve and validate the destination config file path.""" + config_file_name = os.getenv( + "INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml" + ).strip() + if ( + not config_file_name + or config_file_name in {".", ".."} + or Path(config_file_name).name != config_file_name + ): + logger.error( + "Input 'config-file-name' must be a non-empty file name without directory components.", + ) + sys.exit(1) + + return str(Path(CONFIG_DIR) / config_file_name) + + +def ensure_collector_is_available() -> None: + """Check snap prerequisites needed to configure the collector.""" + if SNAP_CMD is None: + logger.error("Required command is missing: snap") + sys.exit(1) + + snap_list_result = subprocess.run( + [SNAP_CMD, "list", "opentelemetry-collector"], + capture_output=True, + check=False, + ) + if snap_list_result.returncode != 0: + logger.error("opentelemetry-collector snap is not installed on this runner.") + sys.exit(1) + + +def validate_exporter_configuration( + resolved_endpoint: str, exporter_already_exists: bool +) -> None: + """Ensure there is enough exporter information to build a working config.""" + if exporter_already_exists or resolved_endpoint: + return + + logger.error( + "Exporter '%s' was not found in scanned collector config directories " + "and no OTLP endpoint was provided.", + EXPORTER_NAME, + ) + logger.error( + "Set input 'otlp-endpoint', or expose " + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", + ) + sys.exit(1) + + +def write_collector_config(config_content: str, config_path: str) -> None: + """Write generated config to /etc/otelcol/config.d via root privileges.""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False, encoding="utf-8" + ) as tmp: + tmp.write(config_content) + tmp_path = tmp.name + + try: + mkdir_result = run_as_root( + "mkdir", "-p", CONFIG_DIR # create directory if it doesn't exist + ) + if mkdir_result.returncode != 0: + stderr = mkdir_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to create collector config directory '%s': %s", + CONFIG_DIR, + stderr or "unknown error", + ) + sys.exit(1) + + install_result = run_as_root( + "install", "-m", "0644", tmp_path, config_path # rw-r--r-- permissions + ) + if install_result.returncode != 0: + stderr = install_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to install collector config to '%s': %s", + config_path, + stderr or "unknown error", + ) + sys.exit(1) + finally: + os.unlink(tmp_path) + + logger.info("Wrote log-forwarding collector config to: %s", config_path) + + +def restart_collector() -> None: + """Restart collector service so new config is loaded.""" + if SNAP_CMD is None: + logger.error("Required command is missing: snap") + sys.exit(1) + + restart_result = run_as_root(SNAP_CMD, "restart", "opentelemetry-collector") + if restart_result.returncode != 0: + stderr = restart_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to restart opentelemetry-collector: %s", + stderr or "unknown error", + ) + sys.exit(1) + logger.info("Restarted opentelemetry-collector to apply log-forwarding config.") + + +def main(): + """Validate inputs, write collector config, and restart the collector service.""" + files_input = read_files_input() + config_path = resolve_config_path() + ensure_collector_is_available() + + files = parse_files_into_list(files_input) + if not files: + logger.error("Input 'files' must contain at least one path or glob.") + sys.exit(1) + + resolved_endpoint = resolve_endpoint() + exporter_already_exists = check_exporter_exists() + validate_exporter_configuration(resolved_endpoint, exporter_already_exists) + + config_content = build_config(files, resolved_endpoint, exporter_already_exists) + write_collector_config(config_content, config_path) + restart_collector() + + +if __name__ == "__main__": + main() diff --git a/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py new file mode 100644 index 00000000..61404587 --- /dev/null +++ b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py @@ -0,0 +1,115 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +"""Unit tests for the enable_log_forwarding action script.""" + +import importlib.util +import json +import pathlib +import unittest +from unittest import mock + +MODULE_PATH = ( + pathlib.Path(__file__).resolve().parent.parent / "enable_log_forwarding.py" +) + + +def load_module(path: pathlib.Path): + """Load the action module from file for direct function-level unit testing.""" + spec = importlib.util.spec_from_file_location("enable_log_forwarding", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load module from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MODULE = load_module(MODULE_PATH) + + +class TestEnableLogForwarding(unittest.TestCase): + """Test suite for parsing, endpoint resolution, exporter detection, and config generation.""" + + def test_parse_files_into_list(self): + """It parses comma/newline-separated inputs and drops empty entries.""" + files_input = " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n" + parsed = MODULE.parse_files_into_list(files_input) + self.assertEqual( + parsed, ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"] + ) + + def test_resolve_endpoint_prefers_input(self): + """It prefers the explicit action input endpoint over fallback env values.""" + with mock.patch.dict( + MODULE.os.environ, + { + "INPUT_OTLP_ENDPOINT": "input-endpoint:4318", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT": "system-endpoint:4318", + }, + clear=False, + ): + resolved = MODULE.resolve_endpoint() + + self.assertEqual(resolved, "input-endpoint:4318") + + def test_resolve_endpoint_falls_back_to_action_env(self): + """It falls back to the workflow-provided endpoint when input is empty.""" + with mock.patch.dict( + MODULE.os.environ, + { + "INPUT_OTLP_ENDPOINT": "", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT": "system-endpoint:4318", + }, + clear=False, + ): + resolved = MODULE.resolve_endpoint() + + self.assertEqual(resolved, "system-endpoint:4318") + + def test_check_exporter_exists_true(self): + """It returns true when a collector config file defines the exporter name.""" + mock_file = mock.Mock(spec=pathlib.Path) + mock_file.is_file.return_value = True + mock_file.read_text.return_value = "exporters:\n otlp_grpc:\n" + + with mock.patch.object( + pathlib.Path, "is_dir", return_value=True + ), mock.patch.object(pathlib.Path, "rglob", return_value=[mock_file]): + self.assertTrue(MODULE.check_exporter_exists()) + + def test_build_config_adds_exporter_when_missing(self): + """It adds an exporter block when no pre-existing exporter is detected.""" + env = { + "GITHUB_REPOSITORY": "canonical/github-runner-operators", + "RUNNER_NAME": "runner-1", + "GITHUB_WORKFLOW": "CI", + "GITHUB_JOB": "test", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "1", + } + with mock.patch.dict(MODULE.os.environ, env, clear=False): + raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", False) + + config = json.loads(raw) + self.assertEqual( + config["receivers"]["filelog/github_runner_optin"]["include"], + ["/var/log/syslog"], + ) + self.assertEqual( + config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"], + [MODULE.EXPORTER_NAME], + ) + self.assertEqual( + config["exporters"][MODULE.EXPORTER_NAME]["endpoint"], "otel:4318" + ) + + def test_build_config_reuses_existing_exporter(self): + """It omits exporter creation when an exporter already exists elsewhere.""" + with mock.patch.dict(MODULE.os.environ, {}, clear=False): + raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", True) + + config = json.loads(raw) + self.assertNotIn("exporters", config) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/changelog.md b/docs/changelog.md index ae0a4b20..fd7dd68c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Each revision is versioned by the date of the revision. +## 2026-04-22 + +- add action to allow workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. + ## 2026-04-13 - add 5xx error logging to planner routes. diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md new file mode 100644 index 00000000..d7c3faf7 --- /dev/null +++ b/docs/how-to/enable-log-forwarding.md @@ -0,0 +1,61 @@ +# How to enable log forwarding + +The `enable-log-forwarding` action allows workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. + +By default, nothing is forwarded. Log forwarding starts only when this action is used in a workflow. + +## Prerequisites + +- Use a self-hosted Linux runner. +- Install the `opentelemetry-collector` snap on the runner. +- Ensure the workflow can update `/etc/otelcol/config.d` with root privileges. + +## Provide inputs + +To enable log forwarding, set the following inputs in your workflow file as required by your setup: + +- `files` (required): newline or comma separated file paths or glob patterns. +- `config-file-name` (optional, default `90-github-runner-log-forwarding.yaml`): generated config file name. +- `otlp-endpoint` (optional): OTLP/gRPC endpoint used to create the exporter when one is not already configured. + +When `otlp-endpoint` is not set, the action falls back to `ACTION_OTEL_EXPORTER_OTLP_ENDPOINT` from the workflow environment. + +## Use the action + +Add this snippet to a job in your workflow file (for example, `.github/workflows/ci.yaml`): + +```yaml +jobs: + chrony-testing: + runs-on: [self-hosted, linux] + steps: + - uses: canonical/github-runner-operators/actions/enable_log_forwarding@main + with: + files: | + /var/log/chrony/*.log + /var/log/syslog +``` + +Pin the action to a release tag or commit SHA in production workflows. + +Use these checks to confirm forwarding: + +- Check the action step is completed and prints success messages in the workflow logs. +- Generate new log lines after the action step and query Loki to confirm they arrive. + +## Examine Loki queries + +The action adds GitHub context as resource attributes on forwarded logs: + +- `github.job.id` +- `github.repository` +- `github.runner.name` +- `github.workflow` +- `github.run.id` +- `github.run.attempt` + +Example Loki query by workflow run id: + +``` +{github_run_id="123456789"} +``` diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 47992476..27cca196 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -7,3 +7,4 @@ The following guides cover key processes and common tasks for managing and using :maxdepth: 1 Contribute + Enable log forwarding diff --git a/tox.ini b/tox.ini index 7808cf7b..70e7628b 100644 --- a/tox.ini +++ b/tox.ini @@ -4,11 +4,12 @@ [tox] no_package = True skip_missing_interpreters = True -env_list = format, lint, static +env_list = format, lint, static, actions-lint, actions-static, actions-unit min_version = 4.0.0 [vars] tests_path = {tox_root}/charms/tests +actions_path = {tox_root}/actions [testenv] setenv = @@ -69,3 +70,28 @@ commands = --log-cli-level=INFO \ {[vars]tests_path}/integration \ {posargs} + +[testenv:actions-lint] +description = Run formatting and lint checks for Python code under actions/ +deps = + black + ruff +commands = + black --check {[vars]actions_path} + ruff check {[vars]actions_path} + +[testenv:actions-static] +description = Run static analysis for Python code under actions/ +deps = + mypy + pylint +commands = + mypy {[vars]actions_path} + pylint {[vars]actions_path} + +[testenv:actions-unit] +description = Run unit tests for Python code under actions/ +deps = + pytest +commands = + pytest -v -s {[vars]actions_path}