diff --git a/docs/reference/organization/repository/branch-or-tag.md b/docs/reference/organization/repository/branch-or-tag.md index 0fda1415..e21f0169 100644 --- a/docs/reference/organization/repository/branch-or-tag.md +++ b/docs/reference/organization/repository/branch-or-tag.md @@ -1,4 +1,4 @@ -A BranchOrTag represents either a branch or tag pattern to use within an [Environment](environment.md). +A BranchOrTag represents either a branch or tag pattern to use within an [Environment](environment/index.md). The following format is used to distinguish between tags and branches: | Type | Format | Example | diff --git a/docs/reference/organization/repository/environment/index.md b/docs/reference/organization/repository/environment/index.md new file mode 100644 index 00000000..cd2b53a1 --- /dev/null +++ b/docs/reference/organization/repository/environment/index.md @@ -0,0 +1,46 @@ +Definition of an `Environment` on repository level, the following properties are supported: + +| Key | Value | Description | Notes | +|----------------------------|--------------------------------------------|---------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| _name_ | string | The name of the environment | | +| _wait_timer_ | int | The amount of time to wait before allowing deployments to proceed | | +| _reviewers_ | list\[[Actor](../actor.md)\] | Users or Teams that may approve workflow runs that access this environment | | +| _deployment_branch_policy_ | string | Limit which branches can deploy to this environment based on rules or naming patterns | `all`, `protected` or `selected` | +| _branch_policies_ | list\[[BranchOrTag](../branch-or-tag.md)\] | List of branch or tag patterns which can deploy to this environment | only applicable if `deployment_branch_policy` is set to `selected` | + +## Jsonnet Function + +``` jsonnet +orgs.newEnvironment('') { + : +} +``` + +## Validation rules + +- specifying a non-empty list of `branch_policies` while `deployment_branch_policy` is not set to `selected` triggers a warning + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + environments: [ + orgs.newEnvironment('linux') { + deployment_branch_policy: "protected", + reviewers+: [ + "@OtterdogTest/eclipsefdn-security", + "@netomi" + ], + wait_timer: 30, + }, + ] + } + ] + } + ``` diff --git a/docs/reference/organization/repository/environment/secret.md b/docs/reference/organization/repository/environment/secret.md new file mode 100644 index 00000000..8d2ce270 --- /dev/null +++ b/docs/reference/organization/repository/environment/secret.md @@ -0,0 +1,64 @@ +Definition of a `Secret` on repository environment level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|------------------------------------------------|------| +| _name_ | string | The name of the secret | | +| _value_ | string | The secret value | | + +The secret value can be resolved via a credential provider. The supported format is `:`. + +- Bitwarden: `bitwarden:@` + + ``` json + "secret": "bitwarden:118276ad-158c-4720-b68d-af8c00fe3481@secret" + ``` + +- Pass: `pass:` + + ``` json + "secret": "pass:path/to/repo/secret" + ``` + +!!! note + + After executing an `import` operation, the secret will be set to `********` as GitHub will not disclose the + secret value anymore via its API. You will need to update the configuration with the real secret value, either + by entering the secret value (not advised), or referencing it via a credential provider. + + Secrets which have a redacted value defined will be skipped during processing. + +## Jsonnet Function + +``` jsonnet +orgs.newEnvSecret('') { + : +} +``` + +## Validation rules + +- redacted secret values (`********`) trigger a validation info and will skip the secret during processing + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + environments: [ + orgs.newEnvironment('Environment') { + secrets: [ + orgs.newEnvSecret('TEST_SECRET') { + value: "pass:path/to/secret", + }, + ], + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/repository/environment/variable.md b/docs/reference/organization/repository/environment/variable.md new file mode 100644 index 00000000..0432573b --- /dev/null +++ b/docs/reference/organization/repository/environment/variable.md @@ -0,0 +1,42 @@ +Definition of a `Variable` on repository environment level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|--------------------------|------| +| _name_ | string | The name of the variable | | +| _value_ | string | The variable value | | + +## Jsonnet Function + +``` jsonnet +orgs.newEnvVariable('') { + : +} +``` + +## Validation rules + +- None + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + environments: [ + orgs.newEnvironment('Environment') { + variables: [ + orgs.newEnvVariable('TEST_VARIABLE') { + value: "TESTVALUE", + }, + ], + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/repository/index.md b/docs/reference/organization/repository/index.md index f60d3bb5..a7f49fc5 100644 --- a/docs/reference/organization/repository/index.md +++ b/docs/reference/organization/repository/index.md @@ -51,8 +51,9 @@ Definition of a Repository for a GitHub organization, the following properties a | _webhooks_ | list\[[Webhook](webhook.md)\] | webhooks defined for this repo, see section above for details | | | _secrets_ | list\[[RepositorySecret](secret.md)\] | secrets defined for this repo, see section below for details | | | _variables_ | list\[[RepositoryVariable](variable.md)\] | variables defined for this repo, see section below for details | | -| _environments_ | list\[[Environment](environment.md)\] | environments defined for this repo, see section below for details | | +| _environments_ | list\[[Environment](environment/index.md)\] | environments defined for this repo, see section below for details | | | _branch_protection_rules_ | list\[[BranchProtectionRule](branch-protection-rule.md)\] | branch protection rules of the repo, see section below for details | | +| _team_permissions_ | list\[[TeamPermission](team-permission.md)\] | team permissions defined for this repo, see section below for details | allowed are the following: `pull`, `triage`, `push`, `maintain`, `admin` or `READ`, `WRITE`, `MAINTAIN`, `TRIAGE`, `ADMIN` (The latter values come from github graphql) | ## Embedded Models @@ -139,6 +140,12 @@ Definition of a Repository for a GitHub organization, the following properties a branch_protection_rules: [ orgs.newBranchProtectionRule('main'), ], + team_permissions: [ + orgs.newTeamPermission('team') { + permission: "maintain", + }, + ], }, + } ``` diff --git a/docs/reference/organization/repository/team-permission.md b/docs/reference/organization/repository/team-permission.md new file mode 100644 index 00000000..0cf7e846 --- /dev/null +++ b/docs/reference/organization/repository/team-permission.md @@ -0,0 +1,38 @@ +Definition of a `Team Permission`, the following properties are supported: + +| Key | Value | Description | Notes | +|----------------------------|-----------------------------------------|---------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| _name_ | string | The name of the team in the organization. | | +| _permission_ | string | The name of the permission. | allowed are the following: `pull`, `triage`, `push`, `maintain`, `admin` or `READ`, `WRITE`, `MAINTAIN`, `TRIAGE`, `ADMIN` (The latter values come from github graphql) | + +## Jsonnet Function + +``` jsonnet +orgs.newTeamPermission('') { + : +} +``` + +## Validation rules + +- allowed values are the following `pull`, `triage`, `push`, `maintain`, `admin` or `READ`, `WRITE`, `MAINTAIN`, `TRIAGE`, `ADMIN`. + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + team_permissions: [ + orgs.newTeamPermission('team') { + permission: "maintain", + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/team.md b/docs/reference/organization/team.md index 45cc7338..71b1cbce 100644 --- a/docs/reference/organization/team.md +++ b/docs/reference/organization/team.md @@ -9,6 +9,8 @@ Definition of an organization `Team`, the following properties are supported: | _members_ | list[string] | List of users that should be a member of the team | | | _skip_members_ | boolean | If `true`, team members will be ignored | | | _skip_non_organization_members_ | boolean | If `true`, users which are not yet organization members can not be added to the team | | +| _team_sync_ | string | The id of an IdP group which is connected to a team on Github Enterprise Cloud | | +| _external_groups_ | string | The id of an external group which is provisioned on the enterprise | | ## Jsonnet Function diff --git a/examples/template/otterdog-defaults.libsonnet b/examples/template/otterdog-defaults.libsonnet index 486353c9..7632a1f0 100644 --- a/examples/template/otterdog-defaults.libsonnet +++ b/examples/template/otterdog-defaults.libsonnet @@ -109,7 +109,10 @@ local newRepo(name) = { branch_protection_rules: [], # rulesets - rulesets: [] + rulesets: [], + + # team permissions + team_permissions: [] }; # Function to extend an existing repo with the same name. @@ -223,6 +226,12 @@ local newOrgWebhook(url) = { # Function to create a new repository webhook with default settings. local newRepoWebhook(url) = newOrgWebhook(url); +# Function to create a new environment secret with default settings. +local newEnvSecret(name) = { + name: name, + value: null +}; + # Function to create a new repository secret with default settings. local newRepoSecret(name) = { name: name, @@ -235,6 +244,12 @@ local newOrgSecret(name) = newRepoSecret(name) { selected_repositories: [], }; +# Function to create a new environment variable with default settings. +local newEnvVariable(name) = { + name: name, + value: null +}; + # Function to create a new repository variable with default settings. local newRepoVariable(name) = { name: name, @@ -261,9 +276,19 @@ local newTeam(name) = { description: "", privacy: "visible", notifications: true, - members: [], + #members: [], skip_members: false, skip_non_organization_members: false, + team_sync_id: null, + team_sync_name: null, + team_sync_description: null, + external_groups: null, +}; + +# Function to create a new term permission with default settings. +local newTeamPermission(name) = { + name: name, + permission: "pull", }; # Function to create a new environment with default settings. @@ -274,6 +299,10 @@ local newEnvironment(name) = { # Can be one of: all, protected_branches, branch_policies deployment_branch_policy: "all", branch_policies: [], + # environment secrets + secrets: [], + # environment variables + variables: [], }; # Function to create a new custom property with default settings. @@ -423,7 +452,6 @@ local newOrg(name, id=name) = { { newOrg:: newOrg, newOrgRole:: newOrgRole, - newTeam:: newTeam, newOrgWebhook:: newOrgWebhook, newOrgSecret:: newOrgSecret, newOrgVariable:: newOrgVariable, @@ -434,10 +462,14 @@ local newOrg(name, id=name) = { newRepoWebhook:: newRepoWebhook, newRepoSecret:: newRepoSecret, newRepoVariable:: newRepoVariable, + newEnvSecret:: newEnvSecret, + newEnvVariable:: newEnvVariable, newBranchProtectionRule:: newBranchProtectionRule, newRepoRuleset:: newRepoRuleset, newEnvironment:: newEnvironment, newPullRequest:: newPullRequest, newStatusChecks:: newStatusChecks, + newTeam:: newTeam, + newTeamPermission:: newTeamPermission, newMergeQueue:: newMergeQueue, } diff --git a/mkdocs.yml b/mkdocs.yml index a6228613..2ebd42c1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,9 +58,12 @@ nav: - Repository Webhook: reference/organization/repository/webhook.md - Repository Secret: reference/organization/repository/secret.md - Repository Variable: reference/organization/repository/variable.md - - Environment: reference/organization/repository/environment.md - Branch Protection Rule: reference/organization/repository/branch-protection-rule.md - Repository Ruleset: reference/organization/repository/ruleset.md + - Environment: + - reference/organization/repository/environment/index.md + - Environment Secret: reference/organization/repository/environment/secret.md + - Environment Variable: reference/organization/repository/environment/variable.md - Referenced Types: - Actor: reference/organization/repository/actor.md - Branch or Tag: reference/organization/repository/branch-or-tag.md @@ -68,6 +71,7 @@ nav: - Repo Name Matcher: reference/organization/repo-name-matcher.md - Ref Matcher: reference/organization/repository/ref-matcher.md - Status Check: reference/organization/repository/status-check.md + - Team Permission: reference/organization/repository/team-permission.md - Policies: - reference/policies/index.md - Upload SBOM data to dependency track: reference/policies/dependency-track-upload.md diff --git a/otterdog/jsonnet.py b/otterdog/jsonnet.py index 793942c7..327755fa 100644 --- a/otterdog/jsonnet.py +++ b/otterdog/jsonnet.py @@ -12,8 +12,8 @@ from shutil import ignore_patterns from typing import Any -import aiofiles.os -import aiofiles.ospath +import aiofiles.os # type: ignore +import aiofiles.ospath # type: ignore from .logging import get_logger from .utils import jsonnet_evaluate_snippet, parse_github_url, parse_template_url @@ -29,6 +29,7 @@ class JsonnetConfig: create_org = "newOrg" create_org_role = "newOrgRole" create_org_team = "newTeam" + create_org_team_permission = "newTeamPermission" create_org_custom_property = "newCustomProperty" create_org_webhook = "newOrgWebhook" create_org_secret = "newOrgSecret" @@ -45,6 +46,9 @@ class JsonnetConfig: create_pull_request = "newPullRequest" create_status_checks = "newStatusChecks" create_merge_queue = "newMergeQueue" + create_team_permission = "newTeamPermission" + create_env_secret = "newEnvSecret" + create_env_variable = "newEnvVariable" def __init__( self, @@ -229,6 +233,26 @@ def default_repo_variable_config(self): _logger.debug("no default repo variable config found, variables will be skipped") return None + @cached_property + def default_env_secret_config(self): + try: + # load the default repo env secret config + env_secret_snippet = f"(import '{self.template_file}').{self.create_env_secret}('default')" + return jsonnet_evaluate_snippet(env_secret_snippet) + except RuntimeError: + _logger.debug("no default repo env secret config found, secrets will be skipped") + return None + + @cached_property + def default_env_variable_config(self): + try: + # load the default repo env variable config + env_variable_snippet = f"(import '{self.template_file}').{self.create_env_variable}('default')" + return jsonnet_evaluate_snippet(env_variable_snippet) + except RuntimeError: + _logger.debug("no default repo env variable config found, variables will be skipped") + return None + @cached_property def default_branch_protection_rule_config(self): try: @@ -261,6 +285,16 @@ def default_environment_config(self): _logger.debug("no default environment config found, environments will be skipped") return None + @cached_property + def default_team_permission_config(self): + try: + # load the default team permission config + teampermission_snippet = f"(import '{self.template_file}').{self.create_team_permission}('default')" + return jsonnet_evaluate_snippet(teampermission_snippet) + except RuntimeError: + _logger.debug("no default team permission config found, team permissions will be skipped") + return None + @cached_property def default_pull_request_config(self): try: diff --git a/otterdog/models/__init__.py b/otterdog/models/__init__.py index 18456bf6..8432d7f0 100644 --- a/otterdog/models/__init__.py +++ b/otterdog/models/__init__.py @@ -90,17 +90,30 @@ class LivePatch(Generic[MT]): current_object: MT | None changes: dict[str, Change] | None parent_object: ModelObject | None + grandparent_object: ModelObject | None forced_update: bool fn: LivePatchApplyFn changes_object_to_readonly: bool = False @classmethod - def of_addition(cls, expected_object: MT, parent_object: ModelObject | None, fn: LivePatchApplyFn[MT]) -> LivePatch: - return LivePatch(LivePatchType.ADD, expected_object, None, None, parent_object, False, fn) + def of_addition( + cls, + expected_object: MT, + parent_object: ModelObject | None, + grandparent_object: ModelObject | None, + fn: LivePatchApplyFn[MT], + ) -> LivePatch: + return LivePatch(LivePatchType.ADD, expected_object, None, None, parent_object, grandparent_object, False, fn) @classmethod - def of_deletion(cls, current_object: MT, parent_object: ModelObject | None, fn: LivePatchApplyFn[MT]) -> LivePatch: - return LivePatch(LivePatchType.REMOVE, None, current_object, None, parent_object, False, fn) + def of_deletion( + cls, + current_object: MT, + parent_object: ModelObject | None, + grandparent_object: ModelObject | None, + fn: LivePatchApplyFn[MT], + ) -> LivePatch: + return LivePatch(LivePatchType.REMOVE, None, current_object, None, parent_object, grandparent_object, False, fn) @classmethod def of_changes( @@ -109,6 +122,7 @@ def of_changes( current_object: MT, changes: dict[str, Change], parent_object: ModelObject | None, + grandparent_object: ModelObject | None, forced_update: bool, fn: LivePatchApplyFn[MT], changes_object_to_readonly: bool = False, @@ -119,6 +133,7 @@ def of_changes( current_object, changes, parent_object, + grandparent_object, forced_update, fn, changes_object_to_readonly, @@ -197,7 +212,7 @@ class EmbeddedModelObject(ABC): """ @abstractmethod - def validate(self, context: ValidationContext, parent_object: Any) -> None: ... + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: ... def get_difference_from(self, other: Self) -> Change | None: if not isinstance(other, self.__class__): @@ -383,7 +398,7 @@ def get_all_key_values(self) -> list[Any]: return [self.get_key_value()] @abstractmethod - def validate(self, context: ValidationContext, parent_object: Any) -> None: ... + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: ... # noinspection PyMethodMayBeStatic def execute_custom_validation_if_present(self, context: ValidationContext, filename: str) -> None: @@ -521,7 +536,9 @@ def is_embedded_model(field: dataclasses.Field) -> bool: def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject]]: yield from [] - def get_model_header(self, parent_object: ModelObject | None = None) -> str: + def get_model_header( + self, parent_object: ModelObject | None = None, grandparent_object: ModelObject | None = None + ) -> str: header = f"[bold]{self.model_object_name}[/]" if self.is_keyed(): @@ -534,13 +551,27 @@ def get_model_header(self, parent_object: ModelObject | None = None) -> str: + f", {parent_object.model_object_name}=" + f"[bold]{escape(parent_object.get_key_value())}[/]" ) + if isinstance(grandparent_object, ModelObject) and grandparent_object.is_keyed(): + header = ( + header + + f", {grandparent_object.model_object_name}=" + + f"[bold]{escape(grandparent_object.get_key_value())}[/]" + ) header = header + "]" - elif isinstance(parent_object, ModelObject) and parent_object.is_keyed(): - header = header + "\\[" - header = ( - header + f"{parent_object.model_object_name}=" + f"[bold]{escape(parent_object.get_key_value())}[/]" - ) + else: + if isinstance(parent_object, ModelObject) and parent_object.is_keyed(): + header = header + "\\[" + header = ( + header + f"{parent_object.model_object_name}=" + f"[bold]{escape(parent_object.get_key_value())}[/]" + ) + if isinstance(grandparent_object, ModelObject) and grandparent_object.is_keyed(): + header = header + "\\[" + header = ( + header + + f"{grandparent_object.model_object_name}=" + + f"[bold]{escape(grandparent_object.get_key_value())}[/]" + ) header = header + "]" return header @@ -605,7 +636,9 @@ def include_for_live_patch(self, context: LivePatchContext) -> bool: """ return True - def include_existing_object_for_live_patch(self, org_id: str, parent_object: ModelObject | None) -> bool: + def include_existing_object_for_live_patch( + self, org_id: str, parent_object: ModelObject | None, grandparent_object: ModelObject | None + ) -> bool: """ Indicates if this live ModelObject should be considered when generating a live patch. @@ -722,17 +755,26 @@ def generate_live_patch( expected_object: MT | None, current_object: MT | None, parent_object: ModelObject | None, + grandparent_object: ModelObject | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: if current_object is None: expected_object = unwrap(expected_object) - handler(LivePatch.of_addition(expected_object, parent_object, expected_object.apply_live_patch)) + handler( + LivePatch.of_addition( + expected_object, parent_object, grandparent_object, expected_object.apply_live_patch + ) + ) return if expected_object is None: current_object = unwrap(current_object) - handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + handler( + LivePatch.of_deletion( + current_object, parent_object, grandparent_object, current_object.apply_live_patch + ) + ) return modified_rule: dict[str, Change[Any]] = expected_object.get_difference_from(current_object) @@ -744,6 +786,7 @@ def generate_live_patch( current_object, modified_rule, parent_object, + grandparent_object, False, expected_object.apply_live_patch, ) @@ -755,6 +798,7 @@ def generate_live_patch_of_list( expected_objects: Sequence[MT], current_objects: Sequence[MT], parent_object: MT | None, + grandparent_object: MT | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: @@ -775,12 +819,16 @@ def generate_live_patch_of_list( break if expected_object is None: - if current_object.include_existing_object_for_live_patch(context.org_id, parent_object): - cls.generate_live_patch(None, current_object, parent_object, context, handler) + if current_object.include_existing_object_for_live_patch( + context.org_id, parent_object, grandparent_object + ): + cls.generate_live_patch(None, current_object, parent_object, grandparent_object, context, handler) continue if expected_object.include_for_live_patch(context): - cls.generate_live_patch(expected_object, current_object, parent_object, context, handler) + cls.generate_live_patch( + expected_object, current_object, parent_object, grandparent_object, context, handler + ) for k in expected_object.get_all_key_values(): expected_objects_by_all_keys.pop(k) @@ -788,7 +836,7 @@ def generate_live_patch_of_list( for _, expected_object in expected_objects_by_key.items(): if expected_object.include_for_live_patch(context): - cls.generate_live_patch(expected_object, None, parent_object, context, handler) + cls.generate_live_patch(expected_object, None, parent_object, grandparent_object, context, handler) @classmethod @abstractmethod diff --git a/otterdog/models/branch_protection_rule.py b/otterdog/models/branch_protection_rule.py index 7b084d45..769ce442 100644 --- a/otterdog/models/branch_protection_rule.py +++ b/otterdog/models/branch_protection_rule.py @@ -35,8 +35,6 @@ from otterdog.jsonnet import JsonnetConfig from otterdog.providers.github import GitHubProvider - from .repository import Repository - @dataclasses.dataclass class BranchProtectionRule(ModelObject): @@ -85,7 +83,7 @@ class BranchProtectionRule(ModelObject): def model_object_name(self) -> str: return "branch_protection_rule" - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: # when requires_approving_reviews is false, issue a warning if dependent settings # are still set to non default values. @@ -187,6 +185,9 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: ) if self.requires_deployments is True and len(self.required_deployment_environments) > 0: + if TYPE_CHECKING: + from .repository import Repository + environments = cast("Repository", parent_object).environments environments_by_name = associate_by_key(environments, lambda x: x.name) diff --git a/otterdog/models/custom_property.py b/otterdog/models/custom_property.py index b4b4ae75..97e45d9b 100644 --- a/otterdog/models/custom_property.py +++ b/otterdog/models/custom_property.py @@ -46,7 +46,7 @@ class CustomProperty(ModelObject): def model_object_name(self) -> str: return "custom_property" - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if is_set_and_valid(self.value_type): if self.value_type not in {"string", "single_select", "multi_select", "true_false"}: context.add_failure( @@ -171,16 +171,25 @@ def generate_live_patch( expected_object: CustomProperty | None, current_object: CustomProperty | None, parent_object: ModelObject | None, + grandparent_object: ModelObject | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: if current_object is None: expected_object = unwrap(expected_object) - handler(LivePatch.of_addition(expected_object, parent_object, expected_object.apply_live_patch)) + handler( + LivePatch.of_addition( + expected_object, parent_object, grandparent_object, expected_object.apply_live_patch + ) + ) return if expected_object is None: - handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + handler( + LivePatch.of_deletion( + current_object, parent_object, grandparent_object, current_object.apply_live_patch + ) + ) return modified_property: dict[str, Change[Any]] = expected_object.get_difference_from(current_object) @@ -198,6 +207,7 @@ def generate_live_patch( current_object, modified_property, parent_object, + grandparent_object, False, expected_object.apply_live_patch, ) diff --git a/otterdog/models/env_secret.py b/otterdog/models/env_secret.py new file mode 100644 index 00000000..7b0a912e --- /dev/null +++ b/otterdog/models/env_secret.py @@ -0,0 +1,73 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from otterdog.models import LivePatch, LivePatchType +from otterdog.models.secret import Secret +from otterdog.utils import expect_type, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class EnvironmentSecret(Secret): + """ + Represents a Secret defined on environment level. + """ + + @property + def model_object_name(self) -> str: + return "env_secret" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_env_secret}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[EnvironmentSecret], + org_id: str, + provider: GitHubProvider, + ) -> None: + from .environment import Environment + from .repository import Repository + + environment = expect_type(patch.parent_object, Environment) + repository = expect_type(patch.grandparent_object, Repository) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_env_secret( + org_id, + repository.name, + environment.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_env_secret( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_env_secret( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/env_variable.py b/otterdog/models/env_variable.py new file mode 100644 index 00000000..f6313105 --- /dev/null +++ b/otterdog/models/env_variable.py @@ -0,0 +1,72 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from otterdog.models import LivePatch, LivePatchType +from otterdog.models.variable import Variable +from otterdog.utils import expect_type, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class EnvironmentVariable(Variable): + """ + Represents a Variable defined on environment level. + """ + + @property + def model_object_name(self) -> str: + return "env_variable" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_env_variable}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[EnvironmentVariable], + org_id: str, + provider: GitHubProvider, + ) -> None: + from .environment import Environment + from .repository import Repository + + environment = expect_type(patch.parent_object, Environment) + repository = expect_type(patch.grandparent_object, Repository) + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_env_variable( + org_id, + repository.name, + environment.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_env_variable( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_env_variable( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/environment.py b/otterdog/models/environment.py index c4361898..b59720ab 100644 --- a/otterdog/models/environment.py +++ b/otterdog/models/environment.py @@ -9,23 +9,39 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar from jsonbender import F, Filter, Forall, If, K, OptionalS, S # type: ignore from otterdog.models import ( FailureType, LivePatch, + LivePatchContext, + LivePatchHandler, LivePatchType, ModelObject, + PatchContext, ValidationContext, ) -from otterdog.utils import expect_type, is_set_and_valid, is_unset, unwrap +from otterdog.utils import ( + Change, + IndentingPrinter, + expect_type, + is_set_and_valid, + is_unset, + unwrap, + write_patch_object_as_json, +) + +from .env_secret import EnvironmentSecret +from .env_variable import EnvironmentVariable if TYPE_CHECKING: from otterdog.jsonnet import JsonnetConfig from otterdog.providers.github import GitHubProvider +ET = TypeVar("ET", bound="Environment") + @dataclasses.dataclass class Environment(ModelObject): @@ -40,12 +56,32 @@ class Environment(ModelObject): reviewers: list[str] deployment_branch_policy: str branch_policies: list[str] + secrets: list[EnvironmentSecret] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) + variables: list[EnvironmentVariable] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) + + def add_secret(self, secret: EnvironmentSecret) -> None: + self.secrets.append(secret) + + def get_secret(self, name: str) -> EnvironmentSecret | None: + return next(filter(lambda x: x.name == name, self.secrets), None) # type: ignore + + def set_secrets(self, secrets: list[EnvironmentSecret]) -> None: + self.secrets = secrets + + def add_variable(self, variable: EnvironmentVariable) -> None: + self.variables.append(variable) + + def get_variable(self, name: str) -> EnvironmentVariable | None: + return next(filter(lambda x: x.name == name, self.variables), None) # type: ignore + + def set_variables(self, variables: list[EnvironmentVariable]) -> None: + self.variables = variables @property def model_object_name(self) -> str: return "environment" - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if not is_unset(self.wait_timer) and not (0 <= self.wait_timer <= 43200): context.add_failure( FailureType.ERROR, @@ -69,6 +105,11 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: f"'{self.deployment_branch_policy}', " f"but 'branch_policies' is set to '{self.branch_policies}', setting will be ignored.", ) + for secret in self.secrets: + secret.validate(context, self, parent_object) + + for variable in self.variables: + variable.validate(context, self, parent_object) def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: if self.deployment_branch_policy != "selected": @@ -80,7 +121,9 @@ def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool: return True - def include_existing_object_for_live_patch(self, org_id: str, parent_object: ModelObject | None) -> bool: + def include_existing_object_for_live_patch( + self, org_id: str, parent_object: ModelObject | None, grandparent_object: ModelObject | None + ) -> bool: from .repository import Repository parent_object = expect_type(parent_object, Repository) @@ -94,6 +137,19 @@ def include_existing_object_for_live_patch(self, org_id: str, parent_object: Mod else: return True + @classmethod + def get_mapping_from_model(cls) -> dict[str, Any]: + mapping = super().get_mapping_from_model() + + mapping.update( + { + "secrets": OptionalS("secrets", default=[]) >> Forall(lambda x: EnvironmentSecret.from_model_data(x)), + "variables": OptionalS("variables", default=[]) + >> Forall(lambda x: EnvironmentVariable.from_model_data(x)), + } + ) + return mapping + @classmethod def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: mapping = super().get_mapping_from_provider(org_id, data) @@ -143,6 +199,8 @@ def transform_branch_policy(x): >> Forall(lambda x: transform_reviewers(x)), "deployment_branch_policy": OptionalS("deployment_branch_policy") >> F(transform_policy), "branch_policies": OptionalS("branch_policies", default=[]) >> Forall(transform_branch_policy), + "secrets": K([]), + "variables": K([]), } ) return mapping @@ -192,6 +250,65 @@ async def get_mapping_to_provider( def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: return f"orgs.{jsonnet_config.create_environment}" + @classmethod + def generate_live_patch( + cls, + expected_object: Environment | None, + current_object: Environment | None, + parent_object: ModelObject | None, + grandparent_object: ModelObject | None, + context: LivePatchContext, + handler: LivePatchHandler, + ) -> None: + if expected_object is None: + current_object = unwrap(current_object) + handler( + LivePatch.of_deletion( + current_object, parent_object, grandparent_object, current_object.apply_live_patch + ) + ) + return + + if current_object is None: + handler( + LivePatch.of_addition( + expected_object, parent_object, grandparent_object, expected_object.apply_live_patch + ) + ) + else: + modified_rule: dict[str, Change[Any]] = expected_object.get_difference_from(current_object) + + if len(modified_rule) > 0: + handler( + LivePatch.of_changes( + expected_object, + current_object, + modified_rule, + parent_object, + grandparent_object, + False, + expected_object.apply_live_patch, + ) + ) + + EnvironmentSecret.generate_live_patch_of_list( + expected_object.secrets, + current_object.secrets if current_object is not None else [], + expected_object, + parent_object, + context, + handler, + ) + + EnvironmentVariable.generate_live_patch_of_list( + expected_object.variables, + current_object.variables if current_object is not None else [], + expected_object, + parent_object, + context, + handler, + ) + @classmethod async def apply_live_patch(cls, patch: LivePatch[Environment], org_id: str, provider: GitHubProvider) -> None: from .repository import Repository @@ -221,3 +338,54 @@ async def apply_live_patch(cls, patch: LivePatch[Environment], org_id: str, prov current_object.name, await cls.changes_to_provider(org_id, unwrap(patch.changes), provider), ) + + def to_jsonnet( + self, + printer: IndentingPrinter, + jsonnet_config: JsonnetConfig, + context: PatchContext, + extend: bool, + default_object: ModelObject, + ) -> None: + patch = self.get_patch_to(default_object) + + has_secrets = len(self.secrets) > 0 + has_variables = len(self.variables) > 0 + + if "name" in patch: + patch.pop("name") + + function = self.get_jsonnet_template_function(jsonnet_config, extend) + printer.print(f"{function}('{self.name}')") + + write_patch_object_as_json(patch, printer, close_object=False) + + # FIXME: support overriding secrets for repos coming from the default configuration. + if has_secrets: + default_env_secret = EnvironmentSecret.from_model_data(jsonnet_config.default_env_secret_config) + + printer.println("secrets: [") + printer.level_up() + + for secret in self.secrets: + secret.to_jsonnet(printer, jsonnet_config, context, False, default_env_secret) + + printer.level_down() + printer.println("],") + + # FIXME: support overriding variables for repos coming from the default configuration. + if has_variables: + default_env_variable = EnvironmentVariable.from_model_data(jsonnet_config.default_env_variable_config) + + printer.println("variables: [") + printer.level_up() + + for variable in self.variables: + variable.to_jsonnet(printer, jsonnet_config, context, False, default_env_variable) + + printer.level_down() + printer.println("],") + + # close the repo object + printer.level_down() + printer.println("},") diff --git a/otterdog/models/github_organization.py b/otterdog/models/github_organization.py index 0e22a2e4..624e777d 100644 --- a/otterdog/models/github_organization.py +++ b/otterdog/models/github_organization.py @@ -31,6 +31,8 @@ ) from otterdog.models.branch_protection_rule import BranchProtectionRule from otterdog.models.custom_property import CustomProperty +from otterdog.models.env_secret import EnvironmentSecret +from otterdog.models.env_variable import EnvironmentVariable from otterdog.models.environment import Environment from otterdog.models.organization_role import OrganizationRole from otterdog.models.organization_ruleset import OrganizationRuleset @@ -46,6 +48,7 @@ from otterdog.models.repo_workflow_settings import RepositoryWorkflowSettings from otterdog.models.repository import Repository from otterdog.models.team import Team +from otterdog.models.team_permission import TeamPermission from otterdog.utils import IndentingPrinter, associate_by_key, debug_times, jsonnet_evaluate_file if TYPE_CHECKING: @@ -172,7 +175,7 @@ async def validate( config.exclude_teams_pattern, provider=provider, ) - self.settings.validate(context, self) + self.settings.validate(context, self, None) enterprise_plan = self.settings.plan == "enterprise" @@ -184,16 +187,16 @@ async def validate( ) else: for role in self.roles: - role.validate(context, self) + role.validate(context, self, None) for team in self.teams: - team.validate(context, self) + team.validate(context, self, None) for webhook in self.webhooks: - webhook.validate(context, self) + webhook.validate(context, self, None) for secret in self.secrets: - secret.validate(context, self) + secret.validate(context, self, None) if len(self.rulesets) > 0 and not enterprise_plan: context.add_failure( @@ -203,12 +206,12 @@ async def validate( ) else: for ruleset in self.rulesets: - ruleset.validate(context, self) + ruleset.validate(context, self, None) # Run synchronous validations and collect repos needing API codescaning validation repos_needing_codescaning_language_validation = [] for repo in self.repositories: - repo.validate(context, self) + repo.validate(context, self, None) if repo.requires_language_validation(): repos_needing_codescaning_language_validation.append(repo) @@ -225,7 +228,7 @@ async def validate( @staticmethod def _validate_org_config(data: dict[str, Any]) -> None: - from jsonschema import Draft202012Validator + from jsonschema import Draft202012Validator # type: ignore from referencing import Registry, Resource from referencing.exceptions import NoSuchResource @@ -474,23 +477,27 @@ def to_jsonnet(self, config: JsonnetConfig, context: PatchContext) -> str: def generate_live_patch( self, current_organization: GitHubOrganization, context: LivePatchContext, handler: LivePatchHandler ) -> None: - OrganizationRole.generate_live_patch_of_list(self.roles, current_organization.roles, None, context, handler) - Team.generate_live_patch_of_list(self.teams, current_organization.teams, None, context, handler) - OrganizationSettings.generate_live_patch(self.settings, current_organization.settings, None, context, handler) + OrganizationRole.generate_live_patch_of_list( + self.roles, current_organization.roles, None, None, context, handler + ) + Team.generate_live_patch_of_list(self.teams, current_organization.teams, None, None, context, handler) + OrganizationSettings.generate_live_patch( + self.settings, current_organization.settings, None, None, context, handler + ) OrganizationWebhook.generate_live_patch_of_list( - self.webhooks, current_organization.webhooks, None, context, handler + self.webhooks, current_organization.webhooks, None, None, context, handler ) OrganizationSecret.generate_live_patch_of_list( - self.secrets, current_organization.secrets, None, context, handler + self.secrets, current_organization.secrets, None, None, context, handler ) OrganizationVariable.generate_live_patch_of_list( - self.variables, current_organization.variables, None, context, handler + self.variables, current_organization.variables, None, None, context, handler ) OrganizationRuleset.generate_live_patch_of_list( - self.rulesets, current_organization.rulesets, None, context, handler + self.rulesets, current_organization.rulesets, None, None, context, handler ) Repository.generate_live_patch_of_list( - self.repositories, current_organization.repositories, None, context, handler + self.repositories, current_organization.repositories, None, None, context, handler ) @classmethod @@ -583,6 +590,19 @@ async def _load_teams() -> None: continue team_members = await provider.get_org_team_members(github_id, team_slug) team["members"] = team_members + # Do the team-sync + sync_groups = await provider.get_org_team_sync_groups(github_id, team_slug) + if sync_groups: + team["team_sync_id"] = sync_groups[0].get("group_id", None) + team["team_sync_name"] = sync_groups[0].get("group_name", None) + team["team_sync_description"] = sync_groups[0].get("group_description", None) + else: + team["team_sync_id"] = None + team["team_sync_name"] = None + team["team_sync_description"] = None + # External Groups + external_groups = await provider.get_org_team_external_groups(github_id, team_slug) + team["external_groups"] = external_groups org.add_team(Team.from_provider_data(github_id, team)) else: _logger.debug("not reading teams, no default config available") @@ -672,6 +692,7 @@ async def _process_single_repo( repo_name: str, jsonnet_config: JsonnetConfig, teams: dict[str, Any], + repo_permissions: dict[str, list[dict[str, Any]]], app_installations: dict[str, str], ) -> tuple[str, Repository]: rest_api = gh_client.rest_api @@ -750,15 +771,60 @@ async def _process_single_repo( # get environments of the repo environments = await rest_api.repo.get_environments(github_id, repo_name) for github_environment in environments: - repo.add_environment(Environment.from_provider_data(github_id, github_environment)) + environment = Environment.from_provider_data(github_id, github_environment) + repo.add_environment(environment) + if jsonnet_config.default_env_variable_config is not None: + # get variables of the repo environment + variables = await rest_api.env.get_variables(github_id, repo.name, environment.name) + for github_variable in variables: + environment.add_variable(EnvironmentVariable.from_provider_data(github_id, github_variable)) + else: + _logger.debug("not reading repo env variables, no default config available") + if jsonnet_config.default_env_secret_config is not None: + # get secrets of the repo environment + secrets = await rest_api.env.get_secrets(github_id, repo.name, environment.name) + for github_secret in secrets: + environment.add_secret(EnvironmentSecret.from_provider_data(github_id, github_secret)) + else: + _logger.debug("not reading repo env secrets, no default config available") else: _logger.debug("not reading environments, no default config available") + if jsonnet_config.default_team_permission_config is not None: + team_permissions = repo_permissions.get(repo_name, []) + for github_team_permission in team_permissions: + repo.add_team_permission(TeamPermission.from_provider_data(github_id, github_team_permission)) + else: + _logger.debug("not reading team permissions, no default config available") _logger.debug("done retrieving data for repo '%s'", repo_name) return repo_name, repo +def build_repo_permissions(teams: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + """ + Convert the output from the graphql call, which is team-centric, to a repository + centric structure. + """ + + repo_permissions: dict[str, list[dict[str, Any]]] = {} + for team in teams: + team_slug = team["slug"] + + # List of repo edges of this team + edges = team.get("repositories", {}).get("edges", []) + + for edge in edges: + repo_name = edge["node"]["name"] + permission = edge["permission"] + + if repo_name not in repo_permissions: + repo_permissions[repo_name] = [] + + repo_permissions[repo_name].append({"name": team_slug, "permission": permission}) + return repo_permissions + + async def _load_repos_from_provider( github_id: str, org_settings: OrganizationSettings, @@ -776,6 +842,7 @@ async def _load_repos_from_provider( repo_names = fnmatch.filter(repo_names, repo_filter) teams = {str(team["id"]): f"{github_id}/{team['slug']}" for team in await provider.get_org_teams(github_id)} + repo_permissions = build_repo_permissions(await provider.get_team_permissions(github_id)) # limit the number of repos that are processed concurrently to avoid hitting secondary rate limits sem = asyncio.Semaphore(50 if concurrency is None else concurrency) @@ -789,6 +856,7 @@ async def safe_process(repo_name): repo_name, jsonnet_config, teams, + repo_permissions, app_installations, ) diff --git a/otterdog/models/organization_ruleset.py b/otterdog/models/organization_ruleset.py index 23964597..c297e090 100644 --- a/otterdog/models/organization_ruleset.py +++ b/otterdog/models/organization_ruleset.py @@ -20,7 +20,6 @@ if TYPE_CHECKING: from otterdog.jsonnet import JsonnetConfig - from otterdog.models.github_organization import GitHubOrganization from otterdog.providers.github import GitHubProvider @@ -41,9 +40,11 @@ def model_object_name(self) -> str: def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: return f"orgs.{jsonnet_config.create_org_ruleset}" - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + if TYPE_CHECKING: + from otterdog.models.github_organization import GitHubOrganization - super().validate(context, parent_object) + super().validate(context, parent_object, grandparent_object) repositories = cast("GitHubOrganization", context.root_object).repositories all_repo_names = (x.name for x in repositories) diff --git a/otterdog/models/organization_secret.py b/otterdog/models/organization_secret.py index 412f83bb..1cbc7251 100644 --- a/otterdog/models/organization_secret.py +++ b/otterdog/models/organization_secret.py @@ -21,8 +21,6 @@ from otterdog.jsonnet import JsonnetConfig from otterdog.providers.github import GitHubProvider - from .github_organization import GitHubOrganization - @dataclasses.dataclass class OrganizationSecret(Secret): @@ -37,10 +35,13 @@ class OrganizationSecret(Secret): def model_object_name(self) -> str: return "org_secret" - def validate(self, context: ValidationContext, parent_object: Any) -> None: - super().validate(context, parent_object) + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + super().validate(context, parent_object, grandparent_object) if is_set_and_valid(self.visibility): + if TYPE_CHECKING: + from .github_organization import GitHubOrganization + org = cast("GitHubOrganization", parent_object) if self.visibility == "private" and org.settings.plan == "free": context.add_failure( diff --git a/otterdog/models/organization_settings.py b/otterdog/models/organization_settings.py index 0a76ba1d..b6280810 100644 --- a/otterdog/models/organization_settings.py +++ b/otterdog/models/organization_settings.py @@ -100,7 +100,7 @@ def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool: return True - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: # execute custom validation rules if present self.execute_custom_validation_if_present(context, "validate-org-settings.py") @@ -143,10 +143,10 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: if is_set_and_present(self.custom_properties): for custom_property in self.custom_properties: - custom_property.validate(context, self) + custom_property.validate(context, self, None) if is_set_and_present(self.workflows): - self.workflows.validate(context, self) + self.workflows.validate(context, self, None) def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject]]: if is_set_and_present(self.custom_properties): @@ -254,6 +254,7 @@ def generate_live_patch( expected_object: OrganizationSettings | None, current_object: OrganizationSettings | None, parent_object: ModelObject | None, + grandparent_object: ModelObject | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: @@ -278,6 +279,7 @@ def generate_live_patch( current_object, modified_settings, parent_object, + grandparent_object, False, cls.apply_live_patch, ) @@ -290,6 +292,7 @@ def generate_live_patch( expected_object.custom_properties, current_object.custom_properties, expected_object, + None, context, handler, ) diff --git a/otterdog/models/organization_variable.py b/otterdog/models/organization_variable.py index a015a960..56924d11 100644 --- a/otterdog/models/organization_variable.py +++ b/otterdog/models/organization_variable.py @@ -21,8 +21,6 @@ from otterdog.jsonnet import JsonnetConfig from otterdog.providers.github import GitHubProvider - from .github_organization import GitHubOrganization - @dataclasses.dataclass class OrganizationVariable(Variable): @@ -37,10 +35,13 @@ class OrganizationVariable(Variable): def model_object_name(self) -> str: return "org_variable" - def validate(self, context: ValidationContext, parent_object: Any) -> None: - super().validate(context, parent_object) + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + super().validate(context, parent_object, grandparent_object) if is_set_and_valid(self.visibility): + if TYPE_CHECKING: + from .github_organization import GitHubOrganization + org = cast("GitHubOrganization", parent_object) if self.visibility == "private" and org.settings.plan == "free": context.add_failure( diff --git a/otterdog/models/organization_workflow_settings.py b/otterdog/models/organization_workflow_settings.py index ec18461c..b9e3748e 100644 --- a/otterdog/models/organization_workflow_settings.py +++ b/otterdog/models/organization_workflow_settings.py @@ -48,8 +48,8 @@ def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: return super().include_field_for_diff_computation(field) - def validate(self, context: ValidationContext, parent_object: Any) -> None: - super().validate(context, parent_object) + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + super().validate(context, parent_object, grandparent_object) if is_set_and_valid(self.enabled_repositories): if self.enabled_repositories not in {"all", "none", "selected"}: diff --git a/otterdog/models/repo_workflow_settings.py b/otterdog/models/repo_workflow_settings.py index d12cdb0c..8a724bdd 100644 --- a/otterdog/models/repo_workflow_settings.py +++ b/otterdog/models/repo_workflow_settings.py @@ -25,9 +25,6 @@ from otterdog.models.organization_workflow_settings import OrganizationWorkflowSettings from otterdog.providers.github import GitHubProvider - from .github_organization import GitHubOrganization - from .repository import Repository - @dataclasses.dataclass class RepositoryWorkflowSettings(WorkflowSettings): @@ -45,6 +42,9 @@ def coerce_from_org_settings( if org_workflow_settings.enabled_repositories == "none": copy.enabled = UNSET # type: ignore + if TYPE_CHECKING: + from otterdog.models.repository import Repository + repository_name = cast("Repository", parent_object).name if ( @@ -66,13 +66,19 @@ def coerce_from_org_settings( return copy - def validate(self, context: ValidationContext, parent_object: Any) -> None: - super().validate(context, parent_object) + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + if TYPE_CHECKING: + from .repository import Repository + + super().validate(context, parent_object, grandparent_object) repo = cast("Repository", parent_object) actions_enabled = None if is_set_and_valid(self.enabled) and self.enabled is True: + if TYPE_CHECKING: + from .github_organization import GitHubOrganization + actions_enabled = True org_workflow_settings = cast("GitHubOrganization", context.root_object).settings.workflows diff --git a/otterdog/models/repository.py b/otterdog/models/repository.py index a4c3a235..f77f059c 100644 --- a/otterdog/models/repository.py +++ b/otterdog/models/repository.py @@ -42,6 +42,7 @@ from .repo_variable import RepositoryVariable from .repo_webhook import RepositoryWebhook from .repo_workflow_settings import RepositoryWorkflowSettings +from .team_permission import TeamPermission if TYPE_CHECKING: from collections.abc import Callable, Iterator @@ -122,6 +123,7 @@ class Repository(ModelObject): ) rulesets: list[RepositoryRuleset] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) environments: list[Environment] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) + team_permissions: list[TeamPermission] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) _security_properties: ClassVar[list[str]] = [ "secret_scanning", @@ -245,6 +247,12 @@ def add_environment(self, environment: Environment) -> None: def set_environments(self, environments: list[Environment]) -> None: self.environments = environments + def add_team_permission(self, team_permission: TeamPermission) -> None: + self.team_permissions.append(team_permission) + + def set_team_permisions(self, team_permissions: list[TeamPermission]) -> None: + self.team_permissions = team_permissions + def coerce_from_org_settings(self, org_settings: OrganizationSettings, for_patch: bool = False) -> Repository: copy = dataclasses.replace(self) @@ -342,7 +350,7 @@ async def validate_code_scanning_languages(self, context: ValidationContext, par f"{self.get_model_header(parent_object)} could not validate 'code_scanning_default_languages': {e}", ) - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: github_id = cast("GitHubOrganization", parent_object).github_id org_settings = cast("GitHubOrganization", parent_object).settings @@ -455,7 +463,7 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: ) for webhook in self.webhooks: - webhook.validate(context, self) + webhook.validate(context, self, None) if self.archived is True: if len(self.branch_protection_rules) > 0: @@ -659,22 +667,25 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: ) if is_set_and_present(self.workflows): - self.workflows.validate(context, self) + self.workflows.validate(context, self, None) for secret in self.secrets: - secret.validate(context, self) + secret.validate(context, self, None) for variable in self.variables: - variable.validate(context, self) + variable.validate(context, self, None) for bpr in self.branch_protection_rules: - bpr.validate(context, self) + bpr.validate(context, self, None) for rule in self.rulesets: - rule.validate(context, self) + rule.validate(context, self, None) for env in self.environments: - env.validate(context, self) + env.validate(context, self, None) + + for tp in self.team_permissions: + tp.validate(context, self, None) @staticmethod def _valid_topic(topic, search=re.compile(r"[^a-z0-9\-]").search): @@ -776,6 +787,10 @@ def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject]]: yield env, self yield from env.get_model_objects() + for tp in self.team_permissions: + yield tp, self + yield from tp.get_model_objects() + @classmethod def get_mapping_from_model(cls) -> dict[str, Any]: mapping = super().get_mapping_from_model() @@ -796,6 +811,8 @@ def get_mapping_from_model(cls) -> dict[str, Any]: K(UNSET), S("workflows") >> F(lambda x: RepositoryWorkflowSettings.from_model_data(x)), ), + "team_permissions": OptionalS("team_permissions", default=[]) + >> Forall(lambda x: TeamPermission.from_model_data(x)), } ) @@ -863,6 +880,7 @@ def property_list_to_map(properties): "branch_protection_rules": K([]), "rulesets": K([]), "environments": K([]), + "team_permissions": K([]), "secret_scanning": OptionalS("security_and_analysis", "secret_scanning", "status", default=UNSET), "secret_scanning_push_protection": OptionalS( "security_and_analysis", @@ -1023,6 +1041,7 @@ def to_jsonnet( has_branch_protection_rules = len(self.branch_protection_rules) > 0 has_rulesets = len(self.rulesets) > 0 has_environments = len(self.environments) > 0 + has_team_permissions = len(self.team_permissions) > 0 if "name" in patch: patch.pop("name") @@ -1139,6 +1158,20 @@ def to_jsonnet( printer.level_down() printer.println("],") + # FIXME: support overrding team permissions for repos coming from + # the default configuration. + if has_team_permissions and not extend: + default_teampermission = TeamPermission.from_model_data(jsonnet_config.default_team_permission_config) + + printer.println("team_permissions: [") + printer.level_up() + + for tp in self.team_permissions: + tp.to_jsonnet(printer, jsonnet_config, context, False, default_teampermission) + + printer.level_down() + printer.println("],") + # close the repo object printer.level_down() printer.println("},") @@ -1149,12 +1182,17 @@ def generate_live_patch( expected_object: Repository | None, current_object: Repository | None, parent_object: ModelObject | None, + grandparent_object: ModelObject | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: if expected_object is None: current_object = unwrap(current_object) - handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + handler( + LivePatch.of_deletion( + current_object, parent_object, grandparent_object, current_object.apply_live_patch + ) + ) return expected_object = unwrap(expected_object) @@ -1170,7 +1208,11 @@ def generate_live_patch( changes_object_to_readonly = False if current_object is None: - handler(LivePatch.of_addition(coerced_object, parent_object, coerced_object.apply_live_patch)) + handler( + LivePatch.of_addition( + coerced_object, parent_object, grandparent_object, coerced_object.apply_live_patch + ) + ) else: if context.current_org_settings is not None: current_org_settings = cast("OrganizationSettings", context.current_org_settings) @@ -1212,6 +1254,7 @@ def generate_live_patch( current_object, modified_repo, parent_object, + grandparent_object, False, coerced_object.apply_live_patch, changes_object_to_readonly, @@ -1222,6 +1265,7 @@ def generate_live_patch( coerced_object.webhooks, current_object.webhooks if current_object is not None else [], coerced_object, + None, context, handler, ) @@ -1230,6 +1274,7 @@ def generate_live_patch( coerced_object.secrets, current_object.secrets if current_object is not None else [], coerced_object, + None, context, handler, ) @@ -1238,6 +1283,7 @@ def generate_live_patch( coerced_object.variables, current_object.variables if current_object is not None else [], coerced_object, + None, context, handler, ) @@ -1246,6 +1292,7 @@ def generate_live_patch( coerced_object.environments, current_object.environments if current_object is not None else [], coerced_object, + None, context, handler, ) @@ -1258,6 +1305,7 @@ def generate_live_patch( coerced_object.branch_protection_rules, (current_object.branch_protection_rules if current_object is not None else []), coerced_object, + None, context, handler, ) @@ -1266,10 +1314,20 @@ def generate_live_patch( coerced_object.rulesets, current_object.rulesets if current_object is not None else [], coerced_object, + None, context, handler, ) + TeamPermission.generate_live_patch_of_list( + coerced_object.team_permissions, + current_object.team_permissions if current_object is not None else [], + coerced_object, + None, + context, + handler, + ) + @staticmethod def _include_squash_merge_patch_required_properties( patch: LivePatch[Repository], diff --git a/otterdog/models/role.py b/otterdog/models/role.py index 7a982091..2e23f43c 100644 --- a/otterdog/models/role.py +++ b/otterdog/models/role.py @@ -37,7 +37,7 @@ class Role(ModelObject, abc.ABC): permissions: list[str] base_role: str - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if is_set_and_valid(self.base_role): if self.base_role not in {"none", "read", "triage", "write", "maintain", "admin"}: context.add_failure( diff --git a/otterdog/models/ruleset.py b/otterdog/models/ruleset.py index c9018f0b..f37fb41f 100644 --- a/otterdog/models/ruleset.py +++ b/otterdog/models/ruleset.py @@ -38,9 +38,6 @@ from otterdog.jsonnet import JsonnetConfig from otterdog.providers.github import GitHubProvider - from .github_organization import GitHubOrganization - from .repository import Repository - RS = TypeVar("RS", bound="Ruleset") _logger = get_logger(__name__) @@ -54,7 +51,7 @@ class PullRequestSettings(EmbeddedModelObject): requires_last_push_approval: bool requires_review_thread_resolution: bool - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: for key in self.keys(False): value = self.__getattribute__(key) if is_unset(value): @@ -125,7 +122,7 @@ class StatusCheckSettings(EmbeddedModelObject): strict: bool status_checks: list[str] - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: for key in ["strict", "status_checks"]: value = self.__getattribute__(key) if is_unset(value): @@ -179,7 +176,7 @@ async def get_app_ids(checks) -> dict[str, str]: for check in checks: if ":" in check: - app_slug, _context = re.split(":", check, maxsplit=1) + app_slug, _ = re.split(":", check, maxsplit=1) if app_slug != "any" and " " not in app_slug: app_slugs.add(app_slug) @@ -235,7 +232,7 @@ class MergeQueueSettings(EmbeddedModelObject): status_check_timeout: int requires_all_group_entries_to_pass_required_checks: bool - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: for key in self.keys(False): value = self.__getattribute__(key) if is_unset(value): @@ -358,7 +355,9 @@ class Ruleset(ModelObject, abc.ABC): _roles: ClassVar[dict[str, str]] = {"5": "RepositoryAdmin", "4": "Write", "2": "Maintain", "1": "OrganizationAdmin"} _inverted_roles: ClassVar[dict[str, str]] = {v: k for k, v in _roles.items()} - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + if TYPE_CHECKING: + from .github_organization import GitHubOrganization org_settings = cast("GitHubOrganization", context.root_object).settings @@ -436,6 +435,9 @@ def valid_tag_condition_pattern(pattern: str) -> bool: ) if self.requires_deployments is True and len(self.required_deployment_environments) > 0: + if TYPE_CHECKING: + from .repository import Repository + environments = cast("Repository", parent_object).environments environments_by_name = associate_by_key(environments, lambda x: x.name) @@ -448,10 +450,10 @@ def valid_tag_condition_pattern(pattern: str) -> bool: ) if is_set_and_present(self.required_pull_request): - self.required_pull_request.validate(context, parent_object) + self.required_pull_request.validate(context, parent_object, None) if is_set_and_present(self.required_merge_queue): - self.required_merge_queue.validate(context, parent_object) + self.required_merge_queue.validate(context, parent_object, None) def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: if self.requires_deployments is False: diff --git a/otterdog/models/secret.py b/otterdog/models/secret.py index 7335f259..f0da802b 100644 --- a/otterdog/models/secret.py +++ b/otterdog/models/secret.py @@ -42,7 +42,7 @@ class Secret(ModelObject, abc.ABC): name: str = dataclasses.field(metadata={"key": True}) value: str - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if self.has_dummy_secret(): context.add_failure( FailureType.INFO, f"{self.get_model_header()} only has a dummy value, resource will be skipped." @@ -116,17 +116,26 @@ def generate_live_patch( expected_object: ST | None, current_object: ST | None, parent_object: ModelObject | None, + grandparent_object: ModelObject | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: if current_object is None: expected_object = unwrap(expected_object) - handler(LivePatch.of_addition(expected_object, parent_object, expected_object.apply_live_patch)) + handler( + LivePatch.of_addition( + expected_object, parent_object, grandparent_object, expected_object.apply_live_patch + ) + ) return if expected_object is None: current_object = unwrap(current_object) - handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + handler( + LivePatch.of_deletion( + current_object, parent_object, grandparent_object, current_object.apply_live_patch + ) + ) return # if secrets shall be updated and the secret contains a valid secret perform a forced update. @@ -140,6 +149,7 @@ def generate_live_patch( current_object, modified_secret, parent_object, + grandparent_object, True, expected_object.apply_live_patch, ) @@ -170,6 +180,7 @@ def has_valid_secret(secret: Secret): current_object, modified_secret, parent_object, + grandparent_object, False, expected_object.apply_live_patch, ) diff --git a/otterdog/models/team.py b/otterdog/models/team.py index 288e0eab..b197a1f9 100644 --- a/otterdog/models/team.py +++ b/otterdog/models/team.py @@ -21,7 +21,7 @@ ModelObject, ValidationContext, ) -from otterdog.utils import UNSET, is_set_and_valid, unwrap +from otterdog.utils import UNSET, is_set_and_valid, is_unset, unwrap if TYPE_CHECKING: from otterdog.jsonnet import JsonnetConfig @@ -43,6 +43,10 @@ class Team(ModelObject, abc.ABC): privacy: str notifications: bool members: list[str] + team_sync_id: str | None + team_sync_name: str | None + team_sync_description: str | None + external_groups: str | None skip_members: bool = dataclasses.field(metadata={"model_only": True}, default=False) skip_non_organization_members: bool = dataclasses.field(metadata={"model_only": True}, default=False) @@ -59,7 +63,7 @@ def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: return True - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: # execute custom validation rules if present self.execute_custom_validation_if_present(context, "validate-team.py") @@ -98,6 +102,23 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: f"but 'members' contains user '{member}' who is not an organization member.", ) + values = [ + self.team_sync_id, + self.team_sync_name, + self.team_sync_description, + ] + + all_strings = all(is_set_and_valid(v) for v in values) + all_unset_or_none = all(is_unset(v) or v is None for v in values) + + if not all_strings and not all_unset_or_none: + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has inconsistent team sync configuration: " + f"all of 'team_sync_id', 'team_sync_name', and 'team_sync_description' must either " + f"all be unset/None or all be valid strings.", + ) + @classmethod def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: mapping = super().get_mapping_from_provider(org_id, data) @@ -113,11 +134,17 @@ def transform_notification_setting(x: str | None): def transform_team_members(member): return member["login"] + def transform_external_groups(value): + if isinstance(value, list) and value: + return value[0]["group_id"] + return None + mapping.update( { "privacy": OptionalS("privacy") >> F(lambda x: "visible" if x == "closed" else x), "notifications": OptionalS("notification_setting") >> F(transform_notification_setting), "members": OptionalS("members", default=[]) >> Forall(transform_team_members), + "external_groups": OptionalS("external_groups") >> F(transform_external_groups), } ) return mapping diff --git a/otterdog/models/team_permission.py b/otterdog/models/team_permission.py new file mode 100644 index 00000000..4663d9de --- /dev/null +++ b/otterdog/models/team_permission.py @@ -0,0 +1,123 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING, Any + +from jsonbender import F, OptionalS # type: ignore + +from otterdog.models import ( + FailureType, + LivePatch, + LivePatchType, + ModelObject, + ValidationContext, +) +from otterdog.utils import ( + expect_type, + is_set_and_valid, + unwrap, +) + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class TeamPermission(ModelObject): + """ + Represents a Team Permission on a Repository. + """ + + name: str = dataclasses.field(metadata={"key": True}) + permission: str + + @property + def model_object_name(self) -> str: + return "team_permission" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_org_team_permission}" + + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: + if is_set_and_valid(self.permission): + if self.permission not in { + "pull", + "triage", + "push", + "maintain", + "admin", + "READ", + "WRITE", + "MAINTAIN", + "TRIAGE", + "ADMIN", + }: + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has 'permission' of value '{self.permission}', " + f"while only values ('read/pull' | 'triage' | 'write/push' | 'maintain' | 'admin') are allowed.", + ) + + @classmethod + def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: + mapping = super().get_mapping_from_provider(org_id, data) + + def transform_permission(x): + to_provider = { + "READ": "pull", + "TRIAGE": "triage", + "WRITE": "push", + "MAINTAIN": "maintain", + "ADMIN": "admin", + } + return to_provider[x] + + mapping.update({"permission": OptionalS("permission") >> F(transform_permission)}) + return mapping + + @classmethod + async def get_mapping_to_provider( + cls, org_id: str, data: dict[str, Any], provider: GitHubProvider + ) -> dict[str, Any]: + mapping = await super().get_mapping_to_provider(org_id, data, provider) + + return mapping + + @classmethod + async def apply_live_patch(cls, patch: LivePatch[TeamPermission], org_id: str, provider: GitHubProvider) -> None: + from .repository import Repository + + repository = expect_type(patch.parent_object, Repository) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_team_permission( + org_id, + repository.name, + unwrap(patch.expected_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_team_permission( + org_id, + repository.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_team_permission( + org_id, + repository.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/variable.py b/otterdog/models/variable.py index cf7f2351..328d2939 100644 --- a/otterdog/models/variable.py +++ b/otterdog/models/variable.py @@ -26,10 +26,10 @@ class Variable(ModelObject, abc.ABC): name: str = dataclasses.field(metadata={"key": True}) value: str - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if self.name.startswith("GITHUB_"): context.add_failure( FailureType.ERROR, - f"{self.get_model_header(parent_object)} starts with prefix 'GITHUB_', " + f"{self.get_model_header(parent_object, grandparent_object)} starts with prefix 'GITHUB_', " f"which is not allowed for variables.", ) diff --git a/otterdog/models/webhook.py b/otterdog/models/webhook.py index f1f0e69c..65b32b4c 100644 --- a/otterdog/models/webhook.py +++ b/otterdog/models/webhook.py @@ -80,7 +80,7 @@ def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool: def include_for_live_patch(self, context: LivePatchContext) -> bool: return not self.has_dummy_secret() - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if self.has_dummy_secret(): context.add_failure( FailureType.INFO, @@ -162,17 +162,26 @@ def generate_live_patch( expected_object: WT | None, current_object: WT | None, parent_object: ModelObject | None, + grandparent_object: ModelObject | None, context: LivePatchContext, handler: LivePatchHandler, ) -> None: if current_object is None: expected_object = unwrap(expected_object) - handler(LivePatch.of_addition(expected_object, parent_object, expected_object.apply_live_patch)) + handler( + LivePatch.of_addition( + expected_object, parent_object, grandparent_object, expected_object.apply_live_patch + ) + ) return if expected_object is None: current_object = unwrap(current_object) - handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + handler( + LivePatch.of_deletion( + current_object, parent_object, grandparent_object, current_object.apply_live_patch + ) + ) return # if webhooks shall be updated and the webhook contains a valid secret perform a forced update. @@ -190,6 +199,7 @@ def generate_live_patch( current_object, modified_webhook, parent_object, + grandparent_object, True, expected_object.apply_live_patch, ) @@ -229,6 +239,7 @@ def has_valid_secret(webhook: Webhook): current_object, modified_webhook, parent_object, + grandparent_object, False, expected_object.apply_live_patch, ) diff --git a/otterdog/models/workflow_settings.py b/otterdog/models/workflow_settings.py index 104036d8..95ea08f7 100644 --- a/otterdog/models/workflow_settings.py +++ b/otterdog/models/workflow_settings.py @@ -89,7 +89,7 @@ def are_actions_more_or_equally_restricted(self, repo_allowed_actions: str) -> b return org_level >= repo_level return False - def validate(self, context: ValidationContext, parent_object: Any) -> None: + def validate(self, context: ValidationContext, parent_object: Any, grandparent_object: Any) -> None: if is_set_and_valid(self.allowed_actions): if self.allowed_actions not in {"all", "local_only", "selected"}: context.add_failure( diff --git a/otterdog/operations/apply.py b/otterdog/operations/apply.py index fc407d7c..bb2de473 100644 --- a/otterdog/operations/apply.py +++ b/otterdog/operations/apply.py @@ -63,8 +63,9 @@ def handle_add_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: - super().handle_add_object(org_id, model_object, parent_object) + super().handle_add_object(org_id, model_object, parent_object, grandparent_object) self.execute_custom_hook_if_present_with_model_object(self.org_config, model_object, "pre-add-object-hook.py") def handle_delete_object( @@ -72,8 +73,9 @@ def handle_delete_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: - super().handle_delete_object(org_id, model_object, parent_object) + super().handle_delete_object(org_id, model_object, parent_object, grandparent_object) def handle_modified_object( self, @@ -83,6 +85,7 @@ def handle_modified_object( current_object: ModelObject, expected_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> int: modified = super().handle_modified_object( org_id, @@ -91,6 +94,7 @@ def handle_modified_object( current_object, expected_object, parent_object, + grandparent_object, ) return modified diff --git a/otterdog/operations/check_status.py b/otterdog/operations/check_status.py index 2c963317..078c0bcc 100644 --- a/otterdog/operations/check_status.py +++ b/otterdog/operations/check_status.py @@ -85,6 +85,7 @@ def handle_add_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: ... def handle_delete_object( @@ -92,6 +93,7 @@ def handle_delete_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: ... def handle_modified_object( @@ -102,6 +104,7 @@ def handle_modified_object( current_object: ModelObject, expected_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> int: settings_to_change = 0 for k, _v in modified_object.items(): diff --git a/otterdog/operations/diff_operation.py b/otterdog/operations/diff_operation.py index af4fbc94..f51cc576 100644 --- a/otterdog/operations/diff_operation.py +++ b/otterdog/operations/diff_operation.py @@ -188,11 +188,15 @@ def handle(patch: LivePatch) -> None: match patch.patch_type: case LivePatchType.ADD: - self.handle_add_object(github_id, unwrap(patch.expected_object), patch.parent_object) + self.handle_add_object( + github_id, unwrap(patch.expected_object), patch.parent_object, patch.grandparent_object + ) diff_status.additions += 1 case LivePatchType.REMOVE: - self.handle_delete_object(github_id, unwrap(patch.current_object), patch.parent_object) + self.handle_delete_object( + github_id, unwrap(patch.current_object), patch.parent_object, patch.grandparent_object + ) diff_status.deletions += 1 case LivePatchType.CHANGE: @@ -270,6 +274,7 @@ def handle_add_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: ... @abstractmethod @@ -278,6 +283,7 @@ def handle_delete_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: ... @abstractmethod @@ -289,6 +295,7 @@ def handle_modified_object( current_object: ModelObject, expected_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> int: ... @abstractmethod diff --git a/otterdog/operations/plan.py b/otterdog/operations/plan.py index 410bf333..d55c030f 100644 --- a/otterdog/operations/plan.py +++ b/otterdog/operations/plan.py @@ -62,9 +62,10 @@ def handle_add_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: self.printer.println() - model_header = model_object.get_model_header(parent_object) + model_header = model_object.get_model_header(parent_object, grandparent_object) self.print_dict( model_object.to_model_dict(for_diff=True, include_model_only_fields=True, exclude_none_values=True), f"add {model_header}", @@ -77,9 +78,10 @@ def handle_delete_object( org_id: str, model_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> None: self.printer.println() - model_header = model_object.get_model_header(parent_object) + model_header = model_object.get_model_header(parent_object, grandparent_object) self.print_dict( model_object.to_model_dict(for_diff=True), f"remove {model_header}", @@ -95,9 +97,10 @@ def handle_modified_object( current_object: ModelObject, expected_object: ModelObject, parent_object: ModelObject | None = None, + grandparent_object: ModelObject | None = None, ) -> int: self.printer.println() - model_header = expected_object.get_model_header(parent_object) + model_header = expected_object.get_model_header(parent_object, grandparent_object) self.print_modified_dict(modified_object, model_header, forced_update) # FIXME: this code should be moved to the Webhook model class. diff --git a/otterdog/providers/github/__init__.py b/otterdog/providers/github/__init__.py index ee2c8555..92e1e18d 100644 --- a/otterdog/providers/github/__init__.py +++ b/otterdog/providers/github/__init__.py @@ -166,6 +166,12 @@ async def get_org_teams(self, org_id: str) -> list[dict[str, Any]]: async def get_org_team_members(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: return await self.rest_api.team.get_team_members(org_id, team_slug) + async def get_org_team_sync_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + return await self.rest_api.team.get_team_sync_groups(org_id, team_slug) + + async def get_org_team_external_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + return await self.rest_api.team.get_team_external_groups(org_id, team_slug) + async def add_org_team(self, org_id: str, team_name: str, data: dict[str, str]) -> None: await self.rest_api.team.add_team(org_id, team_name, data) @@ -422,6 +428,53 @@ async def add_repo_variable(self, org_id: str, repo_name: str, data: dict[str, s async def delete_repo_variable(self, org_id: str, repo_name: str, variable_name: str) -> None: await self.rest_api.repo.delete_variable(org_id, repo_name, variable_name) + async def get_env_secrets(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + return await self.rest_api.env.get_secrets(org_id, repo_name, env_name) + + async def update_env_secret( + self, org_id: str, repo_name: str, env_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + if len(secret) > 0: + await self.rest_api.env.update_secret(org_id, repo_name, env_name, secret_name, secret) + + async def add_env_secret(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + await self.rest_api.env.add_secret(org_id, repo_name, env_name, data) + + async def delete_env_secret(self, org_id: str, repo_name: str, env_name: str, secret_name: str) -> None: + await self.rest_api.env.delete_secret(org_id, repo_name, env_name, secret_name) + + async def get_env_variables(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + return await self.rest_api.env.get_variables(org_id, repo_name, env_name) + + async def update_env_variable( + self, org_id: str, repo_name: str, env_name: str, variable_name: str, variable: dict[str, Any] + ) -> None: + if len(variable) > 0: + await self.rest_api.env.update_variable(org_id, repo_name, env_name, variable_name, variable) + + async def add_env_variable(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + await self.rest_api.env.add_variable(org_id, repo_name, env_name, data) + + async def delete_env_variable(self, org_id: str, repo_name: str, env_name: str, variable_name: str) -> None: + await self.rest_api.env.delete_variable(org_id, repo_name, env_name, variable_name) + + async def get_team_permissions(self, org_id: str) -> list[dict[str, Any]]: + return await self.graphql_client.get_team_permissions(org_id) + + async def update_team_permission( + self, org_id: str, repo_name: str, team_name: str, team_permission: dict[str, str] + ) -> None: + if len(team_permission) > 0: + await self.rest_api.repo.update_team_permission(org_id, repo_name, team_name, team_permission) + + async def add_team_permission( + self, org_id: str, repo_name: str, team_name: str, team_permission: dict[str, str] + ) -> None: + await self.rest_api.repo.add_team_permission(org_id, repo_name, team_name, team_permission) + + async def delete_team_permission(self, org_id: str, repo_name: str, team_name: str) -> None: + await self.rest_api.repo.delete_team_permission(org_id, repo_name, team_name) + async def dispatch_workflow(self, org_id: str, repo_name: str, workflow_name: str) -> bool: return await self.rest_api.repo.dispatch_workflow(org_id, repo_name, workflow_name) diff --git a/otterdog/providers/github/graphql.py b/otterdog/providers/github/graphql.py index db51c594..ebd55ea0 100644 --- a/otterdog/providers/github/graphql.py +++ b/otterdog/providers/github/graphql.py @@ -255,11 +255,46 @@ async def get_team_membership(self, org_id: str, user_login: str) -> list[dict[s variables = {"owner": org_id, "user": user_login} return await self._run_paged_query(variables, "get-team-membership.gql", "data.organization.teams") + async def get_team_permissions(self, org_id: str) -> list[dict[str, Any]]: + _logger.debug(f"retrieving team permissions in org '{org_id}'") + + variables = {"org": org_id} + # Run the graphql query with a limit of 100 for teams and repositories. If there are more than + # 100 teams available this gets handled in _run_paged_query. If there are more than 100 + # permissions to repositories available then these are handled in the subsequent loop, where the + # pageInfo is used to get the missing repository entries. + teams = await self._run_paged_query( + input_variables=variables, + query_file="get-team-permissions-repositories.gql", + prefix_selector="data.organization.teams", + ) + for team in teams: + repos = team["repositories"]["edges"] + page_info = team["repositories"]["pageInfo"] + if not page_info["hasNextPage"]: + continue + repo_cursor = page_info["endCursor"] + sub_vars = { + "org": org_id, + "teamSlug": team["slug"], + "endCursor": repo_cursor, + } + sub_result = await self._run_paged_query( + input_variables=sub_vars, + query_file="get-repository-permissions-of-team.gql", + prefix_selector="data.organization.team.repositories", + selector_type=".edges", + ) + repos.extend(sub_result) + + return teams + async def _run_paged_query( self, input_variables: dict[str, Any], query_file: str, prefix_selector: str = "data.repository.branchProtectionRules", + selector_type: str = ".nodes", ) -> list[dict[str, Any]]: _logger.debug(f"running graphql query '{query_file}' with input '{json.dumps(input_variables)}'") @@ -280,7 +315,7 @@ async def _run_paged_query( _logger.trace("graphql result = %s", json.dumps(json_data, indent=2)) if status < 400 and "data" in json_data: - rules_result = query_json(prefix_selector + ".nodes", json_data) + rules_result = query_json(prefix_selector + selector_type, json_data) for rule in rules_result: result.append(rule) diff --git a/otterdog/providers/github/rest/__init__.py b/otterdog/providers/github/rest/__init__.py index 2bddd162..31c2a6d9 100644 --- a/otterdog/providers/github/rest/__init__.py +++ b/otterdog/providers/github/rest/__init__.py @@ -136,6 +136,12 @@ def meta(self): return MetaClient(self) + @cached_property + def env(self): + from .env_client import EnvClient + + return EnvClient(self) + class RestClient: def __init__(self, rest_api: RestApi): diff --git a/otterdog/providers/github/rest/env_client.py b/otterdog/providers/github/rest/env_client.py new file mode 100644 index 00000000..927351f5 --- /dev/null +++ b/otterdog/providers/github/rest/env_client.py @@ -0,0 +1,169 @@ +# ******************************************************************************* +# Copyright (c) 2024-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +import json +from typing import Any + +from otterdog.logging import get_logger +from otterdog.providers.github.exception import GitHubException +from otterdog.providers.github.rest import RestApi, RestClient, encrypt_value + +_logger = get_logger(__name__) + + +class EnvClient(RestClient): + def __init__(self, rest_api: RestApi): + super().__init__(rest_api) + + async def get_secrets(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving secrets for repo env '%s/%s:%s'", org_id, repo_name, env_name) + + try: + status, body = await self.requester.request_raw( + "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets" + ) + if status == 200: + return json.loads(body)["secrets"] + else: + return [] + except GitHubException as ex: + raise RuntimeError( + f"failed retrieving secrets for repo env '{org_id}/{repo_name}:{env_name}':\n{ex}" + ) from ex + + async def update_secret( + self, org_id: str, repo_name: str, env_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + _logger.debug("updating repo env secret '%s' for repo env '%s/%s:%s'", secret_name, org_id, repo_name, env_name) + + if "name" in secret: + secret.pop("name") + + await self._encrypt_secret_inplace(org_id, repo_name, env_name, secret) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/{secret_name}", + json.dumps(secret), + ) + + if status != 204: + raise RuntimeError(f"failed to update repo env secret '{secret_name}'") + + _logger.debug("updated repo env secret '%s'", secret_name) + + async def add_secret(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + secret_name = data.pop("name") + _logger.debug("adding repo env secret '%s' for repo env '%s/%s:%s'", secret_name, org_id, repo_name, env_name) + + await self._encrypt_secret_inplace(org_id, repo_name, env_name, data) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/{secret_name}", + json.dumps(data), + ) + + if status != 201: + raise RuntimeError(f"failed to add repo env secret '{secret_name}'") + + _logger.debug("added repo env secret '%s'", secret_name) + + async def _encrypt_secret_inplace(self, org_id: str, repo_name: str, env_name: str, data: dict[str, Any]) -> None: + value = data.pop("value") + key_id, public_key = await self.get_public_key(org_id, repo_name, env_name) + data["encrypted_value"] = encrypt_value(public_key, value) + data["key_id"] = key_id + + async def delete_secret(self, org_id: str, repo_name: str, env_name: str, secret_name: str) -> None: + _logger.debug("deleting repo env secret '%s' for repo env '%s/%s:%s'", secret_name, org_id, repo_name, env_name) + + status, _ = await self.requester.request_raw( + "DELETE", f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/{secret_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete repo env secret '{secret_name}'") + + _logger.debug("removed repo env secret '%s'", secret_name) + + async def get_public_key(self, org_id: str, repo_name: str, env_name: str) -> tuple[str, str]: + _logger.debug("retrieving repo public key for repo env '%s/%s:%s'", org_id, repo_name, env_name) + + try: + response = await self.requester.request_json( + "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/public-key" + ) + return response["key_id"], response["key"] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving repo env public key:\n{ex}") from ex + + async def get_variables(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving variables for repo env '%s/%s:%s'", org_id, repo_name, env_name) + + try: + status, body = await self.requester.request_raw( + "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables" + ) + if status == 200: + return json.loads(body)["variables"] + else: + return [] + except GitHubException as ex: + raise RuntimeError( + f"failed retrieving variables for repo env'{org_id}/{repo_name}:{env_name}':\n{ex}" + ) from ex + + async def update_variable( + self, org_id: str, repo_name: str, env_name: str, variable_name: str, variable: dict[str, Any] + ) -> None: + _logger.debug("updating repo env variable '%s' for repo '%s/%s:%s'", variable_name, org_id, repo_name, env_name) + + if "name" in variable: + variable.pop("name") + + status, body = await self.requester.request_raw( + "PATCH", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables/{variable_name}", + json.dumps(variable), + ) + if status != 204: + raise RuntimeError(f"failed to update repo env variable '{variable_name}': {body}") + + _logger.debug("updated repo env variable '%s'", variable_name) + + async def add_variable(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + variable_name = data.get("name") + _logger.debug( + "adding repo env variable '%s' for repo env '%s/%s:%s'", variable_name, org_id, repo_name, env_name + ) + + status, body = await self.requester.request_raw( + "POST", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables", + json.dumps(data), + ) + + if status != 201: + raise RuntimeError(f"failed to add repo env variable '{variable_name}': {body}") + + _logger.debug("added repo env variable '%s'", variable_name) + + async def delete_variable(self, org_id: str, repo_name: str, env_name: str, variable_name: str) -> None: + _logger.debug( + "deleting repo env variable '%s' for repo env '%s/%s'", variable_name, org_id, repo_name, env_name + ) + + status, _ = await self.requester.request_raw( + "DELETE", f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables/{variable_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete repo env variable '{variable_name}'") + + _logger.debug("removed repo env variable '%s'", variable_name) diff --git a/otterdog/providers/github/rest/repo_client.py b/otterdog/providers/github/rest/repo_client.py index 072a09bd..d805478c 100644 --- a/otterdog/providers/github/rest/repo_client.py +++ b/otterdog/providers/github/rest/repo_client.py @@ -15,8 +15,8 @@ import zipfile from typing import Any -import aiofiles -import chevron +import aiofiles # type: ignore +import chevron # type: ignore from otterdog.logging import is_trace_enabled from otterdog.providers.github.exception import GitHubException @@ -741,6 +741,47 @@ async def delete_environment(self, org_id: str, repo_name: str, env_name: str) - _logger.debug("removed repo environment '%s'", env_name) + async def get_team_permissions(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving teams with permissions for repo '%s/%s'", org_id, repo_name) + + try: + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/teams") + except GitHubException as ex: + raise RuntimeError(f"failed getting team permissions for repo '{org_id}/{repo_name}':\n{ex}") from ex + + async def update_team_permission( + self, org_id: str, repo_name: str, team_name: str, team_permission: dict[str, Any] + ) -> None: + if "name" in team_permission: + team_permission.pop("name") + + status, _ = await self.requester.request_raw( + "PUT", f"/orgs/{org_id}/teams/{team_name}/repos/{org_id}/{repo_name}", data=json.dumps(team_permission) + ) + + if status != 204: + raise RuntimeError(f"failed to update team permission for team {team_name} on repo {repo_name}") + + _logger.debug(f"updated team permission for team {team_name} on repo {repo_name}") + + async def add_team_permission( + self, org_id: str, repo_name: str, team_name: str, team_permission: dict[str, Any] + ) -> None: + _logger.debug(f"adding team permission for team {team_name} on repo {repo_name}") + await self.update_team_permission(org_id, repo_name, team_name, team_permission) + _logger.debug("added team permisson for team {team_name} on repo {repo_name}") + + async def delete_team_permission(self, org_id: str, repo_name: str, team_name: str) -> None: + _logger.debug(f"deleting team permission for team {team_name} on repo {repo_name}") + status, _ = await self.requester.request_raw( + "DELETE", f"/orgs/{org_id}/teams/{team_name}/repos/{org_id}/{repo_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete team permission for team {team_name} on repo {repo_name}") + + _logger.debug(f"removed team permission for team {team_name} on repo {repo_name}") + async def _get_deployment_branch_policies(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: _logger.debug("retrieving deployment branch policies for env '%s'", env_name) diff --git a/otterdog/providers/github/rest/team_client.py b/otterdog/providers/github/rest/team_client.py index fdd4d728..fcfefe3e 100644 --- a/otterdog/providers/github/rest/team_client.py +++ b/otterdog/providers/github/rest/team_client.py @@ -82,6 +82,15 @@ async def add_team(self, org_id: str, team_name: str, data: dict[str, str]) -> s members = data["members"] for user in members: await self.add_member_to_team(org_id, team_slug, user) + if all(k in data and data[k] is not None for k in ("team_sync_id", "team_sync_name", "team_sync_description")): + tsdata = { + "group_id": data["team_sync_id"], + "group_name": data["team_sync_name"], + "group_description": data["team_sync_description"], + } + await self.update_team_sync_groups(org_id, team_slug, tsdata) + if "external_groups" in data and data["external_groups"] is not None: + await self.update_team_external_groups(org_id, team_slug, data["external_groups"]) _logger.debug("added team '%s'", team_name) return team_slug @@ -94,6 +103,19 @@ async def update_team(self, org_id: str, team_slug: str, team: dict[str, Any]) - if "members" in team: await self.update_team_members(org_id, team_slug, team["members"]) + if all( + k in team and team[k] is not None for k in ("team_sync_id", "team_sync_name", "team_sync_description") + ): + tsdata = { + "group_id": team["team_sync_id"], + "group_name": team["team_sync_name"], + "group_description": team["team_sync_description"], + } + await self.update_team_sync_groups(org_id, team_slug, tsdata) + elif all(k in team for k in ("team_sync_id", "team_sync_name", "team_sync_description")): + await self.update_team_sync_groups(org_id, team_slug, None) + if "external_groups" in team: + await self.update_team_external_groups(org_id, team_slug, team["external_groups"]) _logger.debug("updated team '%s'", team_slug) except GitHubException as ex: @@ -180,3 +202,73 @@ async def get_membership(self, org_id: str, user_name: str) -> dict[str, Any]: return await self.requester.request_json("GET", f"/orgs/{org_id}/memberships/{user_name}") except GitHubException as ex: raise RuntimeError(f"failed retrieving membership for user '{user_name}' in org '{org_id}':\n{ex}") from ex + + async def get_team_sync_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + _logger.debug("retrieving team sync groups for team '%s/%s'", org_id, team_slug) + + try: + response = await self.requester.request_json( + "GET", f"/orgs/{org_id}/teams/{team_slug}/team-sync/group-mappings" + ) + except GitHubException as ex: + # Only suppress 404 (endpoint not available) + if ex.status in (403, 404): + _logger.debug("team sync endpoint not available for team '%s/%s' (404)", org_id, team_slug) + return [] + # All other errors must be raised + raise RuntimeError(f"failed retrieving team sync groups for team '{org_id}/{team_slug}':\n{ex}") from ex + + return response.get("groups", []) + + async def update_team_sync_groups(self, org_id: str, team_slug: str, group: dict[str, str] | None) -> None: + _logger.debug("updating sync_groups for team '%s' in org '%s'", team_slug, org_id) + data = {"groups": []} if group is None else {"groups": [group]} + status, body = await self.requester.request_raw( + "PATCH", f"/orgs/{org_id}/teams/{team_slug}/team-sync/group-mappings", data=json.dumps(data) + ) + + if status == 200: + _logger.debug("updated team-sync '%s' of team '%s' for org '%s'", group, team_slug, org_id) + else: + raise RuntimeError( + f"failed updating team-sync '{group}' to team '{team_slug}' in org '{org_id}'\n{status}: {body}" + ) + + async def get_team_external_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + _logger.debug("retrieving external groups for team '%s/%s'", org_id, team_slug) + + try: + response = await self.requester.request_json("GET", f"/orgs/{org_id}/teams/{team_slug}/external-groups") + except GitHubException as ex: + # Only suppress 404 (endpoint not available) + if ex.status in (400, 404): + _logger.debug("external groups endpoint not available for team '%s/%s' (404)", org_id, team_slug) + return [] + raise RuntimeError(f"failed retrieving external groups for team '{org_id}/{team_slug}':\n{ex}") from ex + + return response + + async def update_team_external_groups(self, org_id: str, team_slug: str, group: str | None) -> None: + _logger.debug("updating external_groups for team '%s' in org '%s'", team_slug, org_id) + if group is None: + status, body = await self.requester.request_raw( + "DELETE", f"/orgs/{org_id}/teams/{team_slug}/external-groups" + ) + if status != 204: + raise RuntimeError( + f"failed updating external groups from team '{team_slug}' in org '{org_id}'\n{status}: {body}" + ) + + _logger.debug("updated external groups from team '%s' in org '%s'", team_slug, org_id) + else: + data = {"group_id": f"{group}"} + status, body = await self.requester.request_raw( + "PATCH", f"/orgs/{org_id}/teams/{team_slug}/external-groups", data=json.dumps(data) + ) + + if status == 200: + _logger.debug("updated external groups '%s' of team '%s' for org '%s'", group, team_slug, org_id) + else: + raise RuntimeError( + f"failed updating external groups '{group}' to team '{team_slug}' in org '{org_id}'\n{status}: {body}" + ) diff --git a/otterdog/resources/graphql/get-repository-permissions-of-team.gql b/otterdog/resources/graphql/get-repository-permissions-of-team.gql new file mode 100644 index 00000000..f4a12412 --- /dev/null +++ b/otterdog/resources/graphql/get-repository-permissions-of-team.gql @@ -0,0 +1,23 @@ +query TeamRepositoriesQuery( + $org: String!, + $teamSlug: String!, + $endCursor: String +) { + organization(login: $org) { + team(slug: $teamSlug) { + repositories(first: 100, after: $endCursor) { + edges { + permission + node { + name + id + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} diff --git a/otterdog/resources/graphql/get-team-permissions-repositories.gql b/otterdog/resources/graphql/get-team-permissions-repositories.gql new file mode 100644 index 00000000..64bbfde3 --- /dev/null +++ b/otterdog/resources/graphql/get-team-permissions-repositories.gql @@ -0,0 +1,28 @@ +query ($org: String!, $endCursor: String) { + organization(login: $org) { + teams(first: 100, after: $endCursor) { + nodes { + id + slug + name + repositories(first: 100) { + edges { + permission + node { + name + id + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } +} diff --git a/otterdog/resources/schemas/env-secret.json b/otterdog/resources/schemas/env-secret.json new file mode 100644 index 00000000..d9c1d3ce --- /dev/null +++ b/otterdog/resources/schemas/env-secret.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + + "$ref": "secret.json", + "type": "object", + "required": [ "name", "value" ], + "unevaluatedProperties": false +} diff --git a/otterdog/resources/schemas/env-variable.json b/otterdog/resources/schemas/env-variable.json new file mode 100644 index 00000000..7dcc582f --- /dev/null +++ b/otterdog/resources/schemas/env-variable.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + + "$ref": "variable.json", + "type": "object", + "required": [ "name", "value" ], + "unevaluatedProperties": false +} diff --git a/otterdog/resources/schemas/environment.json b/otterdog/resources/schemas/environment.json index ba1bea7a..8ce58941 100644 --- a/otterdog/resources/schemas/environment.json +++ b/otterdog/resources/schemas/environment.json @@ -13,6 +13,14 @@ "branch_policies": { "type": "array", "items": { "type": "string" } + }, + "secrets": { + "type": "array", + "items": { "$ref": "env-secret.json" } + }, + "variables": { + "type": "array", + "items": { "$ref": "env-variable.json" } } }, diff --git a/otterdog/resources/schemas/repository.json b/otterdog/resources/schemas/repository.json index e017e17c..19a8f879 100644 --- a/otterdog/resources/schemas/repository.json +++ b/otterdog/resources/schemas/repository.json @@ -85,6 +85,10 @@ "environments": { "type": "array", "items": { "$ref": "environment.json" } + }, + "team_permissions": { + "type": "array", + "items": { "$ref": "team-permission.json" } } }, diff --git a/otterdog/resources/schemas/team-permission.json b/otterdog/resources/schemas/team-permission.json new file mode 100644 index 00000000..80bec638 --- /dev/null +++ b/otterdog/resources/schemas/team-permission.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + + "type": "object", + "properties": { + "name": { "type": "string" }, + "permission": { "type": "string" } + }, + "required": ["name", "permission"], + "additionalProperties": false +} diff --git a/otterdog/resources/schemas/team.json b/otterdog/resources/schemas/team.json index c247d5f9..04221d5c 100644 --- a/otterdog/resources/schemas/team.json +++ b/otterdog/resources/schemas/team.json @@ -1,20 +1,25 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "properties": { "name": { "type": "string" }, - "description": { "$ref": "types.json#/$defs/string-or-null"}, + "description": { "$ref": "types.json#/$defs/string-or-null" }, + "privacy": { "type": "string" }, + "notifications": { "type": "boolean" }, + "skip_members": { "type": "boolean" }, + "skip_non_organization_members": { "type": "boolean" }, + "members": { "type": "array", "items": { "type": "string" } }, - "privacy": { "type": "string" }, - "notifications": { "type": "boolean" }, - "skip_members": { "type": "boolean" }, - "skip_non_organization_members": { "type": "boolean" } - }, - "required": [ "name", "privacy" ], + "team_sync_id": { "$ref": "types.json#/$defs/string-or-null" }, + "team_sync_name": { "$ref": "types.json#/$defs/string-or-null" }, + "team_sync_description": { "$ref": "types.json#/$defs/string-or-null" }, + "external_groups": { "$ref": "types.json#/$defs/string-or-null" } + }, + "required": ["name", "privacy"], "additionalProperties": false } diff --git a/otterdog/utils.py b/otterdog/utils.py index cb0ba8b0..9d3bde0d 100644 --- a/otterdog/utils.py +++ b/otterdog/utils.py @@ -476,7 +476,7 @@ def sort_jsonnet(lines: list[str]) -> list[str]: def _sort_node(node): - _line, context = node + _, context = node if context is not None: last = context.pop() @@ -639,7 +639,7 @@ def wrapper_timed(*args, **kwargs): def render_chevron(content: str, context: dict[str, Any]) -> str: - import chevron + import chevron # type: ignore # need to escape ${{ sequences as used at GitHub escaped_content = content.replace("${{", "$\\{\\{") diff --git a/tests/models/resources/github-env-secret.json b/tests/models/resources/github-env-secret.json new file mode 100644 index 00000000..36339d61 --- /dev/null +++ b/tests/models/resources/github-env-secret.json @@ -0,0 +1,4 @@ +{ + "name": "TEST-SECRET", + "visibility": "selected" +} diff --git a/tests/models/resources/github-team-permission.json b/tests/models/resources/github-team-permission.json new file mode 100644 index 00000000..25e2b47b --- /dev/null +++ b/tests/models/resources/github-team-permission.json @@ -0,0 +1,4 @@ +{ + "name": "TEAM", + "permission": "READ" +} diff --git a/tests/models/resources/otterdog-env-secret.json b/tests/models/resources/otterdog-env-secret.json new file mode 100644 index 00000000..59da4df1 --- /dev/null +++ b/tests/models/resources/otterdog-env-secret.json @@ -0,0 +1,5 @@ +{ + "name": "TEST-SECRET", + "visibility": "selected", + "value": "5678" +} diff --git a/tests/models/resources/otterdog-team-permission.json b/tests/models/resources/otterdog-team-permission.json new file mode 100644 index 00000000..545da5e2 --- /dev/null +++ b/tests/models/resources/otterdog-team-permission.json @@ -0,0 +1,4 @@ +{ + "name": "TEAM", + "permission": "pull" +} diff --git a/tests/models/resources/test-org/vendor/github-env-secret.json b/tests/models/resources/test-org/vendor/github-env-secret.json new file mode 100644 index 00000000..b9244285 --- /dev/null +++ b/tests/models/resources/test-org/vendor/github-env-secret.json @@ -0,0 +1,4 @@ +{ + "name": "TEST-SECRET", + "visibility": "selected" + } diff --git a/tests/models/test_env_secret.py b/tests/models/test_env_secret.py new file mode 100644 index 00000000..22119eb4 --- /dev/null +++ b/tests/models/test_env_secret.py @@ -0,0 +1,67 @@ +# ******************************************************************************* +# Copyright (c) 2023-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from collections.abc import Mapping +from typing import Any + +from otterdog.jsonnet import JsonnetConfig +from otterdog.models import ModelObject +from otterdog.models.env_secret import EnvironmentSecret +from otterdog.utils import Change + +from . import ModelTest + + +class EnvironmentSecretTest(ModelTest): + def create_model(self, data: Mapping[str, Any]) -> ModelObject: + return EnvironmentSecret.from_model_data(data) + + @property + def template_function(self) -> str: + return JsonnetConfig.create_org_secret + + @property + def model_data(self): + return self.load_json_resource("otterdog-env-secret.json") + + @property + def provider_data(self): + return self.load_json_resource("github-env-secret.json") + + def test_load_from_model(self): + secret = EnvironmentSecret.from_model_data(self.model_data) + + assert secret.name == "TEST-SECRET" + assert secret.value == "5678" + + def test_load_from_provider(self): + secret = EnvironmentSecret.from_provider_data(self.org_id, self.provider_data) + + assert secret.name == "TEST-SECRET" + assert secret.value == "********" + + def test_patch(self): + current = EnvironmentSecret.from_model_data(self.model_data) + default = EnvironmentSecret.from_model_data(self.model_data) + + default.value = "8765" + patch = current.get_patch_to(default) + + assert len(patch) == 1 + assert patch["value"] == current.value + + def test_difference(self): + current = EnvironmentSecret.from_model_data(self.model_data) + other = EnvironmentSecret.from_model_data(self.model_data) + + other.value = "8765" + + diff = current.get_difference_from(other) + + assert len(diff) == 1 + assert diff["value"] == Change(other.value, current.value) diff --git a/tests/models/test_env_variable.py b/tests/models/test_env_variable.py new file mode 100644 index 00000000..50190945 --- /dev/null +++ b/tests/models/test_env_variable.py @@ -0,0 +1,33 @@ +# ******************************************************************************* +# Copyright (c) 2023-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from collections.abc import Mapping +from typing import Any + +from otterdog.jsonnet import JsonnetConfig +from otterdog.models import ModelObject +from otterdog.models.env_variable import EnvironmentVariable + +from . import ModelTest + + +class EnvironmentVariableTest(ModelTest): + def create_model(self, data: Mapping[str, Any]) -> ModelObject: + return EnvironmentVariable.from_model_data(data) + + @property + def template_function(self) -> str: + return JsonnetConfig.create_env_variable + + @property + def model_data(self): + raise NotImplementedError + + @property + def provider_data(self): + raise NotImplementedError diff --git a/tests/models/test_repository.py b/tests/models/test_repository.py index 08f98a1c..ce58e930 100644 --- a/tests/models/test_repository.py +++ b/tests/models/test_repository.py @@ -187,6 +187,7 @@ def test__include_gh_pages_patch_required_properties( current_object=default, changes=changes, parent_object=None, + grandparent_object=None, forced_update=False, fn=pretend.stub(), changes_object_to_readonly=False, @@ -257,6 +258,7 @@ def test__include_squash_merge_patch_required_properties( current_object=default, changes=changes, parent_object=None, + grandparent_object=None, forced_update=False, fn=pretend.stub(), changes_object_to_readonly=False, @@ -314,14 +316,14 @@ def test_gh_pages_visibility_validation(self, repository_test): repo.private = True repo.gh_pages_visibility = "public" - repo.validate(context, mock_org_enterprise) + repo.validate(context, mock_org_enterprise, None) failures = [ f for f in context.validation_failures if f[0] == FailureType.ERROR and "gh_pages_visibility" in f[1] ] assert len(failures) == initial_failures repo.gh_pages_visibility = "private" - repo.validate(context, mock_org_enterprise) + repo.validate(context, mock_org_enterprise, None) failures = [ f for f in context.validation_failures if f[0] == FailureType.ERROR and "gh_pages_visibility" in f[1] ] @@ -329,7 +331,7 @@ def test_gh_pages_visibility_validation(self, repository_test): # Test invalid value - should add a failure repo.gh_pages_visibility = "invalid" - repo.validate(context, mock_org_enterprise) + repo.validate(context, mock_org_enterprise, None) # Check that validation failed for invalid value failures = [ @@ -344,7 +346,7 @@ def test_gh_pages_visibility_validation(self, repository_test): # Test with public repository (should fail even with enterprise plan) repo.private = False - repo.validate(context, mock_org_enterprise) + repo.validate(context, mock_org_enterprise, None) failures = [ f for f in context.validation_failures if f[0] == FailureType.ERROR and "gh_pages_visibility" in f[1] @@ -358,7 +360,7 @@ def test_gh_pages_visibility_validation(self, repository_test): repo.gh_pages_visibility = "private" # Test with enterprise plan but members_can_create_private_pages disabled - should fail - repo.validate(context, mock_org_enterprise_no_private_pages) + repo.validate(context, mock_org_enterprise_no_private_pages, None) failures = [ f for f in context.validation_failures if f[0] == FailureType.ERROR and "gh_pages_visibility" in f[1] diff --git a/tests/models/test_team_permission.py b/tests/models/test_team_permission.py new file mode 100644 index 00000000..6a097d27 --- /dev/null +++ b/tests/models/test_team_permission.py @@ -0,0 +1,68 @@ +# ******************************************************************************* +# Copyright (c) 2023-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from collections.abc import Mapping +from typing import Any + +from otterdog.jsonnet import JsonnetConfig +from otterdog.models import ModelObject +from otterdog.models.team_permission import TeamPermission +from otterdog.utils import Change + +from . import ModelTest + + +class TeamPermissionTest(ModelTest): + def create_model(self, data: Mapping[str, Any]) -> ModelObject: + return TeamPermission.from_model_data(data) + + @property + def template_function(self) -> str: + return JsonnetConfig.create_org_secret + + @property + def model_data(self): + return self.load_json_resource("otterdog-team-permission.json") + + @property + def provider_data(self): + return self.load_json_resource("github-team-permission.json") + + def test_load_from_model(self): + team_permission = TeamPermission.from_model_data(self.model_data) + + assert team_permission.name == "TEAM" + assert team_permission.permission == "pull" + + def test_load_from_provider(self): + team_permission = TeamPermission.from_provider_data(self.org_id, self.provider_data) + + assert team_permission.name == "TEAM" + assert team_permission.permission == "pull" + + def test_patch(self): + current = TeamPermission.from_model_data(self.model_data) + default = TeamPermission.from_model_data(self.model_data) + + default.permission = "admin" + + patch = current.get_patch_to(default) + + assert len(patch) == 1 + assert patch["permission"] == current.permission + + def test_difference(self): + current = TeamPermission.from_model_data(self.model_data) + other = TeamPermission.from_model_data(self.model_data) + + other.permission = "triage" + + diff = current.get_difference_from(other) + + assert len(diff) == 1 + assert diff["permission"] == Change(other.permission, current.permission) diff --git a/tests/providers/github/integration/helpers/model.py b/tests/providers/github/integration/helpers/model.py index f1a3dd66..ba9d6da1 100644 --- a/tests/providers/github/integration/helpers/model.py +++ b/tests/providers/github/integration/helpers/model.py @@ -8,6 +8,8 @@ from otterdog.models import LivePatch, LivePatchContext, ModelObject from otterdog.models.custom_property import CustomProperty +from otterdog.models.env_secret import EnvironmentSecret +from otterdog.models.env_variable import EnvironmentVariable from otterdog.models.environment import Environment from otterdog.models.organization_secret import OrganizationSecret from otterdog.models.organization_settings import OrganizationSettings @@ -85,6 +87,7 @@ def generate_live_patch( expected_object=new, current_object=old, parent_object=self.get_parent_object(old, new), + grandparent_object=self.get_grandparent_object(old, new), context=self.live_patch_context, handler=lambda p: patches.append(p), # pyright: ignore[reportArgumentType] ) @@ -98,8 +101,19 @@ def get_parent_object(self, old: ModelObject | None, new: ModelObject | None) -> """ model_cls = determine_model_object(old, new) - if model_cls in {RepositorySecret, RepositoryVariable}: + if model_cls in {RepositorySecret, RepositoryVariable, Environment}: return self.repository if model_cls in {OrganizationSecret, OrganizationVariable, CustomProperty}: return None # Organization-level, no parent object raise ValueError(f"Unknown model class for parent: {model_cls}") + + def get_grandparent_object(self, old: ModelObject | None, new: ModelObject | None) -> ModelObject | None: + """ + Based on provided old/new objects and test context, return the correct parent object. + Objects do not store their parents directly, so we need to reconstruct them here. + """ + + model_cls = determine_model_object(old, new) + if model_cls in {EnvironmentVariable, EnvironmentSecret}: + return self.repository + return None