From 9e6980a56d5a80ec86a91d51c1e68b4137ea5076 Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Wed, 4 Mar 2026 12:47:37 +0100 Subject: [PATCH 1/5] feat: Extend team by team-sync and external groups --- docs/reference/organization/team.md | 2 + otterdog/models/github_organization.py | 13 +++ otterdog/models/team.py | 27 ++++++ otterdog/providers/github/__init__.py | 6 ++ otterdog/providers/github/rest/team_client.py | 92 +++++++++++++++++++ otterdog/resources/schemas/team.json | 21 +++-- 6 files changed, 153 insertions(+), 8 deletions(-) diff --git a/docs/reference/organization/team.md b/docs/reference/organization/team.md index 45cc7338..71b1cbce 100644 --- a/docs/reference/organization/team.md +++ b/docs/reference/organization/team.md @@ -9,6 +9,8 @@ Definition of an organization `Team`, the following properties are supported: | _members_ | list[string] | List of users that should be a member of the team | | | _skip_members_ | boolean | If `true`, team members will be ignored | | | _skip_non_organization_members_ | boolean | If `true`, users which are not yet organization members can not be added to the team | | +| _team_sync_ | string | The id of an IdP group which is connected to a team on Github Enterprise Cloud | | +| _external_groups_ | string | The id of an external group which is provisioned on the enterprise | | ## Jsonnet Function diff --git a/otterdog/models/github_organization.py b/otterdog/models/github_organization.py index 0e22a2e4..150878e1 100644 --- a/otterdog/models/github_organization.py +++ b/otterdog/models/github_organization.py @@ -583,6 +583,19 @@ async def _load_teams() -> None: continue team_members = await provider.get_org_team_members(github_id, team_slug) team["members"] = team_members + # Do the team-sync + sync_groups = await provider.get_org_team_sync_groups(github_id, team_slug) + if sync_groups: + team["team_sync_id"] = sync_groups[0].get("group_id", None) + team["team_sync_name"] = sync_groups[0].get("group_name", None) + team["team_sync_description"] = sync_groups[0].get("group_description", None) + else: + team["team_sync_id"] = None + team["team_sync_name"] = None + team["team_sync_description"] = None + # External Groups + external_groups = await provider.get_org_team_external_groups(github_id, team_slug) + team["external_groups"] = external_groups org.add_team(Team.from_provider_data(github_id, team)) else: _logger.debug("not reading teams, no default config available") diff --git a/otterdog/models/team.py b/otterdog/models/team.py index 288e0eab..30442c8a 100644 --- a/otterdog/models/team.py +++ b/otterdog/models/team.py @@ -43,6 +43,10 @@ class Team(ModelObject, abc.ABC): privacy: str notifications: bool members: list[str] + team_sync_id: str | None + team_sync_name: str | None + team_sync_description: str | None + external_groups: str | None skip_members: bool = dataclasses.field(metadata={"model_only": True}, default=False) skip_non_organization_members: bool = dataclasses.field(metadata={"model_only": True}, default=False) @@ -98,6 +102,23 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: f"but 'members' contains user '{member}' who is not an organization member.", ) + values = [ + self.team_sync_id, + self.team_sync_name, + self.team_sync_description, + ] + + all_strings = all(is_set_and_valid(v) for v in values) + all_unset_or_none = all(is_unset(v) or v is None for v in values) + + if not all_strings and not all_unset_or_none: + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has inconsistent team sync configuration: " + f"all of 'team_sync_id', 'team_sync_name', and 'team_sync_description' must either " + f"all be unset/None or all be valid strings.", + ) + @classmethod def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: mapping = super().get_mapping_from_provider(org_id, data) @@ -113,11 +134,17 @@ def transform_notification_setting(x: str | None): def transform_team_members(member): return member["login"] + def transform_external_groups(value): + if isinstance(value, list) and value: + return value[0]["group_id"] + return None + mapping.update( { "privacy": OptionalS("privacy") >> F(lambda x: "visible" if x == "closed" else x), "notifications": OptionalS("notification_setting") >> F(transform_notification_setting), "members": OptionalS("members", default=[]) >> Forall(transform_team_members), + "external_groups": OptionalS("external_groups") >> F(transform_external_groups), } ) return mapping diff --git a/otterdog/providers/github/__init__.py b/otterdog/providers/github/__init__.py index ee2c8555..31963ac5 100644 --- a/otterdog/providers/github/__init__.py +++ b/otterdog/providers/github/__init__.py @@ -166,6 +166,12 @@ async def get_org_teams(self, org_id: str) -> list[dict[str, Any]]: async def get_org_team_members(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: return await self.rest_api.team.get_team_members(org_id, team_slug) + async def get_org_team_sync_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + return await self.rest_api.team.get_team_sync_groups(org_id, team_slug) + + async def get_org_team_external_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + return await self.rest_api.team.get_team_external_groups(org_id, team_slug) + async def add_org_team(self, org_id: str, team_name: str, data: dict[str, str]) -> None: await self.rest_api.team.add_team(org_id, team_name, data) diff --git a/otterdog/providers/github/rest/team_client.py b/otterdog/providers/github/rest/team_client.py index fdd4d728..fcfefe3e 100644 --- a/otterdog/providers/github/rest/team_client.py +++ b/otterdog/providers/github/rest/team_client.py @@ -82,6 +82,15 @@ async def add_team(self, org_id: str, team_name: str, data: dict[str, str]) -> s members = data["members"] for user in members: await self.add_member_to_team(org_id, team_slug, user) + if all(k in data and data[k] is not None for k in ("team_sync_id", "team_sync_name", "team_sync_description")): + tsdata = { + "group_id": data["team_sync_id"], + "group_name": data["team_sync_name"], + "group_description": data["team_sync_description"], + } + await self.update_team_sync_groups(org_id, team_slug, tsdata) + if "external_groups" in data and data["external_groups"] is not None: + await self.update_team_external_groups(org_id, team_slug, data["external_groups"]) _logger.debug("added team '%s'", team_name) return team_slug @@ -94,6 +103,19 @@ async def update_team(self, org_id: str, team_slug: str, team: dict[str, Any]) - if "members" in team: await self.update_team_members(org_id, team_slug, team["members"]) + if all( + k in team and team[k] is not None for k in ("team_sync_id", "team_sync_name", "team_sync_description") + ): + tsdata = { + "group_id": team["team_sync_id"], + "group_name": team["team_sync_name"], + "group_description": team["team_sync_description"], + } + await self.update_team_sync_groups(org_id, team_slug, tsdata) + elif all(k in team for k in ("team_sync_id", "team_sync_name", "team_sync_description")): + await self.update_team_sync_groups(org_id, team_slug, None) + if "external_groups" in team: + await self.update_team_external_groups(org_id, team_slug, team["external_groups"]) _logger.debug("updated team '%s'", team_slug) except GitHubException as ex: @@ -180,3 +202,73 @@ async def get_membership(self, org_id: str, user_name: str) -> dict[str, Any]: return await self.requester.request_json("GET", f"/orgs/{org_id}/memberships/{user_name}") except GitHubException as ex: raise RuntimeError(f"failed retrieving membership for user '{user_name}' in org '{org_id}':\n{ex}") from ex + + async def get_team_sync_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + _logger.debug("retrieving team sync groups for team '%s/%s'", org_id, team_slug) + + try: + response = await self.requester.request_json( + "GET", f"/orgs/{org_id}/teams/{team_slug}/team-sync/group-mappings" + ) + except GitHubException as ex: + # Only suppress 404 (endpoint not available) + if ex.status in (403, 404): + _logger.debug("team sync endpoint not available for team '%s/%s' (404)", org_id, team_slug) + return [] + # All other errors must be raised + raise RuntimeError(f"failed retrieving team sync groups for team '{org_id}/{team_slug}':\n{ex}") from ex + + return response.get("groups", []) + + async def update_team_sync_groups(self, org_id: str, team_slug: str, group: dict[str, str] | None) -> None: + _logger.debug("updating sync_groups for team '%s' in org '%s'", team_slug, org_id) + data = {"groups": []} if group is None else {"groups": [group]} + status, body = await self.requester.request_raw( + "PATCH", f"/orgs/{org_id}/teams/{team_slug}/team-sync/group-mappings", data=json.dumps(data) + ) + + if status == 200: + _logger.debug("updated team-sync '%s' of team '%s' for org '%s'", group, team_slug, org_id) + else: + raise RuntimeError( + f"failed updating team-sync '{group}' to team '{team_slug}' in org '{org_id}'\n{status}: {body}" + ) + + async def get_team_external_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: + _logger.debug("retrieving external groups for team '%s/%s'", org_id, team_slug) + + try: + response = await self.requester.request_json("GET", f"/orgs/{org_id}/teams/{team_slug}/external-groups") + except GitHubException as ex: + # Only suppress 404 (endpoint not available) + if ex.status in (400, 404): + _logger.debug("external groups endpoint not available for team '%s/%s' (404)", org_id, team_slug) + return [] + raise RuntimeError(f"failed retrieving external groups for team '{org_id}/{team_slug}':\n{ex}") from ex + + return response + + async def update_team_external_groups(self, org_id: str, team_slug: str, group: str | None) -> None: + _logger.debug("updating external_groups for team '%s' in org '%s'", team_slug, org_id) + if group is None: + status, body = await self.requester.request_raw( + "DELETE", f"/orgs/{org_id}/teams/{team_slug}/external-groups" + ) + if status != 204: + raise RuntimeError( + f"failed updating external groups from team '{team_slug}' in org '{org_id}'\n{status}: {body}" + ) + + _logger.debug("updated external groups from team '%s' in org '%s'", team_slug, org_id) + else: + data = {"group_id": f"{group}"} + status, body = await self.requester.request_raw( + "PATCH", f"/orgs/{org_id}/teams/{team_slug}/external-groups", data=json.dumps(data) + ) + + if status == 200: + _logger.debug("updated external groups '%s' of team '%s' for org '%s'", group, team_slug, org_id) + else: + raise RuntimeError( + f"failed updating external groups '{group}' to team '{team_slug}' in org '{org_id}'\n{status}: {body}" + ) diff --git a/otterdog/resources/schemas/team.json b/otterdog/resources/schemas/team.json index c247d5f9..04221d5c 100644 --- a/otterdog/resources/schemas/team.json +++ b/otterdog/resources/schemas/team.json @@ -1,20 +1,25 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "properties": { "name": { "type": "string" }, - "description": { "$ref": "types.json#/$defs/string-or-null"}, + "description": { "$ref": "types.json#/$defs/string-or-null" }, + "privacy": { "type": "string" }, + "notifications": { "type": "boolean" }, + "skip_members": { "type": "boolean" }, + "skip_non_organization_members": { "type": "boolean" }, + "members": { "type": "array", "items": { "type": "string" } }, - "privacy": { "type": "string" }, - "notifications": { "type": "boolean" }, - "skip_members": { "type": "boolean" }, - "skip_non_organization_members": { "type": "boolean" } - }, - "required": [ "name", "privacy" ], + "team_sync_id": { "$ref": "types.json#/$defs/string-or-null" }, + "team_sync_name": { "$ref": "types.json#/$defs/string-or-null" }, + "team_sync_description": { "$ref": "types.json#/$defs/string-or-null" }, + "external_groups": { "$ref": "types.json#/$defs/string-or-null" } + }, + "required": ["name", "privacy"], "additionalProperties": false } From dac28f8354dccbd08e8096b950ff67e410af19fe Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Wed, 4 Mar 2026 14:53:59 +0100 Subject: [PATCH 2/5] feat: Added the description of team_sync entries and the validation rule --- docs/reference/organization/team.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/reference/organization/team.md b/docs/reference/organization/team.md index 71b1cbce..a9896e15 100644 --- a/docs/reference/organization/team.md +++ b/docs/reference/organization/team.md @@ -1,16 +1,18 @@ Definition of an organization `Team`, the following properties are supported: -| Key | Value | Description | Note | -|---------------------------------|--------------|--------------------------------------------------------------------------------------|-----------------------| -| _name_ | string | The name of the team | | -| _description_ | string | The description of the team | | -| _privacy_ | string | The level of privacy this team should have | `visible` or `secret` | -| _notifications_ | boolean | Whether the team members receive notifications when the team is @mentioned | | -| _members_ | list[string] | List of users that should be a member of the team | | -| _skip_members_ | boolean | If `true`, team members will be ignored | | -| _skip_non_organization_members_ | boolean | If `true`, users which are not yet organization members can not be added to the team | | -| _team_sync_ | string | The id of an IdP group which is connected to a team on Github Enterprise Cloud | | -| _external_groups_ | string | The id of an external group which is provisioned on the enterprise | | +| Key | Value | Description | Note | +|---------------------------------|--------------|-----------------------------------------------------------------------------------------|-----------------------| +| _name_ | string | The name of the team | | +| _description_ | string | The description of the team | | +| _privacy_ | string | The level of privacy this team should have | `visible` or `secret` | +| _notifications_ | boolean | Whether the team members receive notifications when the team is @mentioned | | +| _members_ | list[string] | List of users that should be a member of the team | | +| _skip_members_ | boolean | If `true`, team members will be ignored | | +| _skip_non_organization_members_ | boolean | If `true`, users which are not yet organization members can not be added to the team | | +| _team_sync_id_ | string | The id of an IdP group which is connected to a team on Github Enterprise Cloud | | +| _team_sync_description_ | string | The description of an IdP group which is connected to a team on Github Enterprise Cloud | | +| _team_sync_name_ | string | The name of an IdP group which is connected to a team on Github Enterprise Cloud | | +| _external_groups_ | string | The id of an external group which is provisioned on the enterprise | | ## Jsonnet Function @@ -26,6 +28,7 @@ orgs.newTeam('') { - setting `privacy` must be one of `visible` or `secret`, any other value triggers an error - specifying a non-empty list of `members` while `skip_members` is enabled, triggers an error - specifying a user in `members` that is not yet an organization member while `skip_non_organization_members` is enabled, triggers an error +- specifying a team-sync that is missing either `team_sync_id`, `team_sync_name` or `team_sync_description`, triggers an error ## Example usage From 19a16f6a0ec72d9c5358e941c553b15e219f1d4b Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Thu, 5 Mar 2026 08:03:06 +0100 Subject: [PATCH 3/5] fix: import is_unset from otterdog.utils --- otterdog/models/team.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otterdog/models/team.py b/otterdog/models/team.py index 30442c8a..b2ffed9d 100644 --- a/otterdog/models/team.py +++ b/otterdog/models/team.py @@ -21,7 +21,7 @@ ModelObject, ValidationContext, ) -from otterdog.utils import UNSET, is_set_and_valid, unwrap +from otterdog.utils import UNSET, is_set_and_valid, unwrap, is_unset if TYPE_CHECKING: from otterdog.jsonnet import JsonnetConfig From c04ed3b8a57afe234f533624a50f6fc64a1a36d3 Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Thu, 5 Mar 2026 08:07:34 +0100 Subject: [PATCH 4/5] chore: ruff findings --- docs/reference/organization/team.md | 2 +- otterdog/models/team.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/organization/team.md b/docs/reference/organization/team.md index a9896e15..fca1c1a5 100644 --- a/docs/reference/organization/team.md +++ b/docs/reference/organization/team.md @@ -28,7 +28,7 @@ orgs.newTeam('') { - setting `privacy` must be one of `visible` or `secret`, any other value triggers an error - specifying a non-empty list of `members` while `skip_members` is enabled, triggers an error - specifying a user in `members` that is not yet an organization member while `skip_non_organization_members` is enabled, triggers an error -- specifying a team-sync that is missing either `team_sync_id`, `team_sync_name` or `team_sync_description`, triggers an error +- specifying a team-sync that is missing either `team_sync_id`, `team_sync_name` or `team_sync_description`, triggers an error ## Example usage diff --git a/otterdog/models/team.py b/otterdog/models/team.py index b2ffed9d..2d549766 100644 --- a/otterdog/models/team.py +++ b/otterdog/models/team.py @@ -21,7 +21,7 @@ ModelObject, ValidationContext, ) -from otterdog.utils import UNSET, is_set_and_valid, unwrap, is_unset +from otterdog.utils import UNSET, is_set_and_valid, is_unset, unwrap if TYPE_CHECKING: from otterdog.jsonnet import JsonnetConfig From 82452a800ebdb3e2c8f1cfdb30d9cbcda48e143f Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Tue, 14 Apr 2026 14:53:29 +0200 Subject: [PATCH 5/5] fix: serveral team-syncs possible --- .../organization/{team.md => team/index.md} | 35 +++-- docs/reference/organization/team/team-sync.md | 46 +++++++ examples/template/otterdog-defaults.libsonnet | 10 ++ mkdocs.yml | 4 +- otterdog/jsonnet.py | 11 ++ otterdog/models/github_organization.py | 18 +-- otterdog/models/team.py | 129 +++++++++++++++--- otterdog/models/team_sync.py | 106 ++++++++++++++ otterdog/providers/github/__init__.py | 51 +++++++ otterdog/providers/github/rest/team_client.py | 28 +--- otterdog/resources/schemas/team.json | 9 +- otterdog/resources/schemas/team_sync.json | 27 ++++ poetry.lock | 24 ++-- pyproject.toml | 2 + 14 files changed, 415 insertions(+), 85 deletions(-) rename docs/reference/organization/{team.md => team/index.md} (67%) create mode 100644 docs/reference/organization/team/team-sync.md create mode 100644 otterdog/models/team_sync.py create mode 100644 otterdog/resources/schemas/team_sync.json diff --git a/docs/reference/organization/team.md b/docs/reference/organization/team/index.md similarity index 67% rename from docs/reference/organization/team.md rename to docs/reference/organization/team/index.md index fca1c1a5..8a28b2a9 100644 --- a/docs/reference/organization/team.md +++ b/docs/reference/organization/team/index.md @@ -1,18 +1,25 @@ Definition of an organization `Team`, the following properties are supported: -| Key | Value | Description | Note | -|---------------------------------|--------------|-----------------------------------------------------------------------------------------|-----------------------| -| _name_ | string | The name of the team | | -| _description_ | string | The description of the team | | -| _privacy_ | string | The level of privacy this team should have | `visible` or `secret` | -| _notifications_ | boolean | Whether the team members receive notifications when the team is @mentioned | | -| _members_ | list[string] | List of users that should be a member of the team | | -| _skip_members_ | boolean | If `true`, team members will be ignored | | -| _skip_non_organization_members_ | boolean | If `true`, users which are not yet organization members can not be added to the team | | -| _team_sync_id_ | string | The id of an IdP group which is connected to a team on Github Enterprise Cloud | | -| _team_sync_description_ | string | The description of an IdP group which is connected to a team on Github Enterprise Cloud | | -| _team_sync_name_ | string | The name of an IdP group which is connected to a team on Github Enterprise Cloud | | -| _external_groups_ | string | The id of an external group which is provisioned on the enterprise | | +| Key | Value | Description | Note | +|---------------------------------|--------------|--------------------------------------------------------------------------------------|-----------------------| +| _name_ | string | The name of the team | | +| _description_ | string | The description of the team | | +| _privacy_ | string | The level of privacy this team should have | `visible` or `secret` | +| _notifications_ | boolean | Whether the team members receive notifications when the team is @mentioned | | +| _members_ | list[string] | List of users that should be a member of the team | | +| _skip_members_ | boolean | If `true`, team members will be ignored | | +| _skip_non_organization_members_ | boolean | If `true`, users which are not yet organization members can not be added to the team | | +| _team_sync_ | list\[[TeamSync](team-sync.md)\] | List of IdP groups which are connected to the team via GitHub Enterprise Cloud Team Sync | | +| _external_groups_ | string | The id of an external group which is provisioned on the enterprise | Exclusive with `team_sync` | + +## Identity provider integration + +Teams can be connected to an identity provider either via +GitHub Enterprise Cloud **Team Sync** (`team_sync`) or via +enterprise-wide **external groups** (`external_groups`). + +These two mechanisms are **mutually exclusive**. +A team must not define both `team_sync` and `external_groups` at the same time. ## Jsonnet Function @@ -28,7 +35,7 @@ orgs.newTeam('') { - setting `privacy` must be one of `visible` or `secret`, any other value triggers an error - specifying a non-empty list of `members` while `skip_members` is enabled, triggers an error - specifying a user in `members` that is not yet an organization member while `skip_non_organization_members` is enabled, triggers an error -- specifying a team-sync that is missing either `team_sync_id`, `team_sync_name` or `team_sync_description`, triggers an error +- `team_sync` and `external_groups` are mutually exclusive and must not be used together ## Example usage diff --git a/docs/reference/organization/team/team-sync.md b/docs/reference/organization/team/team-sync.md new file mode 100644 index 00000000..8e84c4c7 --- /dev/null +++ b/docs/reference/organization/team/team-sync.md @@ -0,0 +1,46 @@ +Definition of a `TeamSync` for a team, the following properties are supported: + +| Key | Value | Description | Note | +|----------------|----------|------------------------------------------------------|----------------------| +| _id_ | string |The unique identifier of the IdP group | | +| _name_ | string |The name of the IdP group (informational, obligatory) | | +| _description_ | string |Human‑readable description of the IdP group | | + + +## Jsonnet Function + +``` jsonnet +orgs.newTeamSync('') { + : +} +``` + +## Validation rules + +- each `TeamSync` entry **must define `id`, `name` and `description`** +- `id`, `name` and `description` **must be non-empty strings** +- omitting any of these fields prevents successful synchronization with GitHub Enterprise Cloud + + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + teams+: [ + orgs.newTeam('admins') { + description: "The project admins", + privacy: "secret", + + team_sync: [ + orgs.newTeamSync('git-admin') { + id: "1234567890", + description: "Admin access via IdP", + }, + ], + }, + ], + ... + } + ``` diff --git a/examples/template/otterdog-defaults.libsonnet b/examples/template/otterdog-defaults.libsonnet index 486353c9..b2c3baaa 100644 --- a/examples/template/otterdog-defaults.libsonnet +++ b/examples/template/otterdog-defaults.libsonnet @@ -255,6 +255,13 @@ local newOrgRole(name) = { base_role: "none", }; +# Function to create a new team_sync with default settings. +local newTeamSync(name) = { + name: name, + description: "", + id: "", +}; + # Function to create a new team with default settings. local newTeam(name) = { name: name, @@ -262,6 +269,8 @@ local newTeam(name) = { privacy: "visible", notifications: true, members: [], + team_sync: [], + external_groups: null, skip_members: false, skip_non_organization_members: false, }; @@ -424,6 +433,7 @@ local newOrg(name, id=name) = { newOrg:: newOrg, newOrgRole:: newOrgRole, newTeam:: newTeam, + newTeamSync:: newTeamSync, newOrgWebhook:: newOrgWebhook, newOrgSecret:: newOrgSecret, newOrgVariable:: newOrgVariable, diff --git a/mkdocs.yml b/mkdocs.yml index a6228613..aa9f6de4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,7 +52,9 @@ nav: - Organization Variable: reference/organization/variable.md - Organization Ruleset: reference/organization/ruleset.md - Custom Property: reference/organization/custom-property.md - - Team: reference/organization/team.md + - Team: + - reference/organization/team/index.md + - TeamSync: reference/organization/team/team-sync.md - Repository: - reference/organization/repository/index.md - Repository Webhook: reference/organization/repository/webhook.md diff --git a/otterdog/jsonnet.py b/otterdog/jsonnet.py index 793942c7..47ff6fb5 100644 --- a/otterdog/jsonnet.py +++ b/otterdog/jsonnet.py @@ -29,6 +29,7 @@ class JsonnetConfig: create_org = "newOrg" create_org_role = "newOrgRole" create_org_team = "newTeam" + create_org_team_sync = "newTeamSync" create_org_custom_property = "newCustomProperty" create_org_webhook = "newOrgWebhook" create_org_secret = "newOrgSecret" @@ -137,6 +138,16 @@ def default_team_config(self): _logger.debug("no default team config found, teams will be skipped") return None + @cached_property + def default_team_sync_config(self): + try: + # load the default team sync config + team_sync_snippet = f"(import '{self.template_file}').{self.create_org_team_sync}('default')" + return jsonnet_evaluate_snippet(team_sync_snippet) + except RuntimeError: + _logger.debug("no default team sync config found, team syncs will be skipped") + return None + @cached_property def default_org_custom_property_config(self): try: diff --git a/otterdog/models/github_organization.py b/otterdog/models/github_organization.py index 150878e1..f745a4a6 100644 --- a/otterdog/models/github_organization.py +++ b/otterdog/models/github_organization.py @@ -46,6 +46,7 @@ from otterdog.models.repo_workflow_settings import RepositoryWorkflowSettings from otterdog.models.repository import Repository from otterdog.models.team import Team +from otterdog.models.team_sync import TeamSync from otterdog.utils import IndentingPrinter, associate_by_key, debug_times, jsonnet_evaluate_file if TYPE_CHECKING: @@ -583,20 +584,15 @@ async def _load_teams() -> None: continue team_members = await provider.get_org_team_members(github_id, team_slug) team["members"] = team_members - # Do the team-sync - sync_groups = await provider.get_org_team_sync_groups(github_id, team_slug) - if sync_groups: - team["team_sync_id"] = sync_groups[0].get("group_id", None) - team["team_sync_name"] = sync_groups[0].get("group_name", None) - team["team_sync_description"] = sync_groups[0].get("group_description", None) - else: - team["team_sync_id"] = None - team["team_sync_name"] = None - team["team_sync_description"] = None # External Groups external_groups = await provider.get_org_team_external_groups(github_id, team_slug) team["external_groups"] = external_groups - org.add_team(Team.from_provider_data(github_id, team)) + tm = Team.from_provider_data(github_id, team) + # Do the team-sync + sync_groups = await provider.get_org_team_sync_groups(github_id, team_slug) + for sg in sync_groups: + tm.add_team_sync(TeamSync.from_provider_data(github_id, sg)) + org.add_team(tm) else: _logger.debug("not reading teams, no default config available") diff --git a/otterdog/models/team.py b/otterdog/models/team.py index 2d549766..f7357018 100644 --- a/otterdog/models/team.py +++ b/otterdog/models/team.py @@ -17,13 +17,26 @@ from otterdog.models import ( FailureType, LivePatch, + LivePatchContext, + LivePatchHandler, LivePatchType, ModelObject, + PatchContext, ValidationContext, ) -from otterdog.utils import UNSET, is_set_and_valid, is_unset, unwrap +from otterdog.models.team_sync import TeamSync +from otterdog.utils import ( + UNSET, + Change, + IndentingPrinter, + is_set_and_valid, + unwrap, + write_patch_object_as_json, +) if TYPE_CHECKING: + from collections.abc import Iterator + from otterdog.jsonnet import JsonnetConfig from otterdog.providers.github import GitHubProvider @@ -43,10 +56,8 @@ class Team(ModelObject, abc.ABC): privacy: str notifications: bool members: list[str] - team_sync_id: str | None - team_sync_name: str | None - team_sync_description: str | None external_groups: str | None + team_sync: list[TeamSync] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) skip_members: bool = dataclasses.field(metadata={"model_only": True}, default=False) skip_non_organization_members: bool = dataclasses.field(metadata={"model_only": True}, default=False) @@ -54,6 +65,9 @@ class Team(ModelObject, abc.ABC): def model_object_name(self) -> str: return "team" + def add_team_sync(self, ts: TeamSync) -> None: + self.team_sync.append(ts) + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: return f"orgs.{jsonnet_config.create_org_team}" @@ -102,22 +116,8 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: f"but 'members' contains user '{member}' who is not an organization member.", ) - values = [ - self.team_sync_id, - self.team_sync_name, - self.team_sync_description, - ] - - all_strings = all(is_set_and_valid(v) for v in values) - all_unset_or_none = all(is_unset(v) or v is None for v in values) - - if not all_strings and not all_unset_or_none: - context.add_failure( - FailureType.ERROR, - f"{self.get_model_header(parent_object)} has inconsistent team sync configuration: " - f"all of 'team_sync_id', 'team_sync_name', and 'team_sync_description' must either " - f"all be unset/None or all be valid strings.", - ) + for ts in self.team_sync: + ts.validate(context, self) @classmethod def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: @@ -149,6 +149,11 @@ def transform_external_groups(value): ) return mapping + def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject]]: + for ts in self.team_sync: + yield ts, self + yield from ts.get_model_objects() + @classmethod async def get_mapping_to_provider( cls, org_id: str, data: dict[str, Any], provider: GitHubProvider @@ -166,6 +171,90 @@ async def get_mapping_to_provider( return mapping + @classmethod + def get_mapping_from_model(cls) -> dict[str, Any]: + mapping = super().get_mapping_from_model() + + mapping.update( + { + "team_sync": OptionalS("team_sync", default=[]) >> Forall(lambda x: TeamSync.from_model_data(x)), + } + ) + return mapping + + def to_jsonnet( + self, + printer: IndentingPrinter, + jsonnet_config: JsonnetConfig, + context: PatchContext, + extend: bool, + default_object: ModelObject, + ) -> None: + + has_team_sync = len(self.team_sync) > 0 + + patch = self.get_patch_to(default_object) + + template_function = self.get_jsonnet_template_function(jsonnet_config, False) + + printer.print(f"{unwrap(template_function)}('{self.name}')") + + write_patch_object_as_json(patch, printer, False) + + if has_team_sync and not extend: + default_team_sync = TeamSync.from_model_data(jsonnet_config.default_team_sync_config) + printer.println("team_sync: [") + printer.level_up() + + for ts in self.team_sync: + ts.to_jsonnet(printer, jsonnet_config, context, False, default_team_sync) + + printer.level_down() + printer.println("],") + + # close the team object + printer.level_down() + printer.println("},") + + @classmethod + def generate_live_patch( + cls, + expected_object: Team | None, + current_object: Team | None, + parent_object: ModelObject | None, + context: LivePatchContext, + handler: LivePatchHandler, + ) -> None: + if expected_object is None: + current_object = unwrap(current_object) + handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + return + + if current_object is None: + handler(LivePatch.of_addition(expected_object, parent_object, expected_object.apply_live_patch)) + else: + modified_rule: dict[str, Change[Any]] = expected_object.get_difference_from(current_object) + + if len(modified_rule) > 0: + handler( + LivePatch.of_changes( + expected_object, + current_object, + modified_rule, + parent_object, + False, + expected_object.apply_live_patch, + ) + ) + + TeamSync.generate_live_patch_of_list( + expected_object.team_sync, + current_object.team_sync if current_object is not None else [], + expected_object, + context, + handler, + ) + @classmethod async def apply_live_patch( cls, diff --git a/otterdog/models/team_sync.py b/otterdog/models/team_sync.py new file mode 100644 index 00000000..7c659db9 --- /dev/null +++ b/otterdog/models/team_sync.py @@ -0,0 +1,106 @@ +# ******************************************************************************* +# Copyright (c) 2023-2026 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + + +import abc +import dataclasses +from typing import Any + +from jsonbender import S # type: ignore + +from otterdog.jsonnet import JsonnetConfig +from otterdog.models import FailureType, LivePatch, LivePatchType, ModelObject, ValidationContext +from otterdog.providers.github import GitHubProvider +from otterdog.utils import expect_type, unwrap + + +@dataclasses.dataclass +class TeamSync(ModelObject, abc.ABC): + """ + Represents a team sync to an IdP provider + """ + + name: str = dataclasses.field(metadata={"key": True}) + description: str + id: str + + @property + def model_object_name(self) -> str: + return "team_sync" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_org_team_sync}" + + def validate(self, context: ValidationContext, parent_object: Any) -> None: + missing: list[str] = [] + + if not self.id or not self.id.strip(): + missing.append("id") + if not self.name or not self.name.strip(): + missing.append("name") + if not self.description or not self.description.strip(): + missing.append("description") + + if missing: + context.add_failure( + FailureType.ERROR, + ( + f"{self.get_model_header(parent_object)} is missing required fields: " + f"{', '.join(missing)}. " + "Fields must not be empty or whitespace-only." + ), + ) + + @classmethod + def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: + mapping = super().get_mapping_from_provider(org_id, data) + mapping.update({"id": S("group_id"), "name": S("group_name"), "description": S("group_description")}) + return mapping + + @classmethod + async def get_mapping_to_provider( + cls, org_id: str, data: dict[str, Any], provider: GitHubProvider + ) -> dict[str, Any]: + mapping = await super().get_mapping_to_provider(org_id, data, provider) + mapping.update( + { + "group_id": S("id"), + "group_name": S("name"), + "group_description": S("description"), + } + ) + + return {k: v for k, v in mapping.items() if k.startswith("group_")} + + @classmethod + async def apply_live_patch(cls, patch: LivePatch["TeamSync"], org_id: str, provider: GitHubProvider) -> None: + from .team import Team + + team = expect_type(patch.parent_object, Team) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_org_team_sync_group( + org_id, + team.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_org_team_sync_group( + org_id, + team.name, + await unwrap(patch.current_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.CHANGE: + await provider.update_org_team_sync_group( + org_id, + team.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/providers/github/__init__.py b/otterdog/providers/github/__init__.py index 31963ac5..1ee2d8b9 100644 --- a/otterdog/providers/github/__init__.py +++ b/otterdog/providers/github/__init__.py @@ -40,6 +40,10 @@ def is_org_settings_key_retrieved_via_web_ui(key: str) -> bool: return key in _SETTINGS_WEB_KEYS +def _same_group(a: dict[str, Any], b: dict[str, Any]) -> bool: + return a.get("group_id") == b.get("group_id") + + class GitHubProvider: def __init__(self, credentials: Credentials | None): self._credentials = credentials @@ -169,6 +173,53 @@ async def get_org_team_members(self, org_id: str, team_slug: str) -> list[dict[s async def get_org_team_sync_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: return await self.rest_api.team.get_team_sync_groups(org_id, team_slug) + async def add_org_team_sync_group(self, org_id: str, team_name: str, data: dict[str, Any]) -> None: + team_slug = await self.rest_api.team.get_team_slug(org_id, team_name) + groups = await self.get_org_team_sync_groups(org_id, team_slug) + + if any(_same_group(g, data) for g in groups): + _logger.debug( + "team-sync group '%s' already present for team '%s/%s'", + data.get("group_id"), + org_id, + team_slug, + ) + return + groups.append(data) + await self.rest_api.team.update_team_sync_groups(org_id, team_slug, groups) + + async def delete_org_team_sync_group(self, org_id: str, team_name: str, data: dict[str, Any]) -> None: + team_slug = await self.rest_api.team.get_team_slug(org_id, team_name) + groups = await self.get_org_team_sync_groups(org_id, team_slug) + + new_groups = [g for g in groups if not _same_group(g, data)] + if len(groups) == len(new_groups): + _logger.debug( + "team-sync group '%s' not present for team '%s/%s'", + data.get("group_id"), + org_id, + team_slug, + ) + return + await self.rest_api.team.update_team_sync_groups(org_id, team_slug, new_groups) + + async def update_org_team_sync_group(self, org_id: str, team_name: str, data: dict[str, Any]) -> None: + team_slug = await self.rest_api.team.get_team_slug(org_id, team_name) + groups = await self.get_org_team_sync_groups(org_id, team_slug) + + replaced = False + new_groups: list[dict[str, Any]] = [] + + for g in groups: + if _same_group(g, data): + new_groups.append(data) + replaced = True + else: + new_groups.append(g) + if not replaced: + new_groups.append(data) + await self.rest_api.team.update_team_sync_groups(org_id, team_slug, new_groups) + async def get_org_team_external_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: return await self.rest_api.team.get_team_external_groups(org_id, team_slug) diff --git a/otterdog/providers/github/rest/team_client.py b/otterdog/providers/github/rest/team_client.py index fcfefe3e..b4e98b17 100644 --- a/otterdog/providers/github/rest/team_client.py +++ b/otterdog/providers/github/rest/team_client.py @@ -82,13 +82,6 @@ async def add_team(self, org_id: str, team_name: str, data: dict[str, str]) -> s members = data["members"] for user in members: await self.add_member_to_team(org_id, team_slug, user) - if all(k in data and data[k] is not None for k in ("team_sync_id", "team_sync_name", "team_sync_description")): - tsdata = { - "group_id": data["team_sync_id"], - "group_name": data["team_sync_name"], - "group_description": data["team_sync_description"], - } - await self.update_team_sync_groups(org_id, team_slug, tsdata) if "external_groups" in data and data["external_groups"] is not None: await self.update_team_external_groups(org_id, team_slug, data["external_groups"]) @@ -103,17 +96,6 @@ async def update_team(self, org_id: str, team_slug: str, team: dict[str, Any]) - if "members" in team: await self.update_team_members(org_id, team_slug, team["members"]) - if all( - k in team and team[k] is not None for k in ("team_sync_id", "team_sync_name", "team_sync_description") - ): - tsdata = { - "group_id": team["team_sync_id"], - "group_name": team["team_sync_name"], - "group_description": team["team_sync_description"], - } - await self.update_team_sync_groups(org_id, team_slug, tsdata) - elif all(k in team for k in ("team_sync_id", "team_sync_name", "team_sync_description")): - await self.update_team_sync_groups(org_id, team_slug, None) if "external_groups" in team: await self.update_team_external_groups(org_id, team_slug, team["external_groups"]) @@ -220,19 +202,17 @@ async def get_team_sync_groups(self, org_id: str, team_slug: str) -> list[dict[s return response.get("groups", []) - async def update_team_sync_groups(self, org_id: str, team_slug: str, group: dict[str, str] | None) -> None: + async def update_team_sync_groups(self, org_id: str, team_slug: str, groups: list[dict[str, str]] | None) -> None: _logger.debug("updating sync_groups for team '%s' in org '%s'", team_slug, org_id) - data = {"groups": []} if group is None else {"groups": [group]} + data = {"groups": []} if groups is None else {"groups": groups} status, body = await self.requester.request_raw( "PATCH", f"/orgs/{org_id}/teams/{team_slug}/team-sync/group-mappings", data=json.dumps(data) ) if status == 200: - _logger.debug("updated team-sync '%s' of team '%s' for org '%s'", group, team_slug, org_id) + _logger.debug("updated team-syncs of team '%s' for org '%s'", team_slug, org_id) else: - raise RuntimeError( - f"failed updating team-sync '{group}' to team '{team_slug}' in org '{org_id}'\n{status}: {body}" - ) + raise RuntimeError(f"failed updating team-syncs to team '{team_slug}' in org '{org_id}'\n{status}: {body}") async def get_team_external_groups(self, org_id: str, team_slug: str) -> list[dict[str, Any]]: _logger.debug("retrieving external groups for team '%s/%s'", org_id, team_slug) diff --git a/otterdog/resources/schemas/team.json b/otterdog/resources/schemas/team.json index 04221d5c..17299a5b 100644 --- a/otterdog/resources/schemas/team.json +++ b/otterdog/resources/schemas/team.json @@ -15,9 +15,12 @@ "items": { "type": "string" } }, - "team_sync_id": { "$ref": "types.json#/$defs/string-or-null" }, - "team_sync_name": { "$ref": "types.json#/$defs/string-or-null" }, - "team_sync_description": { "$ref": "types.json#/$defs/string-or-null" }, + "team_sync": { + "type": "array", + "items": { + "$ref": "team_sync.json" + } + }, "external_groups": { "$ref": "types.json#/$defs/string-or-null" } }, "required": ["name", "privacy"], diff --git a/otterdog/resources/schemas/team_sync.json b/otterdog/resources/schemas/team_sync.json new file mode 100644 index 00000000..5ea57e42 --- /dev/null +++ b/otterdog/resources/schemas/team_sync.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "team_sync.json", + "title": "Team Sync Entry Schema", + "type": "object", + + "properties": { + "id": { + "type": "string", + "minLength": 1, + "pattern": "^\\S.*" + }, + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S.*" + }, + "description": { + "type": "string", + "minLength": 1, + "pattern": "^\\S.*" + } + }, + + "required": ["id", "name", "description"], + "additionalProperties": false +} diff --git a/poetry.lock b/poetry.lock index f6a01409..e7b22961 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2761,14 +2761,14 @@ dev = ["black", "build", "flake8", "flake8-black", "isort", "jupyter-console", " [[package]] name = "pygments" -version = "2.20.0" +version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" groups = ["main", "docs", "test"] files = [ - {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, - {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, ] [package.extras] @@ -3710,14 +3710,14 @@ files = [ [[package]] name = "types-hvac" -version = "2.4.0.20251115" +version = "2.4.0.20260408" description = "Typing stubs for hvac" optional = false -python-versions = ">=3.9" -groups = ["typing"] +python-versions = ">=3.10" +groups = ["dev", "typing"] files = [ - {file = "types_hvac-2.4.0.20251115-py3-none-any.whl", hash = "sha256:f92bcbb2eced81ce77cdbc865bff911e20341afaa6f2126e925cb8c062003f82"}, - {file = "types_hvac-2.4.0.20251115.tar.gz", hash = "sha256:a7b5d0e86961e4d7f07e51b60b2eedcac5f8db8b98b031d4a4b716215c21df3a"}, + {file = "types_hvac-2.4.0.20260408-py3-none-any.whl", hash = "sha256:ee8d3528eaa6a2b480dbdde04fec06bcd26d13c9dfd97837b62cca98f175d70a"}, + {file = "types_hvac-2.4.0.20260408.tar.gz", hash = "sha256:291eeeaa0af0bc5d02960fc8aaa6365ad53a05715d5b055b3c64023e74d738c1"}, ] [package.dependencies] @@ -3800,7 +3800,7 @@ version = "2.32.4.20260107" description = "Typing stubs for requests" optional = false python-versions = ">=3.9" -groups = ["typing"] +groups = ["dev", "typing"] files = [ {file = "types_requests-2.32.4.20260107-py3-none-any.whl", hash = "sha256:b703fe72f8ce5b31ef031264fe9395cac8f46a04661a79f7ed31a80fb308730d"}, {file = "types_requests-2.32.4.20260107.tar.gz", hash = "sha256:018a11ac158f801bfa84857ddec1650750e393df8a004a8a9ae2a9bec6fcb24f"}, @@ -3873,7 +3873,7 @@ version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main", "app", "docs", "typing"] +groups = ["main", "app", "dev", "docs", "typing"] files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, @@ -4140,4 +4140,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "1c419f8c8e9cbb69c76384b2382c31e3598f4ea89f095b202c497590f1aa065e" +content-hash = "d1fd5671958f8c249b949de73182ae87da003462444fb9ba532464639273bce7" diff --git a/pyproject.toml b/pyproject.toml index bee993db..fe3b4a0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,6 +106,7 @@ python-slugify = "^8.0" [tool.poetry.group.dev.dependencies] ruff = ">=0.15" prek = ">=0.3.0" +types-hvac = "^2.4.0.20260408" [tool.poetry.group.test.dependencies] tox = ">4.22" @@ -135,6 +136,7 @@ mkdocs-exclude = "^1.0" pymdown-extensions = "^10.7" mkdocs-include-markdown-plugin = "^7.1.6" mkdocs-mermaid2-plugin = "^1.2.3" +pygments = ">=2.16,<2.20" [tool.poetry-dynamic-versioning] enable = true