From 137ef10c116994f89a3385806b45c920578c5b2d Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Thu, 12 Mar 2026 14:40:39 +0100 Subject: [PATCH 1/3] feat: Added variables and secrets --- .../organization/codespaces_secret.md | 63 +++++++ .../organization/dependabot_secret.md | 63 +++++++ docs/reference/organization/index.md | 26 ++- .../organization/repository/branch-or-tag.md | 2 +- .../repository/codespaces_secret.md | 60 +++++++ .../repository/dependabot_secret.md | 60 +++++++ .../organization/repository/environment.md | 46 ----- .../repository/environment/index.md | 57 ++++++ .../repository/environment/secret.md | 64 +++++++ .../repository/environment/variable.md | 42 +++++ .../organization/repository/index.md | 4 +- examples/template/otterdog-defaults.libsonnet | 58 ++++++ mkdocs.yml | 9 +- otterdog/jsonnet.py | 70 ++++++++ otterdog/models/__init__.py | 25 ++- otterdog/models/env_secret.py | 75 ++++++++ otterdog/models/env_variable.py | 75 ++++++++ otterdog/models/environment.py | 162 ++++++++++++++++- otterdog/models/github_organization.py | 153 +++++++++++++++- .../models/organization_codespaces_secret.py | 122 +++++++++++++ .../models/organization_dependabot_secret.py | 122 +++++++++++++ otterdog/models/repo_codespaces_secret.py | 68 +++++++ otterdog/models/repo_dependabot_secret.py | 68 +++++++ otterdog/models/repository.py | 116 ++++++++++++ otterdog/providers/github/__init__.py | 86 +++++++++ otterdog/providers/github/rest/__init__.py | 6 + otterdog/providers/github/rest/env_client.py | 169 ++++++++++++++++++ otterdog/providers/github/rest/org_client.py | 160 +++++++++++++++++ otterdog/providers/github/rest/repo_client.py | 158 ++++++++++++++++ otterdog/resources/schemas/env-secret.json | 8 + otterdog/resources/schemas/env-variable.json | 8 + otterdog/resources/schemas/environment.json | 8 + otterdog/resources/schemas/organization.json | 8 + otterdog/resources/schemas/repository.json | 8 + tests/models/resources/github-env-secret.json | 4 + .../models/resources/otterdog-env-secret.json | 5 + .../test-org/vendor/github-env-secret.json | 4 + tests/models/test_env_secret.py | 67 +++++++ tests/models/test_env_variable.py | 33 ++++ .../github/integration/helpers/model.py | 2 +- 40 files changed, 2284 insertions(+), 60 deletions(-) create mode 100644 docs/reference/organization/codespaces_secret.md create mode 100644 docs/reference/organization/dependabot_secret.md create mode 100644 docs/reference/organization/repository/codespaces_secret.md create mode 100644 docs/reference/organization/repository/dependabot_secret.md delete mode 100644 docs/reference/organization/repository/environment.md create mode 100644 docs/reference/organization/repository/environment/index.md create mode 100644 docs/reference/organization/repository/environment/secret.md create mode 100644 docs/reference/organization/repository/environment/variable.md create mode 100644 otterdog/models/env_secret.py create mode 100644 otterdog/models/env_variable.py create mode 100644 otterdog/models/organization_codespaces_secret.py create mode 100644 otterdog/models/organization_dependabot_secret.py create mode 100644 otterdog/models/repo_codespaces_secret.py create mode 100644 otterdog/models/repo_dependabot_secret.py create mode 100644 otterdog/providers/github/rest/env_client.py create mode 100644 otterdog/resources/schemas/env-secret.json create mode 100644 otterdog/resources/schemas/env-variable.json create mode 100644 tests/models/resources/github-env-secret.json create mode 100644 tests/models/resources/otterdog-env-secret.json create mode 100644 tests/models/resources/test-org/vendor/github-env-secret.json create mode 100644 tests/models/test_env_secret.py create mode 100644 tests/models/test_env_variable.py diff --git a/docs/reference/organization/codespaces_secret.md b/docs/reference/organization/codespaces_secret.md new file mode 100644 index 00000000..b7add538 --- /dev/null +++ b/docs/reference/organization/codespaces_secret.md @@ -0,0 +1,63 @@ +Definition of a `Codespaces Secret` on organization level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|------------------------------------------------|------------------------------------------------------| +| _name_ | string | The name of the secret | | +| _selected_repositories_ | list[string] | List of repositories that can use the codespaces secret | only applicable if `visibility` is set to `selected` | +| _value_ | string | The codespaces secret value | | +| _visibility_ | string | Controls which repositories can use the codespaces secret | `public`, `private` or `selected` | + +The codespaces secret value can be resolved via a credential provider. The supported format is `:`. + +- Bitwarden: `bitwarden:@` + + ``` json + "secret": "bitwarden:118276ad-158c-4720-b68d-af8c00fe3481@secret" + ``` + +- Pass: `pass:` + + ``` json + "secret": "pass:path/to/org/secret" + ``` + +!!! note + + After executing an `import` operation, the codespaces secret will be set to `********` as GitHub will not disclose the + secret value anymore via its API. You will need to update the configuration with the real secret value, either + by entering the secret value (not advised), or referencing it via a credential provider. + + Secrets which have a redacted value defined will be skipped during processing. + +## Jsonnet Function + +``` jsonnet +orgs.newOrgCodespacesSecret('') { + : +} +``` + +## Validation rules + +- redacted codespaces secret values (`********`) trigger a validation info and will skip the secret during processing +- `visibility` of `private` is not supported by GitHub with a billing plan of type `free` +- specifying a non-empty list of `selected_repositories` while `visibility` is not set to `selected` triggers a warning + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + secrets+: [ + orgs.newOrgCodespacesSecret('TEST_CODESPACES_SECRET') { + selected_repositories+: [ + "test-repo" + ], + value: "pass:path/to/my/secret/value", + visibility: "selected", + }, + ], + ... + } + ``` diff --git a/docs/reference/organization/dependabot_secret.md b/docs/reference/organization/dependabot_secret.md new file mode 100644 index 00000000..40748697 --- /dev/null +++ b/docs/reference/organization/dependabot_secret.md @@ -0,0 +1,63 @@ +Definition of a `Dependabot Secret` on organization level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|------------------------------------------------|------------------------------------------------------| +| _name_ | string | The name of the secret | | +| _selected_repositories_ | list[string] | List of repositories that can use the dependabot secret | only applicable if `visibility` is set to `selected` | +| _value_ | string | The dependabot secret value | | +| _visibility_ | string | Controls which repositories can use the dependabot secret | `public`, `private` or `selected` | + +The dependabot secret value can be resolved via a credential provider. The supported format is `:`. + +- Bitwarden: `bitwarden:@` + + ``` json + "secret": "bitwarden:118276ad-158c-4720-b68d-af8c00fe3481@secret" + ``` + +- Pass: `pass:` + + ``` json + "secret": "pass:path/to/org/secret" + ``` + +!!! note + + After executing an `import` operation, the dependabot secret will be set to `********` as GitHub will not disclose the + secret value anymore via its API. You will need to update the configuration with the real secret value, either + by entering the secret value (not advised), or referencing it via a credential provider. + + Secrets which have a redacted value defined will be skipped during processing. + +## Jsonnet Function + +``` jsonnet +orgs.newOrgDependabotSecret('') { + : +} +``` + +## Validation rules + +- redacted dependabot secret values (`********`) trigger a validation info and will skip the secret during processing +- `visibility` of `private` is not supported by GitHub with a billing plan of type `free` +- specifying a non-empty list of `selected_repositories` while `visibility` is not set to `selected` triggers a warning + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + secrets+: [ + orgs.newOrgDependabotSecret('TEST_DEPENDABOT_SECRET') { + selected_repositories+: [ + "test-repo" + ], + value: "pass:path/to/my/secret/value", + visibility: "selected", + }, + ], + ... + } + ``` diff --git a/docs/reference/organization/index.md b/docs/reference/organization/index.md index 720e034c..00902db2 100644 --- a/docs/reference/organization/index.md +++ b/docs/reference/organization/index.md @@ -8,18 +8,22 @@ This resource represents a GitHub organization with all supported settings and n settings+: { ... }, // (1)! webhooks+: [ ... ], // (2)! secrets+: [ ... ], // (3)! - variables+: [ ... ], // (4)! - rulesets+: [ ... ], // (5)! - _repositories+:: [ ... ], // (6)! + dependabot_secrets+: [ ... ], // (4)! + codespaces_secrets+: [ ... ], // (5)! + variables+: [ ... ], // (6)! + rulesets+: [ ... ], // (7)! + _repositories+:: [ ... ], // (8)! } ``` 1. see [Organization Settings](settings.md) 2. see [Organization Webhook](webhook.md) 3. see [Organization Secret](secret.md) - 4. see [Organization Variable](variable.md) - 5. see [Organization Ruleset](ruleset.md) - 6. see [Repository](repository/index.md) + 4. see [Organization Dependabot Secret](dependabot_secret.md) + 5. see [Organization Codespaces Secret](codespaces_secret.md) + 6. see [Organization Variable](variable.md) + 7. see [Organization Ruleset](ruleset.md) + 8. see [Repository](repository/index.md) !!! note @@ -70,6 +74,16 @@ The configuration of a GitHub Organization is considered to be valid if all nest value: "pass:bots/adoptium.aqavit/github.com/project-token", }, ], + dependabot_secrets+: [ + orgs.newOrgDependabotSecret('DEPENDABOT_ADOPTIUM_AQAVIT_BOT_TOKEN') { + value: "pass:bots/adoptium.aqavit/github.com/dependabot-token", + }, + ], + codespaces_secrets+: [ + orgs.newOrgCodespacesSecret('CODESPACES_ADOPTIUM_AQAVIT_BOT_TOKEN') { + value: "pass:bots/adoptium.aqavit/github.com/codespaces-token", + }, + ], variables+: [ orgs.newOrgVariable('SONAR_USERNAME') { value: "xxxxx", diff --git a/docs/reference/organization/repository/branch-or-tag.md b/docs/reference/organization/repository/branch-or-tag.md index 0fda1415..e21f0169 100644 --- a/docs/reference/organization/repository/branch-or-tag.md +++ b/docs/reference/organization/repository/branch-or-tag.md @@ -1,4 +1,4 @@ -A BranchOrTag represents either a branch or tag pattern to use within an [Environment](environment.md). +A BranchOrTag represents either a branch or tag pattern to use within an [Environment](environment/index.md). The following format is used to distinguish between tags and branches: | Type | Format | Example | diff --git a/docs/reference/organization/repository/codespaces_secret.md b/docs/reference/organization/repository/codespaces_secret.md new file mode 100644 index 00000000..48a246e2 --- /dev/null +++ b/docs/reference/organization/repository/codespaces_secret.md @@ -0,0 +1,60 @@ +Definition of a `Codespaces Secret` on repository level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|------------------------------------------------|------| +| _name_ | string | The name of the secret | | +| _value_ | string | The secret value | | + +The codespaces secret value can be resolved via a credential provider. The supported format is `:`. + +- Bitwarden: `bitwarden:@` + + ``` json + "secret": "bitwarden:118276ad-158c-4720-b68d-af8c00fe3481@secret" + ``` + +- Pass: `pass:` + + ``` json + "secret": "pass:path/to/repo/secret" + ``` + +!!! note + + After executing an `import` operation, the codespaces secret will be set to `********` as GitHub will not disclose the + secret value anymore via its API. You will need to update the configuration with the real secret value, either + by entering the secret value (not advised), or referencing it via a credential provider. + + Secrets which have a redacted value defined will be skipped during processing. + +## Jsonnet Function + +``` jsonnet +orgs.newRepoDependabotSecret('') { + : +} +``` + +## Validation rules + +- redacted dependabot secret values (`********`) trigger a validation info and will skip the secret during processing + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + codespaces_secrets+: [ + orgs.newRepoCodespacesSecret('TEST_CODESPACES_SECRET') { + value: "pass:path/to/secret", + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/repository/dependabot_secret.md b/docs/reference/organization/repository/dependabot_secret.md new file mode 100644 index 00000000..1620f373 --- /dev/null +++ b/docs/reference/organization/repository/dependabot_secret.md @@ -0,0 +1,60 @@ +Definition of a `Dependabot Secret` on repository level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|------------------------------------------------|------| +| _name_ | string | The name of the secret | | +| _value_ | string | The secret value | | + +The dependabot secret value can be resolved via a credential provider. The supported format is `:`. + +- Bitwarden: `bitwarden:@` + + ``` json + "secret": "bitwarden:118276ad-158c-4720-b68d-af8c00fe3481@secret" + ``` + +- Pass: `pass:` + + ``` json + "secret": "pass:path/to/repo/secret" + ``` + +!!! note + + After executing an `import` operation, the dependabot secret will be set to `********` as GitHub will not disclose the + secret value anymore via its API. You will need to update the configuration with the real secret value, either + by entering the secret value (not advised), or referencing it via a credential provider. + + Secrets which have a redacted value defined will be skipped during processing. + +## Jsonnet Function + +``` jsonnet +orgs.newRepoCodespacesSecret('') { + : +} +``` + +## Validation rules + +- redacted codespaces secret values (`********`) trigger a validation info and will skip the secret during processing + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + dependabot_secrets+: [ + orgs.newRepoDependabotSecret('TEST_DEPENDABOT_SECRET') { + value: "pass:path/to/secret", + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/repository/environment.md b/docs/reference/organization/repository/environment.md deleted file mode 100644 index 40afa927..00000000 --- a/docs/reference/organization/repository/environment.md +++ /dev/null @@ -1,46 +0,0 @@ -Definition of an `Environment` on repository level, the following properties are supported: - -| Key | Value | Description | Notes | -|----------------------------|-----------------------------------------|---------------------------------------------------------------------------------------|--------------------------------------------------------------------| -| _name_ | string | The name of the environment | | -| _wait_timer_ | int | The amount of time to wait before allowing deployments to proceed | | -| _reviewers_ | list\[[Actor](actor.md)\] | Users or Teams that may approve workflow runs that access this environment | | -| _deployment_branch_policy_ | string | Limit which branches can deploy to this environment based on rules or naming patterns | `all`, `protected` or `selected` | -| _branch_policies_ | list\[[BranchOrTag](branch-or-tag.md)\] | List of branch or tag patterns which can deploy to this environment | only applicable if `deployment_branch_policy` is set to `selected` | - -## Jsonnet Function - -``` jsonnet -orgs.newEnvironment('') { - : -} -``` - -## Validation rules - -- specifying a non-empty list of `branch_policies` while `deployment_branch_policy` is not set to `selected` triggers a warning - -## Example usage - -=== "jsonnet" - ``` jsonnet - orgs.newOrg('OtterdogTest') { - ... - _repositories+:: [ - ... - orgs.newRepo('test-repo') { - ... - environments: [ - orgs.newEnvironment('linux') { - deployment_branch_policy: "protected", - reviewers+: [ - "@OtterdogTest/eclipsefdn-security", - "@netomi" - ], - wait_timer: 30, - }, - ] - } - ] - } - ``` diff --git a/docs/reference/organization/repository/environment/index.md b/docs/reference/organization/repository/environment/index.md new file mode 100644 index 00000000..3f2e6687 --- /dev/null +++ b/docs/reference/organization/repository/environment/index.md @@ -0,0 +1,57 @@ +Definition of an `Environment` on repository level, the following properties are supported: + +| Key | Value | Description | Notes | +|----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| _name_ | string | The name of the environment | | +| _wait_timer_ | int | The amount of time to wait before allowing deployments to proceed | | +| _reviewers_ | list\[[Actor](../actor.md)\] | Users or Teams that may approve workflow runs that access this environment | | +| _deployment_branch_policy_ | string | Limit which branches can deploy to this environment based on rules or naming patterns | `all`, `protected` or `selected` | +| _branch_policies_ | list\[[BranchOrTag](../branch-or-tag.md)\] | List of branch or tag patterns which can deploy to this environment | only applicable if `deployment_branch_policy` is set to `selected` | +| _secrets_ | list\[[EnvironmentSecret](secret.md)\] | environment secrets defined for this repo, see section below for details | | +| _variables_ | list\[[EnvironmentVariable](variable.md)\] | environment variables defined for this repo, see section below for details | | + +## Jsonnet Function + +``` jsonnet +orgs.newEnvironment('') { + : +} +``` + +## Validation rules + +- specifying a non-empty list of `branch_policies` while `deployment_branch_policy` is not set to `selected` triggers a warning + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + environments: [ + orgs.newEnvironment('linux') { + deployment_branch_policy: "protected", + reviewers+: [ + "@OtterdogTest/eclipsefdn-security", + "@netomi" + ], + wait_timer: 30, + secrets: [ + orgs.newEnvSecret('ENVIRONMENT_ADOPTIUM_AQAVIT_BOT_TOKEN') { + value: "pass:bots/adoptium.aqavit/github.com/environment-token", + }, + variables: [ + orgs.newEnvVariable('ENVIRONMENT_USERNAME') { + value: "xxxxxxxx", + }, + ], + }, + ] + } + ] + } + ``` diff --git a/docs/reference/organization/repository/environment/secret.md b/docs/reference/organization/repository/environment/secret.md new file mode 100644 index 00000000..8d2ce270 --- /dev/null +++ b/docs/reference/organization/repository/environment/secret.md @@ -0,0 +1,64 @@ +Definition of a `Secret` on repository environment level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|------------------------------------------------|------| +| _name_ | string | The name of the secret | | +| _value_ | string | The secret value | | + +The secret value can be resolved via a credential provider. The supported format is `:`. + +- Bitwarden: `bitwarden:@` + + ``` json + "secret": "bitwarden:118276ad-158c-4720-b68d-af8c00fe3481@secret" + ``` + +- Pass: `pass:` + + ``` json + "secret": "pass:path/to/repo/secret" + ``` + +!!! note + + After executing an `import` operation, the secret will be set to `********` as GitHub will not disclose the + secret value anymore via its API. You will need to update the configuration with the real secret value, either + by entering the secret value (not advised), or referencing it via a credential provider. + + Secrets which have a redacted value defined will be skipped during processing. + +## Jsonnet Function + +``` jsonnet +orgs.newEnvSecret('') { + : +} +``` + +## Validation rules + +- redacted secret values (`********`) trigger a validation info and will skip the secret during processing + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + environments: [ + orgs.newEnvironment('Environment') { + secrets: [ + orgs.newEnvSecret('TEST_SECRET') { + value: "pass:path/to/secret", + }, + ], + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/repository/environment/variable.md b/docs/reference/organization/repository/environment/variable.md new file mode 100644 index 00000000..0432573b --- /dev/null +++ b/docs/reference/organization/repository/environment/variable.md @@ -0,0 +1,42 @@ +Definition of a `Variable` on repository environment level, the following properties are supported: + +| Key | Value | Description | Note | +|-------------------------|----------------|--------------------------|------| +| _name_ | string | The name of the variable | | +| _value_ | string | The variable value | | + +## Jsonnet Function + +``` jsonnet +orgs.newEnvVariable('') { + : +} +``` + +## Validation rules + +- None + +## Example usage + +=== "jsonnet" + ``` jsonnet + orgs.newOrg('OtterdogTest') { + ... + _repositories+:: [ + ... + orgs.newRepo('test-repo') { + ... + environments: [ + orgs.newEnvironment('Environment') { + variables: [ + orgs.newEnvVariable('TEST_VARIABLE') { + value: "TESTVALUE", + }, + ], + }, + ], + } + ] + } + ``` diff --git a/docs/reference/organization/repository/index.md b/docs/reference/organization/repository/index.md index b15364c1..ad3916df 100644 --- a/docs/reference/organization/repository/index.md +++ b/docs/reference/organization/repository/index.md @@ -50,8 +50,10 @@ Definition of a Repository for a GitHub organization, the following properties a | _workflows_ | [Workflow Settings](#workflow-settings) | Workflow settings on organizational level | | | _webhooks_ | list\[[Webhook](webhook.md)\] | webhooks defined for this repo, see section above for details | | | _secrets_ | list\[[RepositorySecret](secret.md)\] | secrets defined for this repo, see section below for details | | +| _dependabot_secrets_ | list\[[RepositoryDependabotSecret](dependabot_secret.md)\] | dependabot secrets defined for this repo, see section below for details | | +| _codespaces_secrets_ | list\[[RepositoryCodespacesSecret](codespaces_secret.md)\] | codespaces secrets defined for this repo, see section below for details | | | _variables_ | list\[[RepositoryVariable](variable.md)\] | variables defined for this repo, see section below for details | | -| _environments_ | list\[[Environment](environment.md)\] | environments defined for this repo, see section below for details | | +| _environments_ | list\[[Environment](environment/index.md)\] | environments defined for this repo, see section below for details | | | _branch_protection_rules_ | list\[[BranchProtectionRule](branch-protection-rule.md)\] | branch protection rules of the repo, see section below for details | | ## Embedded Models diff --git a/examples/template/otterdog-defaults.libsonnet b/examples/template/otterdog-defaults.libsonnet index 4c2ec91e..f5f5cb46 100644 --- a/examples/template/otterdog-defaults.libsonnet +++ b/examples/template/otterdog-defaults.libsonnet @@ -94,6 +94,12 @@ local newRepo(name) = { # repository secrets secrets: [], + # repository dependabot secrets + dependabot_secrets: [], + + # repository codespaces secrets + codespaces_secrets: [], + # repository variables variables: [], @@ -218,6 +224,24 @@ local newOrgWebhook(url) = { # Function to create a new repository webhook with default settings. local newRepoWebhook(url) = newOrgWebhook(url); +# Function to create a new environment secret with default settings. +local newEnvSecret(name) = { + name: name, + value: null +}; + +# Function to create a new dependabot secret with default settings. +local newRepoDependabotSecret(name) = { + name: name, + value: null, +}; + +# Function to create a new codespaces secret with default settings. +local newRepoCodespacesSecret(name) = { + name: name, + value: null, +}; + # Function to create a new repository secret with default settings. local newRepoSecret(name) = { name: name, @@ -230,6 +254,24 @@ local newOrgSecret(name) = newRepoSecret(name) { selected_repositories: [], }; +# Function to create a new organization dependabot secret with default settings. +local newOrgDependabotSecret(name) = newRepoSecret(name) { + visibility: "public", + selected_repositories: [], +}; + +# Function to create a new organization codespaces secret with default settings. +local newOrgCodespacesSecret(name) = newRepoSecret(name) { + visibility: "public", + selected_repositories: [], +}; + +# Function to create a new environment variable with default settings. +local newEnvVariable(name) = { + name: name, + value: null +}; + # Function to create a new repository variable with default settings. local newRepoVariable(name) = { name: name, @@ -269,6 +311,10 @@ local newEnvironment(name) = { # Can be one of: all, protected_branches, branch_policies deployment_branch_policy: "all", branch_policies: [], + # environment secrets + secrets: [], + # environment variables + variables: [], }; # Function to create a new custom property with default settings. @@ -394,6 +440,12 @@ local newOrg(name, id=name) = { # organization secrets secrets: [], + # organization dependabot secrets + dependabot_secrets: [], + + # organization codespaces secrets + codespaces_secrets: [], + # organization variables variables: [], @@ -421,6 +473,8 @@ local newOrg(name, id=name) = { newTeam:: newTeam, newOrgWebhook:: newOrgWebhook, newOrgSecret:: newOrgSecret, + newOrgDependabotSecret:: newOrgDependabotSecret, + newOrgCodespacesSecret:: newOrgCodespacesSecret, newOrgVariable:: newOrgVariable, newOrgRuleset:: newOrgRuleset, newCustomProperty:: newCustomProperty, @@ -428,10 +482,14 @@ local newOrg(name, id=name) = { extendRepo:: extendRepo, newRepoWebhook:: newRepoWebhook, newRepoSecret:: newRepoSecret, + newRepoDependabotSecret:: newRepoDependabotSecret, + newRepoCodespacesSecret:: newRepoCodespacesSecret, newRepoVariable:: newRepoVariable, newBranchProtectionRule:: newBranchProtectionRule, newRepoRuleset:: newRepoRuleset, newEnvironment:: newEnvironment, + newEnvSecret:: newEnvSecret, + newEnvVariable:: newEnvVariable, newPullRequest:: newPullRequest, newStatusChecks:: newStatusChecks, newMergeQueue:: newMergeQueue, diff --git a/mkdocs.yml b/mkdocs.yml index a6228613..4b0dd10b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -49,6 +49,8 @@ nav: - Organization Role: reference/organization/role.md - Organization Webhook: reference/organization/webhook.md - Organization Secret: reference/organization/secret.md + - Organization Dependabot Secret: reference/organization/dependabot_secret.md + - Organization Codespaces Secret: reference/organization/codespaces_secret.md - Organization Variable: reference/organization/variable.md - Organization Ruleset: reference/organization/ruleset.md - Custom Property: reference/organization/custom-property.md @@ -57,10 +59,15 @@ nav: - reference/organization/repository/index.md - Repository Webhook: reference/organization/repository/webhook.md - Repository Secret: reference/organization/repository/secret.md + - Repository Dependabot Secret: reference/organization/repository/dependabot_secret.md + - Repository Codespaces Secret: reference/organization/repository/codespaces_secret.md - Repository Variable: reference/organization/repository/variable.md - - Environment: reference/organization/repository/environment.md - Branch Protection Rule: reference/organization/repository/branch-protection-rule.md - Repository Ruleset: reference/organization/repository/ruleset.md + - Environment: + - reference/organization/repository/environment/index.md + - Environment Secret: reference/organization/repository/environment/secret.md + - Environment Variable: reference/organization/repository/environment/variable.md - Referenced Types: - Actor: reference/organization/repository/actor.md - Branch or Tag: reference/organization/repository/branch-or-tag.md diff --git a/otterdog/jsonnet.py b/otterdog/jsonnet.py index 793942c7..c4aba115 100644 --- a/otterdog/jsonnet.py +++ b/otterdog/jsonnet.py @@ -32,12 +32,16 @@ class JsonnetConfig: create_org_custom_property = "newCustomProperty" create_org_webhook = "newOrgWebhook" create_org_secret = "newOrgSecret" + create_org_dependabot_secret = "newOrgDependabotSecret" + create_org_codespaces_secret = "newOrgCodespacesSecret" create_org_variable = "newOrgVariable" create_org_ruleset = "newOrgRuleset" create_repo = "newRepo" extend_repo = "extendRepo" create_repo_webhook = "newRepoWebhook" create_repo_secret = "newRepoSecret" + create_repo_dependabot_secret = "newRepoDependabotSecret" + create_repo_codespaces_secret = "newRepoCodespacesSecret" create_repo_variable = "newRepoVariable" create_branch_protection_rule = "newBranchProtectionRule" create_repo_ruleset = "newRepoRuleset" @@ -45,6 +49,8 @@ class JsonnetConfig: create_pull_request = "newPullRequest" create_status_checks = "newStatusChecks" create_merge_queue = "newMergeQueue" + create_env_secret = "newEnvSecret" + create_env_variable = "newEnvVariable" def __init__( self, @@ -72,11 +78,15 @@ def __init__( self._default_org_custom_property_config: dict[str, Any] | None = None self._default_org_webhook_config: dict[str, Any] | None = None self._default_org_secret_config: dict[str, Any] | None = None + self._default_org_secret_dependabot_config: dict[str, Any] | None = None + self._default_org_secret_codespaces_config: dict[str, Any] | None = None self._default_org_variable_config: dict[str, Any] | None = None self._default_org_ruleset_config: dict[str, Any] | None = None self._default_repo_config: dict[str, Any] | None = None self._default_repo_webhook_config: dict[str, Any] | None = None self._default_repo_secret_config: dict[str, Any] | None = None + self._default_repo_dependabot_secret_config: dict[str, Any] | None = None + self._default_repo_codespaces_secret_config: dict[str, Any] | None = None self._default_repo_variable_config: dict[str, Any] | None = None self._default_branch_protection_rule_config: dict[str, Any] | None = None self._default_repo_ruleset_config: dict[str, Any] | None = None @@ -84,6 +94,8 @@ def __init__( self._default_pull_request_config: dict[str, Any] | None = None self._default_status_checks_config: dict[str, Any] | None = None self._default_merge_queue_config: dict[str, Any] | None = None + self._default_env_secret_config: dict[str, Any] | None = None + self._default_env_variable_config: dict[str, Any] | None = None self._initialized = False @@ -169,6 +181,26 @@ def default_org_secret_config(self): _logger.debug("no default org secret config found, secrets will be skipped") return None + @cached_property + def default_org_dependabot_secret_config(self): + try: + # load the default org dependabot secret config + snippet = f"(import '{self.template_file}').{self.create_org_dependabot_secret}('default')" + return jsonnet_evaluate_snippet(snippet) + except RuntimeError: + _logger.debug("no default org dependabot secret config found, dependabot secrets will be skipped") + return None + + @cached_property + def default_org_codespaces_secret_config(self): + try: + # load the default org codespaces secret config + snippet = f"(import '{self.template_file}').{self.create_org_codespaces_secret}('default')" + return jsonnet_evaluate_snippet(snippet) + except RuntimeError: + _logger.debug("no default org codespaces secret config found, codespaces secrets will be skipped") + return None + @cached_property def default_org_variable_config(self): try: @@ -219,6 +251,24 @@ def default_repo_secret_config(self): _logger.debug("no default repo secret config found, secrets will be skipped") return None + @cached_property + def default_repo_dependabot_secret_config(self): + try: + snippet = f"(import '{self.template_file}').{self.create_repo_dependabot_secret}('default')" + return jsonnet_evaluate_snippet(snippet) + except RuntimeError: + _logger.debug("no default repo dependabot secret config found, dependabot secrets will be skipped") + return None + + @cached_property + def default_repo_codespaces_secret_config(self): + try: + snippet = f"(import '{self.template_file}').{self.create_repo_codespaces_secret}('default')" + return jsonnet_evaluate_snippet(snippet) + except RuntimeError: + _logger.debug("no default repo codespaces secret config found, codespaces secrets will be skipped") + return None + @cached_property def default_repo_variable_config(self): try: @@ -229,6 +279,26 @@ def default_repo_variable_config(self): _logger.debug("no default repo variable config found, variables will be skipped") return None + @cached_property + def default_env_secret_config(self): + try: + # load the default repo env secret config + env_secret_snippet = f"(import '{self.template_file}').{self.create_env_secret}('default')" + return jsonnet_evaluate_snippet(env_secret_snippet) + except RuntimeError: + _logger.debug("no default repo env secret config found, secrets will be skipped") + return None + + @cached_property + def default_env_variable_config(self): + try: + # load the default repo env variable config + env_variable_snippet = f"(import '{self.template_file}').{self.create_env_variable}('default')" + return jsonnet_evaluate_snippet(env_variable_snippet) + except RuntimeError: + _logger.debug("no default repo env variable config found, variables will be skipped") + return None + @cached_property def default_branch_protection_rule_config(self): try: diff --git a/otterdog/models/__init__.py b/otterdog/models/__init__.py index 1665643f..7df0d40c 100644 --- a/otterdog/models/__init__.py +++ b/otterdog/models/__init__.py @@ -329,6 +329,14 @@ class ModelObject(ABC): The abstract base class for any model object. """ + parent_object: ModelObject | None = dataclasses.field( + default=None, + kw_only=True, + repr=False, + compare=False, + metadata={"model_only": True}, + ) + def __post_init__(self): """ Assigns to all field which are UNSET their default value, if one is available. @@ -521,13 +529,28 @@ def get_model_header(self, parent_object: ModelObject | None = None) -> str: + f", {parent_object.model_object_name}=" + f"[bold]{escape(parent_object.get_key_value())}[/]" ) - + if ( + isinstance(parent_object, ModelObject) + and isinstance(parent_object.parent_object, ModelObject) + and parent_object.parent_object.is_keyed() + ): + header = ( + header + + f", {parent_object.parent_object.model_object_name}=" + + f"[bold]{escape(parent_object.parent_object.get_key_value())}[/]" + ) header = header + "]" elif isinstance(parent_object, ModelObject) and parent_object.is_keyed(): header = header + "\\[" header = ( header + f"{parent_object.model_object_name}=" + f"[bold]{escape(parent_object.get_key_value())}[/]" ) + if isinstance(parent_object.parent_object, ModelObject) and parent_object.parent_object.is_keyed(): + header = ( + header + + f", {parent_object.parent_object.model_object_name}=" + + f"[bold]{escape(parent_object.parent_object.get_key_value())}[/]" + ) header = header + "]" return header diff --git a/otterdog/models/env_secret.py b/otterdog/models/env_secret.py new file mode 100644 index 00000000..fff16992 --- /dev/null +++ b/otterdog/models/env_secret.py @@ -0,0 +1,75 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from otterdog.models import LivePatch, LivePatchType +from otterdog.models.secret import Secret +from otterdog.utils import expect_type, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class EnvironmentSecret(Secret): + """ + Represents a Secret defined on environment level. + """ + + @property + def model_object_name(self) -> str: + return "env_secret" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_env_secret}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[EnvironmentSecret], + org_id: str, + provider: GitHubProvider, + ) -> None: + from .environment import Environment + from .repository import Repository + + if patch.parent_object is None or patch.parent_object.parent_object is None: + raise ValueError("invalid parent_object chain") + environment = expect_type(patch.parent_object, Environment) + repository = expect_type(patch.parent_object.parent_object, Repository) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_env_secret( + org_id, + repository.name, + environment.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_env_secret( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_env_secret( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/env_variable.py b/otterdog/models/env_variable.py new file mode 100644 index 00000000..9a764847 --- /dev/null +++ b/otterdog/models/env_variable.py @@ -0,0 +1,75 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from otterdog.models import LivePatch, LivePatchType +from otterdog.models.variable import Variable +from otterdog.utils import expect_type, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class EnvironmentVariable(Variable): + """ + Represents a Variable defined on environment level. + """ + + @property + def model_object_name(self) -> str: + return "env_variable" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_env_variable}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[EnvironmentVariable], + org_id: str, + provider: GitHubProvider, + ) -> None: + from .environment import Environment + from .repository import Repository + + if patch.parent_object is None or patch.parent_object.parent_object is None: + raise ValueError("invalid parent_object chain") + environment = expect_type(patch.parent_object, Environment) + repository = expect_type(patch.parent_object.parent_object, Repository) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_env_variable( + org_id, + repository.name, + environment.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_env_variable( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_env_variable( + org_id, + repository.name, + environment.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/environment.py b/otterdog/models/environment.py index c4361898..a89ac92d 100644 --- a/otterdog/models/environment.py +++ b/otterdog/models/environment.py @@ -16,11 +16,25 @@ from otterdog.models import ( FailureType, LivePatch, + LivePatchContext, + LivePatchHandler, LivePatchType, ModelObject, + PatchContext, ValidationContext, ) -from otterdog.utils import expect_type, is_set_and_valid, is_unset, unwrap +from otterdog.utils import ( + Change, + IndentingPrinter, + expect_type, + is_set_and_valid, + is_unset, + unwrap, + write_patch_object_as_json, +) + +from .env_secret import EnvironmentSecret +from .env_variable import EnvironmentVariable if TYPE_CHECKING: from otterdog.jsonnet import JsonnetConfig @@ -40,6 +54,26 @@ class Environment(ModelObject): reviewers: list[str] deployment_branch_policy: str branch_policies: list[str] + secrets: list[EnvironmentSecret] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) + variables: list[EnvironmentVariable] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) + + def add_secret(self, secret: EnvironmentSecret) -> None: + self.secrets.append(secret) + + def get_secret(self, name: str) -> EnvironmentSecret | None: + return next(filter(lambda x: x.name == name, self.secrets), None) # type: ignore + + def set_secrets(self, secrets: list[EnvironmentSecret]) -> None: + self.secrets = secrets + + def add_variable(self, variable: EnvironmentVariable) -> None: + self.variables.append(variable) + + def get_variable(self, name: str) -> EnvironmentVariable | None: + return next(filter(lambda x: x.name == name, self.variables), None) # type: ignore + + def set_variables(self, variables: list[EnvironmentVariable]) -> None: + self.variables = variables @property def model_object_name(self) -> str: @@ -69,6 +103,11 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: f"'{self.deployment_branch_policy}', " f"but 'branch_policies' is set to '{self.branch_policies}', setting will be ignored.", ) + for secret in self.secrets: + secret.validate(context, self) + + for variable in self.variables: + variable.validate(context, self) def include_field_for_diff_computation(self, field: dataclasses.Field) -> bool: if self.deployment_branch_policy != "selected": @@ -94,6 +133,19 @@ def include_existing_object_for_live_patch(self, org_id: str, parent_object: Mod else: return True + @classmethod + def get_mapping_from_model(cls) -> dict[str, Any]: + mapping = super().get_mapping_from_model() + + mapping.update( + { + "secrets": OptionalS("secrets", default=[]) >> Forall(lambda x: EnvironmentSecret.from_model_data(x)), + "variables": OptionalS("variables", default=[]) + >> Forall(lambda x: EnvironmentVariable.from_model_data(x)), + } + ) + return mapping + @classmethod def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: mapping = super().get_mapping_from_provider(org_id, data) @@ -143,6 +195,8 @@ def transform_branch_policy(x): >> Forall(lambda x: transform_reviewers(x)), "deployment_branch_policy": OptionalS("deployment_branch_policy") >> F(transform_policy), "branch_policies": OptionalS("branch_policies", default=[]) >> Forall(transform_branch_policy), + "secrets": K([]), + "variables": K([]), } ) return mapping @@ -192,6 +246,61 @@ async def get_mapping_to_provider( def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: return f"orgs.{jsonnet_config.create_environment}" + @classmethod + def generate_live_patch( + cls, + expected_object: Environment | None, + current_object: Environment | None, + parent_object: ModelObject | None, + context: LivePatchContext, + handler: LivePatchHandler, + ) -> None: + if expected_object is None: + current_object = unwrap(current_object) + handler(LivePatch.of_deletion(current_object, parent_object, current_object.apply_live_patch)) + return + + if current_object is None: + handler(LivePatch.of_addition(expected_object, parent_object, expected_object.apply_live_patch)) + else: + modified_rule: dict[str, Change[Any]] = expected_object.get_difference_from(current_object) + + if len(modified_rule) > 0: + handler( + LivePatch.of_changes( + expected_object, + current_object, + modified_rule, + parent_object, + False, + expected_object.apply_live_patch, + ) + ) + + # need to create the context for environment variables and secrets + # only a parent is available. Therefore the parent is then the starting + # point to the linked list of parents. But nevertheless bad design, + # either parent, grandparent or no parameter and linking among model + # objects. Currently both approaches are in which makes it hard to + # understand. + expected_object.parent_object = parent_object + + EnvironmentSecret.generate_live_patch_of_list( + expected_object.secrets, + current_object.secrets if current_object is not None else [], + expected_object, + context, + handler, + ) + + EnvironmentVariable.generate_live_patch_of_list( + expected_object.variables, + current_object.variables if current_object is not None else [], + expected_object, + context, + handler, + ) + @classmethod async def apply_live_patch(cls, patch: LivePatch[Environment], org_id: str, provider: GitHubProvider) -> None: from .repository import Repository @@ -221,3 +330,54 @@ async def apply_live_patch(cls, patch: LivePatch[Environment], org_id: str, prov current_object.name, await cls.changes_to_provider(org_id, unwrap(patch.changes), provider), ) + + def to_jsonnet( + self, + printer: IndentingPrinter, + jsonnet_config: JsonnetConfig, + context: PatchContext, + extend: bool, + default_object: ModelObject, + ) -> None: + patch = self.get_patch_to(default_object) + + has_secrets = len(self.secrets) > 0 + has_variables = len(self.variables) > 0 + + if "name" in patch: + patch.pop("name") + + function = self.get_jsonnet_template_function(jsonnet_config, extend) + printer.print(f"{function}('{self.name}')") + + write_patch_object_as_json(patch, printer, close_object=False) + + # FIXME: support overriding secrets for repos coming from the default configuration. + if has_secrets: + default_env_secret = EnvironmentSecret.from_model_data(jsonnet_config.default_env_secret_config) + + printer.println("secrets: [") + printer.level_up() + + for secret in self.secrets: + secret.to_jsonnet(printer, jsonnet_config, context, False, default_env_secret) + + printer.level_down() + printer.println("],") + + # FIXME: support overriding variables for repos coming from the default configuration. + if has_variables: + default_env_variable = EnvironmentVariable.from_model_data(jsonnet_config.default_env_variable_config) + + printer.println("variables: [") + printer.level_up() + + for variable in self.variables: + variable.to_jsonnet(printer, jsonnet_config, context, False, default_env_variable) + + printer.level_down() + printer.println("],") + + # close the repo object + printer.level_down() + printer.println("},") diff --git a/otterdog/models/github_organization.py b/otterdog/models/github_organization.py index 0e22a2e4..3a314e75 100644 --- a/otterdog/models/github_organization.py +++ b/otterdog/models/github_organization.py @@ -1,4 +1,4 @@ -# ******************************************************************************* +# ********************************************************************************* # Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 @@ -31,7 +31,11 @@ ) from otterdog.models.branch_protection_rule import BranchProtectionRule from otterdog.models.custom_property import CustomProperty +from otterdog.models.env_secret import EnvironmentSecret +from otterdog.models.env_variable import EnvironmentVariable from otterdog.models.environment import Environment +from otterdog.models.organization_codespaces_secret import OrganizationCodespacesSecret +from otterdog.models.organization_dependabot_secret import OrganizationDependabotSecret from otterdog.models.organization_role import OrganizationRole from otterdog.models.organization_ruleset import OrganizationRuleset from otterdog.models.organization_secret import OrganizationSecret @@ -39,6 +43,8 @@ from otterdog.models.organization_variable import OrganizationVariable from otterdog.models.organization_webhook import OrganizationWebhook from otterdog.models.organization_workflow_settings import OrganizationWorkflowSettings +from otterdog.models.repo_codespaces_secret import RepositoryCodespacesSecret +from otterdog.models.repo_dependabot_secret import RepositoryDependabotSecret from otterdog.models.repo_ruleset import RepositoryRuleset from otterdog.models.repo_secret import RepositorySecret from otterdog.models.repo_variable import RepositoryVariable @@ -73,6 +79,8 @@ class GitHubOrganization: teams: list[Team] = dataclasses.field(default_factory=list) webhooks: list[OrganizationWebhook] = dataclasses.field(default_factory=list) secrets: list[OrganizationSecret] = dataclasses.field(default_factory=list) + dependabot_secrets: list[OrganizationDependabotSecret] = dataclasses.field(default_factory=list) + codespaces_secrets: list[OrganizationCodespacesSecret] = dataclasses.field(default_factory=list) variables: list[OrganizationVariable] = dataclasses.field(default_factory=list) rulesets: list[OrganizationRuleset] = dataclasses.field(default_factory=list) repositories: list[Repository] = dataclasses.field(default_factory=list) @@ -119,6 +127,24 @@ def get_secret(self, name: str) -> OrganizationSecret | None: def set_secrets(self, secrets: list[OrganizationSecret]) -> None: self.secrets = secrets + def add_organization_dependabot_secret(self, secret: OrganizationDependabotSecret) -> None: + self.dependabot_secrets.append(secret) + + def get_organization_dependabot_secret(self, name: str) -> OrganizationDependabotSecret | None: + return next(filter(lambda x: x.name == name, self.dependabot_secrets), None) + + def set_organization_dependabot_secrets(self, secrets: list[OrganizationDependabotSecret]) -> None: + self.dependabot_secrets = secrets + + def add_organization_codespaces_secret(self, secret: OrganizationCodespacesSecret) -> None: + self.codespaces_secrets.append(secret) + + def get_organization_codespaces_secret(self, name: str) -> OrganizationCodespacesSecret | None: + return next(filter(lambda x: x.name == name, self.codespaces_secrets), None) + + def set_organization_codespaces_secrets(self, secrets: list[OrganizationCodespacesSecret]) -> None: + self.codespaces_secrets = secrets + def add_variable(self, variable: OrganizationVariable) -> None: self.variables.append(variable) @@ -195,6 +221,12 @@ async def validate( for secret in self.secrets: secret.validate(context, self) + for dependabot_secret in self.dependabot_secrets: + dependabot_secret.validate(context, self) + + for codespaces_secret in self.codespaces_secrets: + codespaces_secret.validate(context, self) + if len(self.rulesets) > 0 and not enterprise_plan: context.add_failure( FailureType.ERROR, @@ -263,6 +295,14 @@ def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject | None]]: yield secret, None yield from secret.get_model_objects() + for dependabot_secret in self.dependabot_secrets: + yield dependabot_secret, None + yield from dependabot_secret.get_model_objects() + + for codespaces_secret in self.codespaces_secrets: + yield codespaces_secret, None + yield from codespaces_secret.get_model_objects() + for variable in self.variables: yield variable, None yield from variable.get_model_objects() @@ -288,6 +328,10 @@ def from_model_data(cls, data: dict[str, Any]) -> GitHubOrganization: "teams": OptionalS("teams", default=[]) >> Forall(lambda x: Team.from_model_data(x)), "webhooks": OptionalS("webhooks", default=[]) >> Forall(lambda x: OrganizationWebhook.from_model_data(x)), "secrets": OptionalS("secrets", default=[]) >> Forall(lambda x: OrganizationSecret.from_model_data(x)), + "dependabot_secrets": OptionalS("dependabot_secrets", default=[]) + >> Forall(lambda x: OrganizationDependabotSecret.from_model_data(x)), + "codespaces_secrets": OptionalS("codespaces_secrets", default=[]) + >> Forall(lambda x: OrganizationCodespacesSecret.from_model_data(x)), "variables": OptionalS("variables", default=[]) >> Forall(lambda x: OrganizationVariable.from_model_data(x)), "rulesets": OptionalS("rulesets", default=[]) >> Forall(lambda x: OrganizationRuleset.from_model_data(x)), @@ -305,6 +349,12 @@ def resolve_secrets(self, secret_resolver: Callable[[str], str]) -> None: for secret in self.secrets: secret.resolve_secrets(secret_resolver) + for dependabot_secret in self.dependabot_secrets: + dependabot_secret.resolve_secrets(secret_resolver) + + for codespaces_secret in self.codespaces_secrets: + codespaces_secret.resolve_secrets(secret_resolver) + for repo in self.repositories: repo.resolve_secrets(secret_resolver) @@ -321,6 +371,16 @@ def copy_secrets(self, other_org: GitHubOrganization) -> None: if other_secret is not None: secret.copy_secrets(other_secret) + for dependabot_secret in self.dependabot_secrets: + other_dependabot_secret = other_org.get_secret(dependabot_secret.name) + if other_dependabot_secret is not None: + dependabot_secret.copy_secrets(other_dependabot_secret) + + for codespaces_secret in self.codespaces_secrets: + other_codespaces_secret = other_org.get_secret(codespaces_secret.name) + if other_codespaces_secret is not None: + codespaces_secret.copy_secrets(other_codespaces_secret) + for repo in self.repositories: other_repo = other_org.get_repository(repo.name) if other_repo is not None: @@ -412,6 +472,36 @@ def to_jsonnet(self, config: JsonnetConfig, context: PatchContext) -> str: printer.level_down() printer.println("],") + # print organization dependabot secrets + if len(self.dependabot_secrets) > 0: + default_org_dependabot_secret = OrganizationDependabotSecret.from_model_data( + config.default_org_dependabot_secret_config + ) + + printer.println("dependabot_secrets+: [") + printer.level_up() + + for dependabot_secret in self.dependabot_secrets: + dependabot_secret.to_jsonnet(printer, config, context, False, default_org_dependabot_secret) + + printer.level_down() + printer.println("],") + + # print organization codespaces secrets + if len(self.codespaces_secrets) > 0: + default_org_codespaces_secret = OrganizationCodespacesSecret.from_model_data( + config.default_org_codespaces_secret_config + ) + + printer.println("codespaces_secrets+: [") + printer.level_up() + + for codespaces_secret in self.codespaces_secrets: + codespaces_secret.to_jsonnet(printer, config, context, False, default_org_codespaces_secret) + + printer.level_down() + printer.println("],") + # print organization variables if len(self.variables) > 0: default_org_variable = OrganizationVariable.from_model_data(config.default_org_variable_config) @@ -483,6 +573,12 @@ def generate_live_patch( OrganizationSecret.generate_live_patch_of_list( self.secrets, current_organization.secrets, None, context, handler ) + OrganizationDependabotSecret.generate_live_patch_of_list( + self.dependabot_secrets, current_organization.dependabot_secrets, None, context, handler + ) + OrganizationCodespacesSecret.generate_live_patch_of_list( + self.codespaces_secrets, current_organization.codespaces_secrets, None, context, handler + ) OrganizationVariable.generate_live_patch_of_list( self.variables, current_organization.variables, None, context, handler ) @@ -605,6 +701,28 @@ async def _load_secrets() -> None: else: _logger.debug("not reading org secrets, no default config available") + @debug_times("dependabot_secrets") + async def _load_dependabot_secrets() -> None: + if jsonnet_config.default_org_dependabot_secret_config is not None: + github_secrets = await provider.get_org_dependabot_secrets(github_id) + for secret in github_secrets: + org.add_organization_dependabot_secret( + OrganizationDependabotSecret.from_provider_data(github_id, secret) + ) + else: + _logger.debug("not reading org dependabot secrets, no default config available") + + @debug_times("codespaces_secrets") + async def _load_codespaces_secrets() -> None: + if jsonnet_config.default_org_codespaces_secret_config is not None: + github_secrets = await provider.get_org_codespaces_secrets(github_id) + for secret in github_secrets: + org.add_organization_codespaces_secret( + OrganizationCodespacesSecret.from_provider_data(github_id, secret) + ) + else: + _logger.debug("not reading org codespaces secrets, no default config available") + @debug_times("variables") async def _load_variables() -> None: if jsonnet_config.default_org_variable_config is not None: @@ -658,6 +776,8 @@ async def _load_repos() -> None: task_group.soonify(_load_teams)() task_group.soonify(_load_webhooks)() task_group.soonify(_load_secrets)() + task_group.soonify(_load_dependabot_secrets)() + task_group.soonify(_load_codespaces_secrets)() task_group.soonify(_load_variables)() task_group.soonify(_load_rulesets)() task_group.soonify(_load_repos)() @@ -738,6 +858,20 @@ async def _process_single_repo( else: _logger.debug("not reading repo secrets, no default config available") + if jsonnet_config.default_repo_dependabot_secret_config is not None: + dependabot_secrets = await rest_api.repo.get_dependabot_secrets(github_id, repo_name) + for github_secret in dependabot_secrets: + repo.add_dependabot_secret(RepositoryDependabotSecret.from_provider_data(github_id, github_secret)) + else: + _logger.debug("not reading repo dependabot secrets, no default config available") + + if jsonnet_config.default_repo_codespaces_secret_config is not None: + codespaces_secrets = await rest_api.repo.get_codespaces_secrets(github_id, repo_name) + for github_secret in codespaces_secrets: + repo.add_codespaces_secret(RepositoryCodespacesSecret.from_provider_data(github_id, github_secret)) + else: + _logger.debug("not reading repo codespaces secrets, no default config available") + if jsonnet_config.default_repo_variable_config is not None: # get variables of the repo variables = await rest_api.repo.get_variables(github_id, repo_name) @@ -750,7 +884,22 @@ async def _process_single_repo( # get environments of the repo environments = await rest_api.repo.get_environments(github_id, repo_name) for github_environment in environments: - repo.add_environment(Environment.from_provider_data(github_id, github_environment)) + environment = Environment.from_provider_data(github_id, github_environment) + repo.add_environment(environment) + if jsonnet_config.default_env_variable_config is not None: + # get variables of the repo environment + variables = await rest_api.env.get_variables(github_id, repo.name, environment.name) + for github_variable in variables: + environment.add_variable(EnvironmentVariable.from_provider_data(github_id, github_variable)) + else: + _logger.debug("not reading repo env variables, no default config available") + if jsonnet_config.default_env_secret_config is not None: + # get secrets of the repo environment + secrets = await rest_api.env.get_secrets(github_id, repo.name, environment.name) + for github_secret in secrets: + environment.add_secret(EnvironmentSecret.from_provider_data(github_id, github_secret)) + else: + _logger.debug("not reading repo env secrets, no default config available") else: _logger.debug("not reading environments, no default config available") diff --git a/otterdog/models/organization_codespaces_secret.py b/otterdog/models/organization_codespaces_secret.py new file mode 100644 index 00000000..7b3a161d --- /dev/null +++ b/otterdog/models/organization_codespaces_secret.py @@ -0,0 +1,122 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING, Any, cast + +from jsonbender import Forall, If, K, OptionalS, S # type: ignore + +from otterdog.models import FailureType, LivePatch, LivePatchType, ValidationContext +from otterdog.models.secret import Secret +from otterdog.utils import UNSET, is_set_and_valid, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + from .github_organization import GitHubOrganization + + +@dataclasses.dataclass +class OrganizationCodespacesSecret(Secret): + """ + Represents a Codespaces Secret defined on organization level. + """ + + visibility: str + selected_repositories: list[str] + + @property + def model_object_name(self) -> str: + return "org_codespaces_secret" + + def validate(self, context: ValidationContext, parent_object: Any) -> None: + super().validate(context, parent_object) + + if is_set_and_valid(self.visibility): + org = cast("GitHubOrganization", parent_object) + if self.visibility == "private" and org.settings.plan == "free": + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has 'visibility' '{self.visibility}', " + f"which is not available for free plan organizations.", + ) + elif self.visibility not in {"public", "private", "selected"}: + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has invalid visibility '{self.visibility}'.", + ) + + if self.visibility != "selected" and len(self.selected_repositories) > 0: + context.add_failure( + FailureType.WARNING, + f"{self.get_model_header(parent_object)} has visibility '{self.visibility}', " + f"but selected_repositories is set and will be ignored.", + ) + + @classmethod + def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: + mapping = super().get_mapping_from_provider(org_id, data) + + mapping.update( + { + "visibility": If( + S("visibility") == K("all"), + K("public"), + OptionalS("visibility", default=UNSET), + ), + "selected_repositories": OptionalS("selected_repositories", default=[]) >> Forall(lambda x: x["name"]), + "value": K("********"), + } + ) + + return mapping + + @classmethod + async def get_mapping_to_provider( + cls, org_id: str, data: dict[str, Any], provider: GitHubProvider + ) -> dict[str, Any]: + mapping = await super().get_mapping_to_provider(org_id, data, provider) + + if "visibility" in mapping: + mapping["visibility"] = If(S("visibility") == K("public"), K("all"), S("visibility")) + + if "selected_repositories" in mapping: + mapping.pop("selected_repositories") + mapping["selected_repository_ids"] = K(await provider.get_repo_ids(org_id, data["selected_repositories"])) + + return mapping + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_org_codespaces_secret}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[OrganizationCodespacesSecret], + org_id: str, + provider: GitHubProvider, + ) -> None: + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_org_codespaces_secret( + org_id, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_org_codespaces_secret(org_id, unwrap(patch.current_object).name) + + case LivePatchType.CHANGE: + await provider.update_org_codespaces_secret( + org_id, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/organization_dependabot_secret.py b/otterdog/models/organization_dependabot_secret.py new file mode 100644 index 00000000..5b25645c --- /dev/null +++ b/otterdog/models/organization_dependabot_secret.py @@ -0,0 +1,122 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING, Any, cast + +from jsonbender import Forall, If, K, OptionalS, S # type: ignore + +from otterdog.models import FailureType, LivePatch, LivePatchType, ValidationContext +from otterdog.models.secret import Secret +from otterdog.utils import UNSET, is_set_and_valid, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + from .github_organization import GitHubOrganization + + +@dataclasses.dataclass +class OrganizationDependabotSecret(Secret): + """ + Represents a Dependabot Secret defined on organization level. + """ + + visibility: str + selected_repositories: list[str] + + @property + def model_object_name(self) -> str: + return "org_dependabot_secret" + + def validate(self, context: ValidationContext, parent_object: Any) -> None: + super().validate(context, parent_object) + + if is_set_and_valid(self.visibility): + org = cast("GitHubOrganization", parent_object) + if self.visibility == "private" and org.settings.plan == "free": + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has 'visibility' '{self.visibility}', " + f"which is not available for free plan organizations.", + ) + elif self.visibility not in {"public", "private", "selected"}: + context.add_failure( + FailureType.ERROR, + f"{self.get_model_header(parent_object)} has invalid visibility '{self.visibility}'.", + ) + + if self.visibility != "selected" and len(self.selected_repositories) > 0: + context.add_failure( + FailureType.WARNING, + f"{self.get_model_header(parent_object)} has visibility '{self.visibility}', " + f"but selected_repositories is set and will be ignored.", + ) + + @classmethod + def get_mapping_from_provider(cls, org_id: str, data: dict[str, Any]) -> dict[str, Any]: + mapping = super().get_mapping_from_provider(org_id, data) + + mapping.update( + { + "visibility": If( + S("visibility") == K("all"), + K("public"), + OptionalS("visibility", default=UNSET), + ), + "selected_repositories": OptionalS("selected_repositories", default=[]) >> Forall(lambda x: x["name"]), + "value": K("********"), + } + ) + + return mapping + + @classmethod + async def get_mapping_to_provider( + cls, org_id: str, data: dict[str, Any], provider: GitHubProvider + ) -> dict[str, Any]: + mapping = await super().get_mapping_to_provider(org_id, data, provider) + + if "visibility" in mapping: + mapping["visibility"] = If(S("visibility") == K("public"), K("all"), S("visibility")) + + if "selected_repositories" in mapping: + mapping.pop("selected_repositories") + mapping["selected_repository_ids"] = K(await provider.get_repo_ids(org_id, data["selected_repositories"])) + + return mapping + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_org_dependabot_secret}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[OrganizationDependabotSecret], + org_id: str, + provider: GitHubProvider, + ) -> None: + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_org_dependabot_secret( + org_id, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_org_dependabot_secret(org_id, unwrap(patch.current_object).name) + + case LivePatchType.CHANGE: + await provider.update_org_dependabot_secret( + org_id, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/repo_codespaces_secret.py b/otterdog/models/repo_codespaces_secret.py new file mode 100644 index 00000000..92e385ec --- /dev/null +++ b/otterdog/models/repo_codespaces_secret.py @@ -0,0 +1,68 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from otterdog.models import LivePatch, LivePatchType +from otterdog.models.secret import Secret +from otterdog.utils import expect_type, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class RepositoryCodespacesSecret(Secret): + """ + Represents a Codespaces Secret defined on repo level. + """ + + @property + def model_object_name(self) -> str: + return "repo_codespaces_secret" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_repo_codespaces_secret}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[RepositoryCodespacesSecret], + org_id: str, + provider: GitHubProvider, + ) -> None: + from .repository import Repository + + repository = expect_type(patch.parent_object, Repository) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_repo_codespaces_secret( + org_id, + repository.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_repo_codespaces_secret( + org_id, + repository.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_repo_codespaces_secret( + org_id, + repository.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/repo_dependabot_secret.py b/otterdog/models/repo_dependabot_secret.py new file mode 100644 index 00000000..dd14f1e7 --- /dev/null +++ b/otterdog/models/repo_dependabot_secret.py @@ -0,0 +1,68 @@ +# ******************************************************************************* +# Copyright (c) 2023-2024 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from otterdog.models import LivePatch, LivePatchType +from otterdog.models.secret import Secret +from otterdog.utils import expect_type, unwrap + +if TYPE_CHECKING: + from otterdog.jsonnet import JsonnetConfig + from otterdog.providers.github import GitHubProvider + + +@dataclasses.dataclass +class RepositoryDependabotSecret(Secret): + """ + Represents a Dependabot Secret defined on repo level. + """ + + @property + def model_object_name(self) -> str: + return "repo_dependabot_secret" + + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: + return f"orgs.{jsonnet_config.create_repo_dependabot_secret}" + + @classmethod + async def apply_live_patch( + cls, + patch: LivePatch[RepositoryDependabotSecret], + org_id: str, + provider: GitHubProvider, + ) -> None: + from .repository import Repository + + repository = expect_type(patch.parent_object, Repository) + + match patch.patch_type: + case LivePatchType.ADD: + await provider.add_repo_dependabot_secret( + org_id, + repository.name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) + + case LivePatchType.REMOVE: + await provider.delete_repo_dependabot_secret( + org_id, + repository.name, + unwrap(patch.current_object).name, + ) + + case LivePatchType.CHANGE: + await provider.update_repo_dependabot_secret( + org_id, + repository.name, + unwrap(patch.current_object).name, + await unwrap(patch.expected_object).to_provider_data(org_id, provider), + ) diff --git a/otterdog/models/repository.py b/otterdog/models/repository.py index a4c3a235..d6eeef62 100644 --- a/otterdog/models/repository.py +++ b/otterdog/models/repository.py @@ -37,6 +37,8 @@ from .branch_protection_rule import BranchProtectionRule from .environment import Environment +from .repo_codespaces_secret import RepositoryCodespacesSecret +from .repo_dependabot_secret import RepositoryDependabotSecret from .repo_ruleset import RepositoryRuleset from .repo_secret import RepositorySecret from .repo_variable import RepositoryVariable @@ -117,6 +119,16 @@ class Repository(ModelObject): webhooks: list[RepositoryWebhook] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) secrets: list[RepositorySecret] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) variables: list[RepositoryVariable] = dataclasses.field(metadata={"nested_model": True}, default_factory=list) + dependabot_secrets: list[RepositoryDependabotSecret] = dataclasses.field( + metadata={"nested_model": True}, + default_factory=list, + ) + + codespaces_secrets: list[RepositoryCodespacesSecret] = dataclasses.field( + metadata={"nested_model": True}, + default_factory=list, + ) + branch_protection_rules: list[BranchProtectionRule] = dataclasses.field( metadata={"nested_model": True}, default_factory=list ) @@ -230,6 +242,24 @@ def get_secret(self, name: str) -> RepositorySecret | None: def set_secrets(self, secrets: list[RepositorySecret]) -> None: self.secrets = secrets + def add_dependabot_secret(self, secret: RepositoryDependabotSecret) -> None: + self.dependabot_secrets.append(secret) + + def get_dependabot_secret(self, name: str) -> RepositoryDependabotSecret | None: + return next(filter(lambda x: x.name == name, self.dependabot_secrets), None) + + def set_dependabot_secrets(self, secrets: list[RepositoryDependabotSecret]) -> None: + self.dependabot_secrets = secrets + + def add_codespaces_secret(self, secret: RepositoryCodespacesSecret) -> None: + self.codespaces_secrets.append(secret) + + def get_codespaces_secret(self, name: str) -> RepositoryCodespacesSecret | None: + return next(filter(lambda x: x.name == name, self.codespaces_secrets), None) + + def set_codespaces_secrets(self, secrets: list[RepositoryCodespacesSecret]) -> None: + self.codespaces_secrets = secrets + def add_variable(self, variable: RepositoryVariable) -> None: self.variables.append(variable) @@ -664,6 +694,12 @@ def validate(self, context: ValidationContext, parent_object: Any) -> None: for secret in self.secrets: secret.validate(context, self) + for dependabot_secret in self.dependabot_secrets: + dependabot_secret.validate(context, self) + + for codespaces_secret in self.codespaces_secrets: + codespaces_secret.validate(context, self) + for variable in self.variables: variable.validate(context, self) @@ -760,6 +796,14 @@ def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject]]: yield secret, self yield from secret.get_model_objects() + for dependabot_secret in self.dependabot_secrets: + yield dependabot_secret, self + yield from dependabot_secret.get_model_objects() + + for codespaces_secret in self.codespaces_secrets: + yield codespaces_secret, self + yield from codespaces_secret.get_model_objects() + for variable in self.variables: yield variable, self yield from variable.get_model_objects() @@ -784,6 +828,10 @@ def get_mapping_from_model(cls) -> dict[str, Any]: { "webhooks": OptionalS("webhooks", default=[]) >> Forall(lambda x: RepositoryWebhook.from_model_data(x)), "secrets": OptionalS("secrets", default=[]) >> Forall(lambda x: RepositorySecret.from_model_data(x)), + "dependabot_secrets": OptionalS("dependabot_secrets", default=[]) + >> Forall(lambda x: RepositoryDependabotSecret.from_model_data(x)), + "codespaces_secrets": OptionalS("codespaces_secrets", default=[]) + >> Forall(lambda x: RepositoryCodespacesSecret.from_model_data(x)), "variables": OptionalS("variables", default=[]) >> Forall(lambda x: RepositoryVariable.from_model_data(x)), "branch_protection_rules": OptionalS("branch_protection_rules", default=[]) @@ -859,6 +907,8 @@ def property_list_to_map(properties): "custom_properties": OptionalS("custom_properties", default={}) >> F(property_list_to_map), "webhooks": K([]), "secrets": K([]), + "dependabot_secrets": K([]), + "codespaces_secrets": K([]), "variables": K([]), "branch_protection_rules": K([]), "rulesets": K([]), @@ -990,6 +1040,12 @@ def resolve_secrets(self, secret_resolver: Callable[[str], str]) -> None: for secret in self.secrets: secret.resolve_secrets(secret_resolver) + for dependabot_secret in self.dependabot_secrets: + dependabot_secret.resolve_secrets(secret_resolver) + + for codespaces_secret in self.codespaces_secrets: + codespaces_secret.resolve_secrets(secret_resolver) + def copy_secrets(self, other_object: ModelObject) -> None: for webhook in self.webhooks: other_repo = cast("Repository", other_object) @@ -1003,6 +1059,18 @@ def copy_secrets(self, other_object: ModelObject) -> None: if other_secret is not None: secret.copy_secrets(other_secret) + for dependabot_secret in self.dependabot_secrets: + other_repo = cast("Repository", other_object) + other_dependabot_secret = other_repo.get_dependabot_secret(dependabot_secret.name) + if other_dependabot_secret is not None: + dependabot_secret.copy_secrets(other_dependabot_secret) + + for codespaces_secret in self.codespaces_secrets: + other_repo = cast("Repository", other_object) + other_codespaces_secret = other_repo.get_codespaces_secret(codespaces_secret.name) + if other_codespaces_secret is not None: + codespaces_secret.copy_secrets(other_codespaces_secret) + def get_jsonnet_template_function(self, jsonnet_config: JsonnetConfig, extend: bool) -> str | None: return f"orgs.{jsonnet_config.extend_repo}" if extend else f"orgs.{jsonnet_config.create_repo}" @@ -1019,6 +1087,8 @@ def to_jsonnet( has_webhooks = len(self.webhooks) > 0 has_secrets = len(self.secrets) > 0 + has_dependabot_secrets = len(self.dependabot_secrets) > 0 + has_codespaces_secrets = len(self.codespaces_secrets) > 0 has_variables = len(self.variables) > 0 has_branch_protection_rules = len(self.branch_protection_rules) > 0 has_rulesets = len(self.rulesets) > 0 @@ -1085,6 +1155,36 @@ def to_jsonnet( printer.level_down() printer.println("],") + # FIXME: support overriding dependabot secrets for repos coming from the default configuration. + if has_dependabot_secrets and not extend: + default_repo_dependabot_secret = RepositoryDependabotSecret.from_model_data( + jsonnet_config.default_repo_dependabot_secret_config + ) + + printer.println("dependabot_secrets: [") + printer.level_up() + + for dependabot_secret in self.dependabot_secrets: + dependabot_secret.to_jsonnet(printer, jsonnet_config, context, False, default_repo_dependabot_secret) + + printer.level_down() + printer.println("],") + + # FIXME: support overriding codespaces secrets for repos coming from the default configuration. + if has_codespaces_secrets and not extend: + default_repo_codespaces_secret = RepositoryCodespacesSecret.from_model_data( + jsonnet_config.default_repo_codespaces_secret_config + ) + + printer.println("codespaces_secrets: [") + printer.level_up() + + for codespaces_secret in self.codespaces_secrets: + codespaces_secret.to_jsonnet(printer, jsonnet_config, context, False, default_repo_codespaces_secret) + + printer.level_down() + printer.println("],") + # FIXME: support overriding variables for repos coming from the default configuration. if has_variables and not extend: default_repo_variable = RepositoryVariable.from_model_data(jsonnet_config.default_repo_variable_config) @@ -1234,6 +1334,22 @@ def generate_live_patch( handler, ) + RepositoryDependabotSecret.generate_live_patch_of_list( + coerced_object.dependabot_secrets, + current_object.dependabot_secrets if current_object is not None else [], + coerced_object, + context, + handler, + ) + + RepositoryCodespacesSecret.generate_live_patch_of_list( + coerced_object.codespaces_secrets, + current_object.codespaces_secrets if current_object is not None else [], + coerced_object, + context, + handler, + ) + RepositoryVariable.generate_live_patch_of_list( coerced_object.variables, current_object.variables if current_object is not None else [], diff --git a/otterdog/providers/github/__init__.py b/otterdog/providers/github/__init__.py index ee2c8555..843bf585 100644 --- a/otterdog/providers/github/__init__.py +++ b/otterdog/providers/github/__init__.py @@ -381,6 +381,32 @@ async def add_org_secret(self, org_id: str, data: dict[str, str]) -> None: async def delete_org_secret(self, org_id: str, secret_name: str) -> None: await self.rest_api.org.delete_secret(org_id, secret_name) + async def get_org_dependabot_secrets(self, org_id: str) -> list[dict[str, Any]]: + return await self.rest_api.org.get_dependabot_secrets(org_id) + + async def update_org_dependabot_secret(self, org_id: str, secret_name: str, secret: dict[str, Any]) -> None: + if len(secret) > 0: + await self.rest_api.org.update_dependabot_secret(org_id, secret_name, secret) + + async def add_org_dependabot_secret(self, org_id: str, data: dict[str, str]) -> None: + await self.rest_api.org.add_dependabot_secret(org_id, data) + + async def delete_org_dependabot_secret(self, org_id: str, secret_name: str) -> None: + await self.rest_api.org.delete_dependabot_secret(org_id, secret_name) + + async def get_org_codespaces_secrets(self, org_id: str) -> list[dict[str, Any]]: + return await self.rest_api.org.get_codespaces_secrets(org_id) + + async def update_org_codespaces_secret(self, org_id: str, secret_name: str, secret: dict[str, Any]) -> None: + if len(secret) > 0: + await self.rest_api.org.update_codespaces_secret(org_id, secret_name, secret) + + async def add_org_codespaces_secret(self, org_id: str, data: dict[str, str]) -> None: + await self.rest_api.org.add_codespaces_secret(org_id, data) + + async def delete_org_codespaces_secret(self, org_id: str, secret_name: str) -> None: + await self.rest_api.org.delete_codespaces_secret(org_id, secret_name) + async def get_org_variables(self, org_id: str) -> list[dict[str, Any]]: return await self.rest_api.org.get_variables(org_id) @@ -407,6 +433,36 @@ async def add_repo_secret(self, org_id: str, repo_name: str, data: dict[str, str async def delete_repo_secret(self, org_id: str, repo_name: str, secret_name: str) -> None: await self.rest_api.repo.delete_secret(org_id, repo_name, secret_name) + async def get_repo_dependabot_secrets(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: + return await self.rest_api.repo.get_dependabot_secrets(org_id, repo_name) + + async def update_repo_dependabot_secret( + self, org_id: str, repo_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + if len(secret) > 0: + await self.rest_api.repo.update_dependabot_secret(org_id, repo_name, secret_name, secret) + + async def add_repo_dependabot_secret(self, org_id: str, repo_name: str, data: dict[str, str]) -> None: + await self.rest_api.repo.add_dependabot_secret(org_id, repo_name, data) + + async def delete_repo_dependabot_secret(self, org_id: str, repo_name: str, secret_name: str) -> None: + await self.rest_api.repo.delete_dependabot_secret(org_id, repo_name, secret_name) + + async def get_repo_codespaces_secrets(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: + return await self.rest_api.repo.get_codespaces_secrets(org_id, repo_name) + + async def update_repo_codespaces_secret( + self, org_id: str, repo_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + if len(secret) > 0: + await self.rest_api.repo.update_codespaces_secret(org_id, repo_name, secret_name, secret) + + async def add_repo_codespaces_secret(self, org_id: str, repo_name: str, data: dict[str, str]) -> None: + await self.rest_api.repo.add_codespaces_secret(org_id, repo_name, data) + + async def delete_repo_codespaces_secret(self, org_id: str, repo_name: str, secret_name: str) -> None: + await self.rest_api.repo.delete_codespaces_secret(org_id, repo_name, secret_name) + async def get_repo_variables(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: return await self.rest_api.repo.get_variables(org_id, repo_name) @@ -422,6 +478,36 @@ async def add_repo_variable(self, org_id: str, repo_name: str, data: dict[str, s async def delete_repo_variable(self, org_id: str, repo_name: str, variable_name: str) -> None: await self.rest_api.repo.delete_variable(org_id, repo_name, variable_name) + async def get_env_secrets(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + return await self.rest_api.env.get_secrets(org_id, repo_name, env_name) + + async def update_env_secret( + self, org_id: str, repo_name: str, env_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + if len(secret) > 0: + await self.rest_api.env.update_secret(org_id, repo_name, env_name, secret_name, secret) + + async def add_env_secret(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + await self.rest_api.env.add_secret(org_id, repo_name, env_name, data) + + async def delete_env_secret(self, org_id: str, repo_name: str, env_name: str, secret_name: str) -> None: + await self.rest_api.env.delete_secret(org_id, repo_name, env_name, secret_name) + + async def get_env_variables(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + return await self.rest_api.env.get_variables(org_id, repo_name, env_name) + + async def update_env_variable( + self, org_id: str, repo_name: str, env_name: str, variable_name: str, variable: dict[str, Any] + ) -> None: + if len(variable) > 0: + await self.rest_api.env.update_variable(org_id, repo_name, env_name, variable_name, variable) + + async def add_env_variable(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + await self.rest_api.env.add_variable(org_id, repo_name, env_name, data) + + async def delete_env_variable(self, org_id: str, repo_name: str, env_name: str, variable_name: str) -> None: + await self.rest_api.env.delete_variable(org_id, repo_name, env_name, variable_name) + async def dispatch_workflow(self, org_id: str, repo_name: str, workflow_name: str) -> bool: return await self.rest_api.repo.dispatch_workflow(org_id, repo_name, workflow_name) diff --git a/otterdog/providers/github/rest/__init__.py b/otterdog/providers/github/rest/__init__.py index 2bddd162..31c2a6d9 100644 --- a/otterdog/providers/github/rest/__init__.py +++ b/otterdog/providers/github/rest/__init__.py @@ -136,6 +136,12 @@ def meta(self): return MetaClient(self) + @cached_property + def env(self): + from .env_client import EnvClient + + return EnvClient(self) + class RestClient: def __init__(self, rest_api: RestApi): diff --git a/otterdog/providers/github/rest/env_client.py b/otterdog/providers/github/rest/env_client.py new file mode 100644 index 00000000..927351f5 --- /dev/null +++ b/otterdog/providers/github/rest/env_client.py @@ -0,0 +1,169 @@ +# ******************************************************************************* +# Copyright (c) 2024-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +import json +from typing import Any + +from otterdog.logging import get_logger +from otterdog.providers.github.exception import GitHubException +from otterdog.providers.github.rest import RestApi, RestClient, encrypt_value + +_logger = get_logger(__name__) + + +class EnvClient(RestClient): + def __init__(self, rest_api: RestApi): + super().__init__(rest_api) + + async def get_secrets(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving secrets for repo env '%s/%s:%s'", org_id, repo_name, env_name) + + try: + status, body = await self.requester.request_raw( + "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets" + ) + if status == 200: + return json.loads(body)["secrets"] + else: + return [] + except GitHubException as ex: + raise RuntimeError( + f"failed retrieving secrets for repo env '{org_id}/{repo_name}:{env_name}':\n{ex}" + ) from ex + + async def update_secret( + self, org_id: str, repo_name: str, env_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + _logger.debug("updating repo env secret '%s' for repo env '%s/%s:%s'", secret_name, org_id, repo_name, env_name) + + if "name" in secret: + secret.pop("name") + + await self._encrypt_secret_inplace(org_id, repo_name, env_name, secret) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/{secret_name}", + json.dumps(secret), + ) + + if status != 204: + raise RuntimeError(f"failed to update repo env secret '{secret_name}'") + + _logger.debug("updated repo env secret '%s'", secret_name) + + async def add_secret(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + secret_name = data.pop("name") + _logger.debug("adding repo env secret '%s' for repo env '%s/%s:%s'", secret_name, org_id, repo_name, env_name) + + await self._encrypt_secret_inplace(org_id, repo_name, env_name, data) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/{secret_name}", + json.dumps(data), + ) + + if status != 201: + raise RuntimeError(f"failed to add repo env secret '{secret_name}'") + + _logger.debug("added repo env secret '%s'", secret_name) + + async def _encrypt_secret_inplace(self, org_id: str, repo_name: str, env_name: str, data: dict[str, Any]) -> None: + value = data.pop("value") + key_id, public_key = await self.get_public_key(org_id, repo_name, env_name) + data["encrypted_value"] = encrypt_value(public_key, value) + data["key_id"] = key_id + + async def delete_secret(self, org_id: str, repo_name: str, env_name: str, secret_name: str) -> None: + _logger.debug("deleting repo env secret '%s' for repo env '%s/%s:%s'", secret_name, org_id, repo_name, env_name) + + status, _ = await self.requester.request_raw( + "DELETE", f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/{secret_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete repo env secret '{secret_name}'") + + _logger.debug("removed repo env secret '%s'", secret_name) + + async def get_public_key(self, org_id: str, repo_name: str, env_name: str) -> tuple[str, str]: + _logger.debug("retrieving repo public key for repo env '%s/%s:%s'", org_id, repo_name, env_name) + + try: + response = await self.requester.request_json( + "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/secrets/public-key" + ) + return response["key_id"], response["key"] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving repo env public key:\n{ex}") from ex + + async def get_variables(self, org_id: str, repo_name: str, env_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving variables for repo env '%s/%s:%s'", org_id, repo_name, env_name) + + try: + status, body = await self.requester.request_raw( + "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables" + ) + if status == 200: + return json.loads(body)["variables"] + else: + return [] + except GitHubException as ex: + raise RuntimeError( + f"failed retrieving variables for repo env'{org_id}/{repo_name}:{env_name}':\n{ex}" + ) from ex + + async def update_variable( + self, org_id: str, repo_name: str, env_name: str, variable_name: str, variable: dict[str, Any] + ) -> None: + _logger.debug("updating repo env variable '%s' for repo '%s/%s:%s'", variable_name, org_id, repo_name, env_name) + + if "name" in variable: + variable.pop("name") + + status, body = await self.requester.request_raw( + "PATCH", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables/{variable_name}", + json.dumps(variable), + ) + if status != 204: + raise RuntimeError(f"failed to update repo env variable '{variable_name}': {body}") + + _logger.debug("updated repo env variable '%s'", variable_name) + + async def add_variable(self, org_id: str, repo_name: str, env_name: str, data: dict[str, str]) -> None: + variable_name = data.get("name") + _logger.debug( + "adding repo env variable '%s' for repo env '%s/%s:%s'", variable_name, org_id, repo_name, env_name + ) + + status, body = await self.requester.request_raw( + "POST", + f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables", + json.dumps(data), + ) + + if status != 201: + raise RuntimeError(f"failed to add repo env variable '{variable_name}': {body}") + + _logger.debug("added repo env variable '%s'", variable_name) + + async def delete_variable(self, org_id: str, repo_name: str, env_name: str, variable_name: str) -> None: + _logger.debug( + "deleting repo env variable '%s' for repo env '%s/%s'", variable_name, org_id, repo_name, env_name + ) + + status, _ = await self.requester.request_raw( + "DELETE", f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables/{variable_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete repo env variable '{variable_name}'") + + _logger.debug("removed repo env variable '%s'", variable_name) diff --git a/otterdog/providers/github/rest/org_client.py b/otterdog/providers/github/rest/org_client.py index 914d6a4d..983214ee 100644 --- a/otterdog/providers/github/rest/org_client.py +++ b/otterdog/providers/github/rest/org_client.py @@ -383,6 +383,166 @@ async def delete_secret(self, org_id: str, secret_name: str) -> None: _logger.debug("removed org secret '%s'", secret_name) + async def get_dependabot_secrets(self, org_id: str) -> list[dict[str, Any]]: + _logger.debug("retrieving dependabot secrets for org '%s'", org_id) + + try: + response = await self.requester.request_json("GET", f"/orgs/{org_id}/dependabot/secrets") + + secrets = response["secrets"] + for secret in secrets: + if secret["visibility"] == "selected": + secret["selected_repositories"] = await self._get_selected_repositories_for_dependabot_secret( + org_id, secret["name"] + ) + return secrets + except GitHubException as ex: + raise RuntimeError(f"failed getting dependabot secrets for org '{org_id}':\n{ex}") from ex + + async def _get_selected_repositories_for_dependabot_secret( + self, org_id: str, secret_name: str + ) -> list[dict[str, Any]]: + _logger.debug("retrieving selected repositories for dependabot secret '%s' in org '%s'", secret_name, org_id) + + try: + url = f"/orgs/{org_id}/dependabot/secrets/{secret_name}/repositories" + response = await self.requester.request_json("GET", url) + return response["repositories"] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving selected repositories:\n{ex}") from ex + + async def add_dependabot_secret(self, org_id: str, data: dict[str, str]) -> None: + secret_name = data.pop("name") + _logger.debug("adding dependabot secret '%s' in org '%s'", secret_name, org_id) + + await self._encrypt_dependabot_secret_inplace(org_id, data) + + status, _ = await self.requester.request_raw( + "PUT", f"/orgs/{org_id}/dependabot/secrets/{secret_name}", json.dumps(data) + ) + + if status != 201: + raise RuntimeError(f"failed to add dependabot secret '{secret_name}'") + + _logger.debug("added dependabot secret '%s'", secret_name) + + async def update_dependabot_secret(self, org_id: str, secret_name: str, secret: dict[str, Any]) -> None: + _logger.debug("updating dependabot secret '%s' in org '%s'", secret_name, org_id) + + if "name" in secret: + secret.pop("name") + + await self._encrypt_dependabot_secret_inplace(org_id, secret) + + status, _ = await self.requester.request_raw( + "PUT", f"/orgs/{org_id}/dependabot/secrets/{secret_name}", json.dumps(secret) + ) + + if status != 204: + raise RuntimeError(f"failed to update dependabot secret '{secret_name}'") + + _logger.debug("updated dependabot secret '%s'", secret_name) + + async def _encrypt_dependabot_secret_inplace(self, org_id: str, data: dict[str, Any]) -> None: + if "value" in data: + value = data.pop("value") + key_id, public_key = await self.get_dependabot_public_key(org_id) + data["encrypted_value"] = encrypt_value(public_key, value) + data["key_id"] = key_id + + async def get_dependabot_public_key(self, org_id: str) -> tuple[str, str]: + response = await self.requester.request_json("GET", f"/orgs/{org_id}/dependabot/secrets/public-key") + return response["key_id"], response["key"] + + async def delete_dependabot_secret(self, org_id: str, secret_name: str) -> None: + _logger.debug("deleting dependabot secret '%s' in org '%s'", secret_name, org_id) + + status, _ = await self.requester.request_raw("DELETE", f"/orgs/{org_id}/dependabot/secrets/{secret_name}") + if status != 204: + raise RuntimeError(f"failed to delete dependabot secret '{secret_name}'") + + _logger.debug("removed dependabot secret '%s'", secret_name) + + async def get_codespaces_secrets(self, org_id: str) -> list[dict[str, Any]]: + _logger.debug("retrieving codespaces secrets for org '%s'", org_id) + + try: + response = await self.requester.request_json("GET", f"/orgs/{org_id}/codespaces/secrets") + + secrets = response["secrets"] + for secret in secrets: + if secret["visibility"] == "selected": + secret["selected_repositories"] = await self._get_selected_repositories_for_codespaces_secret( + org_id, secret["name"] + ) + return secrets + except GitHubException as ex: + raise RuntimeError(f"failed getting codespaces secrets for org '{org_id}':\n{ex}") from ex + + async def _get_selected_repositories_for_codespaces_secret( + self, org_id: str, secret_name: str + ) -> list[dict[str, Any]]: + _logger.debug("retrieving selected repositories for codespaces secret '%s' in org '%s'", secret_name, org_id) + + try: + url = f"/orgs/{org_id}/codespaces/secrets/{secret_name}/repositories" + response = await self.requester.request_json("GET", url) + return response["repositories"] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving selected repositories:\n{ex}") from ex + + async def add_codespaces_secret(self, org_id: str, data: dict[str, str]) -> None: + secret_name = data.pop("name") + _logger.debug("adding codespaces secret '%s' in org '%s'", secret_name, org_id) + + await self._encrypt_codespaces_secret_inplace(org_id, data) + + status, _ = await self.requester.request_raw( + "PUT", f"/orgs/{org_id}/codespaces/secrets/{secret_name}", json.dumps(data) + ) + + if status != 201: + raise RuntimeError(f"failed to add codespaces secret '{secret_name}'") + + _logger.debug("added codespaces secret '%s'", secret_name) + + async def update_codespaces_secret(self, org_id: str, secret_name: str, secret: dict[str, Any]) -> None: + _logger.debug("updating codespaces secret '%s' in org '%s'", secret_name, org_id) + + if "name" in secret: + secret.pop("name") + + await self._encrypt_codespaces_secret_inplace(org_id, secret) + + status, _ = await self.requester.request_raw( + "PUT", f"/orgs/{org_id}/codespaces/secrets/{secret_name}", json.dumps(secret) + ) + + if status != 204: + raise RuntimeError(f"failed to update codespaces secret '{secret_name}'") + + _logger.debug("updated codespaces secret '%s'", secret_name) + + async def _encrypt_codespaces_secret_inplace(self, org_id: str, data: dict[str, Any]) -> None: + if "value" in data: + value = data.pop("value") + key_id, public_key = await self.get_codespaces_public_key(org_id) + data["encrypted_value"] = encrypt_value(public_key, value) + data["key_id"] = key_id + + async def get_codespaces_public_key(self, org_id: str) -> tuple[str, str]: + response = await self.requester.request_json("GET", f"/orgs/{org_id}/codespaces/secrets/public-key") + return response["key_id"], response["key"] + + async def delete_codespaces_secret(self, org_id: str, secret_name: str) -> None: + _logger.debug("deleting codespaces secret '%s' in org '%s'", secret_name, org_id) + + status, _ = await self.requester.request_raw("DELETE", f"/orgs/{org_id}/codespaces/secrets/{secret_name}") + if status != 204: + raise RuntimeError(f"failed to delete codespaces secret '{secret_name}'") + + _logger.debug("removed codespaces secret '%s'", secret_name) + async def get_variables(self, org_id: str) -> list[dict[str, Any]]: _logger.debug("retrieving variables for org '%s'", org_id) diff --git a/otterdog/providers/github/rest/repo_client.py b/otterdog/providers/github/rest/repo_client.py index b560741f..00de441d 100644 --- a/otterdog/providers/github/rest/repo_client.py +++ b/otterdog/providers/github/rest/repo_client.py @@ -879,6 +879,164 @@ async def delete_secret(self, org_id: str, repo_name: str, secret_name: str) -> _logger.debug("removed repo secret '%s'", secret_name) + async def get_dependabot_secrets(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving dependabot secrets for repo '%s/%s'", org_id, repo_name) + + try: + status, body = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/dependabot/secrets") + if status == 200: + return json.loads(body)["secrets"] + else: + return [] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving dependabot secrets for repo '{org_id}/{repo_name}':\n{ex}") from ex + + async def update_dependabot_secret( + self, org_id: str, repo_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + _logger.debug("updating dependabot secret '%s' for repo '%s/%s'", secret_name, org_id, repo_name) + + if "name" in secret: + secret.pop("name") + + await self._encrypt_dependabot_secret_inplace(org_id, repo_name, secret) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/dependabot/secrets/{secret_name}", + json.dumps(secret), + ) + + if status != 204: + raise RuntimeError(f"failed to update dependabot secret '{secret_name}'") + + _logger.debug("updated dependabot secret '%s'", secret_name) + + async def add_dependabot_secret(self, org_id: str, repo_name: str, data: dict[str, str]) -> None: + secret_name = data.pop("name") + _logger.debug("adding dependabot secret '%s' for repo '%s/%s'", secret_name, org_id, repo_name) + + await self._encrypt_dependabot_secret_inplace(org_id, repo_name, data) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/dependabot/secrets/{secret_name}", + json.dumps(data), + ) + + if status != 201: + raise RuntimeError(f"failed to add dependabot secret '{secret_name}'") + + _logger.debug("added dependabot secret '%s'", secret_name) + + async def delete_dependabot_secret(self, org_id: str, repo_name: str, secret_name: str) -> None: + _logger.debug("deleting dependabot secret '%s' for repo '%s/%s'", secret_name, org_id, repo_name) + + status, _ = await self.requester.request_raw( + "DELETE", f"/repos/{org_id}/{repo_name}/dependabot/secrets/{secret_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete dependabot secret '{secret_name}'") + + _logger.debug("removed dependabot secret '%s'", secret_name) + + async def _encrypt_dependabot_secret_inplace(self, org_id: str, repo_name: str, data: dict[str, Any]) -> None: + value = data.pop("value") + key_id, public_key = await self.get_dependabot_public_key(org_id, repo_name) + data["encrypted_value"] = encrypt_value(public_key, value) + data["key_id"] = key_id + + async def get_dependabot_public_key(self, org_id: str, repo_name: str) -> tuple[str, str]: + _logger.debug("retrieving dependabot public key for repo '%s/%s'", org_id, repo_name) + + try: + response = await self.requester.request_json( + "GET", f"/repos/{org_id}/{repo_name}/dependabot/secrets/public-key" + ) + return response["key_id"], response["key"] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving dependabot public key:\n{ex}") from ex + + async def get_codespaces_secrets(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: + _logger.debug("retrieving codespaces secrets for repo '%s/%s'", org_id, repo_name) + + try: + status, body = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/codespaces/secrets") + if status == 200: + return json.loads(body)["secrets"] + else: + return [] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving codespaces secrets for repo '{org_id}/{repo_name}':\n{ex}") from ex + + async def update_codespaces_secret( + self, org_id: str, repo_name: str, secret_name: str, secret: dict[str, Any] + ) -> None: + _logger.debug("updating codespaces secret '%s' for repo '%s/%s'", secret_name, org_id, repo_name) + + if "name" in secret: + secret.pop("name") + + await self._encrypt_codespaces_secret_inplace(org_id, repo_name, secret) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/codespaces/secrets/{secret_name}", + json.dumps(secret), + ) + + if status != 204: + raise RuntimeError(f"failed to update codespaces secret '{secret_name}'") + + _logger.debug("updated codespaces secret '%s'", secret_name) + + async def add_codespaces_secret(self, org_id: str, repo_name: str, data: dict[str, str]) -> None: + secret_name = data.pop("name") + _logger.debug("adding codespaces secret '%s' for repo '%s/%s'", secret_name, org_id, repo_name) + + await self._encrypt_codespaces_secret_inplace(org_id, repo_name, data) + + status, _ = await self.requester.request_raw( + "PUT", + f"/repos/{org_id}/{repo_name}/codespaces/secrets/{secret_name}", + json.dumps(data), + ) + + if status != 201: + raise RuntimeError(f"failed to add codespaces secret '{secret_name}'") + + _logger.debug("added codespaces secret '%s'", secret_name) + + async def delete_codespaces_secret(self, org_id: str, repo_name: str, secret_name: str) -> None: + _logger.debug("deleting codespaces secret '%s' for repo '%s/%s'", secret_name, org_id, repo_name) + + status, _ = await self.requester.request_raw( + "DELETE", f"/repos/{org_id}/{repo_name}/codespaces/secrets/{secret_name}" + ) + + if status != 204: + raise RuntimeError(f"failed to delete codespaces secret '{secret_name}'") + + _logger.debug("removed codespaces secret '%s'", secret_name) + + async def _encrypt_codespaces_secret_inplace(self, org_id: str, repo_name: str, data: dict[str, Any]) -> None: + value = data.pop("value") + key_id, public_key = await self.get_codespaces_public_key(org_id, repo_name) + data["encrypted_value"] = encrypt_value(public_key, value) + data["key_id"] = key_id + + async def get_codespaces_public_key(self, org_id: str, repo_name: str) -> tuple[str, str]: + _logger.debug("retrieving codespaces public key for repo '%s/%s'", org_id, repo_name) + + try: + response = await self.requester.request_json( + "GET", f"/repos/{org_id}/{repo_name}/codespaces/secrets/public-key" + ) + return response["key_id"], response["key"] + except GitHubException as ex: + raise RuntimeError(f"failed retrieving codespaces public key:\n{ex}") from ex + async def get_variables(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: _logger.debug("retrieving variables for repo '%s/%s'", org_id, repo_name) diff --git a/otterdog/resources/schemas/env-secret.json b/otterdog/resources/schemas/env-secret.json new file mode 100644 index 00000000..d9c1d3ce --- /dev/null +++ b/otterdog/resources/schemas/env-secret.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + + "$ref": "secret.json", + "type": "object", + "required": [ "name", "value" ], + "unevaluatedProperties": false +} diff --git a/otterdog/resources/schemas/env-variable.json b/otterdog/resources/schemas/env-variable.json new file mode 100644 index 00000000..7dcc582f --- /dev/null +++ b/otterdog/resources/schemas/env-variable.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + + "$ref": "variable.json", + "type": "object", + "required": [ "name", "value" ], + "unevaluatedProperties": false +} diff --git a/otterdog/resources/schemas/environment.json b/otterdog/resources/schemas/environment.json index ba1bea7a..8ce58941 100644 --- a/otterdog/resources/schemas/environment.json +++ b/otterdog/resources/schemas/environment.json @@ -13,6 +13,14 @@ "branch_policies": { "type": "array", "items": { "type": "string" } + }, + "secrets": { + "type": "array", + "items": { "$ref": "env-secret.json" } + }, + "variables": { + "type": "array", + "items": { "$ref": "env-variable.json" } } }, diff --git a/otterdog/resources/schemas/organization.json b/otterdog/resources/schemas/organization.json index e765c423..262a6c78 100644 --- a/otterdog/resources/schemas/organization.json +++ b/otterdog/resources/schemas/organization.json @@ -22,6 +22,14 @@ "type": "array", "items": { "$ref": "org-secret.json" } }, + "dependabot_secrets": { + "type": "array", + "items": { "$ref": "org-secret.json" } + }, + "codespaces_secrets": { + "type": "array", + "items": { "$ref": "org-secret.json" } + }, "variables": { "type": "array", "items": { "$ref": "org-variable.json" } diff --git a/otterdog/resources/schemas/repository.json b/otterdog/resources/schemas/repository.json index e017e17c..82afbea7 100644 --- a/otterdog/resources/schemas/repository.json +++ b/otterdog/resources/schemas/repository.json @@ -78,6 +78,14 @@ "type": "array", "items": { "$ref": "repo-secret.json" } }, + "dependabot_secrets": { + "type": "array", + "items": { "$ref": "repo-secret.json" } + }, + "codespaces_secrets": { + "type": "array", + "items": { "$ref": "repo-secret.json" } + }, "variables": { "type": "array", "items": { "$ref": "repo-variable.json" } diff --git a/tests/models/resources/github-env-secret.json b/tests/models/resources/github-env-secret.json new file mode 100644 index 00000000..36339d61 --- /dev/null +++ b/tests/models/resources/github-env-secret.json @@ -0,0 +1,4 @@ +{ + "name": "TEST-SECRET", + "visibility": "selected" +} diff --git a/tests/models/resources/otterdog-env-secret.json b/tests/models/resources/otterdog-env-secret.json new file mode 100644 index 00000000..59da4df1 --- /dev/null +++ b/tests/models/resources/otterdog-env-secret.json @@ -0,0 +1,5 @@ +{ + "name": "TEST-SECRET", + "visibility": "selected", + "value": "5678" +} diff --git a/tests/models/resources/test-org/vendor/github-env-secret.json b/tests/models/resources/test-org/vendor/github-env-secret.json new file mode 100644 index 00000000..b9244285 --- /dev/null +++ b/tests/models/resources/test-org/vendor/github-env-secret.json @@ -0,0 +1,4 @@ +{ + "name": "TEST-SECRET", + "visibility": "selected" + } diff --git a/tests/models/test_env_secret.py b/tests/models/test_env_secret.py new file mode 100644 index 00000000..22119eb4 --- /dev/null +++ b/tests/models/test_env_secret.py @@ -0,0 +1,67 @@ +# ******************************************************************************* +# Copyright (c) 2023-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from collections.abc import Mapping +from typing import Any + +from otterdog.jsonnet import JsonnetConfig +from otterdog.models import ModelObject +from otterdog.models.env_secret import EnvironmentSecret +from otterdog.utils import Change + +from . import ModelTest + + +class EnvironmentSecretTest(ModelTest): + def create_model(self, data: Mapping[str, Any]) -> ModelObject: + return EnvironmentSecret.from_model_data(data) + + @property + def template_function(self) -> str: + return JsonnetConfig.create_org_secret + + @property + def model_data(self): + return self.load_json_resource("otterdog-env-secret.json") + + @property + def provider_data(self): + return self.load_json_resource("github-env-secret.json") + + def test_load_from_model(self): + secret = EnvironmentSecret.from_model_data(self.model_data) + + assert secret.name == "TEST-SECRET" + assert secret.value == "5678" + + def test_load_from_provider(self): + secret = EnvironmentSecret.from_provider_data(self.org_id, self.provider_data) + + assert secret.name == "TEST-SECRET" + assert secret.value == "********" + + def test_patch(self): + current = EnvironmentSecret.from_model_data(self.model_data) + default = EnvironmentSecret.from_model_data(self.model_data) + + default.value = "8765" + patch = current.get_patch_to(default) + + assert len(patch) == 1 + assert patch["value"] == current.value + + def test_difference(self): + current = EnvironmentSecret.from_model_data(self.model_data) + other = EnvironmentSecret.from_model_data(self.model_data) + + other.value = "8765" + + diff = current.get_difference_from(other) + + assert len(diff) == 1 + assert diff["value"] == Change(other.value, current.value) diff --git a/tests/models/test_env_variable.py b/tests/models/test_env_variable.py new file mode 100644 index 00000000..50190945 --- /dev/null +++ b/tests/models/test_env_variable.py @@ -0,0 +1,33 @@ +# ******************************************************************************* +# Copyright (c) 2023-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from collections.abc import Mapping +from typing import Any + +from otterdog.jsonnet import JsonnetConfig +from otterdog.models import ModelObject +from otterdog.models.env_variable import EnvironmentVariable + +from . import ModelTest + + +class EnvironmentVariableTest(ModelTest): + def create_model(self, data: Mapping[str, Any]) -> ModelObject: + return EnvironmentVariable.from_model_data(data) + + @property + def template_function(self) -> str: + return JsonnetConfig.create_env_variable + + @property + def model_data(self): + raise NotImplementedError + + @property + def provider_data(self): + raise NotImplementedError diff --git a/tests/providers/github/integration/helpers/model.py b/tests/providers/github/integration/helpers/model.py index f1a3dd66..6cbcf67d 100644 --- a/tests/providers/github/integration/helpers/model.py +++ b/tests/providers/github/integration/helpers/model.py @@ -98,7 +98,7 @@ def get_parent_object(self, old: ModelObject | None, new: ModelObject | None) -> """ model_cls = determine_model_object(old, new) - if model_cls in {RepositorySecret, RepositoryVariable}: + if model_cls in {RepositorySecret, RepositoryVariable, Environment}: return self.repository if model_cls in {OrganizationSecret, OrganizationVariable, CustomProperty}: return None # Organization-level, no parent object From 8c706482d106b346e4d5147b0d789b73cf34d7fb Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Fri, 20 Mar 2026 11:10:23 +0100 Subject: [PATCH 2/3] fix: review findings --- docs/reference/organization/repository/codespaces_secret.md | 2 +- docs/reference/organization/repository/dependabot_secret.md | 2 +- otterdog/models/env_secret.py | 2 +- otterdog/models/env_variable.py | 2 +- otterdog/models/organization_codespaces_secret.py | 2 +- otterdog/models/organization_dependabot_secret.py | 2 +- otterdog/models/repo_codespaces_secret.py | 2 +- otterdog/models/repo_dependabot_secret.py | 2 +- otterdog/providers/github/rest/env_client.py | 2 +- tests/models/test_env_secret.py | 2 +- tests/models/test_env_variable.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/reference/organization/repository/codespaces_secret.md b/docs/reference/organization/repository/codespaces_secret.md index 48a246e2..f9ab2a25 100644 --- a/docs/reference/organization/repository/codespaces_secret.md +++ b/docs/reference/organization/repository/codespaces_secret.md @@ -30,7 +30,7 @@ The codespaces secret value can be resolved via a credential provider. The suppo ## Jsonnet Function ``` jsonnet -orgs.newRepoDependabotSecret('') { +orgs.newRepoCodespacesSecret('') { : } ``` diff --git a/docs/reference/organization/repository/dependabot_secret.md b/docs/reference/organization/repository/dependabot_secret.md index 1620f373..8c4a3b6b 100644 --- a/docs/reference/organization/repository/dependabot_secret.md +++ b/docs/reference/organization/repository/dependabot_secret.md @@ -30,7 +30,7 @@ The dependabot secret value can be resolved via a credential provider. The suppo ## Jsonnet Function ``` jsonnet -orgs.newRepoCodespacesSecret('') { +orgs.newRepoDependabotSecret('') { : } ``` diff --git a/otterdog/models/env_secret.py b/otterdog/models/env_secret.py index fff16992..9899e709 100644 --- a/otterdog/models/env_secret.py +++ b/otterdog/models/env_secret.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/otterdog/models/env_variable.py b/otterdog/models/env_variable.py index 9a764847..2f1dd97d 100644 --- a/otterdog/models/env_variable.py +++ b/otterdog/models/env_variable.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/otterdog/models/organization_codespaces_secret.py b/otterdog/models/organization_codespaces_secret.py index 7b3a161d..3a5b3097 100644 --- a/otterdog/models/organization_codespaces_secret.py +++ b/otterdog/models/organization_codespaces_secret.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/otterdog/models/organization_dependabot_secret.py b/otterdog/models/organization_dependabot_secret.py index 5b25645c..b3ce9453 100644 --- a/otterdog/models/organization_dependabot_secret.py +++ b/otterdog/models/organization_dependabot_secret.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/otterdog/models/repo_codespaces_secret.py b/otterdog/models/repo_codespaces_secret.py index 92e385ec..0672f4ed 100644 --- a/otterdog/models/repo_codespaces_secret.py +++ b/otterdog/models/repo_codespaces_secret.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/otterdog/models/repo_dependabot_secret.py b/otterdog/models/repo_dependabot_secret.py index dd14f1e7..dd80081a 100644 --- a/otterdog/models/repo_dependabot_secret.py +++ b/otterdog/models/repo_dependabot_secret.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/otterdog/providers/github/rest/env_client.py b/otterdog/providers/github/rest/env_client.py index 927351f5..83b5dd76 100644 --- a/otterdog/providers/github/rest/env_client.py +++ b/otterdog/providers/github/rest/env_client.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2024-2025 Eclipse Foundation and others. +# Copyright (c) 2024-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/tests/models/test_env_secret.py b/tests/models/test_env_secret.py index 22119eb4..e843ab66 100644 --- a/tests/models/test_env_secret.py +++ b/tests/models/test_env_secret.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2025 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html diff --git a/tests/models/test_env_variable.py b/tests/models/test_env_variable.py index 50190945..74165148 100644 --- a/tests/models/test_env_variable.py +++ b/tests/models/test_env_variable.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2025 Eclipse Foundation and others. +# Copyright (c) 2023-2026 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html From f6ce4400c874ec3c8fd678ae1bc7825e925bb7e2 Mon Sep 17 00:00:00 2001 From: Wolfgang Fischer Date: Fri, 20 Mar 2026 11:46:16 +0100 Subject: [PATCH 3/3] fix: two warnings added if unexpected results detected for get_secrets and get_variables --- otterdog/providers/github/rest/env_client.py | 22 +++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/otterdog/providers/github/rest/env_client.py b/otterdog/providers/github/rest/env_client.py index 83b5dd76..e54d8868 100644 --- a/otterdog/providers/github/rest/env_client.py +++ b/otterdog/providers/github/rest/env_client.py @@ -30,6 +30,13 @@ async def get_secrets(self, org_id: str, repo_name: str, env_name: str) -> list[ if status == 200: return json.loads(body)["secrets"] else: + _logger.warning( + "unexpected status %s while retrieving secrets for repo env '%s/%s:%s'", + status, + org_id, + repo_name, + env_name, + ) return [] except GitHubException as ex: raise RuntimeError( @@ -110,13 +117,22 @@ async def get_variables(self, org_id: str, repo_name: str, env_name: str) -> lis status, body = await self.requester.request_raw( "GET", f"/repos/{org_id}/{repo_name}/environments/{env_name}/variables" ) + if status == 200: return json.loads(body)["variables"] - else: - return [] + + _logger.warning( + "unexpected status %s while retrieving variables for repo env '%s/%s:%s'", + status, + org_id, + repo_name, + env_name, + ) + return [] + except GitHubException as ex: raise RuntimeError( - f"failed retrieving variables for repo env'{org_id}/{repo_name}:{env_name}':\n{ex}" + f"failed retrieving variables for repo env '{org_id}/{repo_name}:{env_name}':\n{ex}" ) from ex async def update_variable(