diff --git a/flake.lock b/flake.lock index f6db83e9b4..e116db9bab 100644 --- a/flake.lock +++ b/flake.lock @@ -9036,7 +9036,7 @@ }, "kroki-preprocessor": { "inputs": { - "nixpkgs": "nixpkgs_84", + "nixpkgs": "nixpkgs_86", "std": "std_4" }, "locked": { @@ -13803,6 +13803,34 @@ } }, "nixpkgs_84": { + "locked": { + "lastModified": 1655567057, + "narHash": "sha256-Cc5hQSMsTzOHmZnYm8OSJ5RNUp22bd5NADWLHorULWQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e0a42267f73ea52adc061a64650fddc59906fc99", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_85": { + "locked": { + "lastModified": 1655567057, + "narHash": "sha256-Cc5hQSMsTzOHmZnYm8OSJ5RNUp22bd5NADWLHorULWQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e0a42267f73ea52adc061a64650fddc59906fc99", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_86": { "locked": { "lastModified": 1652059086, "narHash": "sha256-CjHSbr6LSFkN4YBdTB6+8ZQmSqhsbiXqAeQ9hQJ/gBI=", @@ -13818,7 +13846,7 @@ "type": "github" } }, - "nixpkgs_85": { + "nixpkgs_87": { "locked": { "lastModified": 1650469885, "narHash": "sha256-BuILRZ6pzMnGey8/irbjGq1oo3vIvZa1pitSdZCmIXA=", @@ -13834,7 +13862,7 @@ "type": "github" } }, - "nixpkgs_86": { + "nixpkgs_88": { "locked": { "lastModified": 1652632955, "narHash": "sha256-cSxiAaS8ozw63rLJHdN6RPwhA4lY2XeC/J+uggodYXw=", @@ -14741,6 +14769,26 @@ "type": "github" } }, + "oura": { + "inputs": { + "nixpkgs": "nixpkgs_84", + "utils": "utils_43" + }, + "locked": { + "lastModified": 1655933675, + "narHash": "sha256-57FYgyhBUptyGPsOZOBTdd4M609UPlhR1SNPWyUKDQI=", + "owner": "pacman99", + "repo": "oura", + "rev": "90dbad0c7e6dced15dcb559ce81f5d562fe2c4af", + "type": "github" + }, + "original": { + "owner": "pacman99", + "ref": "nix-flake", + "repo": "oura", + "type": "github" + } + }, "ouroboros-network": { "flake": false, "locked": { @@ -15364,6 +15412,8 @@ "n2c": "n2c_3", "nix-inclusive": "nix-inclusive", "nixpkgs": "nixpkgs_83", + "oura": "oura", + "scrolls": "scrolls", "std": "std_3" } }, @@ -15612,6 +15662,26 @@ "type": "github" } }, + "scrolls": { + "inputs": { + "nixpkgs": "nixpkgs_85", + "utils": "utils_44" + }, + "locked": { + "lastModified": 1656015488, + "narHash": "sha256-l1P4all+Goo9ckFq0ZsmKY5qnM0Aqhs3mv+nb4lwPL8=", + "owner": "pacman99", + "repo": "scrolls", + "rev": "4ba5242747d665a02341ad30bce92d2758cffdf5", + "type": "github" + }, + "original": { + "owner": "pacman99", + "ref": "nix-flake", + "repo": "scrolls", + "type": "github" + } + }, "stackage": { "flake": false, "locked": { @@ -16056,7 +16126,7 @@ "inputs": { "devshell": "devshell_15", "kroki-preprocessor": "kroki-preprocessor", - "nixpkgs": "nixpkgs_86", + "nixpkgs": "nixpkgs_88", "yants": "yants_7" }, "locked": { @@ -16076,7 +16146,7 @@ "std_4": { "inputs": { "devshell": "devshell_16", - "nixpkgs": "nixpkgs_85", + "nixpkgs": "nixpkgs_87", "yants": "yants_6" }, "locked": { @@ -17014,6 +17084,36 @@ "type": "github" } }, + "utils_43": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "utils_44": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "utils_5": { "locked": { "lastModified": 1638122382, diff --git a/flake.nix b/flake.nix index 764f4da12b..d480805025 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,8 @@ # flake = false; #}; # -------------------------------------------------------------- + oura.url = "github:pacman99/oura/nix-flake"; + scrolls.url = "github:pacman99/scrolls/nix-flake"; }; outputs = inputs: let nomadEnvs = inputs.self.${system}.cloud.nomadEnvs; diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index 390f7a680f..823ee81ede 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -279,6 +279,21 @@ | jq -s 'add' > "$NODE_TOPOLOGY" } ''; + + txpipe-config = '' + NETWORK_MAGIC=$(jq '.networkMagic' "$( + file="$(jq '.ShelleyGenesisFile' "$NODE_CONFIG" )" + folder="$(dirname "$NODE_CONFIG")" + [[ "$file" == /* ]] && echo "$file" || echo "$folder/$file" + )") + + SOURCE_CONFIG=$( jq -n \ + --arg sp "$SOCKET_PATH" \ + --arg nm "$NETWORK_MAGIC" \ + '{metrics: {}, source: {type: "N2C", address: ["Unix", $sp], magic: $nm}}' ) + + ''; + in { cardano-node = writeShellApplication { name = "entrypoint"; @@ -565,4 +580,37 @@ in { exec ${packages.ogmios}/bin/ogmios "''${args[@]}" ''; }; + + oura = writeShellApplication { + runtimeInputs = prelude-runtime; + debugInputs = [packages.oura]; + name = "entrypoint"; + text = '' + ${prelude} + + ${txpipe-config} + + OURA_CONFIG="$DATA_DIR/config/$ENVIRONMENT/oura-config.json" + echo "[$json,$TXPIPE_CONFIG]" | jq '.[0].ouraConfig + .[1]' > "$OURA_CONFIG" + + exec ${packages.oura}/bin/oura daemon --config "$OURA_CONFIG" + ''; + }; + + scrolls = writeShellApplication { + # runtimeInputs = prelude-runtime; + debugInputs = [packages.scrolls]; + name = "entrypoint"; + text = '' + ${prelude} + + ${txpipe-config} + + SCROLLS_CONFIG="$DATA_DIR/config/$ENVIRONMENT/scrolls-config.json" + echo "[$json,$TXPIPE_CONFIG]" | jq '.[0].scrollsConfig + .[1]' > "$SCROLLS_CONFIG" + + exec ${packages.scrolls}/bin/scrolls daemon -c "$SCROLLS_CONFIG" + ''; + }; + } diff --git a/nix/cardano/hydrationProfiles.nix b/nix/cardano/hydrationProfiles.nix index e217c63d93..b9e03d9e0f 100644 --- a/nix/cardano/hydrationProfiles.nix +++ b/nix/cardano/hydrationProfiles.nix @@ -59,6 +59,23 @@ }; }; + # Oura + workload-policies-oura = { + tf.hydrate-cluster.configuration.locals.policies = { + consul.oura = { + # oura also needs to read the cardano config + key_prefix."config/cardano" = { + policy = "read"; + intentions = "deny"; + }; + session_prefix."" = { + policy = "write"; + intentions = "deny"; + }; + }; + }; + }; + # Db Sync workload-policies-db-sync = { tf.hydrate-cluster.configuration.locals.policies = { diff --git a/nix/cardano/nomadJob/oura.nix b/nix/cardano/nomadJob/oura.nix new file mode 100644 index 0000000000..19eea6fbd6 --- /dev/null +++ b/nix/cardano/nomadJob/oura.nix @@ -0,0 +1,142 @@ +{ + inputs, + cell, +}: let + inherit (inputs) data-merge cells nixpkgs; + inherit (inputs.bitte-cells) vector _utils; + inherit (cell) healthChecks constants oci-images; + l = nixpkgs.lib // builtins; + # OCI-Image Namer + ociNamer = oci: builtins.unsafeDiscardStringContext "${oci.imageName}:${oci.imageTag}"; +in + { + jobname ? "oura", + namespace, + datacenters ? ["eu-central-1" "eu-west-1" "us-east-2"], + domain, + nodeClass, + scaling, + } @ args: let + id = jobname; + type = "service"; + priority = 50; + persistanceMount = "/persist"; + # vaultPkiPath = "pki/issue/oura"; + consulRolePath = "consul/creds/oura"; + in + with data-merge; { + job.${id} = { + inherit namespace datacenters id type priority; + # ---------- + # Scheduling + # ---------- + constraint = [ + { + attribute = "\${node.class}"; + operator = "="; + value = "${nodeClass}"; + } + # { + # attribute = "\${meta.cardano}"; + # operator = "is_set"; + # } + { + operator = "distinct_hosts"; + value = "true"; + } + ]; + spread = [{attribute = "\${node.datacenter}";}]; + # ---------- + # Update + # ---------- + update.health_check = "task_states"; + update.healthy_deadline = "5m0s"; + update.max_parallel = 1; + update.min_healthy_time = "10s"; + update.progress_deadline = "60m0s"; + update.stagger = "30s"; + # ---------- + # Migrate + # ---------- + migrate.health_check = "checks"; + migrate.healthy_deadline = "8m20s"; + migrate.max_parallel = 1; + migrate.min_healthy_time = "10s"; + # ---------- + # Reschedule + # ---------- + reschedule.delay = "30s"; + reschedule.delay_function = "exponential"; + reschedule.max_delay = "1h0m0s"; + reschedule.unlimited = true; + # ---------- + # Task Groups + # ---------- + group.oura = let + # work-around: we need to get rid of vector first + node' = (cell.nomadJob.cardano-node (args // {jobname = "node";})).job.node.group.cardano; + group = l.removeAttrs node' ["task"]; + node = group // {task.node = node'.task.node;}; + in + merge + # task.vector ... + (vector.nomadTask.default { + inherit namespace; + endpoints = [ + # prometheus metrics for oura + "http://127.0.0.1:9186/metrics" + # prometheus metrics for cardano-node + # "http://127.0.0.1:12798/metrics" + ]; + }) + ( + merge node + { + count = scaling; + service = append [ + (import ./srv-oura.nix {inherit namespace healthChecks;}) + ]; + network = { + dns = {servers = update [0] ["172.17.0.1"];}; + mode = "bridge"; + port.oura = { + # port from https://github.com/txpipe/oura/blob/main/book/src/advanced/pipeline_metrics.md + to = 9186; + }; + }; + task = { + # ---------- + # Task: Oura + # ---------- + oura = { + env.DATA_DIR = persistanceMount; + inherit (node.task.node.env) SOCKET_PATH; + + /** Oura doesn't need any secrets yet + template = + _utils.nomadFragments.workload-identity-vault {inherit vaultPkiPath;} + ++ _utils.nomadFragments.workload-identity-vault-consul {inherit consulRolePath;}; + env.WORKLOAD_CACERT = "/secrets/tls/ca.pem"; + env.WORKLOAD_CLIENT_KEY = "/secrets/tls/key.pem"; + env.WORKLOAD_CLIENT_CERT = "/secrets/tls/cert.pem"; + **/ + config.image = ociNamer oci-images.oura; + user = "0:0"; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources.cpu = 2000; + resources.memory = 4096; + /** + vault = { + change_mode = "noop"; + env = true; + policies = ["oura"]; + }; + */ + }; + }; + } + ); + }; + } diff --git a/nix/cardano/nomadJob/scrolls.nix b/nix/cardano/nomadJob/scrolls.nix new file mode 100644 index 0000000000..5bff83d7be --- /dev/null +++ b/nix/cardano/nomadJob/scrolls.nix @@ -0,0 +1,139 @@ +{ + inputs, + cell, +}: let + inherit (inputs) data-merge cells nixpkgs; + inherit (inputs.bitte-cells) vector _utils; + inherit (cell) healthChecks constants oci-images; + l = nixpkgs.lib // builtins; + # OCI-Image Namer + ociNamer = oci: builtins.unsafeDiscardStringContext "${oci.imageName}:${oci.imageTag}"; +in + { + jobname ? "scrolls", + namespace, + datacenters ? ["eu-central-1" "eu-west-1" "us-east-2"], + domain, + nodeClass, + scaling, + } @ args: let + id = jobname; + type = "service"; + priority = 50; + persistanceMount = "/persist"; + # vaultPkiPath = "pki/issue/scrolls"; + consulRolePath = "consul/creds/scrolls"; + in + with data-merge; { + job.${id} = { + inherit namespace datacenters id type priority; + # ---------- + # Scheduling + # ---------- + constraint = [ + { + attribute = "\${node.class}"; + operator = "="; + value = "${nodeClass}"; + } + # { + # attribute = "\${meta.cardano}"; + # operator = "is_set"; + # } + { + operator = "distinct_hosts"; + value = "true"; + } + ]; + spread = [{attribute = "\${node.datacenter}";}]; + # ---------- + # Update + # ---------- + update.health_check = "task_states"; + update.healthy_deadline = "5m0s"; + update.max_parallel = 1; + update.min_healthy_time = "10s"; + update.progress_deadline = "60m0s"; + update.stagger = "30s"; + # ---------- + # Migrate + # ---------- + migrate.health_check = "checks"; + migrate.healthy_deadline = "8m20s"; + migrate.max_parallel = 1; + migrate.min_healthy_time = "10s"; + # ---------- + # Reschedule + # ---------- + reschedule.delay = "30s"; + reschedule.delay_function = "exponential"; + reschedule.max_delay = "1h0m0s"; + reschedule.unlimited = true; + # ---------- + # Task Groups + # ---------- + group.scrolls = let + # work-around: we need to get rid of vector first + node' = (cell.nomadJob.cardano-node (args // {jobname = "node";})).job.node.group.cardano; + group = l.removeAttrs node' ["task"]; + node = group // {task.node = node'.task.node;}; + in + merge + # task.vector ... + (vector.nomadTask.default { + inherit namespace; + endpoints = [ + # prometheus metrics for scrolls + "http://127.0.0.1:80/metrics" + # prometheus metrics for cardano-node + # "http://127.0.0.1:12798/metrics" + ]; + }) + ( + merge node + { + count = scaling; + # service = append [ + # (import ./srv-scrolls.nix {inherit namespace healthChecks;}) + # ]; + network = { + dns = {servers = update [0] ["172.17.0.1"];}; + mode = "bridge"; + port.scrolls = {to = 1337;}; + }; + task = { + # ---------- + # Task: scrolls + # ---------- + oura = { + env.DATA_DIR = persistanceMount; + inherit (node.task.node.env) SOCKET_PATH; + + /** + template = + _utils.nomadFragments.workload-identity-vault {inherit vaultPkiPath;} + ++ _utils.nomadFragments.workload-identity-vault-consul {inherit consulRolePath;}; + env.WORKLOAD_CACERT = "/secrets/tls/ca.pem"; + env.WORKLOAD_CLIENT_KEY = "/secrets/tls/key.pem"; + env.WORKLOAD_CLIENT_CERT = "/secrets/tls/cert.pem"; + */ + config.image = ociNamer oci-images.oura; + user = "0:0"; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources.cpu = 2000; + resources.memory = 4096; + /** + vault = { + change_mode = "noop"; + env = true; + policies = ["oura"]; + }; + */ + }; + }; + } + ); + }; + } diff --git a/nix/cardano/nomadJob/srv-oura.nix b/nix/cardano/nomadJob/srv-oura.nix new file mode 100644 index 0000000000..5b027dd398 --- /dev/null +++ b/nix/cardano/nomadJob/srv-oura.nix @@ -0,0 +1,27 @@ +{ + namespace, + healthChecks, +}: { + address_mode = "auto"; + /** TODO: find way to check oura's health + check = [ + { + address_mode = "host"; + args = ["health-check"]; + # FIXME: switch back to fully qualified invocation + # after: https://github.com/nlewo/nix2container/issues/15 + # command = "${healthChecks.cardano-wallet-network-sync}/bin/healthcheck"; + command = "/bin/oura"; + interval = "30s"; + # on_update = "ignore_warnings"; + # check_restart.ignore_warnings = true; + timeout = "10s"; + type = "script"; + } + ]; + */ + task = "oura"; + name = "${namespace}-oura"; + port = "oura"; + tags = ["\${NOMAD_ALLOC_ID}" "${namespace}"]; +} diff --git a/nix/cardano/oci-images.nix b/nix/cardano/oci-images.nix index 6ead02af0e..53c7ed6afc 100644 --- a/nix/cardano/oci-images.nix +++ b/nix/cardano/oci-images.nix @@ -74,4 +74,32 @@ in { ]; config.User = "1000:1000"; }; + oura = buildDebugImage entrypoints.oura { + name = "registry.ci.iog.io/oura"; + maxLayers = 25; + layers = [ + # (n2c.buildLayer {deps = entrypoints.oura.runtimeInputs;}) + (n2c.buildLayer {deps = [packages.oura];}) + ]; + contents = [nixpkgs.bashInteractive]; + config.Cmd = [ + "${entrypoints.oura}/bin/entrypoint" + ]; + config.User = "1000:1000"; + }; + scrolls = buildDebugImage entrypoints.scrolls { + name = "registry.ci.iog.io/scrolls"; + maxLayers = 25; + layers = [ + # (n2c.buildLayer {deps = entrypoints.scrolls.runtimeInputs;}) + (n2c.buildLayer {deps = [packages.scrolls];}) + ]; + contents = [nixpkgs.bashInteractive]; + config.Cmd = [ + "${entrypoints.scrolls}/bin/entrypoint" + ]; + config.User = "nobody:nogroup"; + }; + + } diff --git a/nix/cardano/packages/default.nix b/nix/cardano/packages/default.nix index e5ad294fe6..b33c864eca 100644 --- a/nix/cardano/packages/default.nix +++ b/nix/cardano/packages/default.nix @@ -2,9 +2,10 @@ inputs, cell, }: let - inherit (inputs) nixpkgs cardano-wallet cardano-db-sync cardano-node cardano-ogmios cardano-graphql cardano-explorer-app nix-inclusive; + inherit (inputs) nixpkgs cardano-wallet cardano-db-sync cardano-node cardano-ogmios cardano-graphql cardano-explorer-app nix-inclusive oura scrolls; inherit (inputs.cells) cardano; inherit (nixpkgs) lib; + cardano-node-project = ( cardano-node.legacyPackages.extend ( @@ -47,6 +48,9 @@ in { #cardano-rosetta-server = (import (cardano-rosetta + "/nix/pkgs.nix") {inherit (nixpkgs) system;}).packages.cardano-rosetta-server; bech32 = cardano-node-project.hsPkgs.bech32.components.exes.bech32; ogmios = cardano-ogmios.packages.ogmios; + oura = oura.packages.oura; + scrolls = scrolls.packages.scrolls; + cardano-config-html-public = let publicEnvNames = ["mainnet" "testnet" "shelley_qa" "vasil-dev"]; environments = lib.filterAttrs (n: _: builtins.elem n publicEnvNames) cardano.environments;