From b7621f52a9581aebcb95fefa3756aac8f47513cd Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Wed, 22 Jun 2022 14:44:57 -0700 Subject: [PATCH 1/7] feat: add oura package, entrypoint, and oci-image --- flake.lock | 60 +++++++++++++++++++++++++++++--- flake.nix | 1 + nix/cardano/entrypoints.nix | 12 +++++++ nix/cardano/oci-images.nix | 14 ++++++++ nix/cardano/packages/default.nix | 5 ++- 5 files changed, 86 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index f6db83e9b4..ae5ddddf16 100644 --- a/flake.lock +++ b/flake.lock @@ -9036,7 +9036,7 @@ }, "kroki-preprocessor": { "inputs": { - "nixpkgs": "nixpkgs_84", + "nixpkgs": "nixpkgs_85", "std": "std_4" }, "locked": { @@ -13803,6 +13803,20 @@ } }, "nixpkgs_84": { + "locked": { + "lastModified": 1655567057, + "narHash": "sha256-Cc5hQSMsTzOHmZnYm8OSJ5RNUp22bd5NADWLHorULWQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e0a42267f73ea52adc061a64650fddc59906fc99", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_85": { "locked": { "lastModified": 1652059086, "narHash": "sha256-CjHSbr6LSFkN4YBdTB6+8ZQmSqhsbiXqAeQ9hQJ/gBI=", @@ -13818,7 +13832,7 @@ "type": "github" } }, - "nixpkgs_85": { + "nixpkgs_86": { "locked": { "lastModified": 1650469885, "narHash": "sha256-BuILRZ6pzMnGey8/irbjGq1oo3vIvZa1pitSdZCmIXA=", @@ -13834,7 +13848,7 @@ "type": "github" } }, - "nixpkgs_86": { + "nixpkgs_87": { "locked": { "lastModified": 1652632955, "narHash": "sha256-cSxiAaS8ozw63rLJHdN6RPwhA4lY2XeC/J+uggodYXw=", @@ -14741,6 +14755,26 @@ "type": "github" } }, + "oura": { + "inputs": { + "nixpkgs": "nixpkgs_84", + "utils": "utils_43" + }, + "locked": { + "lastModified": 1655933675, + "narHash": "sha256-57FYgyhBUptyGPsOZOBTdd4M609UPlhR1SNPWyUKDQI=", + "owner": "pacman99", + "repo": "oura", + "rev": "90dbad0c7e6dced15dcb559ce81f5d562fe2c4af", + "type": "github" + }, + "original": { + "owner": "pacman99", + "ref": "nix-flake", + "repo": "oura", + "type": "github" + } + }, "ouroboros-network": { "flake": false, "locked": { @@ -15364,6 +15398,7 @@ "n2c": "n2c_3", "nix-inclusive": "nix-inclusive", "nixpkgs": "nixpkgs_83", + "oura": "oura", "std": "std_3" } }, @@ -16056,7 +16091,7 @@ "inputs": { "devshell": "devshell_15", "kroki-preprocessor": "kroki-preprocessor", - "nixpkgs": "nixpkgs_86", + "nixpkgs": "nixpkgs_87", "yants": "yants_7" }, "locked": { @@ -16076,7 +16111,7 @@ "std_4": { "inputs": { "devshell": "devshell_16", - "nixpkgs": "nixpkgs_85", + "nixpkgs": "nixpkgs_86", "yants": "yants_6" }, "locked": { @@ -17014,6 +17049,21 @@ "type": "github" } }, + "utils_43": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "utils_5": { "locked": { "lastModified": 1638122382, diff --git a/flake.nix b/flake.nix index 764f4da12b..b4a68dba1d 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,7 @@ # flake = false; #}; # -------------------------------------------------------------- + oura.url = "github:pacman99/oura/nix-flake"; }; outputs = inputs: let nomadEnvs = inputs.self.${system}.cloud.nomadEnvs; diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index 390f7a680f..20270c5675 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -565,4 +565,16 @@ in { exec ${packages.ogmios}/bin/ogmios "''${args[@]}" ''; }; + + oura = writeShellApplication { + runtimeInputs = prelude-runtime; + debugInputs = [packages.oura]; + name = "entrypoint"; + text = '' + + ${prelude} + + exec ${packages.oura}/bin/oura daemon "''${args[@]}" + ''; + }; } diff --git a/nix/cardano/oci-images.nix b/nix/cardano/oci-images.nix index 6ead02af0e..6a46f33d57 100644 --- a/nix/cardano/oci-images.nix +++ b/nix/cardano/oci-images.nix @@ -74,4 +74,18 @@ in { ]; config.User = "1000:1000"; }; + oura = buildDebugImage entrypoints.oura { + name = "registry.ci.iog.io/oura"; + maxLayers = 25; + layers = [ + (n2c.buildLayer {deps = entrypoints.oura.runtimeInputs;}) + (n2c.buildLayer {deps = [packages.oura];}) + ]; + contents = [nixpkgs.bashInteractive]; + config.Cmd = [ + "${entrypoints.oura}/bin/entrypoint" + ]; + config.User = "1000:1000"; + }; + } diff --git a/nix/cardano/packages/default.nix b/nix/cardano/packages/default.nix index e5ad294fe6..377750fda2 100644 --- a/nix/cardano/packages/default.nix +++ b/nix/cardano/packages/default.nix @@ -2,9 +2,10 @@ inputs, cell, }: let - inherit (inputs) nixpkgs cardano-wallet cardano-db-sync cardano-node cardano-ogmios cardano-graphql cardano-explorer-app nix-inclusive; + inherit (inputs) nixpkgs cardano-wallet cardano-db-sync cardano-node cardano-ogmios cardano-graphql cardano-explorer-app nix-inclusive oura; inherit (inputs.cells) cardano; inherit (nixpkgs) lib; + cardano-node-project = ( cardano-node.legacyPackages.extend ( @@ -47,6 +48,8 @@ in { #cardano-rosetta-server = (import (cardano-rosetta + "/nix/pkgs.nix") {inherit (nixpkgs) system;}).packages.cardano-rosetta-server; bech32 = cardano-node-project.hsPkgs.bech32.components.exes.bech32; ogmios = cardano-ogmios.packages.ogmios; + oura = oura.packages.oura; + cardano-config-html-public = let publicEnvNames = ["mainnet" "testnet" "shelley_qa" "vasil-dev"]; environments = lib.filterAttrs (n: _: builtins.elem n publicEnvNames) cardano.environments; From c17edc3effef135fa2be8a042c70ee88f10422a1 Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Thu, 23 Jun 2022 13:32:36 -0700 Subject: [PATCH 2/7] feat(oura): add nomad job --- nix/cardano/nomadJob/oura.nix | 135 ++++++++++++++++++++++++++++++ nix/cardano/nomadJob/srv-oura.nix | 27 ++++++ 2 files changed, 162 insertions(+) create mode 100644 nix/cardano/nomadJob/oura.nix create mode 100644 nix/cardano/nomadJob/srv-oura.nix diff --git a/nix/cardano/nomadJob/oura.nix b/nix/cardano/nomadJob/oura.nix new file mode 100644 index 0000000000..5e9bf5d817 --- /dev/null +++ b/nix/cardano/nomadJob/oura.nix @@ -0,0 +1,135 @@ +{ + inputs, + cell, +}: let + inherit (inputs) data-merge cells nixpkgs; + inherit (inputs.bitte-cells) vector _utils; + inherit (cell) healthChecks constants oci-images; + l = nixpkgs.lib // builtins; + # OCI-Image Namer + ociNamer = oci: builtins.unsafeDiscardStringContext "${oci.imageName}:${oci.imageTag}"; +in + { + jobname ? "oura", + namespace, + datacenters ? ["eu-central-1" "eu-west-1" "us-east-2"], + domain, + nodeClass, + scaling, + } @ args: let + id = jobname; + type = "service"; + priority = 50; + persistanceMount = "/persist"; + vaultPkiPath = "pki/issue/oura"; + consulRolePath = "consul/creds/oura"; + in + with data-merge; { + job.${id} = { + inherit namespace datacenters id type priority; + # ---------- + # Scheduling + # ---------- + constraint = [ + { + attribute = "\${node.class}"; + operator = "="; + value = "${nodeClass}"; + } + # { + # attribute = "\${meta.cardano}"; + # operator = "is_set"; + # } + { + operator = "distinct_hosts"; + value = "true"; + } + ]; + spread = [{attribute = "\${node.datacenter}";}]; + # ---------- + # Update + # ---------- + update.health_check = "task_states"; + update.healthy_deadline = "5m0s"; + update.max_parallel = 1; + update.min_healthy_time = "10s"; + update.progress_deadline = "60m0s"; + update.stagger = "30s"; + # ---------- + # Migrate + # ---------- + migrate.health_check = "checks"; + migrate.healthy_deadline = "8m20s"; + migrate.max_parallel = 1; + migrate.min_healthy_time = "10s"; + # ---------- + # Reschedule + # ---------- + reschedule.delay = "30s"; + reschedule.delay_function = "exponential"; + reschedule.max_delay = "1h0m0s"; + reschedule.unlimited = true; + # ---------- + # Task Groups + # ---------- + group.oura = let + # work-around: we need to get rid of vector first + node' = (cell.nomadJob.cardano-node (args // {jobname = "node";})).job.node.group.cardano; + group = l.removeAttrs node' ["task"]; + node = group // {task.node = node'.task.node;}; + in + merge + # task.vector ... + (vector.nomadTask.default { + inherit namespace; + endpoints = [ + # prometheus metrics for oura + "http://127.0.0.1:1586/metrics" + # prometheus metrics for cardano-node + # "http://127.0.0.1:12798/metrics" + ]; + }) + ( + merge node + { + count = scaling; + # service = append [ + # (import ./srv-oura.nix {inherit namespace healthChecks;}) + # ]; + network = { + dns = {servers = update [0] ["172.17.0.1"];}; + mode = "bridge"; + port.oura = {to = 1586;}; + }; + task = { + # ---------- + # Task: Oura + # ---------- + oura = { + env.DATA_DIR = persistanceMount; + env.SOCKET_PATH = "/alloc/tmp/node.socket"; # figure out how to pass this from the cardano group + + template = + _utils.nomadFragments.workload-identity-vault {inherit vaultPkiPath;} + ++ _utils.nomadFragments.workload-identity-vault-consul {inherit consulRolePath;}; + env.WORKLOAD_CACERT = "/secrets/tls/ca.pem"; + env.WORKLOAD_CLIENT_KEY = "/secrets/tls/key.pem"; + env.WORKLOAD_CLIENT_CERT = "/secrets/tls/cert.pem"; + config.image = ociNamer oci-images.oura; + user = "0:0"; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources.cpu = 2000; + resources.memory = 4096; + vault = { + change_mode = "noop"; + env = true; + policies = ["oura"]; + }; + }; + }; + } + ); + }; + } diff --git a/nix/cardano/nomadJob/srv-oura.nix b/nix/cardano/nomadJob/srv-oura.nix new file mode 100644 index 0000000000..5b027dd398 --- /dev/null +++ b/nix/cardano/nomadJob/srv-oura.nix @@ -0,0 +1,27 @@ +{ + namespace, + healthChecks, +}: { + address_mode = "auto"; + /** TODO: find way to check oura's health + check = [ + { + address_mode = "host"; + args = ["health-check"]; + # FIXME: switch back to fully qualified invocation + # after: https://github.com/nlewo/nix2container/issues/15 + # command = "${healthChecks.cardano-wallet-network-sync}/bin/healthcheck"; + command = "/bin/oura"; + interval = "30s"; + # on_update = "ignore_warnings"; + # check_restart.ignore_warnings = true; + timeout = "10s"; + type = "script"; + } + ]; + */ + task = "oura"; + name = "${namespace}-oura"; + port = "oura"; + tags = ["\${NOMAD_ALLOC_ID}" "${namespace}"]; +} From 3e064960001e61d12378eae125c6eec686c8cdda Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Thu, 23 Jun 2022 13:33:02 -0700 Subject: [PATCH 3/7] feat(scrolls): add package, entrypoint, oci-image --- nix/cardano/entrypoints.nix | 15 +++++++++++---- nix/cardano/oci-images.nix | 18 ++++++++++++++++-- nix/cardano/packages/default.nix | 1 + 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index 20270c5675..21b66e0ada 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -567,14 +567,21 @@ in { }; oura = writeShellApplication { - runtimeInputs = prelude-runtime; + # runtimeInputs = prelude-runtime; debugInputs = [packages.oura]; name = "entrypoint"; text = '' - - ${prelude} - exec ${packages.oura}/bin/oura daemon "''${args[@]}" ''; }; + + scrolls = writeShellApplication { + # runtimeInputs = prelude-runtime; + debugInputs = [packages.scrolls]; + name = "entrypoint"; + text = '' + exec ${packages.scrolls}/bin/scrolls daemon "''${args[@]}" + ''; + }; + } diff --git a/nix/cardano/oci-images.nix b/nix/cardano/oci-images.nix index 6a46f33d57..60f09ef4ad 100644 --- a/nix/cardano/oci-images.nix +++ b/nix/cardano/oci-images.nix @@ -78,14 +78,28 @@ in { name = "registry.ci.iog.io/oura"; maxLayers = 25; layers = [ - (n2c.buildLayer {deps = entrypoints.oura.runtimeInputs;}) + # (n2c.buildLayer {deps = entrypoints.oura.runtimeInputs;}) (n2c.buildLayer {deps = [packages.oura];}) ]; contents = [nixpkgs.bashInteractive]; config.Cmd = [ "${entrypoints.oura}/bin/entrypoint" ]; - config.User = "1000:1000"; + config.User = "nobody:nogroup"; }; + scrolls = buildDebugImage entrypoints.scrolls { + name = "registry.ci.iog.io/scrolls"; + maxLayers = 25; + layers = [ + # (n2c.buildLayer {deps = entrypoints.scrolls.runtimeInputs;}) + (n2c.buildLayer {deps = [packages.scrolls];}) + ]; + contents = [nixpkgs.bashInteractive]; + config.Cmd = [ + "${entrypoints.scrolls}/bin/entrypoint" + ]; + config.User = "nobody:nogroup"; + }; + } diff --git a/nix/cardano/packages/default.nix b/nix/cardano/packages/default.nix index 377750fda2..5d1985d706 100644 --- a/nix/cardano/packages/default.nix +++ b/nix/cardano/packages/default.nix @@ -49,6 +49,7 @@ in { bech32 = cardano-node-project.hsPkgs.bech32.components.exes.bech32; ogmios = cardano-ogmios.packages.ogmios; oura = oura.packages.oura; + scrolls = scrolls.packages.scrolls; cardano-config-html-public = let publicEnvNames = ["mainnet" "testnet" "shelley_qa" "vasil-dev"]; From c9938fadc70a47b8248cefe1d8f799ec42ae40e3 Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Thu, 23 Jun 2022 15:14:33 -0700 Subject: [PATCH 4/7] fix: add prelude and scrolls --- flake.lock | 60 ++++++++++++-- flake.nix | 1 + nix/cardano/entrypoints.nix | 7 +- nix/cardano/nomadJob/scrolls.nix | 135 +++++++++++++++++++++++++++++++ nix/cardano/packages/default.nix | 2 +- 5 files changed, 196 insertions(+), 9 deletions(-) create mode 100644 nix/cardano/nomadJob/scrolls.nix diff --git a/flake.lock b/flake.lock index ae5ddddf16..e116db9bab 100644 --- a/flake.lock +++ b/flake.lock @@ -9036,7 +9036,7 @@ }, "kroki-preprocessor": { "inputs": { - "nixpkgs": "nixpkgs_85", + "nixpkgs": "nixpkgs_86", "std": "std_4" }, "locked": { @@ -13817,6 +13817,20 @@ } }, "nixpkgs_85": { + "locked": { + "lastModified": 1655567057, + "narHash": "sha256-Cc5hQSMsTzOHmZnYm8OSJ5RNUp22bd5NADWLHorULWQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e0a42267f73ea52adc061a64650fddc59906fc99", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_86": { "locked": { "lastModified": 1652059086, "narHash": "sha256-CjHSbr6LSFkN4YBdTB6+8ZQmSqhsbiXqAeQ9hQJ/gBI=", @@ -13832,7 +13846,7 @@ "type": "github" } }, - "nixpkgs_86": { + "nixpkgs_87": { "locked": { "lastModified": 1650469885, "narHash": "sha256-BuILRZ6pzMnGey8/irbjGq1oo3vIvZa1pitSdZCmIXA=", @@ -13848,7 +13862,7 @@ "type": "github" } }, - "nixpkgs_87": { + "nixpkgs_88": { "locked": { "lastModified": 1652632955, "narHash": "sha256-cSxiAaS8ozw63rLJHdN6RPwhA4lY2XeC/J+uggodYXw=", @@ -15399,6 +15413,7 @@ "nix-inclusive": "nix-inclusive", "nixpkgs": "nixpkgs_83", "oura": "oura", + "scrolls": "scrolls", "std": "std_3" } }, @@ -15647,6 +15662,26 @@ "type": "github" } }, + "scrolls": { + "inputs": { + "nixpkgs": "nixpkgs_85", + "utils": "utils_44" + }, + "locked": { + "lastModified": 1656015488, + "narHash": "sha256-l1P4all+Goo9ckFq0ZsmKY5qnM0Aqhs3mv+nb4lwPL8=", + "owner": "pacman99", + "repo": "scrolls", + "rev": "4ba5242747d665a02341ad30bce92d2758cffdf5", + "type": "github" + }, + "original": { + "owner": "pacman99", + "ref": "nix-flake", + "repo": "scrolls", + "type": "github" + } + }, "stackage": { "flake": false, "locked": { @@ -16091,7 +16126,7 @@ "inputs": { "devshell": "devshell_15", "kroki-preprocessor": "kroki-preprocessor", - "nixpkgs": "nixpkgs_87", + "nixpkgs": "nixpkgs_88", "yants": "yants_7" }, "locked": { @@ -16111,7 +16146,7 @@ "std_4": { "inputs": { "devshell": "devshell_16", - "nixpkgs": "nixpkgs_86", + "nixpkgs": "nixpkgs_87", "yants": "yants_6" }, "locked": { @@ -17064,6 +17099,21 @@ "type": "github" } }, + "utils_44": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "utils_5": { "locked": { "lastModified": 1638122382, diff --git a/flake.nix b/flake.nix index b4a68dba1d..d480805025 100644 --- a/flake.nix +++ b/flake.nix @@ -33,6 +33,7 @@ #}; # -------------------------------------------------------------- oura.url = "github:pacman99/oura/nix-flake"; + scrolls.url = "github:pacman99/scrolls/nix-flake"; }; outputs = inputs: let nomadEnvs = inputs.self.${system}.cloud.nomadEnvs; diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index 21b66e0ada..578cbec0a7 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -567,11 +567,12 @@ in { }; oura = writeShellApplication { - # runtimeInputs = prelude-runtime; + runtimeInputs = prelude-runtime; debugInputs = [packages.oura]; name = "entrypoint"; text = '' - exec ${packages.oura}/bin/oura daemon "''${args[@]}" + ${prelude} + exec ${packages.oura}/bin/oura daemon; ''; }; @@ -580,7 +581,7 @@ in { debugInputs = [packages.scrolls]; name = "entrypoint"; text = '' - exec ${packages.scrolls}/bin/scrolls daemon "''${args[@]}" + exec ${packages.scrolls}/bin/scrolls daemon ''; }; diff --git a/nix/cardano/nomadJob/scrolls.nix b/nix/cardano/nomadJob/scrolls.nix new file mode 100644 index 0000000000..e7101fdf5b --- /dev/null +++ b/nix/cardano/nomadJob/scrolls.nix @@ -0,0 +1,135 @@ +{ + inputs, + cell, +}: let + inherit (inputs) data-merge cells nixpkgs; + inherit (inputs.bitte-cells) vector _utils; + inherit (cell) healthChecks constants oci-images; + l = nixpkgs.lib // builtins; + # OCI-Image Namer + ociNamer = oci: builtins.unsafeDiscardStringContext "${oci.imageName}:${oci.imageTag}"; +in + { + jobname ? "scrolls", + namespace, + datacenters ? ["eu-central-1" "eu-west-1" "us-east-2"], + domain, + nodeClass, + scaling, + } @ args: let + id = jobname; + type = "service"; + priority = 50; + persistanceMount = "/persist"; + vaultPkiPath = "pki/issue/scrolls"; + consulRolePath = "consul/creds/scrolls"; + in + with data-merge; { + job.${id} = { + inherit namespace datacenters id type priority; + # ---------- + # Scheduling + # ---------- + constraint = [ + { + attribute = "\${node.class}"; + operator = "="; + value = "${nodeClass}"; + } + # { + # attribute = "\${meta.cardano}"; + # operator = "is_set"; + # } + { + operator = "distinct_hosts"; + value = "true"; + } + ]; + spread = [{attribute = "\${node.datacenter}";}]; + # ---------- + # Update + # ---------- + update.health_check = "task_states"; + update.healthy_deadline = "5m0s"; + update.max_parallel = 1; + update.min_healthy_time = "10s"; + update.progress_deadline = "60m0s"; + update.stagger = "30s"; + # ---------- + # Migrate + # ---------- + migrate.health_check = "checks"; + migrate.healthy_deadline = "8m20s"; + migrate.max_parallel = 1; + migrate.min_healthy_time = "10s"; + # ---------- + # Reschedule + # ---------- + reschedule.delay = "30s"; + reschedule.delay_function = "exponential"; + reschedule.max_delay = "1h0m0s"; + reschedule.unlimited = true; + # ---------- + # Task Groups + # ---------- + group.scrolls = let + # work-around: we need to get rid of vector first + node' = (cell.nomadJob.cardano-node (args // {jobname = "node";})).job.node.group.cardano; + group = l.removeAttrs node' ["task"]; + node = group // {task.node = node'.task.node;}; + in + merge + # task.vector ... + (vector.nomadTask.default { + inherit namespace; + endpoints = [ + # prometheus metrics for scrolls + "http://127.0.0.1:80/metrics" + # prometheus metrics for cardano-node + # "http://127.0.0.1:12798/metrics" + ]; + }) + ( + merge node + { + count = scaling; + # service = append [ + # (import ./srv-scrolls.nix {inherit namespace healthChecks;}) + # ]; + network = { + dns = {servers = update [0] ["172.17.0.1"];}; + mode = "bridge"; + port.scrolls = {to = 1337;}; + }; + task = { + # ---------- + # Task: scrolls + # ---------- + oura = { + env.DATA_DIR = persistanceMount; + env.SOCKET_PATH = "/alloc/tmp/node.socket"; # figure out how to pass this from the cardano group + + template = + _utils.nomadFragments.workload-identity-vault {inherit vaultPkiPath;} + ++ _utils.nomadFragments.workload-identity-vault-consul {inherit consulRolePath;}; + env.WORKLOAD_CACERT = "/secrets/tls/ca.pem"; + env.WORKLOAD_CLIENT_KEY = "/secrets/tls/key.pem"; + env.WORKLOAD_CLIENT_CERT = "/secrets/tls/cert.pem"; + config.image = ociNamer oci-images.oura; + user = "0:0"; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources.cpu = 2000; + resources.memory = 4096; + vault = { + change_mode = "noop"; + env = true; + policies = ["oura"]; + }; + }; + }; + } + ); + }; + } diff --git a/nix/cardano/packages/default.nix b/nix/cardano/packages/default.nix index 5d1985d706..b33c864eca 100644 --- a/nix/cardano/packages/default.nix +++ b/nix/cardano/packages/default.nix @@ -2,7 +2,7 @@ inputs, cell, }: let - inherit (inputs) nixpkgs cardano-wallet cardano-db-sync cardano-node cardano-ogmios cardano-graphql cardano-explorer-app nix-inclusive oura; + inherit (inputs) nixpkgs cardano-wallet cardano-db-sync cardano-node cardano-ogmios cardano-graphql cardano-explorer-app nix-inclusive oura scrolls; inherit (inputs.cells) cardano; inherit (nixpkgs) lib; From 3d412bd7ee19b4246f67695518a79f80a5d255ca Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Fri, 1 Jul 2022 10:03:46 -0700 Subject: [PATCH 5/7] feat(oura): build configuration in entrypoint also add tf hydration profile --- nix/cardano/entrypoints.nix | 20 ++++++++++++++++++-- nix/cardano/hydrationProfiles.nix | 17 +++++++++++++++++ nix/cardano/nomadJob/oura.nix | 12 +++++++----- nix/cardano/oci-images.nix | 2 +- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index 578cbec0a7..e726fa7677 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -572,7 +572,23 @@ in { name = "entrypoint"; text = '' ${prelude} - exec ${packages.oura}/bin/oura daemon; + + OURA_CONFIG="$DATA_DIR/config/$ENVIRONMENT/oura-config.json" + + NETWORK_MAGIC=$(jq '.networkMagic' "$( + file="$(jq '.ShelleyGenesisFile' "$NODE_CONFIG" )" + folder="$(dirname "$NODE_CONFIG")" + [[ "$file" == /* ]] && echo "$file" || echo "$folder/$file" + )") + + SOURCE_CONFIG=$( jq -n \ + --arg sp "$SOCKET_PATH" \ + --arg nm "$NETWORK_MAGIC" \ + '{source: {type: "N2C", address: ["Unix", $sp], magic: $nm}}' ) + + echo "[$json,$SOURCE_CONFIG]" | jq '.[0].ouraConfig + .[1]' > "$OURA_CONFIG" + + exec ${packages.oura}/bin/oura daemon --config "$OURA_CONFIG"; ''; }; @@ -581,7 +597,7 @@ in { debugInputs = [packages.scrolls]; name = "entrypoint"; text = '' - exec ${packages.scrolls}/bin/scrolls daemon + exec ${packages.scrolls}/bin/scrolls daemon -c $CONFIG_FILE ''; }; diff --git a/nix/cardano/hydrationProfiles.nix b/nix/cardano/hydrationProfiles.nix index e217c63d93..b9e03d9e0f 100644 --- a/nix/cardano/hydrationProfiles.nix +++ b/nix/cardano/hydrationProfiles.nix @@ -59,6 +59,23 @@ }; }; + # Oura + workload-policies-oura = { + tf.hydrate-cluster.configuration.locals.policies = { + consul.oura = { + # oura also needs to read the cardano config + key_prefix."config/cardano" = { + policy = "read"; + intentions = "deny"; + }; + session_prefix."" = { + policy = "write"; + intentions = "deny"; + }; + }; + }; + }; + # Db Sync workload-policies-db-sync = { tf.hydrate-cluster.configuration.locals.policies = { diff --git a/nix/cardano/nomadJob/oura.nix b/nix/cardano/nomadJob/oura.nix index 5e9bf5d817..59c849e4b5 100644 --- a/nix/cardano/nomadJob/oura.nix +++ b/nix/cardano/nomadJob/oura.nix @@ -21,7 +21,7 @@ in type = "service"; priority = 50; persistanceMount = "/persist"; - vaultPkiPath = "pki/issue/oura"; + # vaultPkiPath = "pki/issue/oura"; consulRolePath = "consul/creds/oura"; in with data-merge; { @@ -93,9 +93,9 @@ in merge node { count = scaling; - # service = append [ - # (import ./srv-oura.nix {inherit namespace healthChecks;}) - # ]; + service = append [ + (import ./srv-oura.nix {inherit namespace healthChecks;}) + ]; network = { dns = {servers = update [0] ["172.17.0.1"];}; mode = "bridge"; @@ -107,14 +107,16 @@ in # ---------- oura = { env.DATA_DIR = persistanceMount; - env.SOCKET_PATH = "/alloc/tmp/node.socket"; # figure out how to pass this from the cardano group + inherit (node.task.node.env) SOCKET_PATH; + /** Oura doesn't need any secrets yet template = _utils.nomadFragments.workload-identity-vault {inherit vaultPkiPath;} ++ _utils.nomadFragments.workload-identity-vault-consul {inherit consulRolePath;}; env.WORKLOAD_CACERT = "/secrets/tls/ca.pem"; env.WORKLOAD_CLIENT_KEY = "/secrets/tls/key.pem"; env.WORKLOAD_CLIENT_CERT = "/secrets/tls/cert.pem"; + **/ config.image = ociNamer oci-images.oura; user = "0:0"; driver = "docker"; diff --git a/nix/cardano/oci-images.nix b/nix/cardano/oci-images.nix index 60f09ef4ad..53c7ed6afc 100644 --- a/nix/cardano/oci-images.nix +++ b/nix/cardano/oci-images.nix @@ -85,7 +85,7 @@ in { config.Cmd = [ "${entrypoints.oura}/bin/entrypoint" ]; - config.User = "nobody:nogroup"; + config.User = "1000:1000"; }; scrolls = buildDebugImage entrypoints.scrolls { name = "registry.ci.iog.io/scrolls"; From 297b6b173e5b915635e30aebbda4e52fd0f18f44 Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Fri, 1 Jul 2022 13:45:03 -0700 Subject: [PATCH 6/7] feat(entrypoint): factor out txpipe source config --- nix/cardano/entrypoints.nix | 40 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index e726fa7677..4c6fa275dc 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -279,6 +279,21 @@ | jq -s 'add' > "$NODE_TOPOLOGY" } ''; + + txpipe-source-config = '' + NETWORK_MAGIC=$(jq '.networkMagic' "$( + file="$(jq '.ShelleyGenesisFile' "$NODE_CONFIG" )" + folder="$(dirname "$NODE_CONFIG")" + [[ "$file" == /* ]] && echo "$file" || echo "$folder/$file" + )") + + SOURCE_CONFIG=$( jq -n \ + --arg sp "$SOCKET_PATH" \ + --arg nm "$NETWORK_MAGIC" \ + '{source: {type: "N2C", address: ["Unix", $sp], magic: $nm}}' ) + + ''; + in { cardano-node = writeShellApplication { name = "entrypoint"; @@ -573,22 +588,12 @@ in { text = '' ${prelude} - OURA_CONFIG="$DATA_DIR/config/$ENVIRONMENT/oura-config.json" - - NETWORK_MAGIC=$(jq '.networkMagic' "$( - file="$(jq '.ShelleyGenesisFile' "$NODE_CONFIG" )" - folder="$(dirname "$NODE_CONFIG")" - [[ "$file" == /* ]] && echo "$file" || echo "$folder/$file" - )") - - SOURCE_CONFIG=$( jq -n \ - --arg sp "$SOCKET_PATH" \ - --arg nm "$NETWORK_MAGIC" \ - '{source: {type: "N2C", address: ["Unix", $sp], magic: $nm}}' ) + ${txpipe-source-config} + OURA_CONFIG="$DATA_DIR/config/$ENVIRONMENT/oura-config.json" echo "[$json,$SOURCE_CONFIG]" | jq '.[0].ouraConfig + .[1]' > "$OURA_CONFIG" - exec ${packages.oura}/bin/oura daemon --config "$OURA_CONFIG"; + exec ${packages.oura}/bin/oura daemon --config "$OURA_CONFIG" ''; }; @@ -597,7 +602,14 @@ in { debugInputs = [packages.scrolls]; name = "entrypoint"; text = '' - exec ${packages.scrolls}/bin/scrolls daemon -c $CONFIG_FILE + ${prelude} + + ${txpipe-source-config} + + SCROLLS_CONFIG="$DATA_DIR/config/$ENVIRONMENT/scrolls-config.json" + echo "[$json,$SOURCE_CONFIG]" | jq '.[0].scrollsConfig + .[1]' > "$SCROLLS_CONFIG" + + exec ${packages.scrolls}/bin/scrolls daemon -c "$SCROLLS_CONFIG" ''; }; From a97cabfe4c580ebb01ff7f2927510e6fce8adb6e Mon Sep 17 00:00:00 2001 From: Parthiv Seetharaman Date: Sat, 2 Jul 2022 05:39:05 -0700 Subject: [PATCH 7/7] fix: metrics for oura and socket path for scrolls --- nix/cardano/entrypoints.nix | 12 ++++++------ nix/cardano/nomadJob/oura.nix | 9 +++++++-- nix/cardano/nomadJob/scrolls.nix | 8 ++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index 4c6fa275dc..823ee81ede 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -280,7 +280,7 @@ } ''; - txpipe-source-config = '' + txpipe-config = '' NETWORK_MAGIC=$(jq '.networkMagic' "$( file="$(jq '.ShelleyGenesisFile' "$NODE_CONFIG" )" folder="$(dirname "$NODE_CONFIG")" @@ -290,7 +290,7 @@ SOURCE_CONFIG=$( jq -n \ --arg sp "$SOCKET_PATH" \ --arg nm "$NETWORK_MAGIC" \ - '{source: {type: "N2C", address: ["Unix", $sp], magic: $nm}}' ) + '{metrics: {}, source: {type: "N2C", address: ["Unix", $sp], magic: $nm}}' ) ''; @@ -588,10 +588,10 @@ in { text = '' ${prelude} - ${txpipe-source-config} + ${txpipe-config} OURA_CONFIG="$DATA_DIR/config/$ENVIRONMENT/oura-config.json" - echo "[$json,$SOURCE_CONFIG]" | jq '.[0].ouraConfig + .[1]' > "$OURA_CONFIG" + echo "[$json,$TXPIPE_CONFIG]" | jq '.[0].ouraConfig + .[1]' > "$OURA_CONFIG" exec ${packages.oura}/bin/oura daemon --config "$OURA_CONFIG" ''; @@ -604,10 +604,10 @@ in { text = '' ${prelude} - ${txpipe-source-config} + ${txpipe-config} SCROLLS_CONFIG="$DATA_DIR/config/$ENVIRONMENT/scrolls-config.json" - echo "[$json,$SOURCE_CONFIG]" | jq '.[0].scrollsConfig + .[1]' > "$SCROLLS_CONFIG" + echo "[$json,$TXPIPE_CONFIG]" | jq '.[0].scrollsConfig + .[1]' > "$SCROLLS_CONFIG" exec ${packages.scrolls}/bin/scrolls daemon -c "$SCROLLS_CONFIG" ''; diff --git a/nix/cardano/nomadJob/oura.nix b/nix/cardano/nomadJob/oura.nix index 59c849e4b5..19eea6fbd6 100644 --- a/nix/cardano/nomadJob/oura.nix +++ b/nix/cardano/nomadJob/oura.nix @@ -84,7 +84,7 @@ in inherit namespace; endpoints = [ # prometheus metrics for oura - "http://127.0.0.1:1586/metrics" + "http://127.0.0.1:9186/metrics" # prometheus metrics for cardano-node # "http://127.0.0.1:12798/metrics" ]; @@ -99,7 +99,10 @@ in network = { dns = {servers = update [0] ["172.17.0.1"];}; mode = "bridge"; - port.oura = {to = 1586;}; + port.oura = { + # port from https://github.com/txpipe/oura/blob/main/book/src/advanced/pipeline_metrics.md + to = 9186; + }; }; task = { # ---------- @@ -124,11 +127,13 @@ in kill_timeout = "30s"; resources.cpu = 2000; resources.memory = 4096; + /** vault = { change_mode = "noop"; env = true; policies = ["oura"]; }; + */ }; }; } diff --git a/nix/cardano/nomadJob/scrolls.nix b/nix/cardano/nomadJob/scrolls.nix index e7101fdf5b..5bff83d7be 100644 --- a/nix/cardano/nomadJob/scrolls.nix +++ b/nix/cardano/nomadJob/scrolls.nix @@ -21,7 +21,7 @@ in type = "service"; priority = 50; persistanceMount = "/persist"; - vaultPkiPath = "pki/issue/scrolls"; + # vaultPkiPath = "pki/issue/scrolls"; consulRolePath = "consul/creds/scrolls"; in with data-merge; { @@ -107,14 +107,16 @@ in # ---------- oura = { env.DATA_DIR = persistanceMount; - env.SOCKET_PATH = "/alloc/tmp/node.socket"; # figure out how to pass this from the cardano group + inherit (node.task.node.env) SOCKET_PATH; + /** template = _utils.nomadFragments.workload-identity-vault {inherit vaultPkiPath;} ++ _utils.nomadFragments.workload-identity-vault-consul {inherit consulRolePath;}; env.WORKLOAD_CACERT = "/secrets/tls/ca.pem"; env.WORKLOAD_CLIENT_KEY = "/secrets/tls/key.pem"; env.WORKLOAD_CLIENT_CERT = "/secrets/tls/cert.pem"; + */ config.image = ociNamer oci-images.oura; user = "0:0"; driver = "docker"; @@ -122,11 +124,13 @@ in kill_timeout = "30s"; resources.cpu = 2000; resources.memory = 4096; + /** vault = { change_mode = "noop"; env = true; policies = ["oura"]; }; + */ }; }; }