From c77166bf48f49a43c67fc8d007f8c09e4d2ea09a Mon Sep 17 00:00:00 2001 From: Conrad Taylor Date: Tue, 5 Sep 2023 04:04:09 -0700 Subject: [PATCH 1/2] Add PromEx. --- config/config.exs | 25 ++++++++ config/prod.exs | 22 +++++++ lib/zero_phoenix/application.ex | 12 ++-- lib/zero_phoenix/prom_ex.ex | 103 +++++++++++++++++++++++++++++++ lib/zero_phoenix_web/endpoint.ex | 1 + mix.exs | 3 +- mix.lock | 3 + 7 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 lib/zero_phoenix/prom_ex.ex diff --git a/config/config.exs b/config/config.exs index bf5e865..9e3666a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -48,6 +48,31 @@ if Mix.env() == :dev do ] end +# Configures prom_ex +config :zero_phoenix, ZeroPhoenix.PromEx, + disabled: false, + manual_metrics_start_delay: :no_delay, + drop_metrics_groups: [], + grafana: [ + host: System.get_env("GRAFANA_HOST", "http://localhost:3000"), + username: System.get_env("GF_SECURITY_ADMIN_USER", "admin"), + password: System.get_env("GF_SECURITY_ADMIN_PASSWORD", "admin"), + folder_name: "zero-to-graphql-using-elixir", + upload_dashboards_on_start: true, + annotate_app_lifecycle: true + ], + metrics_server: [ + protocol: :http, + path: "/metrics", + port: 4021, + pool_size: 5, + cowboy_opts: [], + auth_strategy: :none + ] + +# config :zero_phoenix, ZeroPhoenix.PromEx, +# grafana_datasource_id: System.get_env("GRAFANA_DATASOURCE_ID", "Local Prometheus") + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/config/prod.exs b/config/prod.exs index 5f0096b..3e79032 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -12,3 +12,25 @@ config :logger, level: :info # Runtime production configuration, including reading # of environment variables, is done on config/runtime.exs. + +# Configures prom_ex +config :zero_phoenix, ZeroPhoenix.PromEx, + disabled: false, + manual_metrics_start_delay: :no_delay, + drop_metrics_groups: [], + grafana: [ + host: System.get_env("GRAFANA_HOST", raise("GRAFANA_HOST is required")), + auth_token: System.get_env("GRAFANA_TOKEN", raise("GRAFANA_TOKEN is required")), + folder_name: "zero-to-graphql-using-elixir", + upload_dashboards_on_start: true, + annotate_app_lifecycle: true + ], + metrics_server: [ + protocol: :http, + path: "/metrics", + port: 4021, + pool_size: 5, + cowboy_opts: [] + auth_strategy: :bearer, + auth_token: "", + ] diff --git a/lib/zero_phoenix/application.ex b/lib/zero_phoenix/application.ex index 29904cf..86cb2f4 100644 --- a/lib/zero_phoenix/application.ex +++ b/lib/zero_phoenix/application.ex @@ -7,16 +7,18 @@ defmodule ZeroPhoenix.Application do def start(_type, _args) do children = [ - # Start the Ecto repository - ZeroPhoenix.Repo, + # Start the PromEx + ZeroPhoenix.PromEx, # Start the Telemetry supervisor ZeroPhoenixWeb.Telemetry, + # Start the Ecto repository + ZeroPhoenix.Repo, # Start the PubSub system {Phoenix.PubSub, name: ZeroPhoenix.PubSub}, - # Start the endpoint when the application starts + # Start Finch + {Finch, name: ZeroPhoenix.Finch}, + # Start the Endpoint (http/https) ZeroPhoenixWeb.Endpoint - # Start your own worker by calling: ZeroPhoenix.Worker.start_link(arg1, arg2, arg3) - # worker(ZeroPhoenix.Worker, [arg1, arg2, arg3]), ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/lib/zero_phoenix/prom_ex.ex b/lib/zero_phoenix/prom_ex.ex new file mode 100644 index 0000000..aecf91b --- /dev/null +++ b/lib/zero_phoenix/prom_ex.ex @@ -0,0 +1,103 @@ +defmodule ZeroPhoenix.PromEx do + @moduledoc """ + Be sure to add the following to finish setting up PromEx: + + 1. Update your configuration (config.exs, dev.exs, prod.exs, releases.exs, etc) to + configure the necessary bit of PromEx. Be sure to check out `PromEx.Config` for + more details regarding configuring PromEx: + ``` + config :zero_phoenix, ZeroPhoenix.PromEx, + disabled: false, + manual_metrics_start_delay: :no_delay, + drop_metrics_groups: [], + grafana: :disabled, + metrics_server: :disabled + ``` + + 2. Add this module to your application supervision tree. It should be one of the first + things that is started so that no Telemetry events are missed. For example, if PromEx + is started after your Repo module, you will miss Ecto's init events and the dashboards + will be missing some data points: + ``` + def start(_type, _args) do + children = [ + ZeroPhoenix.PromEx, + + ... + ] + + ... + end + ``` + + 3. Update your `endpoint.ex` file to expose your metrics (or configure a standalone + server using the `:metrics_server` config options). Be sure to put this plug before + your `Plug.Telemetry` entry so that you can avoid having calls to your `/metrics` + endpoint create their own metrics and logs which can pollute your logs/metrics given + that Prometheus will scrape at a regular interval and that can get noisy: + ``` + defmodule ZeroPhoenixWeb.Endpoint do + use Phoenix.Endpoint, otp_app: :zero_phoenix + + ... + + plug PromEx.Plug, prom_ex_module: ZeroPhoenix.PromEx + + ... + end + ``` + + 4. Update the list of plugins in the `plugins/0` function return list to reflect your + application's dependencies. Also update the list of dashboards that are to be uploaded + to Grafana in the `dashboards/0` function. + """ + + use PromEx, otp_app: :zero_phoenix + + alias PromEx.Plugins + + @impl true + def plugins do + [ + # PromEx built in plugins + Plugins.Application, + Plugins.Beam, + {Plugins.Phoenix, router: ZeroPhoenixWeb.Router, endpoint: ZeroPhoenixWeb.Endpoint}, + Plugins.Ecto, + # Plugins.Oban, + Plugins.PhoenixLiveView, + Plugins.Absinthe + # Plugins.Broadway, + + # Add your own PromEx metrics plugins + # ZeroPhoenix.Users.PromExPlugin + ] + end + + @impl true + def dashboard_assigns do + [ + datasource_id: "Local Prometheus", + # datasource_id: Application.get_env(:zero_phoenix, :grafana_datasource_id), + default_selected_interval: "30s" + ] + end + + @impl true + def dashboards do + [ + # PromEx built in Grafana dashboards + {:prom_ex, "application.json"}, + {:prom_ex, "beam.json"}, + {:prom_ex, "phoenix.json"}, + {:prom_ex, "ecto.json"}, + # {:prom_ex, "oban.json"}, + {:prom_ex, "phoenix_live_view.json"}, + {:prom_ex, "absinthe.json"} + # {:prom_ex, "broadway.json"}, + + # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"} + # {:zero_phoenix, "/grafana_dashboards/user_metrics.json"} + ] + end +end diff --git a/lib/zero_phoenix_web/endpoint.ex b/lib/zero_phoenix_web/endpoint.ex index e210d31..39870dc 100644 --- a/lib/zero_phoenix_web/endpoint.ex +++ b/lib/zero_phoenix_web/endpoint.ex @@ -35,6 +35,7 @@ defmodule ZeroPhoenixWeb.Endpoint do cookie_key: "request_logger" plug Plug.RequestId + plug PromEx.Plug, prom_ex_module: ZeroPhoenix.PromEx plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint] plug Plug.Parsers, diff --git a/mix.exs b/mix.exs index 5d5ead6..e5941ad 100644 --- a/mix.exs +++ b/mix.exs @@ -53,7 +53,8 @@ defmodule ZeroPhoenix.Mixfile do {:cors_plug, "~> 3.0.3"}, {:credo, "~> 1.7.0", only: [:dev, :test], runtime: false}, {:mix_test_watch, "~> 1.1.0", only: [:dev, :test], runtime: false}, - {:ecto_psql_extras, "~> 0.7.12"} + {:ecto_psql_extras, "~> 0.7.12"}, + {:prom_ex, "~> 1.8.0"} ] end diff --git a/mix.lock b/mix.lock index f180afe..4e1ba43 100644 --- a/mix.lock +++ b/mix.lock @@ -32,6 +32,7 @@ "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, + "octo_fetch": {:hex, :octo_fetch, "0.3.0", "89ff501d2ac0448556ff1931634a538fe6d6cd358ba827ce1747e6a42a46efbf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "c07e44f2214ab153743b7b3182f380798d0b294b1f283811c1e30cff64096d3d"}, "phoenix": {:hex, :phoenix, "1.7.7", "4cc501d4d823015007ba3cdd9c41ecaaf2ffb619d6fb283199fa8ddba89191e0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "8966e15c395e5e37591b6ed0bd2ae7f48e961f0f60ac4c733f9566b519453085"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.2", "b21bd01fdeffcfe2fab49e4942aa938b6d3e89e93a480d4aee58085560a0bc0d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "70242edd4601d50b69273b057ecf7b684644c19ee750989fd555625ae4ce8f5d"}, "phoenix_html": {:hex, :phoenix_html, "3.3.2", "d6ce982c6d8247d2fc0defe625255c721fb8d5f1942c5ac051f6177bffa5973f", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "44adaf8e667c1c20fb9d284b6b0fa8dc7946ce29e81ce621860aa7e96de9a11d"}, @@ -45,11 +46,13 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, "postgrex": {:hex, :postgrex, "0.17.2", "a3ec9e3239d9b33f1e5841565c4eb200055c52cc0757a22b63ca2d529bbe764c", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "80a918a9e9531d39f7bd70621422f3ebc93c01618c645f2d91306f50041ed90c"}, + "prom_ex": {:hex, :prom_ex, "1.8.0", "662615e1d2f2ab3e0dc13a51c92ad0ccfcab24336a90cb9b114ee1bce9ef88aa", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "3eea763dfa941e25de50decbf17a6a94dbd2270e7b32f88279aa6e9bbb8e23e7"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "swoosh": {:hex, :swoosh, "1.11.5", "429dccde78e2f60c6339e96917efecebca9d1f254d2878a150f580d2f782260b", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "21ee57dcd68d2f56d3bbe11e76d56d142b221bb12b6018c551cc68442b800040"}, "table_rex": {:hex, :table_rex, "3.1.1", "0c67164d1714b5e806d5067c1e96ff098ba7ae79413cc075973e17c38a587caa", [:mix], [], "hexpm", "678a23aba4d670419c23c17790f9dcd635a4a89022040df7d5d772cb21012490"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, "thousand_island": {:hex, :thousand_island, "1.0.0-pre.7", "e74136e6cd4547cdf62da6332d9528915233e081553fed8a398caf5c11700a01", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f8d0a50e6ed95770341fb97f29f96031db94736cb93d267f7495e11ed2f5963a"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, From d1f0ac346dd54b82bde7eb61e8aae41ff5a1b4ae Mon Sep 17 00:00:00 2001 From: Conrad Taylor Date: Wed, 6 Sep 2023 21:02:17 -0700 Subject: [PATCH 2/2] Update dev container Elixir version. (#181) --- .devcontainer/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 51616fc..46117d2 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -7,7 +7,7 @@ services: dockerfile: Dockerfile args: # Elixir Version: 1.9, 1.10, 1.10.4, ... - VARIANT: '1.15.4' + VARIANT: '1.15.5' # Phoenix Version: 1.4.17, 1.5.4, ... PHOENIX_VERSION: '1.7.7' # Node Version: 12, 14, ... @@ -22,7 +22,7 @@ services: command: sleep infinity db: - image: postgres:15.3-bullseye + image: postgres:15.4-bullseye restart: unless-stopped volumes: - postgres-data:/var/lib/postgresql/data