diff --git a/apps/lenra/lib/lenra/application.ex b/apps/lenra/lib/lenra/application.ex index 6f870cd9..842f421c 100644 --- a/apps/lenra/lib/lenra/application.ex +++ b/apps/lenra/lib/lenra/application.ex @@ -12,6 +12,7 @@ defmodule Lenra.Application do def start(_type, _args) do Lenra.MigrationHelper.migrate() + Lenra.Monitor.setup() children = [ # Start the ecto repository @@ -62,7 +63,8 @@ defmodule Lenra.Application do ), {Cluster.Supervisor, [Application.get_env(:libcluster, :topologies), [name: Lenra.ClusterSupervisor]]}, Kubernetes.StatusDynSup, - {Kubernetes.StatusTask, []} + {Kubernetes.StatusTask, []}, + Lenra.Monitor.ApplicationDeploymentMonitor ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/apps/lenra/lib/lenra/apps.ex b/apps/lenra/lib/lenra/apps.ex index 70599a00..308e2f8e 100644 --- a/apps/lenra/lib/lenra/apps.ex +++ b/apps/lenra/lib/lenra/apps.ex @@ -223,6 +223,8 @@ defmodule Lenra.Apps do creator_id |> create_build(app.id, params) |> Repo.transaction() do + Lenra.Monitor.ApplicationDeploymentMonitor.monitor(app_id, build.id) + case create_deployment( preloaded_app.main_env.environment_id, build.id, @@ -405,24 +407,25 @@ defmodule Lenra.Apps do |> Repo.transaction() ApplicationServices.stop_app("#{OpenfaasServices.get_function_name(service_name, build_number)}") + Lenra.Monitor.ApplicationDeploymentMonitor.stop(deployment.build_id) transaction - # Function not found in openfaas, 2 retry (10s), - # To let openfaas deploy in case of overload, after 2 retry -> failure + # Function not found in openfaas, 30 retry (15s), + # To let openfaas deploy in case of overload, after 30 retry -> failure :error404 -> - if retry == 3 do + if retry == 30 do Logger.critical("Function #{service_name} not deploy on openfaas, this should not appens") update_deployement(deployment, status: :failure) else - Process.sleep(5000) + Process.sleep(500) update_deployement_after_deploy(deployment, env, service_name, build_number, retry + 1) end :error500 _any -> - Process.sleep(5000) + Process.sleep(500) update_deployement_after_deploy(deployment, env, service_name, build_number, retry + 1) end end diff --git a/apps/lenra/lib/monitor.ex b/apps/lenra/lib/monitor.ex new file mode 100644 index 00000000..9f6286d5 --- /dev/null +++ b/apps/lenra/lib/monitor.ex @@ -0,0 +1,50 @@ +defmodule Lenra.Monitor do + @moduledoc """ + This module is monitoring requests at different places + Lenra's monitor executes the following events: + * `[:lenra, :app_deployment, :start]` - Executed when the app's deployment is triggered. + #### Measurements + * start_time. + #### Metadata + * `:application_id` - The id of the deploying application. + * `:build_id` - The id of the deploying build. + * `[:lenra, :app_deployment, :stop]` - Executed when the `available_replicas` parameter is not 0 or null on OpenFaaS. + #### Measurements + * end_time. + * `:duration` - The time took by the openfaas function in `:native` unit of time. + """ + + alias Lenra.Monitor.ApplicationDeploymentMeasurement + alias Lenra.Repo + + def setup do + events = [ + [:lenra, :app_deployment, :start], + [:lenra, :app_deployment, :stop] + ] + + :telemetry.attach_many( + "lenra.monitor", + events, + &Lenra.Monitor.handle_event/4, + nil + ) + end + + def handle_event([:lenra, :app_deployment, :start], measurements, metadata, _config) do + application_id = Map.get(metadata, :application_id) + build_id = Map.get(metadata, :build_id) + + Repo.insert(ApplicationDeploymentMeasurement.new(application_id, build_id, measurements)) + end + + def handle_event([:lenra, :app_deployment, :stop], measurements, metadata, _config) do + application_id = Map.get(metadata, :application_id) + build_id = Map.get(metadata, :build_id) + + ApplicationDeploymentMeasurement + |> Repo.get_by!(%{application_id: application_id, build_id: build_id}) + |> ApplicationDeploymentMeasurement.update(measurements) + |> Repo.update() + end +end diff --git a/apps/lenra/lib/monitor/application_deployment_measurement.ex b/apps/lenra/lib/monitor/application_deployment_measurement.ex new file mode 100644 index 00000000..2a3f7f67 --- /dev/null +++ b/apps/lenra/lib/monitor/application_deployment_measurement.ex @@ -0,0 +1,38 @@ +defmodule Lenra.Monitor.ApplicationDeploymentMeasurement do + @moduledoc """ + Lenra.Monitor.ApplicationDeploymentMeasurement is a ecto schema to store measurements of applications deployment. + """ + use Ecto.Schema + import Ecto.Changeset + + alias Lenra.Apps.{App, Build} + + schema "application_deployment_measurement" do + belongs_to(:application, App) + belongs_to(:build, Build) + + field(:start_time, :utc_datetime) + field(:end_time, :utc_datetime) + + field(:duration, :integer) + + timestamps() + end + + def changeset(application_deployment_measurement, params \\ %{}) do + application_deployment_measurement + |> cast(params, [:start_time, :end_time, :duration]) + |> validate_required([:start_time, :application_id, :build_id]) + |> foreign_key_constraint(:application_id) + end + + def new(application_id, build_id, params \\ %{}) do + %__MODULE__{application_id: application_id, build_id: build_id} + |> __MODULE__.changeset(params) + end + + def update(application_deployment_measurement, params) do + application_deployment_measurement + |> changeset(params) + end +end diff --git a/apps/lenra/lib/monitor/application_deployment_monitor.ex b/apps/lenra/lib/monitor/application_deployment_monitor.ex new file mode 100644 index 00000000..974fba8c --- /dev/null +++ b/apps/lenra/lib/monitor/application_deployment_monitor.ex @@ -0,0 +1,55 @@ +defmodule Lenra.Monitor.ApplicationDeploymentMonitor do + @moduledoc """ + The application deployment monitor which monitors the time spent deploying an app. + """ + + use GenServer + use SwarmNamed + + alias Lenra.Telemetry + + require Logger + + def monitor(application_id, build_id) do + GenServer.call(__MODULE__, {:monitor, application_id, build_id}) + rescue + e -> + Logger.error( + "#{__MODULE__} fail in monitor with application_id #{application_id}, build_id #{build_id} and error: #{inspect(e)}" + ) + end + + def stop(build_id) do + GenServer.call(__MODULE__, {:stop, build_id}) + rescue + e -> + Logger.error("#{__MODULE__} fail in stop with build_id #{build_id} and error: #{inspect(e)}") + end + + def start_link(_opts) do + Logger.debug("Start #{__MODULE__}") + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + def init(_args) do + {:ok, %{}} + end + + def handle_call({:monitor, application_id, build_id}, _from, state) do + Logger.debug("#{__MODULE__} monitor #{inspect(application_id)} with build_id #{build_id}") + + start_time = Telemetry.start(:app_deployment, %{build_id: build_id}) + + {:reply, :ok, Map.put(state, build_id, {application_id, start_time})} + end + + def handle_info({:stop, build_id}, state) do + {{application_id, start_time}, new_state} = Map.pop(state, build_id) + + Logger.debug("#{__MODULE__} handle down #{inspect(application_id)} with build_id #{build_id}") + + Telemetry.stop(:app_deployment, start_time, %{build_id: build_id}) + + {:noreply, new_state} + end +end diff --git a/apps/lenra/priv/repo/migrations/20240713140043_application_deployment_measurement.exs b/apps/lenra/priv/repo/migrations/20240713140043_application_deployment_measurement.exs new file mode 100644 index 00000000..7138f7eb --- /dev/null +++ b/apps/lenra/priv/repo/migrations/20240713140043_application_deployment_measurement.exs @@ -0,0 +1,17 @@ +defmodule Lenra.Repo.Migrations.ApplicationDeploymentMeasurement do + use Ecto.Migration + + def change do + create table(:application_deployment_measurement) do + add(:user_id, references(:users), null: false) + add(:build_id, references(:builds), null: false) + + add(:start_time, :timestamp, null: false) + add(:end_time, :timestamp) + + add(:duration, :integer) + + timestamps() + end + end +end