From 4b8699428b61db2655672bb9116dec8fd73b373b Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Thu, 30 May 2024 17:57:14 +0200
Subject: [PATCH 01/28] feat: bulk operations

---
 lib/arke/query_manager.ex | 101 +++++++++++++++---
 lib/arke/validator.ex     | 208 +++++++++++++++++++++++++++++---------
 2 files changed, 244 insertions(+), 65 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index a48f0a4..07fd55e 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -44,7 +44,6 @@ defmodule Arke.QueryManager do
   alias Arke.Utils.DatetimeHandler, as: DatetimeHandler
   alias Arke.Core.{Arke, Unit, Query, Parameter}
 
-
   @persistence Application.get_env(:arke, :persistence)
   @record_fields [:id, :data, :metadata, :inserted_at, :updated_at]
 
@@ -142,17 +141,86 @@ defmodule Arke.QueryManager do
   @spec create(project :: atom(), arke :: Arke.t(), args :: list()) :: func_return()
   def create(project, arke, args) do
     persistence_fn = @persistence[:arke_postgres][:create]
+
     with %Unit{} = unit <- Unit.load(arke, args, :create),
-         {:ok, unit} <- Validator.validate(unit, :create, project),
-         {:ok, unit} <- ArkeManager.call_func(arke, :before_create, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
-         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
-         {:ok, unit} <- persistence_fn.(project, unit),
-         {:ok, unit} <- ArkeManager.call_func(arke, :on_create, [arke, unit]),
-         {:ok, unit} <- handle_link_parameters(unit, %{}),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_create),
-         do: {:ok, unit},
-         else: ({:error, errors} -> {:error, errors})
+         %{valid: [unit], errors: errors} <- Validator.validate(unit, :create, project),
+         {:ok, unit} <- run_persistence_hooks(arke, unit, :create, :before),
+         {:ok, unit} <- persistence_fn.(project, unit, []),
+         {:ok, unit} <- run_persistence_hooks(arke, unit, :create, :after) do
+      {:ok, unit}
+    else
+      {:error, errors} -> {:error, errors}
+      %{errors: [{_, err}], valid: _} -> {:error, err}
+    end
+  end
+
+  @spec create_bulk(project :: atom(), arke :: Arke.t(), data :: list(Arke.t()), args :: list()) ::
+          func_return()
+  def create_bulk(project, arke, data, args) do
+    persistence_fn = @persistence[:arke_postgres][:create]
+
+    with %{valid: valid, errors: errors} <- load_bulk_units(arke, data, :create, args),
+         %{valid: valid, errors: errors} <- Validator.validate(valid, :create, project),
+         %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :before),
+         {:ok, valid, persistence_errors} <- persistence_fn.(project, valid, bulk: true),
+         %{valid: valid, errors: errors} <-
+           process_bulk(valid, errors ++ persistence_errors, arke, :after) do
+      {:ok, valid, errors}
+    else
+      {:error, errors} -> {:error, errors}
+    end
+  end
+
+  defp process_bulk(valid, errors, arke, phase) do
+    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+      case run_persistence_hooks(arke, unit, :create, phase) do
+        {:ok, unit} ->
+          %{acc | valid: [unit | acc.valid]}
+
+        {:error, e} ->
+          %{acc | errors: [{unit, e} | acc.errors]}
+      end
+    end)
+  end
+
+  defp load_bulk_units(arke, units, persistence_fn, args),
+    do:
+      Enum.reduce(units, %{valid: [], errors: []}, fn item, acc ->
+        case Unit.load(arke, data_as_klist(item) ++ args, persistence_fn) do
+          %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
+          {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
+        end
+      end)
+
+  defp run_persistence_hooks(arke, unit, action, phase) do
+    with {:ok, unit} <- call_persistence_fn(arke, unit, action, phase),
+         {:ok, unit} <- call_group_persistence_fn(arke, unit, action, phase),
+         {:ok, unit} <- call_link_parameters_fn(arke, unit, action, phase) do
+      {:ok, unit}
+    end
+  end
+
+  defp call_persistence_fn(arke, unit, :create, :before),
+    do: ArkeManager.call_func(arke, :before_create, [arke, unit])
+
+  defp call_persistence_fn(arke, unit, :create, :after),
+    do: ArkeManager.call_func(arke, :on_create, [arke, unit])
+
+  defp call_group_persistence_fn(arke, unit, :create, :before),
+    do: handle_group_call_func(arke, unit, :before_unit_create)
+
+  defp call_group_persistence_fn(arke, unit, :create, :after),
+    do: handle_group_call_func(arke, unit, :on_unit_create)
+
+  defp call_link_parameters_fn(arke, unit, :create, :before),
+    do: handle_link_parameters_unit(arke, unit)
+
+  defp call_link_parameters_fn(arke, unit, :create, :after),
+    do: handle_link_parameters(unit, %{})
+
+  # todo: remove after atoms removal
+  defp data_as_klist(data) do
+    Enum.map(data, fn {key, value} -> {String.to_existing_atom(key), value} end)
   end
 
   defp handle_link_parameters_unit(%{id: :arke_link} = _, unit), do: {:ok, unit}
@@ -161,12 +229,9 @@ defmodule Arke.QueryManager do
          %{data: parameters} = arke,
          %{metadata: %{project: project}} = unit
        ) do
-
-
     {errors, link_units} =
       Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
       |> Enum.reduce({[], []}, fn p, {errors, link_units} ->
-
         arke = ArkeManager.get(String.to_existing_atom(p.data.arke_or_group_id), project)
 
         case handle_create_on_link_parameters_unit(
@@ -222,7 +287,6 @@ defmodule Arke.QueryManager do
     do: {:ok, parameter, value}
 
   def handle_group_call_func(arke, unit, func) do
-
     GroupManager.get_groups_by_arke(arke)
     |> Enum.reduce_while(unit, fn group, new_unit ->
       with {:ok, new_unit} <- GroupManager.call_func(group, func, [arke, new_unit]),
@@ -256,9 +320,11 @@ defmodule Arke.QueryManager do
   def update(%{arke_id: arke_id, metadata: %{project: project}, data: data} = current_unit, args) do
     persistence_fn = @persistence[:arke_postgres][:update]
     arke = ArkeManager.get(arke_id, project)
+
     with %Unit{} = unit <- Unit.update(current_unit, args),
          {:ok, unit} <- update_at_on_update(unit),
-         {:ok, unit} <- Validator.validate(unit, :update, project),
+         %{valid: valid, errors: errors} <- Validator.validate(unit, :update, project),
+         # todo better valid / error handling
          {:ok, unit} <- ArkeManager.call_func(arke, :before_update, [arke, unit]),
          {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
          {:ok, unit} <- handle_link_parameters_unit(arke, unit),
@@ -269,10 +335,12 @@ defmodule Arke.QueryManager do
          do: {:ok, unit},
          else: ({:error, errors} -> {:error, errors})
   end
+
   defp update_at_on_update(unit) do
     updated_at = DatetimeHandler.now(:datetime)
     {:ok, Unit.update(unit, updated_at: updated_at)}
   end
+
   @doc """
   Function to delete a given unit
   ## Parameters
@@ -524,6 +592,7 @@ defmodule Arke.QueryManager do
 
   defp handle_filter(query, :group_id, :eq, value, negate) do
     %{id: id} = group = get_group(value, query.project)
+
     arke_list =
       Enum.map(GroupManager.get_arke_list(group), fn a ->
         Atom.to_string(a.id)
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index e6d7df6..da90ad2 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -43,34 +43,42 @@ defmodule Arke.Validator do
       %{:error, [message]}
 
   """
-  @spec validate(unit :: Unit.t(), peristence_fn :: (() -> any()), project :: atom()) ::
+  @spec validate(unit :: Unit.t() | [Unit.t()], peristence_fn :: (-> any()), project :: atom()) ::
           func_return()
-  def validate(%{arke_id: arke_id} = unit, persistence_fn, project \\ :arke_system) do
-    with {:ok, unit} <- check_duplicate_unit(unit, project, persistence_fn) do
-      {%{data: data} = unit, errors} = before_validate(unit, project)
-      %{data: arke_data} = arke = ArkeManager.get(arke_id, project)
-
-      unit_parameters =
-        Enum.filter(ArkeManager.get_parameters(arke), fn %{data: %{persistence: persistence}} ->
-          persistence == "arke_parameter"
-        end)
-      res =
-        Enum.reduce(unit_parameters, {unit, errors}, fn p, {new_unit, errors} = _res ->
-          {value, err} = validate_parameter(arke, p, Unit.get_value(data, p.id), project)
-          {Unit.update(new_unit, [{p.id, value}]), errors ++ err}
-        end)
+  def validate(unit, persistence_fn, project \\ :arke_system)
 
-      {new_unit, errors} = res
-      filtered_errors = check_old_values(errors, data, new_unit.data, persistence_fn)
+  def validate(%Unit{} = unit, persistence_fn, project),
+    do: validate([unit], persistence_fn, project)
 
-      get_result({new_unit, filtered_errors})
-    else
-      {:error, errors} -> get_result({unit, errors})
-    end
+  def validate([], _persistence_fn, _project),
+    do: %{valid: [], errors: [{nil, "empty list of units"}]}
+
+  def validate([%Unit{arke_id: arke_id} | _] = unit_list, persistence_fn, project) do
+    %{data: arke_data} = arke = ArkeManager.get(arke_id, project)
 
-    # TODO: handle after validate
+    parameter_list =
+      Enum.filter(ArkeManager.get_parameters(arke), fn %{data: %{persistence: persistence}} ->
+        persistence == "arke_parameter"
+      end)
+
+    check_duplicate_units(unit_list, project, persistence_fn)
+    |> apply_before_validate(project)
+    |> check_bulk_parameters(arke, parameter_list, project)
+    |> check_unique_parameters(arke, parameter_list, project)
   end
 
+  defp apply_before_validate(%{valid: valid, errors: errors}, project),
+    do:
+      Enum.reduce(valid, %{valid: [], errors: errors}, fn u, acc ->
+        case before_validate(u, project) do
+          {unit, []} ->
+            Map.put(acc, :valid, [unit | acc.valid])
+
+          {unit, unit_errors} ->
+            Map.put(errors, :errors, errors ++ {unit, unit_errors})
+        end
+      end)
+
   defp before_validate(%{arke_id: arke_id} = unit, project) do
     arke = ArkeManager.get(arke_id, project)
 
@@ -79,16 +87,29 @@ defmodule Arke.Validator do
          else: ({:error, errors} -> {unit, errors})
   end
 
-  defp check_duplicate_unit(%{id: unit_id} = unit, _project, :create) when is_nil(unit_id),
-    do: {:ok, unit}
+  defp check_duplicate_units(unit_list, project, :create) do
+    ids_to_check =
+      Enum.filter(unit_list, fn u -> not is_nil(u.id) end) |> Enum.map(&to_string(&1.id))
 
-  defp check_duplicate_unit(%{id: unit_id} = unit, project, :create) do
-    with nil <- QueryManager.get_by(%{:id => unit_id, :project => project}),
-         do: {:ok, unit},
-         else: (_ -> {:error, [{"You can not create Unit with same id", unit_id}]})
+    duplicates = QueryManager.filter_by(%{:id__in => ids_to_check, :project => project})
+
+    valid =
+      Enum.reduce(unit_list, [], fn item, acc ->
+        case Enum.find(duplicates, fn d -> d.id == item.id end) do
+          nil -> [item | acc]
+          _ -> acc
+        end
+      end)
+
+    %{
+      valid: valid,
+      errors:
+        Enum.map(duplicates, fn d -> {d, "duplicate values are not allowed for #{d.id}"} end)
+    }
   end
 
-  defp check_duplicate_unit(%{id: unit_id} = unit, _project, _persistence_fn), do: {:ok, unit}
+  defp check_duplicate_units(unit_list, _project, _persistence_fn),
+    do: %{valid: unit_list, errors: []}
 
   defp check_old_values(errors, old_unit_data, new_unit_data, :update) do
     duplicate_list =
@@ -109,6 +130,76 @@ defmodule Arke.Validator do
 
   defp get_result({unit, _errors} = _res), do: {:ok, unit}
 
+  defp check_bulk_parameters(%{valid: valid, errors: errors}, arke, parameter_list, project) do
+    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+      case Enum.reduce(parameter_list, {unit, []}, fn parameter, {unit, errors} ->
+             {value, err} =
+               validate_parameter(
+                 arke,
+                 parameter,
+                 Unit.get_value(unit.data, parameter.id),
+                 project
+               )
+
+             {Unit.update(unit, [{parameter.id, value}]), errors ++ err}
+           end) do
+        {unit, []} ->
+          Map.put(acc, :valid, [unit | acc.valid])
+
+        # todo: manage multiple errors
+        {unit, unit_errors} ->
+          {parameter_id, err} = List.first(unit_errors)
+
+          %{acc | errors: [{unit, "#{parameter_id} #{err}"} | acc.errors]}
+      end
+    end)
+  end
+
+  defp create_unique_map(parameter_list, unit_list) do
+    Map.new(Enum.filter(parameter_list, fn p -> p.data[:unique] end), fn p ->
+      {p.id,
+       Enum.reduce(unit_list, [], fn unit, acc ->
+         case Map.get(unit.data, p.id) do
+           nil -> acc
+           v -> [v | acc]
+         end
+       end)}
+    end)
+  end
+
+  defp check_unique_parameters(%{valid: valid, errors: errors}, arke, parameter_list, project) do
+    unique_map = create_unique_map(parameter_list, valid)
+
+    case map_size(unique_map) do
+      0 ->
+        %{valid: valid, errors: errors}
+
+      _ ->
+        db_units =
+          QueryManager.query(arke: arke.id, project: project)
+          |> QueryManager.or_(
+            false,
+            Enum.map(unique_map, fn {id, values} -> QueryManager.condition(id, :in, values) end)
+          )
+          |> QueryManager.all()
+
+        parameter_already_present = create_unique_map(parameter_list, db_units)
+
+        Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+          has_error =
+            Enum.any?(parameter_already_present, fn {parameter_id, values} ->
+              Enum.member?(values, Map.get(unit.data, parameter_id))
+            end)
+
+          if has_error do
+            %{acc | errors: [{unit, "duplicate values are not allowed"} | acc.errors]}
+          else
+            %{acc | valid: [unit | acc.valid]}
+          end
+        end)
+    end
+  end
+
   @doc """
   Check if the value can be assigned to a given parameter in a specific schema struct.
 
@@ -135,7 +226,7 @@ defmodule Arke.Validator do
 
   def validate_parameter(arke, parameter, value, project) when is_atom(parameter) do
     parameter = get_parameter(arke, parameter, project)
-    check_parameter(parameter, value, project,arke)
+    check_parameter(parameter, value, project, arke)
   end
 
   defp get_parameter(nil, parameter_id, project),
@@ -145,10 +236,10 @@ defmodule Arke.Validator do
     do: ArkeManager.get_parameter(arke, parameter_id)
 
   def validate_parameter(arke, parameter, value, project) do
-    check_parameter(parameter, value, project,arke)
+    check_parameter(parameter, value, project, arke)
   end
 
-  defp check_parameter(parameter, value, project,arke) do
+  defp check_parameter(parameter, value, project, arke) do
     value = get_default_value(parameter, value)
     value = parse_value(parameter, value)
     value = check_whitespace(parameter, value)
@@ -157,7 +248,6 @@ defmodule Arke.Validator do
       []
       |> check_required_parameter(parameter, value)
       |> check_by_type(parameter, value)
-      |> check_duplicate(parameter, value, project,arke)
 
     {value, errors}
   end
@@ -221,15 +311,15 @@ defmodule Arke.Validator do
   defp check_required_parameter(errors, _parameter, _value), do: errors
 
   defp check_duplicate(errors, %{id: id, data: %{unique: true}} = _parameter, nil, project),
-       do: errors ++ [{"value must not be null for", id}]
+    do: errors ++ [{"value must not be null for", id}]
 
-  defp check_duplicate(errors, %{id: id, data: %{unique: true}} = parameter, value, project,arke) do
+  defp check_duplicate(errors, %{id: id, data: %{unique: true}} = parameter, value, project, arke) do
     with nil <- QueryManager.get_by(%{id => value, :project => project, :arke_id => arke.id}),
          do: errors,
          else: (_ -> errors ++ [{"duplicate values are not allowed for", id}])
   end
 
-  defp check_duplicate(errors, _parameter, _value, _project,_arke), do: errors
+  defp check_duplicate(errors, _parameter, _value, _project, _arke), do: errors
 
   defp check_by_type(errors, _parameter, value) when is_nil(value), do: errors
 
@@ -258,11 +348,12 @@ defmodule Arke.Validator do
 
   defp check_values(
          errors,
-         %{arke_id: type, data: %{values: values, label: label,multiple: true}} = parameter,
+         %{arke_id: type, data: %{values: values, label: label, multiple: true}} = parameter,
          value
        )
        when is_list(value) do
     admitted_values = Enum.map(values, fn %{label: _l, value: v} -> v end)
+
     with true <- check_values_type(value, type) do
       with [] <- value -- admitted_values do
         errors
@@ -287,7 +378,8 @@ defmodule Arke.Validator do
          value
        ),
        do: errors ++ [{value, "#{label} must be a list of #{type}}"}]
- defp check_values(errors,_parameter,_value), do: errors
+
+  defp check_values(errors, _parameter, _value), do: errors
 
   defp check_values_type(value, type) do
     condition =
@@ -299,18 +391,32 @@ defmodule Arke.Validator do
 
     Enum.all?(value, &condition.(&1))
   end
+
   # --- end Enum ---
   # --- start Multiple ---
-  defp check_multiple(errors, %{id: id, data: %{multiple: false}} = _parameter, value) when is_list(value), do: errors ++ [{"multiple values are not allowed for", id}]
-  defp check_multiple(errors, %{id: id, data: %{multiple: true}} = parameter, value) when not is_list(value), do: check_multiple(errors, parameter, [value])
-  defp check_multiple(errors, %{id: id, arke_id: type, data: %{ multiple: true}} = parameter, value) do
-    case check_values_type(value,type) do
-      true -> errors
+  defp check_multiple(errors, %{id: id, data: %{multiple: false}} = _parameter, value)
+       when is_list(value),
+       do: errors ++ [{"multiple values are not allowed for", id}]
+
+  defp check_multiple(errors, %{id: id, data: %{multiple: true}} = parameter, value)
+       when not is_list(value),
+       do: check_multiple(errors, parameter, [value])
+
+  defp check_multiple(
+         errors,
+         %{id: id, arke_id: type, data: %{multiple: true}} = parameter,
+         value
+       ) do
+    case check_values_type(value, type) do
+      true ->
+        errors
+
       false ->
-        errors ++ [{"[#{Enum.join(value,",")}]", "#{id} must be a list of #{type} "}]
+        errors ++ [{"[#{Enum.join(value, ",")}]", "#{id} must be a list of #{type} "}]
     end
   end
-  defp check_multiple(errors,_parameter,_value), do: errors
+
+  defp check_multiple(errors, _parameter, _value), do: errors
   # --- end Multiple ---
 
   defp check_whitespace(%{data: %{strip: true}} = parameter, value) when is_atom(value) do
@@ -338,7 +444,8 @@ defmodule Arke.Validator do
          value
        ) do
     # todo: used to parse override in metadata which can be written as string
-    max = parse_value(%{arke_id: :integer, data: %{multiple: false}},max_length)
+    max = parse_value(%{arke_id: :integer, data: %{multiple: false}}, max_length)
+
     if String.length(value) > max do
       errors ++ [{label, "max length is #{max_length}"}]
     else
@@ -356,7 +463,8 @@ defmodule Arke.Validator do
          value
        ) do
     # todo: used to parse override in metadata which can be written as string
-    min = parse_value(%{arke_id: :integer, data: %{multiple: false}},min_length)
+    min = parse_value(%{arke_id: :integer, data: %{multiple: false}}, min_length)
+
     if String.length(value) < min do
       errors ++ [{label, "min length is #{min_length}"}]
     else
@@ -405,7 +513,8 @@ defmodule Arke.Validator do
   defp check_max(errors, %{data: %{max: max}} = parameter, _) when is_nil(max), do: errors
 
   defp check_max(errors, %{data: %{max: max, label: label}} = parameter, value) do
-    parsed_max = parse_value(parameter,max)
+    parsed_max = parse_value(parameter, max)
+
     if value > parsed_max do
       errors ++ [{label, "max is #{max}"}]
     else
@@ -416,7 +525,8 @@ defmodule Arke.Validator do
   defp check_min(errors, %{data: %{min: min}} = parameter, _) when is_nil(min), do: errors
 
   defp check_min(errors, %{data: %{min: min, label: label}} = parameter, value) do
-    parsed_min = parse_value(parameter,min)
+    parsed_min = parse_value(parameter, min)
+
     if value < parsed_min do
       errors ++ [{label, "min is #{min}"}]
     else

From 722e98edfda42ea78e8dcd0369167da0899f66de Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Fri, 31 May 2024 17:48:03 +0200
Subject: [PATCH 02/28] feat: handle bulk update

---
 lib/arke/query_manager.ex | 166 +++++++++++++++++++++++++-------------
 lib/arke/validator.ex     |  52 +++++++-----
 2 files changed, 142 insertions(+), 76 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 07fd55e..af32d2b 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -150,7 +150,7 @@ defmodule Arke.QueryManager do
       {:ok, unit}
     else
       {:error, errors} -> {:error, errors}
-      %{errors: [{_, err}], valid: _} -> {:error, err}
+      %{errors: [{_, err} | _], valid: _} -> {:error, err}
     end
   end
 
@@ -159,65 +159,27 @@ defmodule Arke.QueryManager do
   def create_bulk(project, arke, data, args) do
     persistence_fn = @persistence[:arke_postgres][:create]
 
-    with %{valid: valid, errors: errors} <- load_bulk_units(arke, data, :create, args),
+    with %{valid: valid, errors: errors} <- prepare_create_bulk_units(arke, data, args),
          %{valid: valid, errors: errors} <- Validator.validate(valid, :create, project),
-         %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :before),
+         %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :create, :before),
          {:ok, valid, persistence_errors} <- persistence_fn.(project, valid, bulk: true),
          %{valid: valid, errors: errors} <-
-           process_bulk(valid, errors ++ persistence_errors, arke, :after) do
+           process_bulk(valid, errors ++ persistence_errors, arke, :create, :after) do
       {:ok, valid, errors}
     else
       {:error, errors} -> {:error, errors}
     end
   end
 
-  defp process_bulk(valid, errors, arke, phase) do
-    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-      case run_persistence_hooks(arke, unit, :create, phase) do
-        {:ok, unit} ->
-          %{acc | valid: [unit | acc.valid]}
-
-        {:error, e} ->
-          %{acc | errors: [{unit, e} | acc.errors]}
-      end
-    end)
-  end
-
-  defp load_bulk_units(arke, units, persistence_fn, args),
+  defp prepare_create_bulk_units(arke, units, args),
     do:
       Enum.reduce(units, %{valid: [], errors: []}, fn item, acc ->
-        case Unit.load(arke, data_as_klist(item) ++ args, persistence_fn) do
+        case Unit.load(arke, data_as_klist(item) ++ args, :create) do
           %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
           {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
         end
       end)
 
-  defp run_persistence_hooks(arke, unit, action, phase) do
-    with {:ok, unit} <- call_persistence_fn(arke, unit, action, phase),
-         {:ok, unit} <- call_group_persistence_fn(arke, unit, action, phase),
-         {:ok, unit} <- call_link_parameters_fn(arke, unit, action, phase) do
-      {:ok, unit}
-    end
-  end
-
-  defp call_persistence_fn(arke, unit, :create, :before),
-    do: ArkeManager.call_func(arke, :before_create, [arke, unit])
-
-  defp call_persistence_fn(arke, unit, :create, :after),
-    do: ArkeManager.call_func(arke, :on_create, [arke, unit])
-
-  defp call_group_persistence_fn(arke, unit, :create, :before),
-    do: handle_group_call_func(arke, unit, :before_unit_create)
-
-  defp call_group_persistence_fn(arke, unit, :create, :after),
-    do: handle_group_call_func(arke, unit, :on_unit_create)
-
-  defp call_link_parameters_fn(arke, unit, :create, :before),
-    do: handle_link_parameters_unit(arke, unit)
-
-  defp call_link_parameters_fn(arke, unit, :create, :after),
-    do: handle_link_parameters(unit, %{})
-
   # todo: remove after atoms removal
   defp data_as_klist(data) do
     Enum.map(data, fn {key, value} -> {String.to_existing_atom(key), value} end)
@@ -323,17 +285,16 @@ defmodule Arke.QueryManager do
 
     with %Unit{} = unit <- Unit.update(current_unit, args),
          {:ok, unit} <- update_at_on_update(unit),
-         %{valid: valid, errors: errors} <- Validator.validate(unit, :update, project),
+         %{valid: [unit], errors: errors} <- Validator.validate(unit, :update, project),
          # todo better valid / error handling
-         {:ok, unit} <- ArkeManager.call_func(arke, :before_update, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
-         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
-         {:ok, unit} <- persistence_fn.(project, unit),
-         {:ok, unit} <- ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit]),
-         {:ok, unit} <- handle_link_parameters(unit, data),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update),
-         do: {:ok, unit},
-         else: ({:error, errors} -> {:error, errors})
+         {:ok, unit} <- run_persistence_hooks(arke, unit, :update, :before),
+         {:ok, unit} <- persistence_fn.(project, unit, []),
+         {:ok, unit} <- run_persistence_hooks(arke, unit, :update, :after, data, current_unit) do
+      {:ok, unit}
+    else
+      {:error, errors} -> {:error, errors}
+      %{errors: [{_, err} | _], valid: _} -> {:error, err}
+    end
   end
 
   defp update_at_on_update(unit) do
@@ -341,6 +302,48 @@ defmodule Arke.QueryManager do
     {:ok, Unit.update(unit, updated_at: updated_at)}
   end
 
+  def update_bulk(_, _, [], _), do: {:ok, [], []}
+
+  def update_bulk(project, arke, unit_list, data) do
+    persistence_fn = @persistence[:arke_postgres][:update]
+
+    with %{valid: valid, errors: errors} <- prepare_update_bulk_units(arke, unit_list, data),
+         %{valid: valid, errors: errors} <- Validator.validate(valid, :update, project),
+         %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :update, :before),
+         {:ok, valid, persistence_errors} <- persistence_fn.(project, valid, bulk: true),
+         %{valid: valid, errors: errors} <-
+           process_bulk(
+             valid,
+             errors ++ persistence_errors,
+             arke,
+             :update,
+             :after,
+             unit_list,
+             data
+           ) do
+      {:ok, valid, errors}
+    else
+      {:error, errors} -> {:error, errors}
+    end
+  end
+
+  defp prepare_update_bulk_units(arke, unit_list, data),
+    do:
+      Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
+        item = Enum.find(data, fn d -> Map.get(d, "id") == to_string(unit.id) end)
+        updated_at = DatetimeHandler.now(:datetime)
+
+        case Unit.update(unit, data_as_klist(item) ++ [updated_at: updated_at]) do
+          %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
+          {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
+        end
+      end)
+
+  defp update_at_on_update(unit) do
+    updated_at = DatetimeHandler.now(:datetime)
+    {:ok, Unit.update(unit, updated_at: updated_at)}
+  end
+
   @doc """
   Function to delete a given unit
   ## Parameters
@@ -870,4 +873,59 @@ defmodule Arke.QueryManager do
   end
 
   defp normalize_value(value), do: to_string(value)
+
+  defp process_bulk(valid, errors, arke, :update, :after, current_units, data) do
+    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+      current_unit = Enum.find(current_units, fn u -> u.id == unit.id end)
+      updated_data = Enum.find(data, fn d -> Map.get(d, "id") == to_string(unit.id) end)
+
+      case run_persistence_hooks(arke, unit, :update, :after, updated_data, current_unit) do
+        {:ok, unit} ->
+          %{acc | valid: [unit | acc.valid]}
+
+        {:error, e} ->
+          %{acc | errors: [{unit, e} | acc.errors]}
+      end
+    end)
+  end
+
+  defp process_bulk(valid, errors, arke, persistence_fn, phase) do
+    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+      case run_persistence_hooks(arke, unit, persistence_fn, phase) do
+        {:ok, unit} ->
+          %{acc | valid: [unit | acc.valid]}
+
+        {:error, e} ->
+          %{acc | errors: [{unit, e} | acc.errors]}
+      end
+    end)
+  end
+
+  defp run_persistence_hooks(arke, unit, :create, :before) do
+    with {:ok, unit} <- ArkeManager.call_func(arke, :before_create, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
+         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
+         do: {:ok, unit}
+  end
+
+  defp run_persistence_hooks(arke, unit, :create, :after) do
+    with {:ok, unit} <- ArkeManager.call_func(arke, :on_create, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_create),
+         {:ok, unit} <- handle_link_parameters(unit, %{}),
+         do: {:ok, unit}
+  end
+
+  defp run_persistence_hooks(arke, unit, :update, :before) do
+    with {:ok, unit} <- ArkeManager.call_func(arke, :before_update, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
+         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
+         do: {:ok, unit}
+  end
+
+  defp run_persistence_hooks(arke, unit, :update, :after, data, current_unit) do
+    with {:ok, unit} <- ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update),
+         {:ok, unit} <- handle_link_parameters(unit, data),
+         do: {:ok, unit}
+  end
 end
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index da90ad2..fa5a604 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -64,7 +64,7 @@ defmodule Arke.Validator do
     check_duplicate_units(unit_list, project, persistence_fn)
     |> apply_before_validate(project)
     |> check_bulk_parameters(arke, parameter_list, project)
-    |> check_unique_parameters(arke, parameter_list, project)
+    |> check_unique_parameters(arke, parameter_list, project, persistence_fn)
   end
 
   defp apply_before_validate(%{valid: valid, errors: errors}, project),
@@ -103,28 +103,14 @@ defmodule Arke.Validator do
 
     %{
       valid: valid,
-      errors:
-        Enum.map(duplicates, fn d -> {d, "duplicate values are not allowed for #{d.id}"} end)
+      errors: Enum.map(duplicates, fn d -> {d, "value not allowed for #{d.id}"} end)
     }
   end
 
   defp check_duplicate_units(unit_list, _project, _persistence_fn),
+    # todo: manage update
     do: %{valid: unit_list, errors: []}
 
-  defp check_old_values(errors, old_unit_data, new_unit_data, :update) do
-    duplicate_list =
-      Enum.filter(errors, fn {k, v} ->
-        is_binary(k) and String.contains?(k, "duplicate") and is_atom(v)
-      end)
-
-    same_value =
-      Enum.filter(duplicate_list, fn {_k, v} -> old_unit_data[v] == new_unit_data[v] end)
-
-    :ordsets.subtract(errors, same_value)
-  end
-
-  defp check_old_values(errors, _old_unit_data, _new_unit_data, :create), do: errors
-
   defp get_result({_unit, errors} = _res) when is_list(errors) and length(errors) > 0,
     do: Error.create(:parameter_validation, errors)
 
@@ -167,7 +153,13 @@ defmodule Arke.Validator do
     end)
   end
 
-  defp check_unique_parameters(%{valid: valid, errors: errors}, arke, parameter_list, project) do
+  defp check_unique_parameters(
+         %{valid: valid, errors: errors},
+         arke,
+         parameter_list,
+         project,
+         :create
+       ) do
     unique_map = create_unique_map(parameter_list, valid)
 
     case map_size(unique_map) do
@@ -186,13 +178,19 @@ defmodule Arke.Validator do
         parameter_already_present = create_unique_map(parameter_list, db_units)
 
         Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-          has_error =
-            Enum.any?(parameter_already_present, fn {parameter_id, values} ->
+          non_unique_parameters =
+            Enum.filter(parameter_already_present, fn {parameter_id, values} ->
               Enum.member?(values, Map.get(unit.data, parameter_id))
             end)
 
-          if has_error do
-            %{acc | errors: [{unit, "duplicate values are not allowed"} | acc.errors]}
+          if length(non_unique_parameters) > 0 do
+            new_errors =
+              Enum.map(non_unique_parameters, fn {parameter_id, _values} ->
+                {unit,
+                 "value not allowed for parameter #{parameter_id}: #{Map.get(unit.data, parameter_id)}"}
+              end)
+
+            %{acc | errors: new_errors ++ acc.errors}
           else
             %{acc | valid: [unit | acc.valid]}
           end
@@ -200,6 +198,16 @@ defmodule Arke.Validator do
     end
   end
 
+  defp check_unique_parameters(
+         %{valid: valid, errors: errors},
+         arke,
+         parameter_list,
+         project,
+         :update
+       ) do
+    %{valid: valid, errors: errors}
+  end
+
   @doc """
   Check if the value can be assigned to a given parameter in a specific schema struct.
 

From 617d54605bc123709eb5580ae5dda8b6e3e8e771 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Mon, 3 Jun 2024 12:54:23 +0200
Subject: [PATCH 03/28] feat: bulk delete

---
 lib/arke/query_manager.ex | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index af32d2b..6a2faac 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -362,15 +362,29 @@ defmodule Arke.QueryManager do
     arke = ArkeManager.get(arke_id, project)
     persistence_fn = @persistence[:arke_postgres][:delete]
 
-    with {:ok, unit} <- ArkeManager.call_func(arke, :before_delete, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_delete),
-         {:ok, nil} <- persistence_fn.(project, unit),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_delete),
-         {:ok, _unit} <- ArkeManager.call_func(arke, :on_delete, [arke, unit]),
+    with {:ok, unit} <- run_persistence_hooks(arke, unit, :delete, :before),
+         {:ok, unit} <- persistence_fn.(project, unit, []),
+         {:ok, _unit} <- run_persistence_hooks(arke, unit, :delete, :after),
          do: {:ok, nil},
          else: ({:error, errors} -> {:error, errors})
   end
 
+  @spec delete_bulk(project :: atom(), [Unit.t()]) :: {:ok, [any()], [any()]}
+  def delete_bulk(project, []), do: {:ok, [], []}
+
+  def delete_bulk(project, [%{arke_id: arke_id} | _] = unit_list) do
+    arke = ArkeManager.get(arke_id, project)
+    persistence_fn = @persistence[:arke_postgres][:delete]
+
+    with %{valid: valid, errors: errors} <-
+           process_bulk(unit_list, [], arke, :delete, :before),
+         {:ok, _, _} <- persistence_fn.(project, valid, bulk: true),
+         %{valid: valid, errors: errors} <-
+           process_bulk(valid, errors, arke, :delete, :after),
+         do: {:ok, valid, errors},
+         else: ({:error, errors} -> {:error, errors})
+  end
+
   @doc """
   Function to get a single element identified by the opts. Use `Arke.QueryManager.filter_by` if more than one element is returned
   ## Parameters
@@ -928,4 +942,16 @@ defmodule Arke.QueryManager do
          {:ok, unit} <- handle_link_parameters(unit, data),
          do: {:ok, unit}
   end
+
+  defp run_persistence_hooks(arke, unit, :delete, :before) do
+    with {:ok, unit} <- ArkeManager.call_func(arke, :before_delete, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_delete),
+         do: {:ok, unit}
+  end
+
+  defp run_persistence_hooks(arke, unit, :delete, :after) do
+    with {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_delete),
+         {:ok, _unit} <- ArkeManager.call_func(arke, :on_delete, [arke, unit]),
+         do: {:ok, nil}
+  end
 end

From a36623414a98faced22a1991250b8b8b408c0545 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Tue, 4 Jun 2024 18:05:11 +0200
Subject: [PATCH 04/28] fix: adjust check uniques

---
 lib/arke/query_manager.ex | 18 +++++++-----
 lib/arke/validator.ex     | 58 ++++++++++++++++++++++-----------------
 2 files changed, 44 insertions(+), 32 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 6a2faac..d613c3a 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -143,7 +143,7 @@ defmodule Arke.QueryManager do
     persistence_fn = @persistence[:arke_postgres][:create]
 
     with %Unit{} = unit <- Unit.load(arke, args, :create),
-         %{valid: [unit], errors: errors} <- Validator.validate(unit, :create, project),
+         %{valid: [unit], errors: _errors} <- Validator.validate(unit, :create, project),
          {:ok, unit} <- run_persistence_hooks(arke, unit, :create, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
          {:ok, unit} <- run_persistence_hooks(arke, unit, :create, :after) do
@@ -162,10 +162,11 @@ defmodule Arke.QueryManager do
     with %{valid: valid, errors: errors} <- prepare_create_bulk_units(arke, data, args),
          %{valid: valid, errors: errors} <- Validator.validate(valid, :create, project),
          %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :create, :before),
-         {:ok, valid, persistence_errors} <- persistence_fn.(project, valid, bulk: true),
+         {:ok, inserted_count, valid, persistence_errors} <-
+           persistence_fn.(project, valid, bulk: true),
          %{valid: valid, errors: errors} <-
            process_bulk(valid, errors ++ persistence_errors, arke, :create, :after) do
-      {:ok, valid, errors}
+      {:ok, inserted_count, errors}
     else
       {:error, errors} -> {:error, errors}
     end
@@ -285,7 +286,7 @@ defmodule Arke.QueryManager do
 
     with %Unit{} = unit <- Unit.update(current_unit, args),
          {:ok, unit} <- update_at_on_update(unit),
-         %{valid: [unit], errors: errors} <- Validator.validate(unit, :update, project),
+         %{valid: [unit], errors: _errors} <- Validator.validate(unit, :update, project),
          # todo better valid / error handling
          {:ok, unit} <- run_persistence_hooks(arke, unit, :update, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
@@ -307,10 +308,13 @@ defmodule Arke.QueryManager do
   def update_bulk(project, arke, unit_list, data) do
     persistence_fn = @persistence[:arke_postgres][:update]
 
+    %{valid: valid, errors: errors} = prepare_update_bulk_units(arke, unit_list, data)
+
     with %{valid: valid, errors: errors} <- prepare_update_bulk_units(arke, unit_list, data),
          %{valid: valid, errors: errors} <- Validator.validate(valid, :update, project),
          %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :update, :before),
-         {:ok, valid, persistence_errors} <- persistence_fn.(project, valid, bulk: true),
+         {:ok, updated_count, valid, persistence_errors} <-
+           persistence_fn.(project, valid, bulk: true),
          %{valid: valid, errors: errors} <-
            process_bulk(
              valid,
@@ -321,7 +325,7 @@ defmodule Arke.QueryManager do
              unit_list,
              data
            ) do
-      {:ok, valid, errors}
+      {:ok, updated_count, errors}
     else
       {:error, errors} -> {:error, errors}
     end
@@ -938,8 +942,8 @@ defmodule Arke.QueryManager do
 
   defp run_persistence_hooks(arke, unit, :update, :after, data, current_unit) do
     with {:ok, unit} <- ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update),
          {:ok, unit} <- handle_link_parameters(unit, data),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update),
          do: {:ok, unit}
   end
 
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index fa5a604..6660ca2 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -132,11 +132,15 @@ defmodule Arke.Validator do
         {unit, []} ->
           Map.put(acc, :valid, [unit | acc.valid])
 
-        # todo: manage multiple errors
         {unit, unit_errors} ->
-          {parameter_id, err} = List.first(unit_errors)
-
-          %{acc | errors: [{unit, "#{parameter_id} #{err}"} | acc.errors]}
+          %{
+            acc
+            | errors: [
+                {unit,
+                 Enum.map(unit_errors, fn {parameter_id, error} -> "#{parameter_id} #{error}" end)}
+                | acc.errors
+              ]
+          }
       end
     end)
   end
@@ -147,7 +151,7 @@ defmodule Arke.Validator do
        Enum.reduce(unit_list, [], fn unit, acc ->
          case Map.get(unit.data, p.id) do
            nil -> acc
-           v -> [v | acc]
+           v -> [{unit.id, v} | acc]
          end
        end)}
     end)
@@ -158,7 +162,7 @@ defmodule Arke.Validator do
          arke,
          parameter_list,
          project,
-         :create
+         persistence_fn
        ) do
     unique_map = create_unique_map(parameter_list, valid)
 
@@ -171,7 +175,9 @@ defmodule Arke.Validator do
           QueryManager.query(arke: arke.id, project: project)
           |> QueryManager.or_(
             false,
-            Enum.map(unique_map, fn {id, values} -> QueryManager.condition(id, :in, values) end)
+            Enum.map(unique_map, fn {id, uniques} ->
+              QueryManager.condition(id, :in, Enum.map(uniques, fn {_, v} -> v end))
+            end)
           )
           |> QueryManager.all()
 
@@ -179,18 +185,30 @@ defmodule Arke.Validator do
 
         Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
           non_unique_parameters =
-            Enum.filter(parameter_already_present, fn {parameter_id, values} ->
-              Enum.member?(values, Map.get(unit.data, parameter_id))
+            Enum.filter(parameter_already_present, fn {parameter_id, uniques} ->
+              case persistence_fn do
+                :create ->
+                  Enum.member?(
+                    Enum.map(uniques, fn {_, v} -> v end),
+                    Map.get(unit.data, parameter_id)
+                  )
+
+                :update ->
+                  Enum.any?(uniques, fn {unit_id, value} ->
+                    to_string(unit_id) != to_string(unit.id) and
+                      value == Map.get(unit.data, parameter_id)
+                  end)
+              end
             end)
 
           if length(non_unique_parameters) > 0 do
-            new_errors =
-              Enum.map(non_unique_parameters, fn {parameter_id, _values} ->
-                {unit,
-                 "value not allowed for parameter #{parameter_id}: #{Map.get(unit.data, parameter_id)}"}
-              end)
+            unit_errors =
+              {unit,
+               Enum.map(non_unique_parameters, fn {parameter_id, _values} ->
+                 "value not allowed for parameter #{parameter_id}: #{Map.get(unit.data, parameter_id)}"
+               end)}
 
-            %{acc | errors: new_errors ++ acc.errors}
+            %{acc | errors: [unit_errors | acc.errors]}
           else
             %{acc | valid: [unit | acc.valid]}
           end
@@ -198,16 +216,6 @@ defmodule Arke.Validator do
     end
   end
 
-  defp check_unique_parameters(
-         %{valid: valid, errors: errors},
-         arke,
-         parameter_list,
-         project,
-         :update
-       ) do
-    %{valid: valid, errors: errors}
-  end
-
   @doc """
   Check if the value can be assigned to a given parameter in a specific schema struct.
 

From a0ce435fe1d620a258bd43e7b3ac7fcbf83c6666 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Wed, 5 Jun 2024 15:58:14 +0200
Subject: [PATCH 05/28] fix: add multiple input unique validation

---
 lib/arke/validator.ex | 139 +++++++++++++++++++++++++++---------------
 1 file changed, 91 insertions(+), 48 deletions(-)

diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index 6660ca2..17e8cb6 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -64,7 +64,7 @@ defmodule Arke.Validator do
     check_duplicate_units(unit_list, project, persistence_fn)
     |> apply_before_validate(project)
     |> check_bulk_parameters(arke, parameter_list, project)
-    |> check_unique_parameters(arke, parameter_list, project, persistence_fn)
+    |> check_unique_parameters(arke, parameter_list, project)
   end
 
   defp apply_before_validate(%{valid: valid, errors: errors}, project),
@@ -158,61 +158,104 @@ defmodule Arke.Validator do
   end
 
   defp check_unique_parameters(
-         %{valid: valid, errors: errors},
+         %{valid: valid, errors: errors} = unit_map,
          arke,
          parameter_list,
-         project,
-         persistence_fn
+         project
        ) do
     unique_map = create_unique_map(parameter_list, valid)
 
-    case map_size(unique_map) do
-      0 ->
-        %{valid: valid, errors: errors}
-
-      _ ->
-        db_units =
-          QueryManager.query(arke: arke.id, project: project)
-          |> QueryManager.or_(
-            false,
-            Enum.map(unique_map, fn {id, uniques} ->
-              QueryManager.condition(id, :in, Enum.map(uniques, fn {_, v} -> v end))
-            end)
-          )
-          |> QueryManager.all()
-
-        parameter_already_present = create_unique_map(parameter_list, db_units)
-
-        Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-          non_unique_parameters =
-            Enum.filter(parameter_already_present, fn {parameter_id, uniques} ->
-              case persistence_fn do
-                :create ->
-                  Enum.member?(
-                    Enum.map(uniques, fn {_, v} -> v end),
-                    Map.get(unit.data, parameter_id)
-                  )
-
-                :update ->
-                  Enum.any?(uniques, fn {unit_id, value} ->
-                    to_string(unit_id) != to_string(unit.id) and
-                      value == Map.get(unit.data, parameter_id)
-                  end)
-              end
-            end)
-
-          if length(non_unique_parameters) > 0 do
-            unit_errors =
-              {unit,
-               Enum.map(non_unique_parameters, fn {parameter_id, _values} ->
-                 "value not allowed for parameter #{parameter_id}: #{Map.get(unit.data, parameter_id)}"
-               end)}
+    valid
+    |> validate_unique_input([], errors, unique_map)
+    |> validate_and_filter_unique_units(unique_map, arke, project, parameter_list)
+  end
 
-            %{acc | errors: [unit_errors | acc.errors]}
-          else
-            %{acc | valid: [unit | acc.valid]}
+  defp validate_and_filter_unique_units(
+         %{valid: valid, errors: errors},
+         unique_map,
+         _arke,
+         _project,
+         _parameter_list
+       )
+       when map_size(unique_map) == 0,
+       do: %{valid: valid, errors: errors}
+
+  defp validate_and_filter_unique_units(
+         %{valid: valid, errors: errors},
+         unique_map,
+         arke,
+         project,
+         parameter_list
+       ) do
+    db_units =
+      QueryManager.query(arke: arke.id, project: project)
+      |> QueryManager.or_(
+        false,
+        Enum.map(unique_map, fn {parameter_id, uniques} ->
+          QueryManager.condition(parameter_id, :in, Enum.map(uniques, fn {_, v} -> v end))
+        end)
+      )
+      |> QueryManager.all()
+
+    parameter_already_present = create_unique_map(parameter_list, db_units)
+
+    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+      {non_unique_parameters, unit_errors} =
+        parameter_already_present
+        |> Enum.reduce({[], []}, fn {parameter_id, uniques}, {non_unique, errors} ->
+          case Enum.find(uniques, fn {unit_id, value} ->
+                 to_string(unit_id) != to_string(unit.id) and
+                   value == Map.get(unit.data, parameter_id)
+               end) do
+            nil ->
+              {non_unique, errors}
+
+            _ ->
+              {[{parameter_id, uniques} | non_unique],
+               [
+                 "value not allowed for parameter #{parameter_id}: #{Map.get(unit.data, parameter_id)}"
+                 | errors
+               ]}
           end
         end)
+
+      if non_unique_parameters == [] do
+        %{acc | valid: [unit | acc.valid]}
+      else
+        %{acc | errors: [{unit, unit_errors} | acc.errors]}
+      end
+    end)
+  end
+
+  defp validate_unique_input(unit_list, valid, errors, unique_map)
+       when map_size(unique_map) == 0,
+       do: %{valid: valid, errors: errors}
+
+  defp validate_unique_input([], valid, errors, _unique_map),
+    do: %{valid: valid, errors: errors}
+
+  defp validate_unique_input([unit | tail], valid, errors, unique_map) do
+    unit_errors =
+      Enum.reduce(unique_map, [], fn {parameter_id, uniques}, acc ->
+        case Map.get(unit.data, parameter_id) do
+          nil ->
+            acc
+
+          value ->
+            if Enum.count(uniques, fn {_, parameter_value} -> parameter_value == value end) > 1 do
+              [
+                "value not allowed for parameter #{parameter_id}: #{value}"
+                | acc
+              ]
+            else
+              acc
+            end
+        end
+      end)
+
+    case unit_errors do
+      [] -> validate_unique_input(tail, [unit | valid], errors, unique_map)
+      _ -> validate_unique_input(tail, valid, [{unit, unit_errors} | errors], unique_map)
     end
   end
 

From 0859c3d287a8481054c6488c9d8e530e6761b218 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Wed, 5 Jun 2024 17:30:05 +0200
Subject: [PATCH 06/28] fix: add unique nil parameter validation

---
 lib/arke/validator.ex | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index 17e8cb6..7915a75 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -307,6 +307,7 @@ defmodule Arke.Validator do
       []
       |> check_required_parameter(parameter, value)
       |> check_by_type(parameter, value)
+      |> check_unique(parameter, value)
 
     {value, errors}
   end
@@ -369,16 +370,10 @@ defmodule Arke.Validator do
 
   defp check_required_parameter(errors, _parameter, _value), do: errors
 
-  defp check_duplicate(errors, %{id: id, data: %{unique: true}} = _parameter, nil, project),
+  defp check_unique(errors, %{id: id, data: %{unique: true}} = _parameter, nil),
     do: errors ++ [{"value must not be null for", id}]
 
-  defp check_duplicate(errors, %{id: id, data: %{unique: true}} = parameter, value, project, arke) do
-    with nil <- QueryManager.get_by(%{id => value, :project => project, :arke_id => arke.id}),
-         do: errors,
-         else: (_ -> errors ++ [{"duplicate values are not allowed for", id}])
-  end
-
-  defp check_duplicate(errors, _parameter, _value, _project, _arke), do: errors
+  defp check_unique(errors, _parameter, _value), do: errors
 
   defp check_by_type(errors, _parameter, value) when is_nil(value), do: errors
 

From 76bfd2b0131420927b61f5f181e493d3838294f6 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Fri, 7 Jun 2024 15:29:50 +0200
Subject: [PATCH 07/28] fix: validate_unique_input

---
 lib/arke/validator.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index 7915a75..7a8038f 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -227,9 +227,9 @@ defmodule Arke.Validator do
     end)
   end
 
-  defp validate_unique_input(unit_list, valid, errors, unique_map)
+  defp validate_unique_input(unit_list, _valid, errors, unique_map)
        when map_size(unique_map) == 0,
-       do: %{valid: valid, errors: errors}
+       do: %{valid: unit_list, errors: errors}
 
   defp validate_unique_input([], valid, errors, _unique_map),
     do: %{valid: valid, errors: errors}

From 4c02e4eedb200fd07cc97e392d733ffa1dcef5f9 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Tue, 18 Jun 2024 11:37:58 +0200
Subject: [PATCH 08/28] fix: delete return

---
 lib/arke/query_manager.ex | 6 +++---
 lib/arke/validator.ex     | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index d613c3a..4408f44 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -367,8 +367,8 @@ defmodule Arke.QueryManager do
     persistence_fn = @persistence[:arke_postgres][:delete]
 
     with {:ok, unit} <- run_persistence_hooks(arke, unit, :delete, :before),
-         {:ok, unit} <- persistence_fn.(project, unit, []),
-         {:ok, _unit} <- run_persistence_hooks(arke, unit, :delete, :after),
+         {:ok, _} <- persistence_fn.(project, unit, []),
+         {:ok, _} <- run_persistence_hooks(arke, unit, :delete, :after),
          do: {:ok, nil},
          else: ({:error, errors} -> {:error, errors})
   end
@@ -382,7 +382,7 @@ defmodule Arke.QueryManager do
 
     with %{valid: valid, errors: errors} <-
            process_bulk(unit_list, [], arke, :delete, :before),
-         {:ok, _, _} <- persistence_fn.(project, valid, bulk: true),
+         {:ok, _} <- persistence_fn.(project, valid, bulk: true),
          %{valid: valid, errors: errors} <-
            process_bulk(valid, errors, arke, :delete, :after),
          do: {:ok, valid, errors},
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index 7a8038f..0a26fcc 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -92,6 +92,7 @@ defmodule Arke.Validator do
       Enum.filter(unit_list, fn u -> not is_nil(u.id) end) |> Enum.map(&to_string(&1.id))
 
     duplicates = QueryManager.filter_by(%{:id__in => ids_to_check, :project => project})
+    # todo: merge with system duplicates
 
     valid =
       Enum.reduce(unit_list, [], fn item, acc ->

From 85245c4978a3b9995b7ecbf2921c402275df297b Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <81801314+ilyichv@users.noreply.github.com>
Date: Mon, 22 Jul 2024 14:02:46 +0200
Subject: [PATCH 09/28] feat: bulk persistence hooks (#100)

---
 lib/arke/query_manager.ex | 140 +++++++++++++++++++++++++-------------
 lib/arke/system.ex        | 134 ++++++++++++++++++++++++------------
 2 files changed, 183 insertions(+), 91 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 4408f44..f64bad9 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -144,9 +144,11 @@ defmodule Arke.QueryManager do
 
     with %Unit{} = unit <- Unit.load(arke, args, :create),
          %{valid: [unit], errors: _errors} <- Validator.validate(unit, :create, project),
-         {:ok, unit} <- run_persistence_hooks(arke, unit, :create, :before),
+         {:ok, unit} <- run_persistence_hook(arke, unit, :create, :before),
+         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :create, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
-         {:ok, unit} <- run_persistence_hooks(arke, unit, :create, :after) do
+         {:ok, unit} <- run_persistence_hook(arke, unit, :create, :after),
+         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :create, :after) do
       {:ok, unit}
     else
       {:error, errors} -> {:error, errors}
@@ -288,9 +290,12 @@ defmodule Arke.QueryManager do
          {:ok, unit} <- update_at_on_update(unit),
          %{valid: [unit], errors: _errors} <- Validator.validate(unit, :update, project),
          # todo better valid / error handling
-         {:ok, unit} <- run_persistence_hooks(arke, unit, :update, :before),
+
+         {:ok, unit} <- run_persistence_hook(arke, unit, :update, :before),
+         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
-         {:ok, unit} <- run_persistence_hooks(arke, unit, :update, :after, data, current_unit) do
+         {:ok, unit} <- run_persistence_hook(arke, unit, :update, :after, current_unit),
+         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :after, data) do
       {:ok, unit}
     else
       {:error, errors} -> {:error, errors}
@@ -366,9 +371,11 @@ defmodule Arke.QueryManager do
     arke = ArkeManager.get(arke_id, project)
     persistence_fn = @persistence[:arke_postgres][:delete]
 
-    with {:ok, unit} <- run_persistence_hooks(arke, unit, :delete, :before),
+    with {:ok, unit} <- run_persistence_hook(arke, unit, :delete, :before),
+         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :delete, :before),
          {:ok, _} <- persistence_fn.(project, unit, []),
-         {:ok, _} <- run_persistence_hooks(arke, unit, :delete, :after),
+         {:ok, _} <- run_group_and_link_hooks(arke, unit, :delete, :after),
+         {:ok, _} <- run_persistence_hook(arke, unit, :delete, :after),
          do: {:ok, nil},
          else: ({:error, errors} -> {:error, errors})
   end
@@ -893,69 +900,106 @@ defmodule Arke.QueryManager do
   defp normalize_value(value), do: to_string(value)
 
   defp process_bulk(valid, errors, arke, :update, :after, current_units, data) do
-    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-      current_unit = Enum.find(current_units, fn u -> u.id == unit.id end)
-      updated_data = Enum.find(data, fn d -> Map.get(d, "id") == to_string(unit.id) end)
-
-      case run_persistence_hooks(arke, unit, :update, :after, updated_data, current_unit) do
-        {:ok, unit} ->
-          %{acc | valid: [unit | acc.valid]}
+    case run_bulk_persistence_hook(valid, errors, arke, :update, :after, current_units) do
+      {:ok, valid, errors} ->
+        Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+          updated_data = Enum.find(data, fn d -> Map.get(d, "id") == to_string(unit.id) end)
+
+          case run_group_and_link_hooks(arke, unit, :update, :after, updated_data) do
+            {:ok, unit} ->
+              %{acc | valid: [unit | acc.valid]}
+
+            {:error, e} ->
+              %{acc | errors: [{unit, e} | acc.errors]}
+          end
+        end)
 
-        {:error, e} ->
-          %{acc | errors: [{unit, e} | acc.errors]}
-      end
-    end)
+      {:error, errors} ->
+        {:error, errors}
+    end
   end
 
   defp process_bulk(valid, errors, arke, persistence_fn, phase) do
-    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-      case run_persistence_hooks(arke, unit, persistence_fn, phase) do
-        {:ok, unit} ->
-          %{acc | valid: [unit | acc.valid]}
-
-        {:error, e} ->
-          %{acc | errors: [{unit, e} | acc.errors]}
-      end
-    end)
+    case run_bulk_persistence_hook(valid, errors, arke, persistence_fn, phase) do
+      {:ok, valid, errors} ->
+        Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+          case run_group_and_link_hooks(arke, unit, persistence_fn, phase) do
+            {:ok, unit} ->
+              %{acc | valid: [unit | acc.valid]}
+
+            {:error, e} ->
+              %{acc | errors: [{unit, e} | acc.errors]}
+          end
+        end)
+
+      {:error, errors} ->
+        {:error, errors}
+    end
   end
 
-  defp run_persistence_hooks(arke, unit, :create, :before) do
-    with {:ok, unit} <- ArkeManager.call_func(arke, :before_create, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
+  defp run_bulk_persistence_hook(valid, errors, arke, :create, :before),
+    do: ArkeManager.call_func(arke, :before_bulk_create, [arke, valid, errors])
+
+  defp run_bulk_persistence_hook(valid, errors, arke, :create, :after),
+    do: ArkeManager.call_func(arke, :on_bulk_create, [arke, valid, errors])
+
+  defp run_bulk_persistence_hook(valid, errors, arke, :update, :before),
+    do: ArkeManager.call_func(arke, :before_bulk_update, [arke, valid, errors])
+
+  defp run_bulk_persistence_hook(valid, errors, arke, :update, :after, current_units),
+    do: ArkeManager.call_func(arke, :on_bulk_update, [arke, current_units, valid, errors])
+
+  defp run_bulk_persistence_hook(valid, errors, arke, :delete, :before),
+    do: ArkeManager.call_func(arke, :before_bulk_delete, [arke, valid, errors])
+
+  defp run_bulk_persistence_hook(valid, errors, arke, :delete, :after),
+    do: ArkeManager.call_func(arke, :on_bulk_delete, [arke, valid, errors])
+
+  defp run_persistence_hook(arke, unit, :create, :before),
+    do: ArkeManager.call_func(arke, :before_create, [arke, unit])
+
+  defp run_persistence_hook(arke, unit, :create, :after),
+    do: ArkeManager.call_func(arke, :on_create, [arke, unit])
+
+  defp run_persistence_hook(arke, unit, :update, :before),
+    do: ArkeManager.call_func(arke, :before_update, [arke, unit])
+
+  defp run_persistence_hook(arke, unit, :update, :after, current_unit),
+    do: ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit])
+
+  defp run_persistence_hook(arke, unit, :delete, :before),
+    do: ArkeManager.call_func(arke, :before_delete, [arke, unit])
+
+  defp run_persistence_hook(arke, unit, :delete, :after),
+    do: ArkeManager.call_func(arke, :on_delete, [arke, unit])
+
+  defp run_group_and_link_hooks(arke, unit, :create, :before) do
+    with {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
          {:ok, unit} <- handle_link_parameters_unit(arke, unit),
          do: {:ok, unit}
   end
 
-  defp run_persistence_hooks(arke, unit, :create, :after) do
-    with {:ok, unit} <- ArkeManager.call_func(arke, :on_create, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_create),
+  defp run_group_and_link_hooks(arke, unit, :create, :after) do
+    with {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_create),
          {:ok, unit} <- handle_link_parameters(unit, %{}),
          do: {:ok, unit}
   end
 
-  defp run_persistence_hooks(arke, unit, :update, :before) do
-    with {:ok, unit} <- ArkeManager.call_func(arke, :before_update, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
+  defp run_group_and_link_hooks(arke, unit, :update, :before) do
+    with {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
          {:ok, unit} <- handle_link_parameters_unit(arke, unit),
          do: {:ok, unit}
   end
 
-  defp run_persistence_hooks(arke, unit, :update, :after, data, current_unit) do
-    with {:ok, unit} <- ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit]),
-         {:ok, unit} <- handle_link_parameters(unit, data),
+  defp run_group_and_link_hooks(arke, unit, :update, :after, data) do
+    with {:ok, unit} <- handle_link_parameters(unit, data),
          {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update),
          do: {:ok, unit}
   end
 
-  defp run_persistence_hooks(arke, unit, :delete, :before) do
-    with {:ok, unit} <- ArkeManager.call_func(arke, :before_delete, [arke, unit]),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_delete),
-         do: {:ok, unit}
-  end
+  defp run_group_and_link_hooks(arke, unit, :delete, :before),
+    do: handle_group_call_func(arke, unit, :before_unit_delete)
 
-  defp run_persistence_hooks(arke, unit, :delete, :after) do
-    with {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_delete),
-         {:ok, _unit} <- ArkeManager.call_func(arke, :on_delete, [arke, unit]),
-         do: {:ok, nil}
-  end
+  defp run_group_and_link_hooks(arke, unit, :delete, :after),
+    do: handle_group_call_func(arke, unit, :on_unit_delete)
 end
diff --git a/lib/arke/system.ex b/lib/arke/system.ex
index 3123740..cb6d35d 100644
--- a/lib/arke/system.ex
+++ b/lib/arke/system.ex
@@ -50,10 +50,20 @@ defmodule Arke.System do
       def on_delete(arke, unit), do: {:ok, unit}
       def before_delete(arke, unit), do: {:ok, unit}
 
+      def before_bulk_create(arke, valid, errors), do: {:ok, valid, errors}
+      def on_bulk_create(arke, valid, errors), do: {:ok, valid, errors}
+      def before_bulk_update(arke, valid, errors), do: {:ok, valid, errors}
+      def on_bulk_update(arke, old_units, valid, errors), do: {:ok, valid, errors}
+      def before_bulk_delete(arke, valid, errors), do: {:ok, valid, errors}
+      def on_bulk_delete(arke, valid, errors), do: {:ok, valid, errors}
+
       def after_get_struct(arke, unit, struct), do: struct
       def after_get_struct(arke, struct), do: struct
 
-      def import(%{runtime_data: %{conn: %{method: "POST"}=conn}, metadata: %{project: project}} = arke) do
+      def import(
+            %{runtime_data: %{conn: %{method: "POST"} = conn}, metadata: %{project: project}} =
+              arke
+          ) do
         member = ArkeAuth.Guardian.Plug.current_resource(conn)
         mode = Map.get(conn.body_params, "mode", "default")
 
@@ -72,34 +82,51 @@ defmodule Arke.System do
         header_file = Enum.at(file_as_list, 0)
         rows = file_as_list |> List.delete_at(0)
 
-        header = get_header_for_import(project, arke, header_file) |> parse_haeder_for_import(header_file)
-
-        {correct_units, error_units} = Enum.with_index(rows) |> Enum.reduce({[], []}, fn {row, index}, {correct_units, error_units} ->
-          case Enum.filter(row, & !is_nil(&1)) do
-            [] -> {correct_units, error_units}
-            _ -> case load_units(project, arke, header, row, all_units, mode) do
-                   {:error, args, errors} ->
-                     m = Enum.reduce(header, %{}, fn {h, index}, acc ->
-                       acc = Map.put(acc, h, parse_cell(Enum.at(row, index)))
-                     end) |> Map.put("errors", errors)
-                     {correct_units, [m | error_units]}
-                   {:ok, unit_args} -> {[unit_args | correct_units], error_units}
-                 end
-          end
-        end)
+        header =
+          get_header_for_import(project, arke, header_file)
+          |> parse_haeder_for_import(header_file)
+
+        {correct_units, error_units} =
+          Enum.with_index(rows)
+          |> Enum.reduce({[], []}, fn {row, index}, {correct_units, error_units} ->
+            case Enum.filter(row, &(!is_nil(&1))) do
+              [] ->
+                {correct_units, error_units}
+
+              _ ->
+                case load_units(project, arke, header, row, all_units, mode) do
+                  {:error, args, errors} ->
+                    m =
+                      Enum.reduce(header, %{}, fn {h, index}, acc ->
+                        acc = Map.put(acc, h, parse_cell(Enum.at(row, index)))
+                      end)
+                      |> Map.put("errors", errors)
+
+                    {correct_units, [m | error_units]}
+
+                  {:ok, unit_args} ->
+                    {[unit_args | correct_units], error_units}
+                end
+            end
+          end)
 
         existing_units = get_existing_units_for_import(project, arke, header, correct_units)
-        units_args = Enum.filter(correct_units, fn u -> check_existing_units_for_import(project, arke, header, u, existing_units) == false end)
+
+        units_args =
+          Enum.filter(correct_units, fn u ->
+            check_existing_units_for_import(project, arke, header, u, existing_units) == false
+          end)
+
         if length(units_args) > 0 do
           Enum.map(Stream.chunk_every(units_args, 5000) |> Enum.to_list(), fn chunk ->
             ArkePostgres.Repo.insert_all("arke_unit", chunk, prefix: Atom.to_string(project))
           end)
         end
 
-        count_inserted =  length(units_args)
-        count_existing =  length(existing_units)
-        count_error =  length(error_units)
-        total_count =  count_inserted + count_error + count_existing
+        count_inserted = length(units_args)
+        count_existing = length(existing_units)
+        count_error = length(error_units)
+        total_count = count_inserted + count_error + count_existing
 
         res = %{
           count_inserted: count_inserted,
@@ -108,6 +135,7 @@ defmodule Arke.System do
           total_count: total_count,
           error_units: error_units
         }
+
         {:ok, res, 201}
       end
 
@@ -122,11 +150,16 @@ defmodule Arke.System do
           end
         end)
       end
+
       defp parse_haeder_for_import(header, header_file) do
         Enum.reduce(Enum.with_index(header_file), [], fn {cell, index}, acc ->
           case cell do
-            nil -> acc
-            "" -> acc
+            nil ->
+              acc
+
+            "" ->
+              acc
+
             cell ->
               case cell in header do
                 nil -> acc
@@ -139,17 +172,22 @@ defmodule Arke.System do
       defp get_all_units_for_import(project), do: []
 
       defp load_units(project, arke, header, row, _, "default") do
-        args = Enum.reduce(header, [], fn {parameter_id, index}, acc ->
-          acc = Keyword.put(acc, String.to_existing_atom(parameter_id), Enum.at(row, index))
-        end)
+        args =
+          Enum.reduce(header, [], fn {parameter_id, index}, acc ->
+            acc = Keyword.put(acc, String.to_existing_atom(parameter_id), Enum.at(row, index))
+          end)
 
         with %Arke.Core.Unit{} = unit <- Arke.Core.Unit.load(arke, args, :create),
              {:ok, unit} <- Arke.Validator.validate(unit, :create, project),
              do: {:ok, args},
              else: ({:error, errors} -> {:error, args, errors})
       end
+
       defp get_existing_units_for_import(project, arke, header, units_args), do: []
-      defp check_existing_units_for_import(project, arke, header, units_args, existing_units), do: true
+
+      defp check_existing_units_for_import(project, arke, header, units_args, existing_units),
+        do: true
+
       defp get_import_value(header, row, column) do
         index = Enum.find(header, fn {k, v} -> k == column end) |> elem(1)
         Enum.at(row, index)
@@ -167,17 +205,23 @@ defmodule Arke.System do
                      before_update: 2,
                      on_delete: 2,
                      before_delete: 2,
+                     before_bulk_create: 3,
+                     on_bulk_create: 3,
+                     before_bulk_update: 3,
+                     on_bulk_update: 4,
+                     before_bulk_delete: 3,
+                     on_bulk_delete: 3,
                      after_get_struct: 2,
                      after_get_struct: 3,
 
-                    # Import
-                      import: 1,
+                     # Import
+                     import: 1,
                      import_units: 5,
-                      get_header_for_import: 3,
+                     get_header_for_import: 3,
                      get_all_units_for_import: 1,
-                      load_units: 6,
+                     load_units: 6,
                      get_existing_units_for_import: 4,
-                      check_existing_units_for_import: 5
+                     check_existing_units_for_import: 5
     end
   end
 
@@ -250,9 +294,8 @@ defmodule Arke.System do
       @arke %{
         id: id,
         data: %{label: label, active: active, type: type, parameters: @parameters},
-        metadata: metadata,
+        metadata: metadata
       }
-
     end
   end
 
@@ -360,15 +403,18 @@ defmodule Arke.System.BaseParameter do
     %{type: type, opts: opts}
   end
 
-  def check_enum(type, opts) when is_binary(type), do: check_enum(String.to_atom(type),opts)
+  def check_enum(type, opts) when is_binary(type), do: check_enum(String.to_atom(type), opts)
+
   def check_enum(type, opts) do
     enum_parameters = [:string, :integer, :float]
+
     case type in enum_parameters do
       true ->
         __enum_parameter__(opts, type)
-      false -> opts
-    end
 
+      false ->
+        opts
+    end
   end
 
   defp parameter_option_common(opts, id) do
@@ -423,10 +469,14 @@ defmodule Arke.System.BaseParameter do
     Keyword.put_new(opts, key, default)
   end
 
-  def __enum_parameter__(opts, type) when is_map(opts), do: __enum_parameter__(Map.to_list(opts),type)
+  def __enum_parameter__(opts, type) when is_map(opts),
+    do: __enum_parameter__(Map.to_list(opts), type)
+
   def __enum_parameter__(opts, type) do
     case Keyword.has_key?(opts, :values) do
-      true ->   __validate_values__(opts, opts[:values], type)
+      true ->
+        __validate_values__(opts, opts[:values], type)
+
       false ->
         opts
     end
@@ -438,9 +488,7 @@ defmodule Arke.System.BaseParameter do
        when not is_nil(value),
        do: __validate_values__(opts, value, type)
 
-
   defp __validate_values__(opts, [h | _t] = values, type) when is_map(h) do
-
     condition =
       cond do
         type == :string ->
@@ -452,11 +500,12 @@ defmodule Arke.System.BaseParameter do
         type == :float ->
           fn l, v -> is_binary(l) and is_number(v) end
       end
+
     case Enum.all?(values, fn map ->
            Enum.map([:label, :value], fn key -> Map.has_key?(map, key) end)
          end) do
       true ->
-       __create_map_values__(__check_map__(values), opts, type, condition)
+        __create_map_values__(__check_map__(values), opts, type, condition)
 
       # FARE RAISE ECCEZIONE DA GESTIRE. CHIAVI DEVONO ESSERE TUTTE UGUALI
       _ ->
@@ -475,7 +524,6 @@ defmodule Arke.System.BaseParameter do
     __values_from_list__(values, opts, condition)
   end
 
-
   # CONVERT ALL STRINGS KEY TO ATOMS (string are received from API)
   defp __check_map__([%{"label" => _l, "value" => _v} | _h] = values) do
     Enum.map(

From 7df3e1684758f055d738a34bad296be054a6a9a8 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <81801314+ilyichv@users.noreply.github.com>
Date: Fri, 30 Aug 2024 14:19:18 +0200
Subject: [PATCH 10/28] fix: unit load link type (#104)

---
 lib/arke/core/unit.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/arke/core/unit.ex b/lib/arke/core/unit.ex
index 167e393..3e1d86e 100644
--- a/lib/arke/core/unit.ex
+++ b/lib/arke/core/unit.ex
@@ -169,8 +169,8 @@ defmodule Arke.Core.Unit do
 
   defp handle_default_value(_), do: nil
 
-  defp get_link(%{depth: depth, link_metadata: link_metadata,starting_unit: starting_unit} = args),
-    do: {%{depth: depth, metadata: link_metadata,starting_unit: starting_unit}, args}
+  defp get_link(%{depth: depth, link_metadata: link_metadata,starting_unit: starting_unit, link_type: link_type} = args),
+    do: {%{depth: depth, metadata: link_metadata,starting_unit: starting_unit, type: link_type}, args}
 
   defp get_link(args), do: {nil, args}
 

From 62206820d97369603b458264cead074219bd159d Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <81801314+ilyichv@users.noreply.github.com>
Date: Fri, 30 Aug 2024 15:12:11 +0200
Subject: [PATCH 11/28] fix: return units in bulk operations (#105)

---
 lib/arke/query_manager.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 83f1077..39f1223 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -170,7 +170,7 @@ defmodule Arke.QueryManager do
            persistence_fn.(project, valid, bulk: true),
          %{valid: valid, errors: errors} <-
            process_bulk(valid, errors ++ persistence_errors, arke, :create, :after) do
-      {:ok, inserted_count, errors}
+      {:ok, inserted_count, valid, errors}
     else
       {:error, errors} -> {:error, errors}
     end
@@ -311,7 +311,7 @@ defmodule Arke.QueryManager do
     {:ok, Unit.update(unit, updated_at: updated_at)}
   end
 
-  def update_bulk(_, _, [], _), do: {:ok, [], []}
+  def update_bulk(_, _, [], _), do: {:ok, 0, [], []}
 
   def update_bulk(project, arke, unit_list, data) do
     persistence_fn = @persistence[:arke_postgres][:update]
@@ -333,7 +333,7 @@ defmodule Arke.QueryManager do
              unit_list,
              data
            ) do
-      {:ok, updated_count, errors}
+      {:ok, updated_count, valid, errors}
     else
       {:error, errors} -> {:error, errors}
     end

From 974cee6bf17b0fc503f83986c3c1f93af1f82398 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Thu, 26 Sep 2024 22:18:53 +0200
Subject: [PATCH 12/28] fix: add mode to get_header_for_import and
 get_all_units_for_import

---
 lib/arke/system.ex | 50 +++++++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/lib/arke/system.ex b/lib/arke/system.ex
index c0219ae..06a8966 100644
--- a/lib/arke/system.ex
+++ b/lib/arke/system.ex
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 defmodule Arke.System do
+  @default_import_mode "default"
+
   defmacro __using__(_) do
     quote do
       #      @after_compile __MODULE__
@@ -66,7 +68,7 @@ defmodule Arke.System do
               arke
           ) do
         member = ArkeAuth.Guardian.Plug.current_resource(conn)
-        mode = Map.get(conn.body_params, "mode", "default")
+        mode = Map.get(conn.body_params, "mode", @default_import_mode)
 
         case Map.get(conn.body_params, "file", nil) do
           nil -> {:error, "file is required", 400}
@@ -76,7 +78,7 @@ defmodule Arke.System do
 
       defp import_units(arke, project, member, file, mode) do
         {:ok, ref} = Enum.at(Xlsxir.multi_extract(file.path), 0)
-        all_units = get_all_units_for_import(project)
+        all_units = get_all_units_for_import(project, mode)
 
         file_as_list = Xlsxir.get_list(ref)
 
@@ -84,7 +86,7 @@ defmodule Arke.System do
         rows = file_as_list |> List.delete_at(0)
 
         header =
-          get_header_for_import(project, arke, header_file)
+          get_header_for_import(project, arke, header_file, mode)
           |> parse_haeder_for_import(header_file)
 
         {correct_units, error_units} =
@@ -119,11 +121,15 @@ defmodule Arke.System do
           end)
 
         if length(units_args) > 0 do
-          {existing_units,units_args,error_units} = before_unit_import(project,existing_units,units_args,error_units)
+          {existing_units, units_args, error_units} =
+            before_unit_import(project, existing_units, units_args, error_units)
+
           Enum.map(Stream.chunk_every(units_args, 5000) |> Enum.to_list(), fn chunk ->
             ArkePostgres.Repo.insert_all("arke_unit", chunk, prefix: Atom.to_string(project))
           end)
-          {existing_units,units_args,error_units} = on_unit_import(project,existing_units,units_args,error_units)
+
+          {existing_units, units_args, error_units} =
+            on_unit_import(project, existing_units, units_args, error_units)
         end
 
         count_inserted = length(units_args)
@@ -145,7 +151,7 @@ defmodule Arke.System do
       defp parse_cell(value) when is_tuple(value), do: Kernel.inspect(value)
       defp parse_cell(value), do: value
 
-      defp get_header_for_import(project, arke, header_file) do
+      defp get_header_for_import(project, arke, header_file, @default_import_mode) do
         Enum.reduce(Enum.with_index(header_file), [], fn {cell, index}, acc ->
           case Arke.Boundary.ArkeManager.get_parameter(arke, project, cell) do
             nil -> acc
@@ -172,9 +178,9 @@ defmodule Arke.System do
         end)
       end
 
-      defp get_all_units_for_import(project), do: []
+      defp get_all_units_for_import(_project, _mode), do: []
 
-      defp load_units(project, arke, header, row, _, "default") do
+      defp load_units(project, arke, header, row, _, @default_import_mode) do
         args =
           Enum.reduce(header, [], fn {parameter_id, index}, acc ->
             acc = Keyword.put(acc, String.to_existing_atom(parameter_id), Enum.at(row, index))
@@ -196,8 +202,11 @@ defmodule Arke.System do
         Enum.at(row, index)
       end
 
-      defp before_unit_import(_project,existing_units,units_args,error_units), do: {existing_units,units_args,error_units}
-      defp on_unit_import(_project,existing_units,units_args,error_units), do: {existing_units,units_args,error_units}
+      defp before_unit_import(_project, existing_units, units_args, error_units),
+        do: {existing_units, units_args, error_units}
+
+      defp on_unit_import(_project, existing_units, units_args, error_units),
+        do: {existing_units, units_args, error_units}
 
       defoverridable on_load: 2,
                      before_load: 2,
@@ -221,20 +230,19 @@ defmodule Arke.System do
                      after_get_struct: 2,
                      after_get_struct: 3,
 
-                    # Import
-                      import: 1,
-                      import_units: 5,
-                      get_header_for_import: 3,
-                      get_all_units_for_import: 1,
-                      load_units: 6,
-                      get_existing_units_for_import: 4,
-                      check_existing_units_for_import: 5,
-                      before_unit_import: 4,
-                      on_unit_import: 4
+                     # Import
+                     import: 1,
+                     import_units: 5,
+                     get_header_for_import: 4,
+                     get_all_units_for_import: 2,
+                     load_units: 6,
+                     get_existing_units_for_import: 4,
+                     check_existing_units_for_import: 5,
+                     before_unit_import: 4,
+                     on_unit_import: 4
     end
   end
 
-
   ######################################################################################################################
   # ARKE MACRO #########################################################################################################
   ######################################################################################################################

From b9812ac432ac672f738aab2876ad0d351eea87ea Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <81801314+ilyichv@users.noreply.github.com>
Date: Wed, 16 Oct 2024 15:16:36 +0200
Subject: [PATCH 13/28] feat: add nested filters and sort (#109)

---
 lib/arke/core/query.ex    | 57 +++++++++++++++++++++++++--------------
 lib/arke/query_manager.ex | 26 +++++++++++++-----
 2 files changed, 57 insertions(+), 26 deletions(-)

diff --git a/lib/arke/core/query.ex b/lib/arke/core/query.ex
index 56926e8..40e57d2 100644
--- a/lib/arke/core/query.ex
+++ b/lib/arke/core/query.ex
@@ -51,11 +51,13 @@ defmodule Arke.Core.Query do
       - parameter => %Arke.Core.Parameter.`ParameterType` => refer to `Arke.Core.Parameter`
       - operator => refer to [operators](#module-operators)
       - value => any => the value that the query will search for
-      - negate => boolean => used to figure out whether the condition is to be denied \n
+      - negate => boolean => used to figure out whether the condition is to be denied
+      - path => [Arke.Core.Parameter.ParameterType] => the path of the parameter
+       \n
       It is used to keep the same logic structure across all the Filter
     """
 
-    defstruct ~w[parameter operator value negate]a
+    defstruct ~w[parameter operator value negate path]a
     @type t() :: %Arke.Core.Query.BaseFilter{}
 
     @doc """
@@ -79,14 +81,16 @@ defmodule Arke.Core.Query do
             parameter :: Arke.Core.Parameter.ParameterType,
             operator :: atom(),
             value :: any,
-            negate :: boolean
+            negate :: boolean(),
+            path :: [Arke.Core.Parameter.ParameterType]
           ) :: Arke.Core.Query.BaseFilter.t()
-    def new(parameter, operator, value, negate) do
+    def new(parameter, operator, value, negate, path) do
       %__MODULE__{
         parameter: parameter,
         operator: operator,
         value: cast_value(parameter, value),
-        negate: negate
+        negate: negate,
+        path: path
       }
     end
 
@@ -111,10 +115,12 @@ defmodule Arke.Core.Query do
     @moduledoc """
       Base struct Order:
       - parameter => %Arke.Core.Parameter.`ParameterType` => refer to `Arke.Core.Parameter`
-      - direction => "child" | "parent" => the direction the query will use to search \n
+      - direction => "child" | "parent" => the direction the query will use to search
+      - path => [Arke.Core.Parameter.ParameterType] => the path of the parameter
+       \n
       It is used to define the return order of a Query
     """
-    defstruct ~w[parameter direction]a
+    defstruct ~w[parameter direction path]a
     @type t() :: %Arke.Core.Query.Order{}
   end
 
@@ -133,18 +139,19 @@ defmodule Arke.Core.Query do
       %Arke.Core.Query{}
 
   """
-  @spec new(arke :: %Arke.Core.Arke{}, project :: atom(), distinct :: atom()) :: Arke.Core.Query.t()
+  @spec new(arke :: %Arke.Core.Arke{}, project :: atom(), distinct :: atom()) ::
+          Arke.Core.Query.t()
   def new(arke, project, distinct \\ nil),
-      do: %__MODULE__{
-        project: project,
-        arke: arke,
-        distinct: distinct,
-        persistence: nil,
-        filters: [],
-        orders: [],
-        offset: nil,
-        limit: nil
-      }
+    do: %__MODULE__{
+      project: project,
+      arke: arke,
+      distinct: distinct,
+      persistence: nil,
+      filters: [],
+      orders: [],
+      offset: nil,
+      limit: nil
+    }
 
   @doc """
   Add a new link filter
@@ -303,8 +310,8 @@ defmodule Arke.Core.Query do
   # TODO: standardize parameter
   #  if it is a string convert it to existing atom and get it from paramater manager
   #  if it is an atom get it from paramater manaager
-  def new_base_filter(parameter, operator, value, negate) do
-    BaseFilter.new(parameter, operator, value, negate)
+  def new_base_filter(parameter, operator, value, negate, path \\ []) do
+    BaseFilter.new(parameter, operator, value, negate, path)
   end
 
   defp parse_base_filters(base_filters) when is_list(base_filters), do: base_filters
@@ -327,6 +334,16 @@ defmodule Arke.Core.Query do
   ## Return
       %Arke.Core.Query{ ... orders: [ %Arke.Core.Query.Order{} ] ... }
   """
+
+  def add_order(query, parameter, direction) when is_list(parameter) do
+    {parameter, path} = List.pop_at(parameter, -1)
+
+    %{
+      query
+      | orders: [%Order{parameter: parameter, direction: direction, path: path} | query.orders]
+    }
+  end
+
   def add_order(query, parameter, direction) do
     %{
       query
diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 39f1223..0ad2de9 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -44,7 +44,6 @@ defmodule Arke.QueryManager do
   alias Arke.Utils.DatetimeHandler, as: DatetimeHandler
   alias Arke.Core.{Arke, Unit, Query, Parameter}
 
-
   @persistence Application.get_env(:arke, :persistence)
   @record_fields [:id, :data, :metadata, :inserted_at, :updated_at]
 
@@ -495,7 +494,7 @@ defmodule Arke.QueryManager do
   defp parse_base_filters(query, filters) do
     Enum.reduce(filters, [], fn f, new_filters ->
       parameter = get_parameter(query, f.parameter)
-      [Query.new_base_filter(parameter, f.operator, f.value, f.negate) | new_filters]
+      [Query.new_base_filter(parameter, f.operator, f.value, f.negate, f.path) | new_filters]
     end)
   end
 
@@ -518,10 +517,11 @@ defmodule Arke.QueryManager do
           parameter :: Arke.t() | atom(),
           negate :: boolean(),
           value :: String.t() | boolean() | nil,
-          negate :: boolean()
+          negate :: boolean(),
+          path :: [Arke.t()]
         ) :: Query.BaseFilter.t()
-  def condition(parameter, operator, value, negate \\ false),
-    do: Query.new_base_filter(parameter, operator, value, negate)
+  def condition(parameter, operator, value, negate \\ false, path \\ []),
+    do: Query.new_base_filter(parameter, operator, value, negate, path)
 
   @doc """
   Create a list of `Arke.Core.Query.BaseFilter`
@@ -656,9 +656,23 @@ defmodule Arke.QueryManager do
   """
   @spec order(
           query :: Query.t(),
-          parameter :: Arke.t() | String.t() | atom(),
+          parameter :: Arke.t() | String.t() | atom() | [Arke.t()] | [String.t()] | [atom()],
           direction :: atom()
         ) :: Query.t()
+  def order(query, parameter, direction) when is_list(parameter) do
+    parameters =
+      Enum.with_index(parameter)
+      |> Enum.map(fn {p, index} ->
+        if index == 0 do
+          get_parameter(query, p)
+        else
+          get_parameter(%{query | arke: nil}, p)
+        end
+      end)
+
+    Query.add_order(query, parameters, direction)
+  end
+
   def order(query, parameter, direction),
     do: Query.add_order(query, get_parameter(query, parameter), direction)
 

From 47c7b5e5fac4caa229ecf503336749ad6e623f55 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Tue, 22 Oct 2024 14:32:25 +0200
Subject: [PATCH 14/28] fix: add current_unit to before_update

---
 lib/arke/query_manager.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 0ad2de9..8853aa4 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -293,7 +293,7 @@ defmodule Arke.QueryManager do
          %{valid: [unit], errors: _errors} <- Validator.validate(unit, :update, project),
          # todo better valid / error handling
 
-         {:ok, unit} <- run_persistence_hook(arke, unit, :update, :before),
+         {:ok, unit} <- run_persistence_hook(arke, unit, :update, :before, current_unit),
          {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
          {:ok, unit} <- run_persistence_hook(arke, unit, :update, :after, current_unit),
@@ -979,8 +979,8 @@ defmodule Arke.QueryManager do
   defp run_persistence_hook(arke, unit, :create, :after),
     do: ArkeManager.call_func(arke, :on_create, [arke, unit])
 
-  defp run_persistence_hook(arke, unit, :update, :before),
-    do: ArkeManager.call_func(arke, :before_update, [arke, unit])
+  defp run_persistence_hook(arke, unit, :update, :before, current_unit),
+    do: ArkeManager.call_func(arke, :before_update, [arke, current_unit, unit])
 
   defp run_persistence_hook(arke, unit, :update, :after, current_unit),
     do: ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit])

From a374502fb5b7d03e50e14598c4a9a3abcf3e59ce Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Mon, 28 Oct 2024 12:16:35 +0100
Subject: [PATCH 15/28] fix: optimize find speed

---
 lib/arke/query_manager.ex | 37 +++++++++++++++++++++++--------------
 lib/arke/validator.ex     |  6 +++---
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 8853aa4..9648d75 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -292,7 +292,6 @@ defmodule Arke.QueryManager do
          {:ok, unit} <- update_at_on_update(unit),
          %{valid: [unit], errors: _errors} <- Validator.validate(unit, :update, project),
          # todo better valid / error handling
-
          {:ok, unit} <- run_persistence_hook(arke, unit, :update, :before, current_unit),
          {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
@@ -315,8 +314,6 @@ defmodule Arke.QueryManager do
   def update_bulk(project, arke, unit_list, data) do
     persistence_fn = @persistence[:arke_postgres][:update]
 
-    %{valid: valid, errors: errors} = prepare_update_bulk_units(arke, unit_list, data)
-
     with %{valid: valid, errors: errors} <- prepare_update_bulk_units(arke, unit_list, data),
          %{valid: valid, errors: errors} <- Validator.validate(valid, :update, project),
          %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :update, :before),
@@ -338,17 +335,27 @@ defmodule Arke.QueryManager do
     end
   end
 
-  defp prepare_update_bulk_units(arke, unit_list, data),
-    do:
-      Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
-        item = Enum.find(data, fn d -> Map.get(d, "id") == to_string(unit.id) end)
-        updated_at = DatetimeHandler.now(:datetime)
+  defp prepare_update_bulk_units(arke, unit_list, data) do
+    data_map =
+      data
+      |> Enum.map(fn d -> {Map.get(d, "id"), d} end)
+      |> Map.new()
 
-        case Unit.update(unit, data_as_klist(item) ++ [updated_at: updated_at]) do
-          %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
-          {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
-        end
-      end)
+    updated_at = DatetimeHandler.now(:datetime)
+
+    Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
+      case data_map[to_string(unit.id)] do
+        nil ->
+          acc
+
+        item ->
+          case Unit.update(unit, data_as_klist(item) ++ [updated_at: updated_at]) do
+            %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
+            {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
+          end
+      end
+    end)
+  end
 
   defp update_at_on_update(unit) do
     updated_at = DatetimeHandler.now(:datetime)
@@ -918,10 +925,12 @@ defmodule Arke.QueryManager do
   defp normalize_value(value), do: to_string(value)
 
   defp process_bulk(valid, errors, arke, :update, :after, current_units, data) do
+    data_by_id = Map.new(data, fn d -> {Map.get(d, "id"), d} end)
+
     case run_bulk_persistence_hook(valid, errors, arke, :update, :after, current_units) do
       {:ok, valid, errors} ->
         Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-          updated_data = Enum.find(data, fn d -> Map.get(d, "id") == to_string(unit.id) end)
+          updated_data = Map.get(data_by_id, to_string(unit.id))
 
           case run_group_and_link_hooks(arke, unit, :update, :after, updated_data) do
             {:ok, unit} ->
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index a6dbf0d..5cfb8d4 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -303,7 +303,7 @@ defmodule Arke.Validator do
     value = get_default_value(parameter, value)
     value = parse_value(parameter, value)
     value = check_whitespace(parameter, value)
-    value = check_lowercase(parameter,value)
+    value = check_lowercase(parameter, value)
 
     errors =
       []
@@ -319,7 +319,7 @@ defmodule Arke.Validator do
 
   defp parse_value(%{arke_id: :integer, data: %{multiple: false} = data} = _, value)
        when not is_integer(value) and not is_nil(value) do
-    case Integer.parse(value) do
+    case to_string(value) |> Integer.parse() do
       :error -> value
       {v, _e} -> v
     end
@@ -327,7 +327,7 @@ defmodule Arke.Validator do
 
   defp parse_value(%{arke_id: :float, data: %{multiple: false} = data} = _, value)
        when not is_number(value) and not is_nil(value) do
-    case Float.parse(value) do
+    case to_string(value) |> Float.parse() do
       :error -> value
       {v, _e} -> v
     end

From 4f29b4d5087602d7b2e22123691f99afcbd1f2c7 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Mon, 4 Nov 2024 18:31:45 +0100
Subject: [PATCH 16/28] fix: utf8

---
 lib/mix/tasks/arke.export_data.ex | 209 +++++++++++++++++++-----------
 1 file changed, 134 insertions(+), 75 deletions(-)

diff --git a/lib/mix/tasks/arke.export_data.ex b/lib/mix/tasks/arke.export_data.ex
index 72c1c8a..c83f258 100644
--- a/lib/mix/tasks/arke.export_data.ex
+++ b/lib/mix/tasks/arke.export_data.ex
@@ -36,7 +36,7 @@ defmodule Mix.Tasks.Arke.ExportData do
     parameter: :boolean,
     splitfile: :boolean,
     group: :boolean,
-    persistence: :string,
+    persistence: :string
   ]
   @aliases [
     p: :project,
@@ -44,25 +44,26 @@ defmodule Mix.Tasks.Arke.ExportData do
     g: :group,
     sf: :splitfile,
     pr: :parameter,
-    ps: :persistence,
+    ps: :persistence
   ]
 
   @impl true
   def run(args) do
     case OptionParser.parse!(args, strict: @switches, aliases: @aliases) do
-      {[], _opts}->
+      {[], _opts} ->
         Mix.Tasks.Help.run(["arke.export_data"])
+
       {opts, []} ->
         persistence = parse_persistence!(opts[:persistence] || "arke_postgres")
 
-        app_to_start(persistence) ++ [:arke]
+        (app_to_start(persistence) ++ [:arke])
         |> Enum.each(&Application.ensure_all_started/1)
 
         repo_module = Application.get_env(:arke, :persistence)[String.to_atom(persistence)][:repo]
         Mix.shell().info("--- Starting repo --- ")
+
         case start_repo!(repo_module) do
           {:ok, pid} ->
-
             opts |> export_data(persistence)
             Process.exit(pid, :normal)
             :ok
@@ -71,116 +72,156 @@ defmodule Mix.Tasks.Arke.ExportData do
             opts |> export_data(persistence)
             :ok
         end
-
     end
-
   end
 
-
   defp app_to_start("arke_postgres"), do: [:ecto_sql, :postgrex]
-  defp parse_persistence!(ps) when ps in  @persistence_repo, do: ps
-  defp parse_persistence!(ps), do: Mix.raise("Invalid persistence: `#{ps}`\nSupported persistence are: #{Enum.join(@persistence_repo, " | ")}")
+  defp parse_persistence!(ps) when ps in @persistence_repo, do: ps
+
+  defp parse_persistence!(ps),
+    do:
+      Mix.raise(
+        "Invalid persistence: `#{ps}`\nSupported persistence are: #{Enum.join(@persistence_repo, " | ")}"
+      )
+
+  defp write_to_file(project, arke_id, data) do
+    {:ok, datetime} =
+      Arke.Utils.DatetimeHandler.now(:datetime)
+      |> Arke.Utils.DatetimeHandler.format("{ISO:Basic:Z}")
 
-  defp write_to_file(project,arke_id,data) do
-    {:ok, datetime} = Arke.Utils.DatetimeHandler.now(:datetime) |> Arke.Utils.DatetimeHandler.format("{ISO:Basic:Z}")
     dir_path = "export/arke_export_data/#{project}/#{datetime}"
     arke_id_str = to_string(arke_id)
     path = "#{dir_path}/#{arke_id_str}.json"
     Mix.shell().info("--- Writing data to #{path} for #{arke_id_str}  --- ")
     File.mkdir_p!(dir_path)
     {:ok, body} = Jason.encode(data)
-    {:ok, file} = File.open(path, [:append])
+
+    IO.inspect(body)
+    {:ok, file} = File.open(path, [:append, :utf8])
     IO.write(file, body)
     File.close(file)
-
   end
 
+  defp start_repo!(nil),
+    do:
+      Mix.raise(
+        "Invalid repo module in arke configuration. Please provide a valid module accordingly to the persistence supported"
+      )
 
-  defp start_repo!(nil), do: Mix.raise("Invalid repo module in arke configuration. Please provide a valid module accordingly to the persistence supported")
   # this is for arke_postgres
   defp start_repo!(repo_module) do
     repo_module.start_link()
   end
 
-  defp start_manager!(nil), do: Mix.raise("Missing `init` function in arke.persistence configuration. Please provide a valid function accordingly to the persistence supported")
+  defp start_manager!(nil),
+    do:
+      Mix.raise(
+        "Missing `init` function in arke.persistence configuration. Please provide a valid function accordingly to the persistence supported"
+      )
+
   # this is for arke_postgres
   defp start_manager!(function), do: function.()
 
-
-  defp export_data(opts,persistence)  do
+  defp export_data(opts, persistence) do
     start_manager!(Application.get_env(:arke, :persistence)[String.to_atom(persistence)][:init])
     project = String.to_atom(opts[:project]) || :arke_system
-    split_file =  opts[:splitfile] || false
-    data = get_data(project,opts)
-    arke = prepare_arke(data.arke,data.arke_parameter)
+    split_file = opts[:splitfile] || false
+    data = get_data(project, opts)
+    arke = prepare_arke(data.arke, data.arke_parameter)
     group = prepare_group(data.group)
     parameter = prepare_parameter(data.parameter)
     permission = prepare_permission(data.permission)
 
     if split_file do
-      write_to_file(project,:arke,%{arke: arke})
-      write_to_file(project,:group,%{group: group})
-      write_to_file(project,:parameter,%{parameter: parameter})
-      write_to_file(project,:link,%{link: permission})
+      write_to_file(project, :arke, %{arke: arke})
+      write_to_file(project, :group, %{group: group})
+      write_to_file(project, :parameter, %{parameter: parameter})
+      write_to_file(project, :link, %{link: permission})
     else
-      write_to_file(project,:all,%{arke: arke,group: group,parameter: parameter,link: permission})
+      write_to_file(project, :all, %{
+        arke: arke,
+        group: group,
+        parameter: parameter,
+        link: permission
+      })
     end
 
-    Mix.shell().info("--- All data has been exported. Keep in mind that if you want to use them in the `arke.seed_project` task
-       you must move them under the `registry` folder --- ")
+    Mix.shell().info(
+      "--- All data has been exported. Keep in mind that if you want to use them in the `arke.seed_project` task
+       you must move them under the `registry` folder --- "
+    )
   end
 
-  def get_data(project,opts) do
-
+  def get_data(project, opts) do
     # if the opts doesn not include any flag then export all
-    if not Enum.any?(Keyword.keys(opts), fn k -> k in [:arke,:parameter,:group] end) do
+    if not Enum.any?(Keyword.keys(opts), fn k -> k in [:arke, :parameter, :group] end) do
       get_all(project)
     else
-      arke =  get_arke(project,opts[:arke])
-      arke_parameter =  get_arke_parameter(project)
-      parameter =  get_parameter(project,opts[:parameter])
-      group =  get_group(project,opts[:group])
+      arke = get_arke(project, opts[:arke])
+      arke_parameter = get_arke_parameter(project)
+      parameter = get_parameter(project, opts[:parameter])
+      group = get_group(project, opts[:group])
       permission = get_permission(project)
-      %{arke: arke, parameter: parameter, group: group, permission: permission, arke_parameter: arke_parameter}
+
+      %{
+        arke: arke,
+        parameter: parameter,
+        group: group,
+        permission: permission,
+        arke_parameter: arke_parameter
+      }
     end
   end
 
   defp get_arke(project, nil), do: []
-  defp get_arke(project,_), do: QueryManager.filter_by(project: project, arke_id: "arke")
+  defp get_arke(project, _), do: QueryManager.filter_by(project: project, arke_id: "arke")
+
+  defp get_parameter(project, nil), do: []
 
-  defp get_parameter(project,nil), do: []
-  defp get_parameter(project,_), do: QueryManager.filter_by(project: project, arke_id__in: Arke.Utils.DefaultData.get_parameters_id())
+  defp get_parameter(project, _),
+    do:
+      QueryManager.filter_by(
+        project: project,
+        arke_id__in: Arke.Utils.DefaultData.get_parameters_id()
+      )
 
-  defp get_group(project,nil), do: []
-  defp get_group(project,_), do: QueryManager.filter_by(project: project, arke_id: "group")
+  defp get_group(project, nil), do: []
+  defp get_group(project, _), do: QueryManager.filter_by(project: project, arke_id: "group")
 
   defp get_permission(project) do
     arke_link = ArkeManager.get(:arke_link, :arke_system)
+
     QueryManager.query(project: project, arke: arke_link.id)
     |> QueryManager.where(type: "permission")
-    |> QueryManager.all
+    |> QueryManager.all()
   end
 
   defp get_arke_parameter(project) do
     arke_link = ArkeManager.get(:arke_link, :arke_system)
+
     QueryManager.query(project: project, arke: arke_link.id)
     |> QueryManager.where(type: "parameter")
-    |> QueryManager.all
+    |> QueryManager.all()
   end
 
   defp get_all(project) do
-    data = QueryManager.filter_by(project: project, arke_id__in: Arke.Utils.DefaultData.get_arke_id())
+    data =
+      QueryManager.filter_by(project: project, arke_id__in: Arke.Utils.DefaultData.get_arke_id())
+
     p_list = Arke.Utils.DefaultData.get_parameters_id()
-    parsed_data = Enum.reduce(data,%{arke: [],group: [],parameter: []}, fn unit,acc ->
-    parse_data(unit,to_string(unit.arke_id),p_list,acc)
-    end)
-    |> Map.put(:permission, get_permission(project))
-    |> Map.put(:arke_parameter, get_arke_parameter(project))
+
+    parsed_data =
+      Enum.reduce(data, %{arke: [], group: [], parameter: []}, fn unit, acc ->
+        parse_data(unit, to_string(unit.arke_id), p_list, acc)
+      end)
+      |> Map.put(:permission, get_permission(project))
+      |> Map.put(:arke_parameter, get_arke_parameter(project))
   end
 
-  defp parse_data(unit,"arke",_p_list,acc), do: %{acc | arke: acc.arke ++ [unit]}
-  defp parse_data(unit,"group",_p_list,acc), do: %{acc | group: acc.group ++ [unit]}
-  defp parse_data(unit,arke_id,p_list,acc) do
+  defp parse_data(unit, "arke", _p_list, acc), do: %{acc | arke: acc.arke ++ [unit]}
+  defp parse_data(unit, "group", _p_list, acc), do: %{acc | group: acc.group ++ [unit]}
+
+  defp parse_data(unit, arke_id, p_list, acc) do
     if arke_id in p_list do
       %{acc | parameter: acc.parameter ++ [unit]}
     else
@@ -189,33 +230,51 @@ defmodule Mix.Tasks.Arke.ExportData do
   end
 
   # parse arke and make them usable for seed_project
-  defp prepare_arke(data,arke_param_list) do
-    Enum.map(data,fn arke ->
-    parameters = Enum.filter(arke_param_list, fn p -> to_string(p.data.parent_id) == to_string(arke.id) end)
-    |> Enum.map(fn p -> %{id: to_string(p.data.child_id), metadata: Map.delete(p.metadata,:project)} end)
-    |>Enum.sort_by(&Map.fetch(&1, :id))
-
-    %{id: to_string(arke.id),label: arke.data.label, parameters: parameters}
+  defp prepare_arke(data, arke_param_list) do
+    Enum.map(data, fn arke ->
+      parameters =
+        Enum.filter(arke_param_list, fn p -> to_string(p.data.parent_id) == to_string(arke.id) end)
+        |> Enum.map(fn p ->
+          %{id: to_string(p.data.child_id), metadata: Map.delete(p.metadata, :project)}
+        end)
+        |> Enum.sort_by(&Map.fetch(&1, :id))
+
+      %{id: to_string(arke.id), label: arke.data.label, parameters: parameters}
     end)
-    |>Enum.sort_by(&Map.fetch(&1, :id))
+    |> Enum.sort_by(&Map.fetch(&1, :id))
   end
 
   defp prepare_group(data) do
-    Enum.map(data,fn group ->
-    ordered_arke  = Enum.sort_by(group.data.arke_list, &(&1))
-      %{id: to_string(group.id),label: group.data.label, description: group.data.description,arke_list: ordered_arke}
+    Enum.map(data, fn group ->
+      ordered_arke = Enum.sort_by(group.data.arke_list, & &1)
+
+      %{
+        id: to_string(group.id),
+        label: group.data.label,
+        description: group.data.description,
+        arke_list: ordered_arke
+      }
     end)
-    |>Enum.sort_by(&Map.fetch(&1, :id))
+    |> Enum.sort_by(&Map.fetch(&1, :id))
   end
 
-  defp prepare_parameter(data), do: Enum.map(data,fn parameter->
-      Map.put(parameter.data, :id, to_string(parameter.id))
-      |> Map.put(:type, to_string(parameter.arke_id))
-    end)
-   |> Enum.sort_by(&Map.fetch(&1, :id))
-
-  defp prepare_permission(data), do: Enum.map(data, fn permission ->
-    %{parent: permission.data.parent_id,child: permission.data.child_id,metadata: Map.delete(permission.metadata,:project), type: permission.data.type}
-  end)
-  |>Enum.sort_by(&Map.fetch(&1, :parent))
-end
\ No newline at end of file
+  defp prepare_parameter(data),
+    do:
+      Enum.map(data, fn parameter ->
+        Map.put(parameter.data, :id, to_string(parameter.id))
+        |> Map.put(:type, to_string(parameter.arke_id))
+      end)
+      |> Enum.sort_by(&Map.fetch(&1, :id))
+
+  defp prepare_permission(data),
+    do:
+      Enum.map(data, fn permission ->
+        %{
+          parent: permission.data.parent_id,
+          child: permission.data.child_id,
+          metadata: Map.delete(permission.metadata, :project),
+          type: permission.data.type
+        }
+      end)
+      |> Enum.sort_by(&Map.fetch(&1, :parent))
+end

From fcf933a9a4cfcfaf2d9365d632935b442beddcbe Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Sat, 30 Nov 2024 13:57:06 +0100
Subject: [PATCH 17/28] wip

---
 lib/arke/link_manager.ex           |  151 ++-
 lib/arke/query_manager.ex          |    7 +-
 lib/arke/validator.ex              |    2 +-
 lib/registry/system/arke.json      | 1956 ++++++++++++++--------------
 lib/registry/system/parameter.json | 1681 ++++++++++++------------
 5 files changed, 1969 insertions(+), 1828 deletions(-)

diff --git a/lib/arke/link_manager.ex b/lib/arke/link_manager.ex
index e3692e2..9da79ce 100644
--- a/lib/arke/link_manager.ex
+++ b/lib/arke/link_manager.ex
@@ -20,11 +20,20 @@ defmodule Arke.LinkManager do
   alias Arke.Utils.ErrorGenerator, as: Error
   alias Arke.QueryManager
   alias Arke.Core.Unit
+  alias Arke.Core.Query
+
+  defmodule LinkData do
+    @enforce_keys [:parent_id, :child_id, :type]
+    defstruct [:parent_id, :child_id, :type, metadata: %{}]
+
+    def to_map(link_data) do
+      Map.take(link_data, [:parent_id, :child_id, :type, :metadata])
+    end
+  end
 
   def add_node(project, %Unit{} = parent, %Unit{} = child, type \\ "link", metadata \\ %{}) do
     arke_link = ArkeManager.get(:arke_link, project)
 
-
     case check_link(project, parent, child, arke_link) do
       {_, nil} ->
         QueryManager.create(project, arke_link,
@@ -41,15 +50,29 @@ defmodule Arke.LinkManager do
 
   def add_node(project, parent, child, type, metadata)
       when is_binary(parent) and is_binary(child) do
-    with %Unit{}=unit_parent <- QueryManager.get_by(id: parent, project: project),
-         %Unit{}=unit_child <- QueryManager.get_by(id: child, project: project) do
+    with %Unit{} = unit_parent <- QueryManager.get_by(id: parent, project: project),
+         %Unit{} = unit_child <- QueryManager.get_by(id: child, project: project) do
       add_node(project, unit_parent, unit_child, type, metadata)
-      else
-      _ ->  Error.create(:link, "parent: `#{parent}` or child: `#{child}` not found")
+    else
+      _ -> Error.create(:link, "parent: `#{parent}` or child: `#{child}` not found")
     end
   end
 
-  def add_node(_project,  _parent, _child, _type, _metadata), do: Error.create(:link, "invalid parameters")
+  def add_node(_project, _parent, _child, _type, _metadata),
+    do: Error.create(:link, "invalid parameters")
+
+  def add_node_bulk(project, link_list) do
+    arke_link = ArkeManager.get(:arke_link, project)
+
+    with {valid, errors} <- validate_units(project, arke_link, link_list),
+         {valid, errors} <- validate_existing(project, arke_link, valid, errors),
+         {:ok, inserted_count, valid, insert_errors} <-
+           QueryManager.create_bulk(project, arke_link, prepare_link_data(valid), []) do
+      {:ok, inserted_count, valid, errors ++ insert_errors}
+    else
+      {:error, errors} -> {:error, errors}
+    end
+  end
 
   def update_node(project, %Unit{} = parent, %Unit{} = child, type, metadata) do
     arke_link = ArkeManager.get(:arke_link, :arke_system)
@@ -91,6 +114,13 @@ defmodule Arke.LinkManager do
   def delete_node(_project, _parent, _child, _type, _metadata),
     do: Error.create(:link, "invalid parameters")
 
+  def delete_node_bulk(project, link_list) do
+    arke_link = ArkeManager.get(:arke_link, :arke_system)
+    {valid, _} = validate_units(project, arke_link, link_list)
+
+    QueryManager.delete_bulk(project, existing_links(project, arke_link, valid))
+  end
+
   defp check_link(project, parent, child, arke_link) do
     with %Arke.Core.Unit{} = link <-
            Arke.QueryManager.query(project: project, arke: arke_link)
@@ -100,4 +130,113 @@ defmodule Arke.LinkManager do
          do: {:ok, link},
          else: (_ -> {:error, nil})
   end
+
+  @spec validate_units(atom(), Arke.t(), list()) :: {list(LinkData.t()), list(map())}
+  defp validate_units(project, arke_link, link_list) do
+    ids =
+      Enum.flat_map(link_list, fn link ->
+        case {link["parent_id"], link["child_id"]} do
+          {parent, child} when is_binary(parent) and is_binary(child) ->
+            [parent, child]
+
+          {parent, _} when is_binary(parent) ->
+            [parent]
+
+          {_, child} when is_binary(child) ->
+            [child]
+
+          _ ->
+            []
+        end
+      end)
+
+    unit_list = QueryManager.filter_by(id__in: ids, project: project)
+    unit_map = Map.new(unit_list, fn unit -> {Atom.to_string(unit.id), unit} end)
+
+    Enum.reduce(link_list, {[], []}, fn link, {valid, errors} ->
+      parent =
+        case link["parent_id"] do
+          %Unit{} = p -> p
+          id -> Map.get(unit_map, id)
+        end
+
+      child =
+        case link["child_id"] do
+          %Unit{} = c -> c
+          id -> Map.get(unit_map, id)
+        end
+
+      type = Map.get(link, "type", "link")
+      metadata = Map.get(link, "metadata", %{})
+
+      case {parent, child} do
+        {nil, nil} ->
+          {valid, [%{link: link, error: "invalid parent and child"} | errors]}
+
+        {nil, _} ->
+          {valid, [%{link: link, error: "invalid parent"} | errors]}
+
+        {_, nil} ->
+          {valid, [%{link: link, error: "invalid child"} | errors]}
+
+        {p, c} ->
+          {[
+             %LinkData{
+               parent_id: Atom.to_string(p.id),
+               child_id: Atom.to_string(c.id),
+               type: type,
+               metadata: metadata
+             }
+             | valid
+           ], errors}
+      end
+    end)
+  end
+
+  @spec existing_links(project :: atom(), arke_link :: Arke.t(), link_list :: list(LinkData.t())) ::
+          list(Arke.Core.Unit.t())
+  defp existing_links(project, arke_link, link_list) do
+    parameters = ArkeManager.get_parameters(arke_link)
+    parent_id = Enum.find(parameters, fn p -> p.id == :parent_id end)
+    child_id = Enum.find(parameters, fn p -> p.id == :child_id end)
+    type = Enum.find(parameters, fn p -> p.id == :type end)
+
+    # todo: handle type default_string ?
+
+    Arke.QueryManager.query(project: project, arke: arke_link)
+    |> Arke.Core.Query.add_filter(
+      :or,
+      false,
+      Enum.map(link_list, fn link ->
+        Arke.Core.Query.new_filter(:and, false, [
+          Arke.QueryManager.condition(parent_id, :eq, link.parent_id, false),
+          Arke.QueryManager.condition(child_id, :eq, link.child_id, false),
+          Arke.QueryManager.condition(type, :eq, link.type, false)
+        ])
+      end)
+    )
+    |> Arke.QueryManager.all()
+  end
+
+  @spec validate_existing(atom(), Arke.t(), list(LinkData.t()), list()) ::
+          {list(LinkData.t()), list()}
+  defp validate_existing(project, arke_link, link_list, errors) do
+    existing =
+      existing_links(project, arke_link, link_list)
+      |> Enum.reduce(MapSet.new(), fn link, acc ->
+        MapSet.put(acc, {link.data.parent_id, link.data.child_id, link.data.type})
+      end)
+
+    Enum.reduce(link_list, {[], errors}, fn link, {valid_acc, errors_acc} ->
+      case MapSet.member?(existing, {link.parent_id, link.child_id, link.type}) do
+        true -> {valid_acc, [LinkData.to_map(link) | errors_acc]}
+        false -> {[LinkData.to_map(link) | valid_acc], errors_acc}
+      end
+    end)
+  end
+
+  @spec prepare_link_data(list(LinkData.t())) :: list(map())
+  defp prepare_link_data(link_list) do
+    Enum.map(link_list, &LinkData.to_map/1)
+  end
 end
diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 9648d75..9375e6f 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -186,7 +186,12 @@ defmodule Arke.QueryManager do
 
   # todo: remove after atoms removal
   defp data_as_klist(data) do
-    Enum.map(data, fn {key, value} -> {String.to_existing_atom(key), value} end)
+    Enum.map(data, fn {key, value} ->
+      case is_atom(key) do
+        true -> {key, value}
+        false -> {String.to_existing_atom(key), value}
+      end
+    end)
   end
 
   defp handle_link_parameters_unit(%{id: :arke_link} = _, unit), do: {:ok, unit}
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index 5cfb8d4..3b4b85d 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -51,7 +51,7 @@ defmodule Arke.Validator do
     do: validate([unit], persistence_fn, project)
 
   def validate([], _persistence_fn, _project),
-    do: %{valid: [], errors: [{nil, "empty list of units"}]}
+    do: %{valid: [], errors: []}
 
   def validate([%Unit{arke_id: arke_id} | _] = unit_list, persistence_fn, project) do
     %{data: arke_data} = arke = ArkeManager.get(arke_id, project)
diff --git a/lib/registry/system/arke.json b/lib/registry/system/arke.json
index bdcdd0b..7983540 100644
--- a/lib/registry/system/arke.json
+++ b/lib/registry/system/arke.json
@@ -1,998 +1,986 @@
-{"arke": [
-  {
-    "id": "arke",
-    
+{
+  "arke": [
+    {
+      "id": "arke",
       "label": "Arke",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "active",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "type",
-        "metadata": {
-          "required": true,
-          "default_string": "arke"
-        }
-      },
-      {
-        "id": "parameters",
-        "metadata": {
-          "multiple": true,
-          "arke_or_group_id": "parameters",
-          "connection_type": "parameters",
-          "direction": "child",
-          "filter_keys": [
-            "arke_id",
-            "id",
-            "label"
-          ],
-          "depth": 0,
-          "default_link": []
-        }
-      }
-    ]
-  },
-  {
-    "id": "arke_link",
-    "label": "Link",
-    "type": "table",
-    "parameters": [
-      {
-        "id": "parent_id",
-        "metadata": {
-          "is_primary": true,
-          "required": true,
-          "persistence": "table_column"
-        }
-      },
-      {
-        "id": "child_id",
-        "metadata": {
-          "is_primary": true,
-          "required": true,
-          "persistence": "table_column"
-        }
-      },
-      {
-        "id": "type",
-        "metadata": {
-          "required": true,
-          "persistence": "table_column"
-        }
-      },
-      {
-        "id": "metadata",
-        "metadata": {
-          "is_primary": true,
-          "default_dict": {},
-          "persistence": "table_column"
-        }
-      }
-    ]
-  },
-  {
-    "id": "integer",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "active",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "type",
+          "metadata": {
+            "required": true,
+            "default_string": "arke"
+          }
+        },
+        {
+          "id": "parameters",
+          "metadata": {
+            "multiple": true,
+            "arke_or_group_id": "parameters",
+            "connection_type": "parameters",
+            "direction": "child",
+            "filter_keys": [
+              "arke_id",
+              "id",
+              "label"
+            ],
+            "depth": 0,
+            "default_link": []
+          }
+        }
+      ]
+    },
+    {
+      "id": "arke_link",
+      "label": "Link",
+      "type": "table",
+      "parameters": [
+        {
+          "id": "parent_id",
+          "metadata": {
+            "is_primary": true,
+            "required": true
+          }
+        },
+        {
+          "id": "child_id",
+          "metadata": {
+            "is_primary": true,
+            "required": true
+          }
+        },
+        {
+          "id": "type",
+          "metadata": {
+            "is_primary": true,
+            "required": true,
+            "persistence": "table_column",
+            "default_string": "link"
+          }
+        },
+        {
+          "id": "metadata",
+          "metadata": {
+            "default_dict": {}
+          }
+        }
+      ]
+    },
+    {
+      "id": "integer",
       "label": "Integer",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "min",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "max",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "values",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "multiple",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "default_integer",
-        "metadata": {
-          "default_integer": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "string",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "min",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "max",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "values",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "multiple",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "default_integer",
+          "metadata": {
+            "default_integer": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "string",
       "label": "String",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "min_length",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "max_length",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "strip",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "values",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "multiple",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "unique",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_string",
-        "metadata": {
-          "default_string": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "dict",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "min_length",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "max_length",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "strip",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "values",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "multiple",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "unique",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_string",
+          "metadata": {
+            "default_string": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "dict",
       "label": "Dict",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_dict",
-        "metadata": {
-          "required": false
-        }
-      }
-    ]
-  },
-  {
-    "id": "date",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_dict",
+          "metadata": {
+            "required": false
+          }
+        }
+      ]
+    },
+    {
+      "id": "date",
       "label": "Date",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_date",
-        "metadata": {
-          "required": false
-        }
-      }
-    ]
-  },
-  {
-    "id": "datetime",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_date",
+          "metadata": {
+            "required": false
+          }
+        }
+      ]
+    },
+    {
+      "id": "datetime",
       "label": "Datetime",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_datetime",
-        "metadata": {
-          "required": false
-        }
-      }
-    ]
-  },
-  {
-    "id": "dynamic",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_datetime",
+          "metadata": {
+            "required": false
+          }
+        }
+      ]
+    },
+    {
+      "id": "dynamic",
       "label": "Dynamic",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_dynamic",
-        "metadata": {
-          "required": false
-        }
-      }
-    ]
-  },
-  {
-    "id": "float",
-
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_dynamic",
+          "metadata": {
+            "required": false
+          }
+        }
+      ]
+    },
+    {
+      "id": "float",
       "label": "Float",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "min",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "max",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "values",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "multiple",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "default_float",
-        "metadata": {
-          "default_float": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "boolean",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "min",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "max",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "values",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "multiple",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "default_float",
+          "metadata": {
+            "default_float": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "boolean",
       "label": "Boolean",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_boolean",
-        "metadata": {
-          "default_boolean": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "list",
-    
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_boolean",
+          "metadata": {
+            "default_boolean": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "list",
       "label": "List",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_list",
-        "metadata": {
-          "default_list": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "time",
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_list",
+          "metadata": {
+            "default_list": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "time",
       "label": "Time",
       "parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_time",
-        "metadata": {
-          "default_time": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "link",
-    "label": "Link",
-    "parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_link",
-        "metadata": {
-          "default_link": null
-        }
-      },
-      {
-        "id": "depth",
-        "metadata": {
-          "default_integer": 0
-        }
-      },
-      {
-        "id": "connection_type",
-        "metadata": {
-          "default_string": "link"
-        }
-      },
-      {
-        "id": "arke_or_group_id",
-        "metadata": {
-          "default_string": "arke"
-        }
-      },
-      {
-        "id": "filter_keys",
-        "metadata": {
-          "multiple": true,
-          "default_string": [
-            "id",
-            "arke_id"
-          ]
-        }
-      },
-      {
-        "id": "multiple",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-
-      {
-        "id": "direction",
-        "metadata": {
-          "default_string": "child"
-        }
-      }
-    ]
-  },
-  {
-    "id": "binary",
-    "label": "Binary",
-    "parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "format",
-        "metadata": {
-          "default_string": "attribute"
-        }
-      },
-      {
-        "id": "is_primary",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "nullable",
-        "metadata": {
-          "default_boolean": true
-        }
-      },
-      {
-        "id": "required",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "default_string": "arke_parameter"
-        }
-      },
-      {
-        "id": "only_run_time",
-        "metadata": {
-          "default_boolean": false
-        }
-      },
-      {
-        "id": "helper_text",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "default_binary",
-        "metadata": {
-          "default_binary": null
-        }
-      }
-    ]
-  },
-  {
-    "id": "group",
-    
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_time",
+          "metadata": {
+            "default_time": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "link",
+      "label": "Link",
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_link",
+          "metadata": {
+            "default_link": null
+          }
+        },
+        {
+          "id": "depth",
+          "metadata": {
+            "default_integer": 0
+          }
+        },
+        {
+          "id": "connection_type",
+          "metadata": {
+            "default_string": "link"
+          }
+        },
+        {
+          "id": "arke_or_group_id",
+          "metadata": {
+            "default_string": "arke"
+          }
+        },
+        {
+          "id": "filter_keys",
+          "metadata": {
+            "multiple": true,
+            "default_string": [
+              "id",
+              "arke_id"
+            ]
+          }
+        },
+        {
+          "id": "multiple",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "direction",
+          "metadata": {
+            "default_string": "child"
+          }
+        }
+      ]
+    },
+    {
+      "id": "binary",
+      "label": "Binary",
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "format",
+          "metadata": {
+            "default_string": "attribute"
+          }
+        },
+        {
+          "id": "is_primary",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "nullable",
+          "metadata": {
+            "default_boolean": true
+          }
+        },
+        {
+          "id": "required",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "default_string": "arke_parameter"
+          }
+        },
+        {
+          "id": "only_run_time",
+          "metadata": {
+            "default_boolean": false
+          }
+        },
+        {
+          "id": "helper_text",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "default_binary",
+          "metadata": {
+            "default_binary": null
+          }
+        }
+      ]
+    },
+    {
+      "id": "group",
       "label": "Group",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "description",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "arke_list",
-        "metadata": {
-          "multiple": true,
-          "arke_or_group_id": "arke",
-          "connection_type": "group",
-          "depth": 0,
-          "default_link": []
-        }
-      }
-    ]
-  },
-  {
-    "id": "arke_project",
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "description",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "arke_list",
+          "metadata": {
+            "multiple": true,
+            "arke_or_group_id": "arke",
+            "connection_type": "group",
+            "depth": 0,
+            "default_link": []
+          }
+        }
+      ]
+    },
+    {
+      "id": "arke_project",
       "label": "Project",
-"parameters": [
-      {
-        "id": "label",
-        "metadata": {
-          "required": true
-        }
-      },
-      {
-        "id": "description",
-        "metadata": {
-          "required": false
-        }
-      },
-      {
-        "id": "persistence",
-        "metadata": {
-          "required": true,
-          "values": null,
-          "default_dict": {}
-        }
-      },
-      {
-        "id": "type",
-        "metadata": {
-          "required": true,
-          "default_string": "postgres_schema"
-        }
-      }
-    ]
-  }
-]}
\ No newline at end of file
+      "parameters": [
+        {
+          "id": "label",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "description",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "persistence",
+          "metadata": {
+            "required": true,
+            "values": null,
+            "default_dict": {}
+          }
+        },
+        {
+          "id": "type",
+          "metadata": {
+            "required": true,
+            "default_string": "postgres_schema"
+          }
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/lib/registry/system/parameter.json b/lib/registry/system/parameter.json
index 04f5bd5..2c8cf0b 100644
--- a/lib/registry/system/parameter.json
+++ b/lib/registry/system/parameter.json
@@ -1,837 +1,846 @@
-{  "parameter": [
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Multiple",
-    "id": "multiple",
-    "is_primary": false,
-    "label": "Multiple",
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Unique",
-    "id": "unique",
-    "is_primary": false,
-    "label": "Unique",
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_boolean",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Nullable",
-    "id": "nullable",
-    "is_primary": false,
-    "label": "Nullable",
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": true
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Is Primary",
-    "id": "is_primary",
-    "is_primary": false,
-    "label": "Is Primary",
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Remove Whitespace",
-    "id": "strip",
-    "is_primary": false,
-    "label": "Remove Whitespace",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Transform to lowercase",
-    "id": "lowercase",
-    "is_primary": false,
-    "label": "Lowercase",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Active",
-    "id": "active",
-    "is_primary": false,
-    "label": "Active",
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Only run time",
-    "id": "only_run_time",
-    "is_primary": false,
-    "label": "Only run time",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_boolean": false,
-    "type": "boolean",
-    "format": "attribute",
-    "helper_text": "Required",
-    "id": "required",
-    "is_primary": false,
-    "label": "Required",
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_datetime": null,
-    "type": "datetime",
-    "format": "attribute",
-    "helper_text": "Inserted at",
-    "id": "inserted_at",
-    "is_primary": false,
-    "label": "Inserted at",
-    "nullable": true,
-    "persistence": "table_column",
-    "required": false
-  },
-  {
-    "default_datetime": null,
-    "type": "datetime",
-    "format": "attribute",
-    "helper_text": "Updated at",
-    "id": "updated_at",
-    "is_primary": false,
-    "label": "Updated at",
-    "nullable": true,
-    "persistence": "table_column",
-    "required": false
-  },
-  {
-    "default_datetime": null,
-    "type": "datetime",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_datetime",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_date": null,
-    "type": "date",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_date",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_time": null,
-    "type": "time",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_time",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "arke_or_group_id": "arke",
-    "connection_type": "link",
-    "default_link": null,
-    "type": "link",
-    "depth": 0,
-    "direction": "child",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_link",
-    "is_primary": false,
-    "label": "Default",
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "filter_keys": ["id","arke_id"]
-  },
-  {
-    "arke_or_group_id": "arke",
-    "connection_type": "group",
-    "default_link": [],
-    "type": "link",
-    "depth": 0,
-    "direction": "child",
-    "format": "attribute",
-    "helper_text": "Arke List",
-    "id": "arke_list",
-    "is_primary": false,
-    "label": "Arke List",
-    "multiple": true,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "filter_keys": ["id","arke_id"]
-  },
-  {
-    "connection_type": "link",
-    "default_link": [],
-    "type": "link",
-    "depth": 0,
-    "direction": "child",
-    "format": "attribute",
-    "helper_text": "Parameters",
-    "id": "parameters",
-    "is_primary": false,
-    "label": "Parameters",
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "arke_or_group_id": "parameter",
-    "required": false
-  },
-  {
-    "arke_or_group_id": "arke_or_group",
-    "connection_type": "link",
-    "default_link": null,
-    "type": "link",
-    "depth": 0,
-    "direction": "child",
-    "format": "attribute",
-    "helper_text": "Arke or Group id",
-    "id": "arke_or_group_id",
-    "is_primary": false,
-    "label": "Arke or Group id",
-    "multiple": false,
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": true,
-    "filter_keys": ["id","arke_id"]
-
-  },
-  {
-    "default_list": null,
-    "type": "dict",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_dict",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_list": null,
-    "type": "dict",
-    "format": "attribute",
-    "helper_text": "Address",
-    "id": "address",
-    "is_primary": false,
-    "label": "Address",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_dict": {},
-    "type": "dict",
-    "format": "attribute",
-    "helper_text": "Metadata",
-    "id": "metadata",
-    "is_primary": false,
-    "label": "Metadata",
-    "nullable": true,
-    "persistence": "table_column",
-    "required": false
-  },
-  {
-    "default_dynamic": null,
-    "type": "dynamic",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_dynamic",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_integer": null,
-    "type": "integer",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_integer",
-    "is_primary": false,
-    "label": "Default",
-    "max": null,
-    "min": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_integer": null,
-    "type": "integer",
-    "format": "attribute",
-    "helper_text": "Min Length",
-    "id": "min_length",
-    "is_primary": false,
-    "label": "Min Length",
-    "max": null,
-    "min": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_integer": null,
-    "type": "integer",
-    "format": "attribute",
-    "helper_text": "Max Length",
-    "id": "max_length",
-    "is_primary": false,
-    "label": "Max Length",
-    "max": null,
-    "min": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_integer": null,
-    "type": "integer",
-    "format": "attribute",
-    "helper_text": "Min",
-    "id": "min",
-    "is_primary": false,
-    "label": "Min",
-    "max": null,
-    "min": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_integer": null,
-    "type": "integer",
-    "format": "attribute",
-    "helper_text": "Max",
-    "id": "max",
-    "is_primary": false,
-    "label": "Max",
-    "max": null,
-    "min": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_integer": 0,
-    "type": "integer",
-    "format": "attribute",
-    "helper_text": "Depth",
-    "id": "depth",
-    "is_primary": false,
-    "label": "Depth",
-    "max": 100,
-    "min": 0,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_float": null,
-    "type": "float",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_float",
-    "is_primary": false,
-    "label": "Default",
-    "max": null,
-    "min": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_list": null,
-    "type": "list",
-    "format": "attribute",
-    "helper_text": "Values",
-    "id": "values",
-    "is_primary": false,
-    "label": "Values",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_list": null,
-    "type": "list",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_list",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_binary": null,
-    "type": "binary",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_binary",
-    "is_primary": false,
-    "label": "Default",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Id",
-    "id": "id",
-    "is_primary": true,
-    "label": "Id",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": false,
-    "persistence": "table_column",
-    "required": true,
-    "strip": true,
-"lowercase": false,
-    "unique": true,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Arke id",
-    "id": "arke_id",
-    "is_primary": false,
-    "label": "Arke id",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": false,
-    "persistence": "table_column",
-    "required": true,
-    "strip": true,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Label",
-    "id": "label",
-    "is_primary": false,
-    "label": "Label",
-    "max_length": 200,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": true,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": "attribute",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Format",
-    "id": "format",
-    "is_primary": false,
-    "label": "Format",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": "arke_parameter",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Persistence",
-    "id": "persistence",
-    "is_primary": false,
-    "label": "Persistence",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": false,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": [
-      {
-        "label": "Arke Parameter",
-        "value": "arke_parameter"
-      },
-      {
-        "label": "Table Column",
-        "value": "table_column"
-      }
-    ]
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Helper text",
-    "id": "helper_text",
-    "is_primary": false,
-    "label": "Helper text",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Default",
-    "id": "default_string",
-    "is_primary": false,
-    "label": "Default",
-    "max_length": null,
-    "min_length": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Description",
-    "id": "description",
-    "is_primary": false,
-    "label": "Description",
-    "max_length": 500,
-    "min_length": 0,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Child Id",
-    "id": "child_id",
-    "is_primary": false,
-    "label": "Child Id",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": true,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Parent Id",
-    "id": "parent_id",
-    "is_primary": false,
-    "label": "Parent Id",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": true,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Type",
-    "id": "type",
-    "is_primary": false,
-    "label": "Type",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": true,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Public Key",
-    "id": "public_key",
-    "is_primary": false,
-    "label": "Public Key",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Vat",
-    "id": "vat",
-    "is_primary": false,
-    "label": "Vat",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Environment",
-    "id": "environment",
-    "is_primary": false,
-    "label": "Environment",
-    "max_length": null,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Fiscal code",
-    "id": "fiscal_code",
-    "is_primary": false,
-    "label": "Fiscal code",
-    "max_length": null,
-    "min_length": 3,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Birth date",
-    "id": "birth_date",
-    "is_primary": false,
-    "label": "Birth date",
-    "max_length": null,
-    "min_length": 1,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": "link",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Connection type",
-    "id": "connection_type",
-    "is_primary": false,
-    "label": "Connection type",
-    "max_length": null,
-    "min_length": 1,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": true,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": "child",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Direction",
-    "id": "direction",
-    "is_primary": false,
-    "label": "Direction",
-    "max_length": null,
-    "min_length": 1,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": true,
-"lowercase": false,
-    "unique": false,
-    "values": [
-      {
-        "label": "Child",
-        "value": "child"
-      },
-      {
-        "label": "Parent",
-        "value": "parent"
-      }
-    ]
-  },
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Filter keys",
-    "id": "filter_keys",
-    "is_primary": false,
-    "label": "Filter keys",
-    "max_length": null,
-    "min_length": null,
-    "multiple": true,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  }
-]
+{
+  "parameter": [
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Multiple",
+      "id": "multiple",
+      "is_primary": false,
+      "label": "Multiple",
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Unique",
+      "id": "unique",
+      "is_primary": false,
+      "label": "Unique",
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_boolean",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Nullable",
+      "id": "nullable",
+      "is_primary": false,
+      "label": "Nullable",
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": true
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Is Primary",
+      "id": "is_primary",
+      "is_primary": false,
+      "label": "Is Primary",
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Remove Whitespace",
+      "id": "strip",
+      "is_primary": false,
+      "label": "Remove Whitespace",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Transform to lowercase",
+      "id": "lowercase",
+      "is_primary": false,
+      "label": "Lowercase",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Active",
+      "id": "active",
+      "is_primary": false,
+      "label": "Active",
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Only run time",
+      "id": "only_run_time",
+      "is_primary": false,
+      "label": "Only run time",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_boolean": false,
+      "type": "boolean",
+      "format": "attribute",
+      "helper_text": "Required",
+      "id": "required",
+      "is_primary": false,
+      "label": "Required",
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_datetime": null,
+      "type": "datetime",
+      "format": "attribute",
+      "helper_text": "Inserted at",
+      "id": "inserted_at",
+      "is_primary": false,
+      "label": "Inserted at",
+      "nullable": true,
+      "persistence": "table_column",
+      "required": false
+    },
+    {
+      "default_datetime": null,
+      "type": "datetime",
+      "format": "attribute",
+      "helper_text": "Updated at",
+      "id": "updated_at",
+      "is_primary": false,
+      "label": "Updated at",
+      "nullable": true,
+      "persistence": "table_column",
+      "required": false
+    },
+    {
+      "default_datetime": null,
+      "type": "datetime",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_datetime",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_date": null,
+      "type": "date",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_date",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_time": null,
+      "type": "time",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_time",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "arke_or_group_id": "arke",
+      "connection_type": "link",
+      "default_link": null,
+      "type": "link",
+      "depth": 0,
+      "direction": "child",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_link",
+      "is_primary": false,
+      "label": "Default",
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "filter_keys": [
+        "id",
+        "arke_id"
+      ]
+    },
+    {
+      "arke_or_group_id": "arke",
+      "connection_type": "group",
+      "default_link": [],
+      "type": "link",
+      "depth": 0,
+      "direction": "child",
+      "format": "attribute",
+      "helper_text": "Arke List",
+      "id": "arke_list",
+      "is_primary": false,
+      "label": "Arke List",
+      "multiple": true,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "filter_keys": [
+        "id",
+        "arke_id"
+      ]
+    },
+    {
+      "connection_type": "link",
+      "default_link": [],
+      "type": "link",
+      "depth": 0,
+      "direction": "child",
+      "format": "attribute",
+      "helper_text": "Parameters",
+      "id": "parameters",
+      "is_primary": false,
+      "label": "Parameters",
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "arke_or_group_id": "parameter",
+      "required": false
+    },
+    {
+      "arke_or_group_id": "arke_or_group",
+      "connection_type": "link",
+      "default_link": null,
+      "type": "link",
+      "depth": 0,
+      "direction": "child",
+      "format": "attribute",
+      "helper_text": "Arke or Group id",
+      "id": "arke_or_group_id",
+      "is_primary": false,
+      "label": "Arke or Group id",
+      "multiple": false,
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": true,
+      "filter_keys": [
+        "id",
+        "arke_id"
+      ]
+    },
+    {
+      "default_list": null,
+      "type": "dict",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_dict",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_list": null,
+      "type": "dict",
+      "format": "attribute",
+      "helper_text": "Address",
+      "id": "address",
+      "is_primary": false,
+      "label": "Address",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_dict": {},
+      "type": "dict",
+      "format": "attribute",
+      "helper_text": "Metadata",
+      "id": "metadata",
+      "is_primary": false,
+      "label": "Metadata",
+      "nullable": true,
+      "persistence": "table_column",
+      "required": false
+    },
+    {
+      "default_dynamic": null,
+      "type": "dynamic",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_dynamic",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_integer": null,
+      "type": "integer",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_integer",
+      "is_primary": false,
+      "label": "Default",
+      "max": null,
+      "min": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_integer": null,
+      "type": "integer",
+      "format": "attribute",
+      "helper_text": "Min Length",
+      "id": "min_length",
+      "is_primary": false,
+      "label": "Min Length",
+      "max": null,
+      "min": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_integer": null,
+      "type": "integer",
+      "format": "attribute",
+      "helper_text": "Max Length",
+      "id": "max_length",
+      "is_primary": false,
+      "label": "Max Length",
+      "max": null,
+      "min": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_integer": null,
+      "type": "integer",
+      "format": "attribute",
+      "helper_text": "Min",
+      "id": "min",
+      "is_primary": false,
+      "label": "Min",
+      "max": null,
+      "min": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_integer": null,
+      "type": "integer",
+      "format": "attribute",
+      "helper_text": "Max",
+      "id": "max",
+      "is_primary": false,
+      "label": "Max",
+      "max": null,
+      "min": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_integer": 0,
+      "type": "integer",
+      "format": "attribute",
+      "helper_text": "Depth",
+      "id": "depth",
+      "is_primary": false,
+      "label": "Depth",
+      "max": 100,
+      "min": 0,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_float": null,
+      "type": "float",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_float",
+      "is_primary": false,
+      "label": "Default",
+      "max": null,
+      "min": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_list": null,
+      "type": "list",
+      "format": "attribute",
+      "helper_text": "Values",
+      "id": "values",
+      "is_primary": false,
+      "label": "Values",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_list": null,
+      "type": "list",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_list",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_binary": null,
+      "type": "binary",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_binary",
+      "is_primary": false,
+      "label": "Default",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Id",
+      "id": "id",
+      "is_primary": true,
+      "label": "Id",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": false,
+      "persistence": "table_column",
+      "required": true,
+      "strip": true,
+      "lowercase": false,
+      "unique": true,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Arke id",
+      "id": "arke_id",
+      "is_primary": false,
+      "label": "Arke id",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": false,
+      "persistence": "table_column",
+      "required": true,
+      "strip": true,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Label",
+      "id": "label",
+      "is_primary": false,
+      "label": "Label",
+      "max_length": 200,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": true,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": "attribute",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Format",
+      "id": "format",
+      "is_primary": false,
+      "label": "Format",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": "arke_parameter",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Persistence",
+      "id": "persistence",
+      "is_primary": false,
+      "label": "Persistence",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": false,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": [
+        {
+          "label": "Arke Parameter",
+          "value": "arke_parameter"
+        },
+        {
+          "label": "Table Column",
+          "value": "table_column"
+        }
+      ]
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Helper text",
+      "id": "helper_text",
+      "is_primary": false,
+      "label": "Helper text",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Default",
+      "id": "default_string",
+      "is_primary": false,
+      "label": "Default",
+      "max_length": null,
+      "min_length": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Description",
+      "id": "description",
+      "is_primary": false,
+      "label": "Description",
+      "max_length": 500,
+      "min_length": 0,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Child Id",
+      "id": "child_id",
+      "is_primary": false,
+      "label": "Child Id",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "table_column",
+      "required": false,
+      "strip": true,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Parent Id",
+      "id": "parent_id",
+      "is_primary": false,
+      "label": "Parent Id",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "table_column",
+      "required": false,
+      "strip": true,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Type",
+      "id": "type",
+      "is_primary": false,
+      "label": "Type",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": true,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Public Key",
+      "id": "public_key",
+      "is_primary": false,
+      "label": "Public Key",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Vat",
+      "id": "vat",
+      "is_primary": false,
+      "label": "Vat",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Environment",
+      "id": "environment",
+      "is_primary": false,
+      "label": "Environment",
+      "max_length": null,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Fiscal code",
+      "id": "fiscal_code",
+      "is_primary": false,
+      "label": "Fiscal code",
+      "max_length": null,
+      "min_length": 3,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Birth date",
+      "id": "birth_date",
+      "is_primary": false,
+      "label": "Birth date",
+      "max_length": null,
+      "min_length": 1,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": "link",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Connection type",
+      "id": "connection_type",
+      "is_primary": false,
+      "label": "Connection type",
+      "max_length": null,
+      "min_length": 1,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": true,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": "child",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Direction",
+      "id": "direction",
+      "is_primary": false,
+      "label": "Direction",
+      "max_length": null,
+      "min_length": 1,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": true,
+      "lowercase": false,
+      "unique": false,
+      "values": [
+        {
+          "label": "Child",
+          "value": "child"
+        },
+        {
+          "label": "Parent",
+          "value": "parent"
+        }
+      ]
+    },
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Filter keys",
+      "id": "filter_keys",
+      "is_primary": false,
+      "label": "Filter keys",
+      "max_length": null,
+      "min_length": null,
+      "multiple": true,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    }
+  ]
 }
\ No newline at end of file

From 1e1f013785e9eaf59c11fa0b7cfe5110466a58fa Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Sat, 30 Nov 2024 14:39:59 +0100
Subject: [PATCH 18/28] fix: single result validator

---
 lib/arke/query_manager.ex |  4 ++--
 lib/arke/validator.ex     | 14 +++++++++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 9375e6f..bff7f80 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -144,7 +144,7 @@ defmodule Arke.QueryManager do
     persistence_fn = @persistence[:arke_postgres][:create]
 
     with %Unit{} = unit <- Unit.load(arke, args, :create),
-         %{valid: [unit], errors: _errors} <- Validator.validate(unit, :create, project),
+         {:ok, unit} <- Validator.validate(unit, :create, project),
          {:ok, unit} <- run_persistence_hook(arke, unit, :create, :before),
          {:ok, unit} <- run_group_and_link_hooks(arke, unit, :create, :before),
          {:ok, unit} <- persistence_fn.(project, unit, []),
@@ -295,7 +295,7 @@ defmodule Arke.QueryManager do
 
     with %Unit{} = unit <- Unit.update(current_unit, args),
          {:ok, unit} <- update_at_on_update(unit),
-         %{valid: [unit], errors: _errors} <- Validator.validate(unit, :update, project),
+         {:ok, unit} <- Validator.validate(unit, :update, project),
          # todo better valid / error handling
          {:ok, unit} <- run_persistence_hook(arke, unit, :update, :before, current_unit),
          {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :before),
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index 3b4b85d..e96e987 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -47,13 +47,21 @@ defmodule Arke.Validator do
           func_return()
   def validate(unit, persistence_fn, project \\ :arke_system)
 
-  def validate(%Unit{} = unit, persistence_fn, project),
-    do: validate([unit], persistence_fn, project)
+  def validate(%Unit{} = unit, persistence_fn, project) do
+    case validate([unit], persistence_fn, project) do
+      %{valid: [unit], errors: []} -> {:ok, unit}
+      %{errors: errors} -> Error.create(:parameter_validation, errors)
+    end
+  end
 
   def validate([], _persistence_fn, _project),
     do: %{valid: [], errors: []}
 
-  def validate([%Unit{arke_id: arke_id} | _] = unit_list, persistence_fn, project) do
+  def validate(
+        [%Unit{arke_id: arke_id} | _] = unit_list,
+        persistence_fn,
+        project
+      ) do
     %{data: arke_data} = arke = ArkeManager.get(arke_id, project)
 
     parameter_list =

From b9e535974eb64ee99f08462b767a453ca93fddf8 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <81801314+ilyichv@users.noreply.github.com>
Date: Mon, 10 Feb 2025 18:00:14 +0100
Subject: [PATCH 19/28] wip: bulk operations on topology (#119)

---
 lib/arke/group.ex         |  26 +++-
 lib/arke/query_manager.ex | 277 +++++++++++++++++++++++++-------------
 2 files changed, 206 insertions(+), 97 deletions(-)

diff --git a/lib/arke/group.ex b/lib/arke/group.ex
index b7dd991..a80acc8 100644
--- a/lib/arke/group.ex
+++ b/lib/arke/group.ex
@@ -26,8 +26,11 @@ defmodule Arke.System.Group do
 
       #      @before_compile unquote(__MODULE__)
 
-      def group_from_attr(), do: Keyword.get(__MODULE__.__info__(:attributes), :group, []) |> List.first()
-      def is_group?(), do: Keyword.get(__MODULE__.__info__(:attributes), :system_group, []) |> List.first()
+      def group_from_attr(),
+        do: Keyword.get(__MODULE__.__info__(:attributes), :group, []) |> List.first()
+
+      def is_group?(),
+        do: Keyword.get(__MODULE__.__info__(:attributes), :system_group, []) |> List.first()
 
       def on_unit_load(arke, data, _persistence_fn), do: {:ok, data}
       def before_unit_load(_arke, data, _persistence_fn), do: {:ok, data}
@@ -41,6 +44,13 @@ defmodule Arke.System.Group do
       def on_unit_delete(_arke, unit), do: {:ok, unit}
       def before_unit_delete(_arke, unit), do: {:ok, unit}
 
+      defp before_unit_bulk_create(arke, valid, errors), do: {:ok, valid, errors}
+      defp on_unit_bulk_create(arke, valid, errors), do: {:ok, valid, errors}
+      defp before_unit_bulk_update(arke, valid, errors), do: {:ok, valid, errors}
+      defp on_unit_bulk_update(arke, valid, errors), do: {:ok, valid, errors}
+      defp before_unit_bulk_delete(arke, valid, errors), do: {:ok, valid, errors}
+      defp on_unit_bulk_delete(arke, valid, errors), do: {:ok, valid, errors}
+
       defoverridable on_unit_load: 3,
                      before_unit_load: 3,
                      on_unit_validate: 2,
@@ -51,7 +61,13 @@ defmodule Arke.System.Group do
                      on_unit_update: 2,
                      before_unit_update: 2,
                      on_unit_delete: 2,
-                     before_unit_delete: 2
+                     before_unit_delete: 2,
+                     before_unit_bulk_create: 3,
+                     on_unit_bulk_create: 3,
+                     before_unit_bulk_update: 3,
+                     on_unit_bulk_update: 3,
+                     before_unit_bulk_delete: 3,
+                     on_unit_bulk_delete: 3
     end
   end
 
@@ -96,7 +112,7 @@ defmodule Arke.System.Group do
       unquote(block)
 
       @group %{
-        id: id,
+        id: id
       }
     end
   end
@@ -114,7 +130,7 @@ defmodule Arke.System.Group do
   See example above `arke/2`
 
   """
-  @spec parameter(id :: atom(), type:: atom(), opts :: list()) :: Macro.t()
+  @spec parameter(id :: atom(), type :: atom(), opts :: list()) :: Macro.t()
   defmacro parameter(id, type, opts \\ []) do
     # parameter_dict = Arke.System.BaseParameter.parameter_options(opts, id, type)
     quote bind_quoted: [id: id, type: type, opts: opts] do
diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 44e3c40..2668309 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -148,10 +148,12 @@ defmodule Arke.QueryManager do
     with %Unit{} = unit <- Unit.load(arke, args, :create),
          {:ok, unit} <- Validator.validate(unit, :create, project),
          {:ok, unit} <- run_persistence_hook(arke, unit, :create, :before),
-         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :create, :before),
+         {:ok, unit} <- run_group_call_func(arke, unit, :create, :before),
+         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
          {:ok, unit} <- persistence_fn.(project, unit, []),
          {:ok, unit} <- run_persistence_hook(arke, unit, :create, :after),
-         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :create, :after) do
+         {:ok, unit} <- run_group_call_func(arke, unit, :create, :after),
+         {:ok, unit} <- handle_link_parameters(unit, %{}) do
       {:ok, unit}
     else
       {:error, errors} -> {:error, errors}
@@ -196,83 +198,31 @@ defmodule Arke.QueryManager do
     end)
   end
 
-  defp handle_link_parameters_unit(%{id: :arke_link} = _, unit), do: {:ok, unit}
-  defp handle_link_parameters_unit(%{id: :parameter_value} = _, unit), do: {:ok, unit}
-
-  defp handle_link_parameters_unit(
-         %{data: parameters} = arke,
-         %{metadata: %{project: project}} = unit
-       ) do
-    {errors, link_units} =
-      Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
-      |> Enum.reduce({[], []}, fn p, {errors, link_units} ->
-        arke = ArkeManager.get(String.to_existing_atom(p.data.arke_or_group_id), project)
-
-        case handle_create_on_link_parameters_unit(
-               project,
-               unit,
-               p,
-               arke,
-               Unit.get_value(unit, p.id)
-             ) do
-          {:ok, parameter, %Unit{} = link_unit} -> {errors, [{parameter, link_unit} | link_units]}
-          {:ok, parameter, link_unit} -> {errors, link_units}
-          {:error, e} -> {[e | errors], link_units}
-        end
-      end)
-
-    case length(errors) > 0 do
-      true ->
-        Enum.map(link_units, fn {p, u} ->
-          delete(project, u)
-        end)
-
-        {:error, errors}
-
-      false ->
-        args =
-          Enum.reduce(link_units, %{}, fn {p, u}, args ->
-            Map.put(args, p.id, Atom.to_string(u.id))
-          end)
-
-        {:ok, Unit.update(unit, args)}
+  defp handle_group_call_func(arke, unit, func) do
+    case GroupManager.get_groups_by_arke(arke)
+         |> Enum.reduce_while(unit, fn group, new_unit ->
+           with {:ok, new_unit} <- GroupManager.call_func(group, func, [arke, new_unit]),
+                do: {:cont, new_unit},
+                else: ({:error, errors} -> {:halt, {:error, errors}})
+         end) do
+      {:error, errors} -> {:error, errors}
+      unit -> {:ok, unit}
     end
   end
 
-  defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
-       when is_nil(value),
-       do: {:ok, parameter, value}
-
-  defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
-       when is_binary(value),
-       do: {:ok, parameter, value}
-
-  defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
-       when is_map(value) do
-    value = Map.put(value, :runtime_data, %{link: unit, link_parameter: parameter})
-
-    case create(project, arke, value) do
-      {:ok, unit} -> {:ok, parameter, unit}
-      {:error, error} -> {:error, error}
+  defp handle_group_call_func(valid, errors, arke, func) do
+    case GroupManager.get_groups_by_arke(arke)
+         |> Enum.reduce_while({:ok, valid, errors}, fn group, {:ok, valid, errors} ->
+           with {:ok, valid, errors} <-
+                  GroupManager.call_func(group, func, [arke, valid, errors]),
+                do: {:cont, {:ok, valid, errors}},
+                else: ({:error, errors} -> {:halt, {:error, errors}})
+         end) do
+      {:error, errors} -> {:error, errors}
+      {:ok, valid, errors} -> %{valid: valid, errors: errors}
     end
   end
 
-  defp handle_create_on_link_parameters_unit(_, _, parameter, _, value),
-    do: {:ok, parameter, value}
-
-  def handle_group_call_func(arke, unit, func) do
-    GroupManager.get_groups_by_arke(arke)
-    |> Enum.reduce_while(unit, fn group, new_unit ->
-      with {:ok, new_unit} <- GroupManager.call_func(group, func, [arke, new_unit]),
-           do: {:cont, new_unit},
-           else: ({:error, errors} -> {:halt, {:error, errors}})
-    end)
-    |> check_group_manager_functions_errors()
-  end
-
-  def check_group_manager_functions_errors({:error, errors} = _), do: {:error, errors}
-  def check_group_manager_functions_errors(unit), do: {:ok, unit}
-
   @doc """
   Function to update an element
 
@@ -323,7 +273,10 @@ defmodule Arke.QueryManager do
 
     with %{valid: valid, errors: errors} <- prepare_update_bulk_units(arke, unit_list, data),
          %{valid: valid, errors: errors} <- Validator.validate(valid, :update, project),
-         %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :update, :before),
+         %{valid: valid, errors: errors} <-
+           run_group_call_func(valid, errors, arke, :update, :before),
+         %{valid: valid, errors: errors} <-
+           handle_link_parameters_v2(valid, errors, arke, :update, unit_list),
          {:ok, updated_count, valid, persistence_errors} <-
            persistence_fn.(project, valid, bulk: true),
          %{valid: valid, errors: errors} <-
@@ -823,6 +776,70 @@ defmodule Arke.QueryManager do
     ArkeManager.get_parameter(arke, project, key)
   end
 
+  defp handle_link_parameters_unit(%{id: :arke_link} = _, unit), do: {:ok, unit}
+  defp handle_link_parameters_unit(%{id: :parameter_value} = _, unit), do: {:ok, unit}
+
+  defp handle_link_parameters_unit(
+         %{data: parameters} = arke,
+         %{metadata: %{project: project}} = unit
+       ) do
+    {errors, link_units} =
+      Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
+      |> Enum.reduce({[], []}, fn p, {errors, link_units} ->
+        arke = ArkeManager.get(String.to_existing_atom(p.data.arke_or_group_id), project)
+
+        case handle_create_on_link_parameters_unit(
+               project,
+               unit,
+               p,
+               arke,
+               Unit.get_value(unit, p.id)
+             ) do
+          {:ok, parameter, %Unit{} = link_unit} -> {errors, [{parameter, link_unit} | link_units]}
+          {:ok, parameter, link_unit} -> {errors, link_units}
+          {:error, e} -> {[e | errors], link_units}
+        end
+      end)
+
+    case length(errors) > 0 do
+      true ->
+        Enum.map(link_units, fn {p, u} ->
+          delete(project, u)
+        end)
+
+        {:error, errors}
+
+      false ->
+        args =
+          Enum.reduce(link_units, %{}, fn {p, u}, args ->
+            Map.put(args, p.id, Atom.to_string(u.id))
+          end)
+
+        {:ok, Unit.update(unit, args)}
+    end
+  end
+
+  defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
+       when is_nil(value),
+       do: {:ok, parameter, value}
+
+  defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
+       when is_binary(value),
+       do: {:ok, parameter, value}
+
+  defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
+       when is_map(value) do
+    value = Map.put(value, :runtime_data, %{link: unit, link_parameter: parameter})
+
+    case create(project, arke, value) do
+      {:ok, unit} -> {:ok, parameter, unit}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  defp handle_create_on_link_parameters_unit(_, _, parameter, _, value),
+    do: {:ok, parameter, value}
+
   defp handle_link_parameters(
          %{arke_id: arke_id, metadata: %{project: project}, data: new_data, id: id} = unit,
          old_data
@@ -934,6 +951,46 @@ defmodule Arke.QueryManager do
     })
   end
 
+  defp handle_link_parameters_v2(valid, errors, arke, :update, current_units) do
+    to_check =
+      Enum.filter(ArkeManager.get_parameters(arke), fn p ->
+        p.arke_id == :link
+      end)
+
+    {to_create, to_delete} =
+      Enum.reduce(valid, {[], []}, fn unit, {create_acc, delete_acc} ->
+        current_unit = Enum.find(current_units, &(&1.id == unit.id))
+
+        Enum.reduce(to_check, {create_acc, delete_acc}, fn p, {create_acc, delete_acc} ->
+          direction = Map.get(p.data, :direction)
+          next_value = normalize_value(Unit.get_value(unit, p.id))
+          previous_value = normalize_value(Unit.get_value(current_unit, p.id))
+
+          IO.inspect({unit.id, previous_value, next_value, p.id})
+
+          case {next_value != previous_value, previous_value} do
+            {true, nil} ->
+              {create_acc, [prepare_link(direction, unit, previous_value) | delete_acc]}
+
+            {true, _} ->
+              {[prepare_link(direction, unit, next_value) | create_acc],
+               [prepare_link(direction, unit, previous_value) | delete_acc]}
+
+            _ ->
+              {create_acc, delete_acc}
+          end
+        end)
+      end)
+
+    IO.inspect(to_create)
+  end
+
+  defp prepare_link("child", starting_unit, linked_unit_id),
+    do: %{parent_id: starting_unit.id, child_id: linked_unit_id}
+
+  defp prepare_link("parent", starting_unit, linked_unit_id),
+    do: %{parent_id: linked_unit_id, child_id: starting_unit.id}
+
   # Function to get only the parameter id from `handle_link_parameter`
   defp normalize_value(nil), do: nil
 
@@ -946,7 +1003,7 @@ defmodule Arke.QueryManager do
   defp process_bulk(valid, errors, arke, :update, :after, current_units, data) do
     data_by_id = Map.new(data, fn d -> {Map.get(d, "id"), d} end)
 
-    case run_bulk_persistence_hook(valid, errors, arke, :update, :after, current_units) do
+    case run_persistence_hook(valid, errors, arke, :update, :after, current_units) do
       {:ok, valid, errors} ->
         Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
           updated_data = Map.get(data_by_id, to_string(unit.id))
@@ -966,7 +1023,7 @@ defmodule Arke.QueryManager do
   end
 
   defp process_bulk(valid, errors, arke, persistence_fn, phase) do
-    case run_bulk_persistence_hook(valid, errors, arke, persistence_fn, phase) do
+    case run_persistence_hook(valid, errors, arke, persistence_fn, phase) do
       {:ok, valid, errors} ->
         Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
           case run_group_and_link_hooks(arke, unit, persistence_fn, phase) do
@@ -983,42 +1040,78 @@ defmodule Arke.QueryManager do
     end
   end
 
-  defp run_bulk_persistence_hook(valid, errors, arke, :create, :before),
+  defp run_persistence_hook(valid, errors, arke, :create, :before),
     do: ArkeManager.call_func(arke, :before_bulk_create, [arke, valid, errors])
 
-  defp run_bulk_persistence_hook(valid, errors, arke, :create, :after),
-    do: ArkeManager.call_func(arke, :on_bulk_create, [arke, valid, errors])
-
-  defp run_bulk_persistence_hook(valid, errors, arke, :update, :before),
-    do: ArkeManager.call_func(arke, :before_bulk_update, [arke, valid, errors])
-
-  defp run_bulk_persistence_hook(valid, errors, arke, :update, :after, current_units),
-    do: ArkeManager.call_func(arke, :on_bulk_update, [arke, current_units, valid, errors])
-
-  defp run_bulk_persistence_hook(valid, errors, arke, :delete, :before),
-    do: ArkeManager.call_func(arke, :before_bulk_delete, [arke, valid, errors])
-
-  defp run_bulk_persistence_hook(valid, errors, arke, :delete, :after),
-    do: ArkeManager.call_func(arke, :on_bulk_delete, [arke, valid, errors])
-
   defp run_persistence_hook(arke, unit, :create, :before),
     do: ArkeManager.call_func(arke, :before_create, [arke, unit])
 
   defp run_persistence_hook(arke, unit, :create, :after),
     do: ArkeManager.call_func(arke, :on_create, [arke, unit])
 
+  defp run_persistence_hook(valid, errors, arke, :create, :after),
+    do: ArkeManager.call_func(arke, :on_bulk_create, [arke, valid, errors])
+
+  defp run_persistence_hook(valid, errors, arke, :update, :before),
+    do: ArkeManager.call_func(arke, :before_bulk_update, [arke, valid, errors])
+
   defp run_persistence_hook(arke, unit, :update, :before, current_unit),
     do: ArkeManager.call_func(arke, :before_update, [arke, current_unit, unit])
 
+  defp run_persistence_hook(valid, errors, arke, :update, :after, current_units),
+    do: ArkeManager.call_func(arke, :on_bulk_update, [arke, current_units, valid, errors])
+
   defp run_persistence_hook(arke, unit, :update, :after, current_unit),
     do: ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit])
 
+  defp run_persistence_hook(valid, errors, arke, :delete, :before),
+    do: ArkeManager.call_func(arke, :before_bulk_delete, [arke, valid, errors])
+
   defp run_persistence_hook(arke, unit, :delete, :before),
     do: ArkeManager.call_func(arke, :before_delete, [arke, unit])
 
+  defp run_persistence_hook(valid, errors, arke, :delete, :after),
+    do: ArkeManager.call_func(arke, :on_bulk_delete, [arke, valid, errors])
+
   defp run_persistence_hook(arke, unit, :delete, :after),
     do: ArkeManager.call_func(arke, :on_delete, [arke, unit])
 
+  defp run_group_call_func(arke, unit, :create, :before),
+    do: handle_group_call_func(arke, unit, :before_unit_create)
+
+  defp run_group_call_func(valid, errors, arke, :create, :before),
+    do: handle_group_call_func(valid, errors, arke, :before_unit_bulk_create)
+
+  defp run_group_call_func(arke, unit, :create, :after),
+    do: handle_group_call_func(arke, unit, :on_unit_create)
+
+  defp run_group_call_func(valid, errors, arke, :create, :after),
+    do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_create)
+
+  defp run_group_call_func(arke, unit, :update, :before),
+    do: handle_group_call_func(arke, unit, :before_unit_update)
+
+  defp run_group_call_func(valid, errors, arke, :update, :before),
+    do: handle_group_call_func(valid, errors, arke, :before_unit_bulk_create)
+
+  defp run_group_call_func(arke, unit, :update, :after),
+    do: handle_group_call_func(arke, unit, :on_unit_update)
+
+  defp run_group_call_func(valid, errors, arke, :update, :after),
+    do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_update)
+
+  defp run_group_call_func(arke, unit, :delete, :before),
+    do: handle_group_call_func(arke, unit, :before_unit_delete)
+
+  defp run_group_call_func(valid, errors, arke, :delete, :before),
+    do: handle_group_call_func(valid, errors, arke, :before_unit_bulk_delete)
+
+  defp run_group_call_func(arke, unit, :delete, :after),
+    do: handle_group_call_func(arke, unit, :on_unit_delete)
+
+  defp run_group_call_func(valid, errors, arke, :delete, :after),
+    do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_delete)
+
   defp run_group_and_link_hooks(arke, unit, :create, :before) do
     with {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
          {:ok, unit} <- handle_link_parameters_unit(arke, unit),

From 67c6cf6761a030f70673af585fe2e60ecf78bdca Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <81801314+ilyichv@users.noreply.github.com>
Date: Mon, 10 Feb 2025 18:04:01 +0100
Subject: [PATCH 20/28] wip: bulk links (#118)

---
 lib/arke/core/link.ex     |  20 ++++
 lib/arke/link_manager.ex  | 186 ++++++++++++++------------------------
 lib/arke/query_manager.ex | 154 ++++++++++++++++++++++++++++---
 3 files changed, 229 insertions(+), 131 deletions(-)

diff --git a/lib/arke/core/link.ex b/lib/arke/core/link.ex
index f984f6f..2e34473 100644
--- a/lib/arke/core/link.ex
+++ b/lib/arke/core/link.ex
@@ -22,6 +22,26 @@ defmodule Arke.Core.Link do
   arke id: :arke_link do
   end
 
+  @enforce_keys [:parent_id, :child_id, :type]
+  defstruct [:parent_id, :child_id, :type, metadata: %{}]
+
+  def new(parent_id, child_id, type, metadata \\ %{}) do
+    __struct__(parent_id: parent_id, child_id: child_id, type: type, metadata: metadata)
+  end
+
+  def load(arke_link, %Arke.Core.Unit{arke_id: :arke_link} = unit) do
+    new(unit.data.parent_id, unit.data.child_id, unit.data.type, unit.metadata)
+  end
+
+  def load(arke_link, opts) do
+    {parent_id, opts} = Map.pop(opts, :parent_id, nil)
+    {child_id, opts} = Map.pop(opts, :child_id, nil)
+    {type, opts} = Map.pop(opts, :type, nil)
+    {metadata, opts} = Map.pop(opts, :metadata, arke_link.metadata)
+
+    new(parent_id, child_id, type, metadata)
+  end
+
   def on_create(
         _,
         %{
diff --git a/lib/arke/link_manager.ex b/lib/arke/link_manager.ex
index 9da79ce..551839a 100644
--- a/lib/arke/link_manager.ex
+++ b/lib/arke/link_manager.ex
@@ -14,48 +14,29 @@
 
 defmodule Arke.LinkManager do
   @moduledoc false
-  @record_fields [:id, :data, :metadata, :inserted_at, :updated_at]
 
   alias Arke.Boundary.ArkeManager
   alias Arke.Utils.ErrorGenerator, as: Error
   alias Arke.QueryManager
   alias Arke.Core.Unit
   alias Arke.Core.Query
+  alias Arke.Core.Link
 
-  defmodule LinkData do
-    @enforce_keys [:parent_id, :child_id, :type]
-    defstruct [:parent_id, :child_id, :type, metadata: %{}]
-
-    def to_map(link_data) do
-      Map.take(link_data, [:parent_id, :child_id, :type, :metadata])
-    end
-  end
-
-  def add_node(project, %Unit{} = parent, %Unit{} = child, type \\ "link", metadata \\ %{}) do
+  def add_node(project, parent, child, type \\ "link", metadata \\ %{}) do
     arke_link = ArkeManager.get(:arke_link, project)
-
-    case check_link(project, parent, child, arke_link) do
-      {_, nil} ->
-        QueryManager.create(project, arke_link,
-          parent_id: Atom.to_string(parent.id),
-          child_id: Atom.to_string(child.id),
-          type: type,
-          metadata: metadata
-        )
-
-      {:ok, _} ->
-        Error.create(:link, "link already exists")
-    end
-  end
-
-  def add_node(project, parent, child, type, metadata)
-      when is_binary(parent) and is_binary(child) do
-    with %Unit{} = unit_parent <- QueryManager.get_by(id: parent, project: project),
-         %Unit{} = unit_child <- QueryManager.get_by(id: child, project: project) do
-      add_node(project, unit_parent, unit_child, type, metadata)
-    else
-      _ -> Error.create(:link, "parent: `#{parent}` or child: `#{child}` not found")
-    end
+    link = %{"parent" => parent, "child" => child, "type" => type, "metadata" => metadata}
+
+    with {valid, _} <-
+           validate_links(project, arke_link, [link]),
+         {[link], _} <- validate_existing(project, arke_link, valid, []),
+         do:
+           QueryManager.create(project, arke_link,
+             parent_id: link.parent_id,
+             child_id: link.child_id,
+             type: link.type,
+             metadata: link.metadata
+           ),
+         else: (_ -> Error.create(:link, "link already exists"))
   end
 
   def add_node(_project, _parent, _child, _type, _metadata),
@@ -64,7 +45,7 @@ defmodule Arke.LinkManager do
   def add_node_bulk(project, link_list) do
     arke_link = ArkeManager.get(:arke_link, project)
 
-    with {valid, errors} <- validate_units(project, arke_link, link_list),
+    with {valid, errors} <- validate_links(project, arke_link, link_list),
          {valid, errors} <- validate_existing(project, arke_link, valid, errors),
          {:ok, inserted_count, valid, insert_errors} <-
            QueryManager.create_bulk(project, arke_link, prepare_link_data(valid), []) do
@@ -74,41 +55,40 @@ defmodule Arke.LinkManager do
     end
   end
 
-  def update_node(project, %Unit{} = parent, %Unit{} = child, type, metadata) do
+  def update_node(project, parent, child, type, metadata) do
     arke_link = ArkeManager.get(:arke_link, :arke_system)
+    link = %{"parent" => parent, "child" => child, "type" => type, "metadata" => metadata}
 
-    case check_link(project, parent, child, arke_link) do
-      {:error, _} -> Error.create(:link, "link not found")
-      {:ok, link} -> QueryManager.update(link, metadata: metadata, type: type)
+    with {[valid_link], _} <- validate_links(project, arke_link, [link]),
+         [existing_link] <- existing_links(project, arke_link, [valid_link]) do
+      QueryManager.update(existing_link, metadata: metadata)
+    else
+      _ -> Error.create(:link, "link not found")
     end
   end
 
-  def update_node(project, parent, child, type, metadata)
-      when is_binary(parent) and is_binary(child) do
-    unit_parent = QueryManager.get_by(id: parent, project: project)
-    unit_child = QueryManager.get_by(id: child, project: project)
-
-    update_node(project, unit_parent, unit_child, type, metadata)
-  end
-
   def update_node(_project, _parent, _child, _type, _metadata),
     do: Error.create(:link, "invalid parameters")
 
-  def delete_node(project, %Unit{} = parent, %Unit{} = child, type, metadata \\ %{}) do
-    arke_link = ArkeManager.get(:arke_link, :arke_system)
+  def update_node_bulk(project, link_list) do
+    arke_link = ArkeManager.get(:arke_link, project)
 
-    case check_link(project, parent, child, arke_link) do
-      {:error, _} -> Error.create(:link, "link not found")
-      {:ok, link} -> QueryManager.delete(project, link)
-    end
+    {valid, errors} = validate_links(project, arke_link, link_list)
+    existing = existing_links(project, arke_link, valid)
+
+    QueryManager.update_bulk(project, arke_link, existing, prepare_link_data(valid))
   end
 
-  def delete_node(project, parent, child, type, metadata)
-      when is_binary(parent) and is_binary(child) do
-    unit_parent = QueryManager.get_by(id: parent, project: project)
-    unit_child = QueryManager.get_by(id: child, project: project)
+  def delete_node(project, parent, child, type, metadata \\ %{}) do
+    arke_link = ArkeManager.get(:arke_link, :arke_system)
+    link = %{"parent" => parent, "child" => child, "type" => type, "metadata" => metadata}
 
-    delete_node(project, unit_parent, unit_child, type, metadata)
+    with {[valid_link], _} <- validate_links(project, arke_link, [link]),
+         [existing_link] <- existing_links(project, arke_link, [valid_link]) do
+      QueryManager.delete(project, existing_link)
+    else
+      _ -> Error.create(:link, "link not found")
+    end
   end
 
   def delete_node(_project, _parent, _child, _type, _metadata),
@@ -116,60 +96,35 @@ defmodule Arke.LinkManager do
 
   def delete_node_bulk(project, link_list) do
     arke_link = ArkeManager.get(:arke_link, :arke_system)
-    {valid, _} = validate_units(project, arke_link, link_list)
+    {valid, _} = validate_links(project, arke_link, link_list)
 
     QueryManager.delete_bulk(project, existing_links(project, arke_link, valid))
   end
 
-  defp check_link(project, parent, child, arke_link) do
-    with %Arke.Core.Unit{} = link <-
-           Arke.QueryManager.query(project: project, arke: arke_link)
-           |> Arke.QueryManager.filter(:parent_id, :eq, Atom.to_string(parent.id), false)
-           |> Arke.QueryManager.filter(:child_id, :eq, Atom.to_string(child.id), false)
-           |> Arke.QueryManager.one(),
-         do: {:ok, link},
-         else: (_ -> {:error, nil})
-  end
-
-  @spec validate_units(atom(), Arke.t(), list()) :: {list(LinkData.t()), list(map())}
-  defp validate_units(project, arke_link, link_list) do
-    ids =
-      Enum.flat_map(link_list, fn link ->
-        case {link["parent_id"], link["child_id"]} do
-          {parent, child} when is_binary(parent) and is_binary(child) ->
-            [parent, child]
-
-          {parent, _} when is_binary(parent) ->
-            [parent]
+  @spec validate_links(atom(), Arke.t(), list()) :: {list(Link.t()), list(map())}
+  defp validate_links(project, arke_link, [%Link{} | _] = link_list),
+    do: {link_list, []}
 
-          {_, child} when is_binary(child) ->
-            [child]
-
-          _ ->
-            []
-        end
+  defp validate_links(project, arke_link, [%Unit{} | _] = unit_list) do
+    links =
+      Enum.map(unit_list, fn unit ->
+        Link.load(project, unit)
       end)
 
-    unit_list = QueryManager.filter_by(id__in: ids, project: project)
-    unit_map = Map.new(unit_list, fn unit -> {Atom.to_string(unit.id), unit} end)
+    {links, []}
+  end
 
+  defp validate_links(project, arke_link, link_list) do
     Enum.reduce(link_list, {[], []}, fn link, {valid, errors} ->
-      parent =
-        case link["parent_id"] do
-          %Unit{} = p -> p
-          id -> Map.get(unit_map, id)
-        end
-
-      child =
-        case link["child_id"] do
-          %Unit{} = c -> c
-          id -> Map.get(unit_map, id)
-        end
-
+      parent = link["parent"]
+      child = link["child"]
       type = Map.get(link, "type", "link")
       metadata = Map.get(link, "metadata", %{})
 
       case {parent, child} do
+        {parent, child} when is_binary(parent) and is_binary(child) ->
+          {[Link.new(parent, child, type, metadata) | valid], errors}
+
         {nil, nil} ->
           {valid, [%{link: link, error: "invalid parent and child"} | errors]}
 
@@ -179,22 +134,17 @@ defmodule Arke.LinkManager do
         {_, nil} ->
           {valid, [%{link: link, error: "invalid child"} | errors]}
 
-        {p, c} ->
-          {[
-             %LinkData{
-               parent_id: Atom.to_string(p.id),
-               child_id: Atom.to_string(c.id),
-               type: type,
-               metadata: metadata
-             }
-             | valid
-           ], errors}
+        _ ->
+          {valid, [%{link: link, error: "invalid parent and child format"} | errors]}
       end
     end)
   end
 
-  @spec existing_links(project :: atom(), arke_link :: Arke.t(), link_list :: list(LinkData.t())) ::
+  @spec existing_links(project :: atom(), arke_link :: Arke.t(), link_list :: list(Link.t())) ::
           list(Arke.Core.Unit.t())
+
+  defp existing_links(_project, _arke_link, []), do: []
+
   defp existing_links(project, arke_link, link_list) do
     parameters = ArkeManager.get_parameters(arke_link)
     parent_id = Enum.find(parameters, fn p -> p.id == :parent_id end)
@@ -218,8 +168,8 @@ defmodule Arke.LinkManager do
     |> Arke.QueryManager.all()
   end
 
-  @spec validate_existing(atom(), Arke.t(), list(LinkData.t()), list()) ::
-          {list(LinkData.t()), list()}
+  @spec validate_existing(atom(), Arke.t(), list(Unit.t()), list()) ::
+          {list(Link.t()), list()}
   defp validate_existing(project, arke_link, link_list, errors) do
     existing =
       existing_links(project, arke_link, link_list)
@@ -229,14 +179,18 @@ defmodule Arke.LinkManager do
 
     Enum.reduce(link_list, {[], errors}, fn link, {valid_acc, errors_acc} ->
       case MapSet.member?(existing, {link.parent_id, link.child_id, link.type}) do
-        true -> {valid_acc, [LinkData.to_map(link) | errors_acc]}
-        false -> {[LinkData.to_map(link) | valid_acc], errors_acc}
+        true -> {valid_acc, [data_to_map(link) | errors_acc]}
+        false -> {[data_to_map(link) | valid_acc], errors_acc}
       end
     end)
   end
 
-  @spec prepare_link_data(list(LinkData.t())) :: list(map())
+  @spec prepare_link_data(list(Link.t())) :: list(map())
   defp prepare_link_data(link_list) do
-    Enum.map(link_list, &LinkData.to_map/1)
+    Enum.map(link_list, &data_to_map/1)
+  end
+
+  defp data_to_map(link) do
+    Map.take(link, [:parent_id, :child_id, :type, :metadata])
   end
 end
diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 2668309..5eb31f2 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -44,7 +44,7 @@ defmodule Arke.QueryManager do
   alias Arke.Utils.DatetimeHandler, as: DatetimeHandler
   alias Arke.Errors.ArkeError
   alias Arke.Utils.ErrorGenerator, as: Error
-  alias Arke.Core.{Arke, Unit, Query, Parameter}
+  alias Arke.Core.{Arke, Unit, Query, Parameter, Link}
 
   @persistence Application.get_env(:arke, :persistence)
   @record_fields [:id, :data, :metadata, :inserted_at, :updated_at]
@@ -151,6 +151,7 @@ defmodule Arke.QueryManager do
          {:ok, unit} <- run_group_call_func(arke, unit, :create, :before),
          {:ok, unit} <- handle_link_parameters_unit(arke, unit),
          {:ok, unit} <- persistence_fn.(project, unit, []),
+         {:ok, unit} <- handle_link_parameters_single(arke, unit, :create, :after),
          {:ok, unit} <- run_persistence_hook(arke, unit, :create, :after),
          {:ok, unit} <- run_group_call_func(arke, unit, :create, :after),
          {:ok, unit} <- handle_link_parameters(unit, %{}) do
@@ -295,6 +296,34 @@ defmodule Arke.QueryManager do
     end
   end
 
+  # todo: enhance for different arke types that acts as tables
+  defp prepare_update_bulk_units(%{id: :arke_link} = arke, unit_list, data) do
+    data_map =
+      data
+      |> Enum.map(fn d ->
+        key = {d.parent_id, d.child_id, d.type}
+        {key, d}
+      end)
+      |> Map.new()
+
+    updated_at = DatetimeHandler.now(:datetime)
+
+    Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
+      key = {unit.data.parent_id, unit.data.child_id, unit.data.type}
+
+      case data_map[key] do
+        nil ->
+          acc
+
+        item ->
+          case Unit.update(unit, data_as_klist(item)) do
+            %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
+            {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
+          end
+      end
+    end)
+  end
+
   defp prepare_update_bulk_units(arke, unit_list, data) do
     data_map =
       data
@@ -321,6 +350,7 @@ defmodule Arke.QueryManager do
     updated_at = DatetimeHandler.now(:datetime)
     {:ok, Unit.update(unit, updated_at: updated_at)}
   end
+
   @doc """
   Function to delete a given unit
   ## Parameters
@@ -856,6 +886,83 @@ defmodule Arke.QueryManager do
     {:ok, unit}
   end
 
+  defp handle_link_parameters_v2([], errors, _arke, _old_data), do: {:ok, [], errors}
+
+  defp handle_link_parameters_v2(
+         [%{metadata: %{project: project}} | _] = valid,
+         errors,
+         arke,
+         old_data \\ %{}
+       ) do
+    link_parameters =
+      Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
+
+    {to_add, to_delete} =
+      Enum.reduce(link_parameters, {[], []}, fn p, {to_add_acc, to_delete_acc} ->
+        Enum.reduce(valid, {to_add_acc, to_delete_acc}, fn unit, {add_acc, del_acc} ->
+          old_value = get_in(old_data, [unit.id, p.id])
+          new_value = get_in(unit.data, [p.id])
+          {new_add, new_del} = build_link_entries(p, unit, old_value, new_value)
+          {add_acc ++ new_add, del_acc ++ new_del}
+        end)
+      end)
+
+    with {:ok, add_count, add_valid, add_errors} <- LinkManager.add_node_bulk(project, to_add),
+         {:ok, del_valid, del_errors} <-
+           LinkManager.delete_node_bulk(project, to_delete) do
+      {:ok, valid ++ add_valid ++ del_valid, errors ++ add_errors ++ del_errors}
+    else
+      {:error, err} -> {:error, err}
+    end
+  end
+
+  defp build_link_entries(%{data: %{multiple: true}} = parameter, unit, old_value, new_value) do
+    Enum.reduce(old_value, {[], []}, fn value, {add_acc, del_acc} ->
+      {[update_link_entry(parameter, unit, value) | add_acc],
+       [update_link_entry(parameter, unit, value) | del_acc]}
+    end)
+  end
+
+  defp build_link_entries(_parameter, _unit, old_value, new_value) when old_value == new_value,
+    do: {[], []}
+
+  defp build_link_entries(parameter, unit, old_value, new_value) when is_nil(new_value) do
+    {[], [update_link_entry(parameter, unit, old_value)]}
+  end
+
+  defp build_link_entries(parameter, unit, old_value, new_value) when is_nil(old_value) do
+    {[update_link_entry(parameter, unit, new_value)], []}
+  end
+
+  defp build_link_entries(parameter, unit, old_value, new_value) do
+    {[update_link_entry(parameter, unit, new_value)],
+     [update_link_entry(parameter, unit, old_value)]}
+  end
+
+  defp update_link_entry(
+         %{data: %{connection_type: type, direction: "child"}} = parameter,
+         unit,
+         new_value
+       ) do
+    Link.new(
+      normalize_value(unit.id),
+      normalize_value(new_value),
+      type
+    )
+  end
+
+  defp update_link_entry(
+         %{data: %{connection_type: type, direction: "parent"}} = parameter,
+         unit,
+         new_value
+       ) do
+    Link.new(
+      normalize_value(new_value),
+      normalize_value(unit.id),
+      type
+    )
+  end
+
   defp handle_link_parameter(_, nil, _, _), do: nil
 
   defp handle_link_parameter(unit, %{data: %{multiple: false}} = parameter, old_value, new_value) do
@@ -1023,20 +1130,21 @@ defmodule Arke.QueryManager do
   end
 
   defp process_bulk(valid, errors, arke, persistence_fn, phase) do
-    case run_persistence_hook(valid, errors, arke, persistence_fn, phase) do
-      {:ok, valid, errors} ->
-        Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-          case run_group_and_link_hooks(arke, unit, persistence_fn, phase) do
-            {:ok, unit} ->
-              %{acc | valid: [unit | acc.valid]}
-
-            {:error, e} ->
-              %{acc | errors: [{unit, e} | acc.errors]}
-          end
-        end)
-
-      {:error, errors} ->
-        {:error, errors}
+    with {:ok, valid, errors} <-
+           run_persistence_hook(valid, errors, arke, persistence_fn, phase),
+         {:ok, valid, errors} <-
+           handle_bulk_link_parameters(valid, errors, arke, persistence_fn, phase) do
+      Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+        case run_group_and_link_hooks(arke, unit, persistence_fn, phase) do
+          {:ok, unit} ->
+            %{acc | valid: [unit | acc.valid]}
+
+          {:error, e} ->
+            %{acc | errors: [{unit, e} | acc.errors]}
+        end
+      end)
+    else
+      {:error, errors} -> {:error, errors}
     end
   end
 
@@ -1112,6 +1220,22 @@ defmodule Arke.QueryManager do
   defp run_group_call_func(valid, errors, arke, :delete, :after),
     do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_delete)
 
+  def handle_bulk_link_parameters(valid, errors, arke, :create, :after),
+    do: handle_link_parameters_v2(valid, errors, arke)
+
+  def handle_bulk_link_parameters(valid, errors, _, _, _), do: {:ok, valid, errors}
+
+  def handle_link_parameters_single(arke, unit, :create, :after) do
+    handle_link_parameters_v2([unit], [], arke)
+    |> case do
+      {:ok, [unit], []} -> {:ok, unit}
+      {:ok, _, errors} -> {:error, errors}
+      {:error, errors} -> {:error, errors}
+    end
+  end
+
+  def handle_link_parameters_single(arke, unit, _phase, _action), do: {:ok, unit}
+
   defp run_group_and_link_hooks(arke, unit, :create, :before) do
     with {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
          {:ok, unit} <- handle_link_parameters_unit(arke, unit),

From e3af42f9f363d7e07319285a1a808ab47c2e0b47 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Tue, 11 Feb 2025 19:13:18 +0100
Subject: [PATCH 21/28] wip

---
 lib/arke/query_manager.ex | 458 +++++++++-----------------------------
 lib/arke/validator.ex     | 154 +++++++------
 2 files changed, 191 insertions(+), 421 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 5eb31f2..7548157 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -147,14 +147,12 @@ defmodule Arke.QueryManager do
 
     with %Unit{} = unit <- Unit.load(arke, args, :create),
          {:ok, unit} <- Validator.validate(unit, :create, project),
-         {:ok, unit} <- run_persistence_hook(arke, unit, :create, :before),
-         {:ok, unit} <- run_group_call_func(arke, unit, :create, :before),
-         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
+         {:ok, unit} <- ArkeManager.call_func(arke, :before_create, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
          {:ok, unit} <- persistence_fn.(project, unit, []),
-         {:ok, unit} <- handle_link_parameters_single(arke, unit, :create, :after),
-         {:ok, unit} <- run_persistence_hook(arke, unit, :create, :after),
-         {:ok, unit} <- run_group_call_func(arke, unit, :create, :after),
-         {:ok, unit} <- handle_link_parameters(unit, %{}) do
+         {:ok, unit} <- handle_link_parameters(arke, unit),
+         {:ok, unit} <- ArkeManager.call_func(arke, :on_create, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_create) do
       {:ok, unit}
     else
       {:error, errors} -> {:error, errors}
@@ -167,21 +165,27 @@ defmodule Arke.QueryManager do
   def create_bulk(project, arke, data, args) do
     persistence_fn = @persistence[:arke_postgres][:create]
 
-    with %{valid: valid, errors: errors} <- prepare_create_bulk_units(arke, data, args),
-         %{valid: valid, errors: errors} <- Validator.validate(valid, :create, project),
-         %{valid: valid, errors: errors} <- process_bulk(valid, errors, arke, :create, :before),
+    with {:ok, valid, errors} <- prepare_create_bulk_units(arke, data, args),
+         {:ok, valid, errors} <- Validator.validate(valid, :create, project),
+         {:ok, valid, errors} <-
+           ArkeManager.call_func(arke, :before_bulk_create, [arke, valid, errors]),
+         {:ok, valid, errors} <-
+           handle_group_call_func(valid, errors, arke, :before_unit_bulk_create),
          {:ok, inserted_count, valid, persistence_errors} <-
            persistence_fn.(project, valid, bulk: true),
-         %{valid: valid, errors: errors} <-
-           process_bulk(valid, errors ++ persistence_errors, arke, :create, :after) do
+         {:ok, valid, errors} <-
+           handle_bulk_link_parameters(valid, errors, arke),
+         {:ok, valid, errors} <-
+           ArkeManager.call_func(arke, :on_bulk_create, [arke, valid, errors]),
+         {:ok, valid, errors} <- handle_group_call_func(valid, errors, arke, :on_unit_bulk_create) do
       {:ok, inserted_count, valid, errors}
     else
       {:error, errors} -> {:error, errors}
     end
   end
 
-  defp prepare_create_bulk_units(arke, units, args),
-    do:
+  defp prepare_create_bulk_units(arke, units, args) do
+    result =
       Enum.reduce(units, %{valid: [], errors: []}, fn item, acc ->
         case Unit.load(arke, data_as_klist(item) ++ args, :create) do
           %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
@@ -189,6 +193,9 @@ defmodule Arke.QueryManager do
         end
       end)
 
+    {:ok, result.valid, result.errors}
+  end
+
   # todo: remove after atoms removal
   defp data_as_klist(data) do
     Enum.map(data, fn {key, value} ->
@@ -212,16 +219,13 @@ defmodule Arke.QueryManager do
   end
 
   defp handle_group_call_func(valid, errors, arke, func) do
-    case GroupManager.get_groups_by_arke(arke)
-         |> Enum.reduce_while({:ok, valid, errors}, fn group, {:ok, valid, errors} ->
-           with {:ok, valid, errors} <-
-                  GroupManager.call_func(group, func, [arke, valid, errors]),
-                do: {:cont, {:ok, valid, errors}},
-                else: ({:error, errors} -> {:halt, {:error, errors}})
-         end) do
-      {:error, errors} -> {:error, errors}
-      {:ok, valid, errors} -> %{valid: valid, errors: errors}
-    end
+    GroupManager.get_groups_by_arke(arke)
+    |> Enum.reduce_while({:ok, valid, errors}, fn group, {:ok, valid, errors} ->
+      with {:ok, valid, errors} <-
+             GroupManager.call_func(group, func, [arke, valid, errors]),
+           do: {:cont, {:ok, valid, errors}},
+           else: ({:error, errors} -> {:halt, {:error, errors}})
+    end)
   end
 
   @doc """
@@ -249,12 +253,12 @@ defmodule Arke.QueryManager do
     with %Unit{} = unit <- Unit.update(current_unit, args),
          {:ok, unit} <- update_at_on_update(unit),
          {:ok, unit} <- Validator.validate(unit, :update, project),
-         # todo better valid / error handling
-         {:ok, unit} <- run_persistence_hook(arke, unit, :update, :before, current_unit),
-         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :before),
+         {:ok, unit} <- ArkeManager.call_func(arke, :before_update, [arke, current_unit, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
          {:ok, unit} <- persistence_fn.(project, unit, []),
-         {:ok, unit} <- run_persistence_hook(arke, unit, :update, :after, current_unit),
-         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :update, :after, data) do
+         {:ok, unit} <- ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit]),
+         {:ok, unit} <- handle_link_parameters(arke, unit, data),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update) do
       {:ok, unit}
     else
       {:error, errors} -> {:error, errors}
@@ -272,24 +276,20 @@ defmodule Arke.QueryManager do
   def update_bulk(project, arke, unit_list, data) do
     persistence_fn = @persistence[:arke_postgres][:update]
 
-    with %{valid: valid, errors: errors} <- prepare_update_bulk_units(arke, unit_list, data),
-         %{valid: valid, errors: errors} <- Validator.validate(valid, :update, project),
-         %{valid: valid, errors: errors} <-
-           run_group_call_func(valid, errors, arke, :update, :before),
-         %{valid: valid, errors: errors} <-
-           handle_link_parameters_v2(valid, errors, arke, :update, unit_list),
+    with {:ok, valid, errors} <- prepare_update_bulk_units(arke, unit_list, data),
+         {:ok, valid, errors} <- Validator.validate(valid, :update, project),
+         {:ok, valid, errors} <-
+           ArkeManager.call_func(arke, :before_bulk_update, [arke, valid, errors]),
+         {:ok, valid, errors} <-
+           handle_group_call_func(valid, errors, arke, :before_unit_bulk_create),
          {:ok, updated_count, valid, persistence_errors} <-
            persistence_fn.(project, valid, bulk: true),
-         %{valid: valid, errors: errors} <-
-           process_bulk(
-             valid,
-             errors ++ persistence_errors,
-             arke,
-             :update,
-             :after,
-             unit_list,
-             data
-           ) do
+         {:ok, valid, errors} <-
+           handle_bulk_link_parameters(valid, errors, arke, unit_list),
+         {:ok, valid, errors} <-
+           ArkeManager.call_func(arke, :on_bulk_update, [arke, unit_list, valid, errors]),
+         {:ok, valid, errors} <-
+           handle_group_call_func(valid, errors, arke, :on_unit_bulk_update) do
       {:ok, updated_count, valid, errors}
     else
       {:error, errors} -> {:error, errors}
@@ -308,20 +308,23 @@ defmodule Arke.QueryManager do
 
     updated_at = DatetimeHandler.now(:datetime)
 
-    Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
-      key = {unit.data.parent_id, unit.data.child_id, unit.data.type}
+    result =
+      Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
+        key = {unit.data.parent_id, unit.data.child_id, unit.data.type}
 
-      case data_map[key] do
-        nil ->
-          acc
+        case data_map[key] do
+          nil ->
+            acc
 
-        item ->
-          case Unit.update(unit, data_as_klist(item)) do
-            %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
-            {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
-          end
-      end
-    end)
+          item ->
+            case Unit.update(unit, data_as_klist(item)) do
+              %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
+              {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
+            end
+        end
+      end)
+
+    {:ok, result.valid, result.errors}
   end
 
   defp prepare_update_bulk_units(arke, unit_list, data) do
@@ -332,18 +335,21 @@ defmodule Arke.QueryManager do
 
     updated_at = DatetimeHandler.now(:datetime)
 
-    Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
-      case data_map[to_string(unit.id)] do
-        nil ->
-          acc
+    result =
+      Enum.reduce(unit_list, %{valid: [], errors: []}, fn unit, acc ->
+        case data_map[to_string(unit.id)] do
+          nil ->
+            acc
+
+          item ->
+            case Unit.update(unit, data_as_klist(item) ++ [updated_at: updated_at]) do
+              %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
+              {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
+            end
+        end
+      end)
 
-        item ->
-          case Unit.update(unit, data_as_klist(item) ++ [updated_at: updated_at]) do
-            %Unit{} = unit -> Map.put(acc, :valid, [unit | acc.valid])
-            {:error, error} -> Map.put(acc, :errors, [error | acc.errors])
-          end
-      end
-    end)
+    {:ok, result.valid, result.errors}
   end
 
   defp update_at_on_update(unit) do
@@ -369,11 +375,11 @@ defmodule Arke.QueryManager do
     arke = ArkeManager.get(arke_id, project)
     persistence_fn = @persistence[:arke_postgres][:delete]
 
-    with {:ok, unit} <- run_persistence_hook(arke, unit, :delete, :before),
-         {:ok, unit} <- run_group_and_link_hooks(arke, unit, :delete, :before),
+    with {:ok, unit} <- ArkeManager.call_func(arke, :before_delete, [arke, unit]),
+         {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_delete),
          {:ok, _} <- persistence_fn.(project, unit, []),
-         {:ok, _} <- run_group_and_link_hooks(arke, unit, :delete, :after),
-         {:ok, _} <- run_persistence_hook(arke, unit, :delete, :after),
+         {:ok, _} <- handle_group_call_func(arke, unit, :on_unit_delete),
+         {:ok, _} <- ArkeManager.call_func(arke, :on_delete, [arke, unit]),
          do: {:ok, nil},
          else: ({:error, errors} -> {:error, errors})
   end
@@ -385,11 +391,15 @@ defmodule Arke.QueryManager do
     arke = ArkeManager.get(arke_id, project)
     persistence_fn = @persistence[:arke_postgres][:delete]
 
-    with %{valid: valid, errors: errors} <-
-           process_bulk(unit_list, [], arke, :delete, :before),
+    with {:ok, valid, errors} <-
+           ArkeManager.call_func(arke, :before_bulk_delete, [arke, unit_list, []]),
+         {:ok, valid, errors} <-
+           handle_group_call_func(valid, errors, arke, :before_unit_bulk_delete),
          {:ok, _} <- persistence_fn.(project, valid, bulk: true),
-         %{valid: valid, errors: errors} <-
-           process_bulk(valid, errors, arke, :delete, :after),
+         {:ok, valid, errors} <-
+           handle_group_call_func(valid, errors, arke, :on_unit_bulk_delete),
+         {:ok, valid, errors} <-
+           ArkeManager.call_func(arke, :on_bulk_delete, [arke, valid, errors]),
          do: {:ok, valid, errors},
          else: ({:error, errors} -> {:error, errors})
   end
@@ -806,49 +816,6 @@ defmodule Arke.QueryManager do
     ArkeManager.get_parameter(arke, project, key)
   end
 
-  defp handle_link_parameters_unit(%{id: :arke_link} = _, unit), do: {:ok, unit}
-  defp handle_link_parameters_unit(%{id: :parameter_value} = _, unit), do: {:ok, unit}
-
-  defp handle_link_parameters_unit(
-         %{data: parameters} = arke,
-         %{metadata: %{project: project}} = unit
-       ) do
-    {errors, link_units} =
-      Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
-      |> Enum.reduce({[], []}, fn p, {errors, link_units} ->
-        arke = ArkeManager.get(String.to_existing_atom(p.data.arke_or_group_id), project)
-
-        case handle_create_on_link_parameters_unit(
-               project,
-               unit,
-               p,
-               arke,
-               Unit.get_value(unit, p.id)
-             ) do
-          {:ok, parameter, %Unit{} = link_unit} -> {errors, [{parameter, link_unit} | link_units]}
-          {:ok, parameter, link_unit} -> {errors, link_units}
-          {:error, e} -> {[e | errors], link_units}
-        end
-      end)
-
-    case length(errors) > 0 do
-      true ->
-        Enum.map(link_units, fn {p, u} ->
-          delete(project, u)
-        end)
-
-        {:error, errors}
-
-      false ->
-        args =
-          Enum.reduce(link_units, %{}, fn {p, u}, args ->
-            Map.put(args, p.id, Atom.to_string(u.id))
-          end)
-
-        {:ok, Unit.update(unit, args)}
-    end
-  end
-
   defp handle_create_on_link_parameters_unit(project, unit, parameter, arke, value)
        when is_nil(value),
        do: {:ok, parameter, value}
@@ -870,29 +837,23 @@ defmodule Arke.QueryManager do
   defp handle_create_on_link_parameters_unit(_, _, parameter, _, value),
     do: {:ok, parameter, value}
 
-  defp handle_link_parameters(
-         %{arke_id: arke_id, metadata: %{project: project}, data: new_data, id: id} = unit,
-         old_data
-       ) do
-    arke = ArkeManager.get(arke_id, project)
-
-    Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
-    |> Enum.each(fn p ->
-      old_value = Map.get(old_data, p.id, nil)
-      new_value = Map.get(new_data, p.id, nil)
-      handle_link_parameter(unit, p, old_value, new_value)
-    end)
+  defp handle_link_parameters(arke, unit, current_units \\ nil) do
+    old_data = if current_units, do: [current_units], else: nil
 
-    {:ok, unit}
+    case handle_bulk_link_parameters([unit], [], arke, old_data) do
+      {:ok, [unit], []} -> {:ok, unit}
+      {:ok, _, errors} -> {:error, errors}
+      {:error, errors} -> {:error, errors}
+    end
   end
 
-  defp handle_link_parameters_v2([], errors, _arke, _old_data), do: {:ok, [], errors}
+  defp handle_bulk_link_parameters([], errors, _arke, _old_data), do: {:ok, [], errors}
 
-  defp handle_link_parameters_v2(
+  defp handle_bulk_link_parameters(
          [%{metadata: %{project: project}} | _] = valid,
          errors,
          arke,
-         old_data \\ %{}
+         current_units \\ []
        ) do
     link_parameters =
       Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
@@ -900,17 +861,17 @@ defmodule Arke.QueryManager do
     {to_add, to_delete} =
       Enum.reduce(link_parameters, {[], []}, fn p, {to_add_acc, to_delete_acc} ->
         Enum.reduce(valid, {to_add_acc, to_delete_acc}, fn unit, {add_acc, del_acc} ->
-          old_value = get_in(old_data, [unit.id, p.id])
+          current_unit = Enum.find(current_units, &(&1.id == unit.id))
+          old_value = if current_unit, do: Unit.get_value(current_unit, p.id), else: nil
           new_value = get_in(unit.data, [p.id])
           {new_add, new_del} = build_link_entries(p, unit, old_value, new_value)
           {add_acc ++ new_add, del_acc ++ new_del}
         end)
       end)
 
-    with {:ok, add_count, add_valid, add_errors} <- LinkManager.add_node_bulk(project, to_add),
-         {:ok, del_valid, del_errors} <-
-           LinkManager.delete_node_bulk(project, to_delete) do
-      {:ok, valid ++ add_valid ++ del_valid, errors ++ add_errors ++ del_errors}
+    with {:ok, del_valid, del_errors} <- LinkManager.delete_node_bulk(project, to_delete),
+         {:ok, _add_count, add_valid, add_errors} <- LinkManager.add_node_bulk(project, to_add) do
+      {:ok, valid ++ add_valid, errors ++ add_errors ++ del_errors}
     else
       {:error, err} -> {:error, err}
     end
@@ -940,27 +901,17 @@ defmodule Arke.QueryManager do
   end
 
   defp update_link_entry(
-         %{data: %{connection_type: type, direction: "child"}} = parameter,
+         %{data: %{connection_type: type, direction: direction}, id: id},
          unit,
          new_value
        ) do
-    Link.new(
-      normalize_value(unit.id),
-      normalize_value(new_value),
-      type
-    )
-  end
+    {parent_id, child_id} =
+      case direction do
+        "child" -> {normalize_value(unit.id), normalize_value(new_value)}
+        "parent" -> {normalize_value(new_value), normalize_value(unit.id)}
+      end
 
-  defp update_link_entry(
-         %{data: %{connection_type: type, direction: "parent"}} = parameter,
-         unit,
-         new_value
-       ) do
-    Link.new(
-      normalize_value(new_value),
-      normalize_value(unit.id),
-      type
-    )
+    Link.new(parent_id, child_id, type, %{parameter_id: id})
   end
 
   defp handle_link_parameter(_, nil, _, _), do: nil
@@ -1058,40 +1009,6 @@ defmodule Arke.QueryManager do
     })
   end
 
-  defp handle_link_parameters_v2(valid, errors, arke, :update, current_units) do
-    to_check =
-      Enum.filter(ArkeManager.get_parameters(arke), fn p ->
-        p.arke_id == :link
-      end)
-
-    {to_create, to_delete} =
-      Enum.reduce(valid, {[], []}, fn unit, {create_acc, delete_acc} ->
-        current_unit = Enum.find(current_units, &(&1.id == unit.id))
-
-        Enum.reduce(to_check, {create_acc, delete_acc}, fn p, {create_acc, delete_acc} ->
-          direction = Map.get(p.data, :direction)
-          next_value = normalize_value(Unit.get_value(unit, p.id))
-          previous_value = normalize_value(Unit.get_value(current_unit, p.id))
-
-          IO.inspect({unit.id, previous_value, next_value, p.id})
-
-          case {next_value != previous_value, previous_value} do
-            {true, nil} ->
-              {create_acc, [prepare_link(direction, unit, previous_value) | delete_acc]}
-
-            {true, _} ->
-              {[prepare_link(direction, unit, next_value) | create_acc],
-               [prepare_link(direction, unit, previous_value) | delete_acc]}
-
-            _ ->
-              {create_acc, delete_acc}
-          end
-        end)
-      end)
-
-    IO.inspect(to_create)
-  end
-
   defp prepare_link("child", starting_unit, linked_unit_id),
     do: %{parent_id: starting_unit.id, child_id: linked_unit_id}
 
@@ -1106,163 +1023,4 @@ defmodule Arke.QueryManager do
   end
 
   defp normalize_value(value), do: to_string(value)
-
-  defp process_bulk(valid, errors, arke, :update, :after, current_units, data) do
-    data_by_id = Map.new(data, fn d -> {Map.get(d, "id"), d} end)
-
-    case run_persistence_hook(valid, errors, arke, :update, :after, current_units) do
-      {:ok, valid, errors} ->
-        Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-          updated_data = Map.get(data_by_id, to_string(unit.id))
-
-          case run_group_and_link_hooks(arke, unit, :update, :after, updated_data) do
-            {:ok, unit} ->
-              %{acc | valid: [unit | acc.valid]}
-
-            {:error, e} ->
-              %{acc | errors: [{unit, e} | acc.errors]}
-          end
-        end)
-
-      {:error, errors} ->
-        {:error, errors}
-    end
-  end
-
-  defp process_bulk(valid, errors, arke, persistence_fn, phase) do
-    with {:ok, valid, errors} <-
-           run_persistence_hook(valid, errors, arke, persistence_fn, phase),
-         {:ok, valid, errors} <-
-           handle_bulk_link_parameters(valid, errors, arke, persistence_fn, phase) do
-      Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-        case run_group_and_link_hooks(arke, unit, persistence_fn, phase) do
-          {:ok, unit} ->
-            %{acc | valid: [unit | acc.valid]}
-
-          {:error, e} ->
-            %{acc | errors: [{unit, e} | acc.errors]}
-        end
-      end)
-    else
-      {:error, errors} -> {:error, errors}
-    end
-  end
-
-  defp run_persistence_hook(valid, errors, arke, :create, :before),
-    do: ArkeManager.call_func(arke, :before_bulk_create, [arke, valid, errors])
-
-  defp run_persistence_hook(arke, unit, :create, :before),
-    do: ArkeManager.call_func(arke, :before_create, [arke, unit])
-
-  defp run_persistence_hook(arke, unit, :create, :after),
-    do: ArkeManager.call_func(arke, :on_create, [arke, unit])
-
-  defp run_persistence_hook(valid, errors, arke, :create, :after),
-    do: ArkeManager.call_func(arke, :on_bulk_create, [arke, valid, errors])
-
-  defp run_persistence_hook(valid, errors, arke, :update, :before),
-    do: ArkeManager.call_func(arke, :before_bulk_update, [arke, valid, errors])
-
-  defp run_persistence_hook(arke, unit, :update, :before, current_unit),
-    do: ArkeManager.call_func(arke, :before_update, [arke, current_unit, unit])
-
-  defp run_persistence_hook(valid, errors, arke, :update, :after, current_units),
-    do: ArkeManager.call_func(arke, :on_bulk_update, [arke, current_units, valid, errors])
-
-  defp run_persistence_hook(arke, unit, :update, :after, current_unit),
-    do: ArkeManager.call_func(arke, :on_update, [arke, current_unit, unit])
-
-  defp run_persistence_hook(valid, errors, arke, :delete, :before),
-    do: ArkeManager.call_func(arke, :before_bulk_delete, [arke, valid, errors])
-
-  defp run_persistence_hook(arke, unit, :delete, :before),
-    do: ArkeManager.call_func(arke, :before_delete, [arke, unit])
-
-  defp run_persistence_hook(valid, errors, arke, :delete, :after),
-    do: ArkeManager.call_func(arke, :on_bulk_delete, [arke, valid, errors])
-
-  defp run_persistence_hook(arke, unit, :delete, :after),
-    do: ArkeManager.call_func(arke, :on_delete, [arke, unit])
-
-  defp run_group_call_func(arke, unit, :create, :before),
-    do: handle_group_call_func(arke, unit, :before_unit_create)
-
-  defp run_group_call_func(valid, errors, arke, :create, :before),
-    do: handle_group_call_func(valid, errors, arke, :before_unit_bulk_create)
-
-  defp run_group_call_func(arke, unit, :create, :after),
-    do: handle_group_call_func(arke, unit, :on_unit_create)
-
-  defp run_group_call_func(valid, errors, arke, :create, :after),
-    do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_create)
-
-  defp run_group_call_func(arke, unit, :update, :before),
-    do: handle_group_call_func(arke, unit, :before_unit_update)
-
-  defp run_group_call_func(valid, errors, arke, :update, :before),
-    do: handle_group_call_func(valid, errors, arke, :before_unit_bulk_create)
-
-  defp run_group_call_func(arke, unit, :update, :after),
-    do: handle_group_call_func(arke, unit, :on_unit_update)
-
-  defp run_group_call_func(valid, errors, arke, :update, :after),
-    do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_update)
-
-  defp run_group_call_func(arke, unit, :delete, :before),
-    do: handle_group_call_func(arke, unit, :before_unit_delete)
-
-  defp run_group_call_func(valid, errors, arke, :delete, :before),
-    do: handle_group_call_func(valid, errors, arke, :before_unit_bulk_delete)
-
-  defp run_group_call_func(arke, unit, :delete, :after),
-    do: handle_group_call_func(arke, unit, :on_unit_delete)
-
-  defp run_group_call_func(valid, errors, arke, :delete, :after),
-    do: handle_group_call_func(valid, errors, arke, :on_unit_bulk_delete)
-
-  def handle_bulk_link_parameters(valid, errors, arke, :create, :after),
-    do: handle_link_parameters_v2(valid, errors, arke)
-
-  def handle_bulk_link_parameters(valid, errors, _, _, _), do: {:ok, valid, errors}
-
-  def handle_link_parameters_single(arke, unit, :create, :after) do
-    handle_link_parameters_v2([unit], [], arke)
-    |> case do
-      {:ok, [unit], []} -> {:ok, unit}
-      {:ok, _, errors} -> {:error, errors}
-      {:error, errors} -> {:error, errors}
-    end
-  end
-
-  def handle_link_parameters_single(arke, unit, _phase, _action), do: {:ok, unit}
-
-  defp run_group_and_link_hooks(arke, unit, :create, :before) do
-    with {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_create),
-         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
-         do: {:ok, unit}
-  end
-
-  defp run_group_and_link_hooks(arke, unit, :create, :after) do
-    with {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_create),
-         {:ok, unit} <- handle_link_parameters(unit, %{}),
-         do: {:ok, unit}
-  end
-
-  defp run_group_and_link_hooks(arke, unit, :update, :before) do
-    with {:ok, unit} <- handle_group_call_func(arke, unit, :before_unit_update),
-         {:ok, unit} <- handle_link_parameters_unit(arke, unit),
-         do: {:ok, unit}
-  end
-
-  defp run_group_and_link_hooks(arke, unit, :update, :after, data) do
-    with {:ok, unit} <- handle_link_parameters(unit, data),
-         {:ok, unit} <- handle_group_call_func(arke, unit, :on_unit_update),
-         do: {:ok, unit}
-  end
-
-  defp run_group_and_link_hooks(arke, unit, :delete, :before),
-    do: handle_group_call_func(arke, unit, :before_unit_delete)
-
-  defp run_group_and_link_hooks(arke, unit, :delete, :after),
-    do: handle_group_call_func(arke, unit, :on_unit_delete)
 end
diff --git a/lib/arke/validator.ex b/lib/arke/validator.ex
index e96e987..4124f90 100644
--- a/lib/arke/validator.ex
+++ b/lib/arke/validator.ex
@@ -49,13 +49,13 @@ defmodule Arke.Validator do
 
   def validate(%Unit{} = unit, persistence_fn, project) do
     case validate([unit], persistence_fn, project) do
-      %{valid: [unit], errors: []} -> {:ok, unit}
-      %{errors: errors} -> Error.create(:parameter_validation, errors)
+      {:ok, [unit], []} -> {:ok, unit}
+      {:ok, [], errors} -> Error.create(:parameter_validation, errors)
     end
   end
 
   def validate([], _persistence_fn, _project),
-    do: %{valid: [], errors: []}
+    do: {:ok, [], []}
 
   def validate(
         [%Unit{arke_id: arke_id} | _] = unit_list,
@@ -69,23 +69,27 @@ defmodule Arke.Validator do
         persistence == "arke_parameter"
       end)
 
-    check_duplicate_units(unit_list, project, persistence_fn)
-    |> apply_before_validate(project)
-    |> check_bulk_parameters(arke, parameter_list, project)
-    |> check_unique_parameters(arke, parameter_list, project)
+    with {:ok, valid, errors} <- check_duplicate_units(unit_list, project, persistence_fn),
+         {:ok, valid, errors} <- apply_before_validate(valid, errors, project),
+         {:ok, valid, errors} <-
+           check_bulk_parameters(valid, errors, arke, parameter_list, project),
+         {:ok, valid, errors} <-
+           check_unique_parameters(valid, errors, arke, parameter_list, project) do
+      {:ok, valid, errors}
+    end
   end
 
-  defp apply_before_validate(%{valid: valid, errors: errors}, project),
-    do:
-      Enum.reduce(valid, %{valid: [], errors: errors}, fn u, acc ->
-        case before_validate(u, project) do
-          {unit, []} ->
-            Map.put(acc, :valid, [unit | acc.valid])
+  defp apply_before_validate(valid, errors, project) do
+    Enum.reduce_while(valid, {:ok, [], errors}, fn u, {:ok, acc_valid, acc_errors} ->
+      case before_validate(u, project) do
+        {unit, []} ->
+          {:cont, {:ok, [unit | acc_valid], acc_errors}}
 
-          {unit, unit_errors} ->
-            Map.put(errors, :errors, errors ++ {unit, unit_errors})
-        end
-      end)
+        {unit, unit_errors} ->
+          {:cont, {:ok, acc_valid, [{unit, unit_errors} | acc_errors]}}
+      end
+    end)
+  end
 
   defp before_validate(%{arke_id: arke_id} = unit, project) do
     arke = ArkeManager.get(arke_id, project)
@@ -100,7 +104,6 @@ defmodule Arke.Validator do
       Enum.filter(unit_list, fn u -> not is_nil(u.id) end) |> Enum.map(&to_string(&1.id))
 
     duplicates = QueryManager.filter_by(%{:id__in => ids_to_check, :project => project})
-    # todo: merge with system duplicates
 
     valid =
       Enum.reduce(unit_list, [], fn item, acc ->
@@ -110,48 +113,51 @@ defmodule Arke.Validator do
         end
       end)
 
-    %{
-      valid: valid,
-      errors: Enum.map(duplicates, fn d -> {d, "value not allowed for #{d.id}"} end)
-    }
+    errors = Enum.map(duplicates, fn d -> {d, "value not allowed for #{d.id}"} end)
+
+    {:ok, valid, errors}
   end
 
   defp check_duplicate_units(unit_list, _project, _persistence_fn),
-    # todo: manage update
-    do: %{valid: unit_list, errors: []}
+    do: {:ok, unit_list, []}
 
   defp get_result({_unit, errors} = _res) when is_list(errors) and length(errors) > 0,
     do: Error.create(:parameter_validation, errors)
 
   defp get_result({unit, _errors} = _res), do: {:ok, unit}
 
-  defp check_bulk_parameters(%{valid: valid, errors: errors}, arke, parameter_list, project) do
-    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
-      case Enum.reduce(parameter_list, {unit, []}, fn parameter, {unit, errors} ->
-             {value, err} =
-               validate_parameter(
-                 arke,
-                 parameter,
-                 Unit.get_value(unit.data, parameter.id),
-                 project
-               )
-
-             {Unit.update(unit, [{parameter.id, value}]), errors ++ err}
-           end) do
-        {unit, []} ->
-          Map.put(acc, :valid, [unit | acc.valid])
+  defp check_bulk_parameters(valid, errors, arke, parameter_list, project) do
+    result =
+      Enum.reduce(valid, {[], errors}, fn unit, {acc_valid, acc_errors} ->
+        case Enum.reduce(parameter_list, {unit, []}, fn parameter, {unit, errors} ->
+               {value, err} =
+                 validate_parameter(
+                   arke,
+                   parameter,
+                   Unit.get_value(unit.data, parameter.id),
+                   project
+                 )
+
+               {Unit.update(unit, [{parameter.id, value}]), errors ++ err}
+             end) do
+          {unit, []} ->
+            {[unit | acc_valid], acc_errors}
 
-        {unit, unit_errors} ->
-          %{
-            acc
-            | errors: [
-                {unit,
-                 Enum.map(unit_errors, fn {parameter_id, error} -> "#{parameter_id} #{error}" end)}
-                | acc.errors
-              ]
-          }
-      end
-    end)
+          {unit, unit_errors} ->
+            {acc_valid,
+             [
+               {unit,
+                Enum.map(unit_errors, fn {parameter_id, error} ->
+                  "#{parameter_id} #{error}"
+                end)}
+               | acc_errors
+             ]}
+        end
+      end)
+
+    case result do
+      {valid, errors} -> {:ok, valid, errors}
+    end
   end
 
   defp create_unique_map(parameter_list, unit_list) do
@@ -166,31 +172,37 @@ defmodule Arke.Validator do
     end)
   end
 
-  defp check_unique_parameters(
-         %{valid: valid, errors: errors} = unit_map,
-         arke,
-         parameter_list,
-         project
-       ) do
+  defp check_unique_parameters(valid, errors, arke, parameter_list, project) do
     unique_map = create_unique_map(parameter_list, valid)
 
-    valid
-    |> validate_unique_input([], errors, unique_map)
-    |> validate_and_filter_unique_units(unique_map, arke, project, parameter_list)
+    with {:ok, valid, errors} <- validate_unique_input(valid, [], errors, unique_map),
+         {:ok, valid, errors} <-
+           validate_and_filter_unique_units(
+             valid,
+             errors,
+             unique_map,
+             arke,
+             project,
+             parameter_list
+           ) do
+      {:ok, valid, errors}
+    end
   end
 
   defp validate_and_filter_unique_units(
-         %{valid: valid, errors: errors},
+         valid,
+         errors,
          unique_map,
-         _arke,
-         _project,
-         _parameter_list
+         arke,
+         project,
+         parameter_list
        )
        when map_size(unique_map) == 0,
-       do: %{valid: valid, errors: errors}
+       do: {:ok, valid, errors}
 
   defp validate_and_filter_unique_units(
-         %{valid: valid, errors: errors},
+         valid,
+         errors,
          unique_map,
          arke,
          project,
@@ -208,7 +220,7 @@ defmodule Arke.Validator do
 
     parameter_already_present = create_unique_map(parameter_list, db_units)
 
-    Enum.reduce(valid, %{valid: [], errors: errors}, fn unit, acc ->
+    Enum.reduce(valid, {:ok, [], errors}, fn unit, {:ok, acc_valid, acc_errors} ->
       {non_unique_parameters, unit_errors} =
         parameter_already_present
         |> Enum.reduce({[], []}, fn {parameter_id, uniques}, {non_unique, errors} ->
@@ -229,19 +241,19 @@ defmodule Arke.Validator do
         end)
 
       if non_unique_parameters == [] do
-        %{acc | valid: [unit | acc.valid]}
+        {:ok, [unit | acc_valid], acc_errors}
       else
-        %{acc | errors: [{unit, unit_errors} | acc.errors]}
+        {:ok, acc_valid, [{unit, unit_errors} | acc_errors]}
       end
     end)
   end
 
   defp validate_unique_input(unit_list, _valid, errors, unique_map)
        when map_size(unique_map) == 0,
-       do: %{valid: unit_list, errors: errors}
+       do: {:ok, unit_list, errors}
 
   defp validate_unique_input([], valid, errors, _unique_map),
-    do: %{valid: valid, errors: errors}
+    do: {:ok, valid, errors}
 
   defp validate_unique_input([unit | tail], valid, errors, unique_map) do
     unit_errors =
@@ -264,7 +276,7 @@ defmodule Arke.Validator do
 
     case unit_errors do
       [] -> validate_unique_input(tail, [unit | valid], errors, unique_map)
-      _ -> validate_unique_input(tail, valid, [{unit, unit_errors} | errors], unique_map)
+      _ -> {:ok, valid, [{unit, unit_errors} | errors]}
     end
   end
 

From 91b885cf1a43b7a0dfa405b78543e07690c5abb3 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Tue, 18 Feb 2025 17:28:25 +0100
Subject: [PATCH 22/28] wip

---
 lib/arke/group.ex         | 12 ++++++------
 lib/arke/query_manager.ex |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/arke/group.ex b/lib/arke/group.ex
index a80acc8..881edbb 100644
--- a/lib/arke/group.ex
+++ b/lib/arke/group.ex
@@ -44,12 +44,12 @@ defmodule Arke.System.Group do
       def on_unit_delete(_arke, unit), do: {:ok, unit}
       def before_unit_delete(_arke, unit), do: {:ok, unit}
 
-      defp before_unit_bulk_create(arke, valid, errors), do: {:ok, valid, errors}
-      defp on_unit_bulk_create(arke, valid, errors), do: {:ok, valid, errors}
-      defp before_unit_bulk_update(arke, valid, errors), do: {:ok, valid, errors}
-      defp on_unit_bulk_update(arke, valid, errors), do: {:ok, valid, errors}
-      defp before_unit_bulk_delete(arke, valid, errors), do: {:ok, valid, errors}
-      defp on_unit_bulk_delete(arke, valid, errors), do: {:ok, valid, errors}
+      def before_unit_bulk_create(_arke, valid, errors), do: {:ok, valid, errors}
+      def on_unit_bulk_create(_arke, valid, errors), do: {:ok, valid, errors}
+      def before_unit_bulk_update(_arke, valid, errors), do: {:ok, valid, errors}
+      def on_unit_bulk_update(_arke, valid, errors), do: {:ok, valid, errors}
+      def before_unit_bulk_delete(_arke, valid, errors), do: {:ok, valid, errors}
+      def on_unit_bulk_delete(_arke, valid, errors), do: {:ok, valid, errors}
 
       defoverridable on_unit_load: 3,
                      before_unit_load: 3,
diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 7548157..5480be8 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -162,7 +162,7 @@ defmodule Arke.QueryManager do
 
   @spec create_bulk(project :: atom(), arke :: Arke.t(), data :: list(Arke.t()), args :: list()) ::
           func_return()
-  def create_bulk(project, arke, data, args) do
+  def create_bulk(project, arke, data, args \\ []) do
     persistence_fn = @persistence[:arke_postgres][:create]
 
     with {:ok, valid, errors} <- prepare_create_bulk_units(arke, data, args),
@@ -281,7 +281,7 @@ defmodule Arke.QueryManager do
          {:ok, valid, errors} <-
            ArkeManager.call_func(arke, :before_bulk_update, [arke, valid, errors]),
          {:ok, valid, errors} <-
-           handle_group_call_func(valid, errors, arke, :before_unit_bulk_create),
+           handle_group_call_func(valid, errors, arke, :before_unit_bulk_update),
          {:ok, updated_count, valid, persistence_errors} <-
            persistence_fn.(project, valid, bulk: true),
          {:ok, valid, errors} <-

From 2a1fa72e0af216ea85165e09b982f04413de8f17 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Fri, 21 Feb 2025 11:37:10 +0100
Subject: [PATCH 23/28] wip

---
 lib/arke/query_manager.ex | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 5480be8..205d88b 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -837,8 +837,8 @@ defmodule Arke.QueryManager do
   defp handle_create_on_link_parameters_unit(_, _, parameter, _, value),
     do: {:ok, parameter, value}
 
-  defp handle_link_parameters(arke, unit, current_units \\ nil) do
-    old_data = if current_units, do: [current_units], else: nil
+  defp handle_link_parameters(arke, unit, current_unit \\ nil) do
+    old_data = if current_unit, do: [Map.put(current_unit, :id, unit.id)], else: nil
 
     case handle_bulk_link_parameters([unit], [], arke, old_data) do
       {:ok, [unit], []} -> {:ok, unit}
@@ -878,10 +878,8 @@ defmodule Arke.QueryManager do
   end
 
   defp build_link_entries(%{data: %{multiple: true}} = parameter, unit, old_value, new_value) do
-    Enum.reduce(old_value, {[], []}, fn value, {add_acc, del_acc} ->
-      {[update_link_entry(parameter, unit, value) | add_acc],
-       [update_link_entry(parameter, unit, value) | del_acc]}
-    end)
+    {Enum.map(new_value, &update_link_entry(parameter, unit, &1)),
+     Enum.map(old_value, &update_link_entry(parameter, unit, &1))}
   end
 
   defp build_link_entries(_parameter, _unit, old_value, new_value) when old_value == new_value,

From 988da395b7fa216583f843fc301318ce5026fffd Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Fri, 28 Feb 2025 12:10:14 +0100
Subject: [PATCH 24/28] fix: import use bulk actions

---
 lib/arke/core/unit.ex     | 24 ++++++++++++++++--------
 lib/arke/query_manager.ex | 10 +++++++---
 lib/arke/system.ex        | 37 ++++++++++++++++++++++++++++---------
 3 files changed, 51 insertions(+), 20 deletions(-)

diff --git a/lib/arke/core/unit.ex b/lib/arke/core/unit.ex
index 3e1d86e..b8233d5 100644
--- a/lib/arke/core/unit.ex
+++ b/lib/arke/core/unit.ex
@@ -169,8 +169,17 @@ defmodule Arke.Core.Unit do
 
   defp handle_default_value(_), do: nil
 
-  defp get_link(%{depth: depth, link_metadata: link_metadata,starting_unit: starting_unit, link_type: link_type} = args),
-    do: {%{depth: depth, metadata: link_metadata,starting_unit: starting_unit, type: link_type}, args}
+  defp get_link(
+         %{
+           depth: depth,
+           link_metadata: link_metadata,
+           starting_unit: starting_unit,
+           link_type: link_type
+         } = args
+       ),
+       do:
+         {%{depth: depth, metadata: link_metadata, starting_unit: starting_unit, type: link_type},
+          args}
 
   defp get_link(args), do: {nil, args}
 
@@ -289,11 +298,11 @@ defmodule Arke.Core.Unit do
   defp update_encoded_unit_data(%{data: %{only_runtime: true}}, data, _), do: data
 
   defp update_encoded_unit_data(%{id: id}, data, value),
-       do:
-         Map.put_new(data, Atom.to_string(id), %{
-           :value => value,
-           :datetime => Arke.DatetimeHandler.now(:datetime)
-         })
+    do:
+      Map.put_new(data, Atom.to_string(id), %{
+        :value => value,
+        :datetime => Arke.DatetimeHandler.now(:datetime)
+      })
 
   defp update_encoded_unit_data(_, data, _), do: data
 
@@ -373,7 +382,6 @@ defmodule Arke.Core.Unit do
   defp parse_value(value, %{arke_id: :atom}) when is_binary(value),
     do: String.to_existing_atom(value)
 
-
   defp parse_value(value, %{arke_id: :date}) do
     with {:ok, date} <- DatetimeHandler.parse_date(value),
          do: date,
diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 205d88b..9027991 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -849,6 +849,9 @@ defmodule Arke.QueryManager do
 
   defp handle_bulk_link_parameters([], errors, _arke, _old_data), do: {:ok, [], errors}
 
+  defp handle_bulk_link_parameters(valid, errors, %Unit{id: :arke_link}, _old_data),
+    do: {:ok, valid, errors}
+
   defp handle_bulk_link_parameters(
          [%{metadata: %{project: project}} | _] = valid,
          errors,
@@ -869,9 +872,10 @@ defmodule Arke.QueryManager do
         end)
       end)
 
-    with {:ok, del_valid, del_errors} <- LinkManager.delete_node_bulk(project, to_delete),
-         {:ok, _add_count, add_valid, add_errors} <- LinkManager.add_node_bulk(project, to_add) do
-      {:ok, valid ++ add_valid, errors ++ add_errors ++ del_errors}
+    with {:ok, del_valid, _del_errors} <- LinkManager.delete_node_bulk(project, to_delete),
+         {:ok, _add_count, _add_valid, []} <- LinkManager.add_node_bulk(project, to_add) do
+      # todo: better error handling
+      {:ok, valid, errors}
     else
       {:error, err} -> {:error, err}
     end
diff --git a/lib/arke/system.ex b/lib/arke/system.ex
index d45d465..c0a39ce 100644
--- a/lib/arke/system.ex
+++ b/lib/arke/system.ex
@@ -114,8 +114,14 @@ defmodule Arke.System do
           end)
 
         existing_units = get_existing_units_for_import(project, arke, header, correct_units)
-        units_args = Enum.filter(correct_units, fn u -> check_existing_units_for_import(project, arke, header, u, existing_units) == false end)
-        {existing_units,units_args,error_units} = handle_insert(project,existing_units,units_args,error_units)
+
+        units_args =
+          Enum.filter(correct_units, fn u ->
+            check_existing_units_for_import(project, arke, header, u, existing_units) == false
+          end)
+
+        {existing_units, units_args, error_units} =
+          handle_insert(project, arke, existing_units, units_args, error_units)
 
         count_inserted = length(units_args)
         count_existing = length(existing_units)
@@ -133,14 +139,27 @@ defmodule Arke.System do
         {:ok, res, 201}
       end
 
-      defp handle_insert(project,existing_units,units_args,error_units) when length(units_args) > 0 do
-        {existing_units,units_args,error_units} = before_unit_import(project,existing_units,units_args,error_units)
-        Enum.map(Stream.chunk_every(units_args, 5000) |> Enum.to_list(), fn chunk ->
-          ArkePostgres.Repo.insert_all("arke_unit", chunk, prefix: Atom.to_string(project))
-        end)
-        on_unit_import(project,existing_units,units_args,error_units)
+      defp handle_insert(project, arke, existing_units, units_args, error_units)
+           when length(units_args) > 0 do
+        {existing_units, units_args, error_units} =
+          before_unit_import(project, existing_units, units_args, error_units)
+
+        units_args_maps =
+          Enum.map(units_args, fn unit_arg ->
+            unit_arg
+            |> Keyword.get(:data, %{})
+            |> Enum.reduce(%{}, fn {key, %{value: value}}, acc ->
+              Map.put(acc, key, value)
+            end)
+          end)
+
+        res = Arke.QueryManager.create_bulk(project, arke, units_args_maps)
+
+        on_unit_import(project, existing_units, units_args, error_units)
       end
-      defp handle_insert(_project,existing_units,units_args,error_units), do: {existing_units,units_args,error_units}
+
+      defp handle_insert(_project, _arke, existing_units, units_args, error_units),
+        do: {existing_units, units_args, error_units}
 
       defp parse_cell(value) when is_tuple(value), do: Kernel.inspect(value)
       defp parse_cell(value), do: value

From d10db7f64283a48840476480c794ea50ed82b261 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Fri, 18 Apr 2025 12:39:54 +0200
Subject: [PATCH 25/28] wip

---
 lib/arke/query_manager.ex | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index 9027991..e8f56de 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -676,15 +676,10 @@ defmodule Arke.QueryManager do
           direction :: atom()
         ) :: Query.t()
   def order(query, parameter, direction) when is_list(parameter) do
+    [head | tail] = parameter
+
     parameters =
-      Enum.with_index(parameter)
-      |> Enum.map(fn {p, index} ->
-        if index == 0 do
-          get_parameter(query, p)
-        else
-          get_parameter(%{query | arke: nil}, p)
-        end
-      end)
+      [get_parameter(query, head)] ++ Enum.map(tail, &get_parameter(%{query | arke: nil}, &1))
 
     Query.add_order(query, parameters, direction)
   end

From 411e46ebca0f56830dfd0820365da00a39038248 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Mon, 23 Jun 2025 10:58:24 +0200
Subject: [PATCH 26/28] chore: version

---
 .github/workflows/publish_hex.yml | 10 +++++-----
 mix.exs                           |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/publish_hex.yml b/.github/workflows/publish_hex.yml
index d7e4d59..2adeaaa 100644
--- a/.github/workflows/publish_hex.yml
+++ b/.github/workflows/publish_hex.yml
@@ -2,7 +2,7 @@ name: Publish package to Hex 📦
 on:
   push:
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]*"
 
 jobs:
   publish:
@@ -14,10 +14,10 @@ jobs:
       - name: Set up Elixir
         uses: erlef/setup-beam@v1
         with:
-          elixir-version: '1.13.4'
-          otp-version: '24.3'
+          elixir-version: "1.13.4"
+          otp-version: "24.3"
       - name: Restore dependencies cache
-        uses: actions/cache@v3.3.1
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
@@ -31,4 +31,4 @@ jobs:
       - name: Release & Publish
         run: mix hex.publish --yes
         env:
-          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
\ No newline at end of file
+          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
diff --git a/mix.exs b/mix.exs
index 09fc44a..73f430a 100644
--- a/mix.exs
+++ b/mix.exs
@@ -1,7 +1,7 @@
 defmodule Arke.MixProject do
   use Mix.Project
 
-  @version "0.3.15"
+  @version "0.4.0-bulk.0"
   @scm_url "https://github.com/arkemishub/arke"
   @site_url "https://arkehub.com"
 
@@ -61,7 +61,7 @@ defmodule Arke.MixProject do
       {:httpoison, "~> 2.0"},
       {:calendar, "~> 1.0.0"},
       {:xlsxir, "~> 1.6"},
-      {:libcluster, "~> 3.3"},
+      {:libcluster, "~> 3.3"}
     ]
   end
 

From 3531bac2fbc97376cb622f447ba77591d2cee845 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Mon, 3 Nov 2025 12:32:27 +0100
Subject: [PATCH 27/28] fix: seed

---
 lib/arke/query_manager.ex          |  10 +-
 lib/registry/shared/arke.json      |  96 ++++++------
 lib/registry/shared/group.json     |   4 +-
 lib/registry/shared/parameter.json | 225 +++++++++++++++--------------
 lib/registry/system/group.json     |  46 ++++--
 5 files changed, 203 insertions(+), 178 deletions(-)

diff --git a/lib/arke/query_manager.ex b/lib/arke/query_manager.ex
index e8f56de..35db08b 100644
--- a/lib/arke/query_manager.ex
+++ b/lib/arke/query_manager.ex
@@ -833,7 +833,7 @@ defmodule Arke.QueryManager do
     do: {:ok, parameter, value}
 
   defp handle_link_parameters(arke, unit, current_unit \\ nil) do
-    old_data = if current_unit, do: [Map.put(current_unit, :id, unit.id)], else: nil
+    old_data = if current_unit, do: [Map.put(current_unit, :id, unit.id)], else: []
 
     case handle_bulk_link_parameters([unit], [], arke, old_data) do
       {:ok, [unit], []} -> {:ok, unit}
@@ -850,9 +850,11 @@ defmodule Arke.QueryManager do
   defp handle_bulk_link_parameters(
          [%{metadata: %{project: project}} | _] = valid,
          errors,
-         arke,
+         %{arke_id: arke_id} = unit,
          current_units \\ []
        ) do
+    arke = ArkeManager.get(arke_id, project)
+
     link_parameters =
       Enum.filter(ArkeManager.get_parameters(arke), fn p -> p.arke_id == :link end)
 
@@ -877,8 +879,8 @@ defmodule Arke.QueryManager do
   end
 
   defp build_link_entries(%{data: %{multiple: true}} = parameter, unit, old_value, new_value) do
-    {Enum.map(new_value, &update_link_entry(parameter, unit, &1)),
-     Enum.map(old_value, &update_link_entry(parameter, unit, &1))}
+    {Enum.map(new_value || [], &update_link_entry(parameter, unit, &1)),
+     Enum.map(old_value || [], &update_link_entry(parameter, unit, &1))}
   end
 
   defp build_link_entries(_parameter, _unit, old_value, new_value) when old_value == new_value,
diff --git a/lib/registry/shared/arke.json b/lib/registry/shared/arke.json
index 7317a39..73cc5da 100644
--- a/lib/registry/shared/arke.json
+++ b/lib/registry/shared/arke.json
@@ -1,50 +1,52 @@
-{"arke": [{
-  "id": "arke_file",
-
-  "label": "Arke File",
-  "parameters": [
+{
+  "arke": [
     {
-      "id": "name",
-      "metadata": {
-        "required": true
-      }
-    },
-    {
-      "id": "path",
-      "metadata": {
-        "required": true
-      }
-    },
-    {
-      "id": "provider",
-      "metadata": {
-        "values": [
-          "local",
-          "gcloud",
-          "aws"
-        ],
-        "default": "gcloud"
-      }
-    },
-    {
-      "id": "size",
-      "metadata": {
-        "required": false
-      }
-    },
-    {
-      "id": "binary_data",
-      "metadata": {
-        "required": true,
-        "only_runtime": true
-      }
-    },
-    {
-      "id": "extension",
-      "metadata": {
-        "required": false
-      }
+      "id": "arke_file",
+      "label": "Arke File",
+      "parameters": [
+        {
+          "id": "name",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "path",
+          "metadata": {
+            "required": true
+          }
+        },
+        {
+          "id": "provider",
+          "metadata": {
+            "values": [
+              "local",
+              "gcloud",
+              "aws"
+            ],
+            "default": "gcloud"
+          }
+        },
+        {
+          "id": "size",
+          "metadata": {
+            "required": false
+          }
+        },
+        {
+          "id": "binary_data",
+          "metadata": {
+            "required": true,
+            "only_runtime": true
+          }
+        },
+        {
+          "id": "extension",
+          "metadata": {
+            "required": false
+          }
+        }
+      ]
     }
   ]
-}
-]}
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/lib/registry/shared/group.json b/lib/registry/shared/group.json
index 597bd02..38ac633 100644
--- a/lib/registry/shared/group.json
+++ b/lib/registry/shared/group.json
@@ -1 +1,3 @@
-{"group": []}
\ No newline at end of file
+{
+    "group": []
+}
\ No newline at end of file
diff --git a/lib/registry/shared/parameter.json b/lib/registry/shared/parameter.json
index 0de854d..ada6fbc 100644
--- a/lib/registry/shared/parameter.json
+++ b/lib/registry/shared/parameter.json
@@ -1,112 +1,113 @@
-{"parameter": [
-  {
-    "default_string": null,
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Name",
-    "id": "name",
-    "is_primary": false,
-    "label": "Name",
-    "max_length": 100,
-    "min_length": 2,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": "",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Path",
-    "id": "path",
-    "is_primary": false,
-    "label": "Path",
-    "max_length": null,
-    "min_length": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_string": "gcloud",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Provider",
-    "id": "provider",
-    "is_primary": false,
-    "label": "Provider",
-    "max_length": null,
-    "min_length": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": [
-      "local",
-      "gcloud",
-      "aws"
-    ]
-  },
-  {
-    "default_float": null,
-    "type": "float",
-    "format": "attribute",
-    "helper_text": "Size",
-    "id": "size",
-    "is_primary": false,
-    "label": "Size",
-    "max": null,
-    "min": 0,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "unique": false,
-    "values": null
-  },
-  {
-    "default_binary": null,
-    "type": "binary",
-    "format": "attribute",
-    "helper_text": "Binary",
-    "id": "binary_data",
-    "is_primary": false,
-    "label": "Binary",
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false
-  },
-  {
-    "default_string": "",
-    "type": "string",
-    "format": "attribute",
-    "helper_text": "Extension",
-    "id": "extension",
-    "is_primary": false,
-    "label": "Extension",
-    "max_length": null,
-    "min_length": null,
-    "multiple": false,
-    "nullable": true,
-    "persistence": "arke_parameter",
-    "required": false,
-    "strip": false,
-"lowercase": false,
-    "unique": false,
-    "values": null
-  }
-
-]}
\ No newline at end of file
+{
+  "parameter": [
+    {
+      "default_string": null,
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Name",
+      "id": "name",
+      "is_primary": false,
+      "label": "Name",
+      "max_length": 100,
+      "min_length": 2,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": "",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Path",
+      "id": "path",
+      "is_primary": false,
+      "label": "Path",
+      "max_length": null,
+      "min_length": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_string": "gcloud",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Provider",
+      "id": "provider",
+      "is_primary": false,
+      "label": "Provider",
+      "max_length": null,
+      "min_length": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": [
+        "local",
+        "gcloud",
+        "aws"
+      ]
+    },
+    {
+      "default_float": null,
+      "type": "float",
+      "format": "attribute",
+      "helper_text": "Size",
+      "id": "size",
+      "is_primary": false,
+      "label": "Size",
+      "max": null,
+      "min": 0,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "unique": false,
+      "values": null
+    },
+    {
+      "default_binary": null,
+      "type": "binary",
+      "format": "attribute",
+      "helper_text": "Binary",
+      "id": "binary_data",
+      "is_primary": false,
+      "label": "Binary",
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false
+    },
+    {
+      "default_string": "",
+      "type": "string",
+      "format": "attribute",
+      "helper_text": "Extension",
+      "id": "extension",
+      "is_primary": false,
+      "label": "Extension",
+      "max_length": null,
+      "min_length": null,
+      "multiple": false,
+      "nullable": true,
+      "persistence": "arke_parameter",
+      "required": false,
+      "strip": false,
+      "lowercase": false,
+      "unique": false,
+      "values": null
+    }
+  ]
+}
\ No newline at end of file
diff --git a/lib/registry/system/group.json b/lib/registry/system/group.json
index 3506bfc..8ee9ba4 100644
--- a/lib/registry/system/group.json
+++ b/lib/registry/system/group.json
@@ -1,14 +1,32 @@
-{ "group": [
-  {
-    "id": "parameter",
-    "label": "Parameter",
-    "description": "Parameter Group",
-    "arke_list": ["string","integer","float","list","dict","date","datetime","time","boolean","dynamic","link","binary"]
-  },
-  {
-    "id": "arke_or_group",
-    "label": "Arke or Group",
-    "description": "Arke or Group",
-    "arke_list": ["arke","group"]
-  }
-]}
\ No newline at end of file
+{
+  "group": [
+    {
+      "id": "parameter",
+      "label": "Parameter",
+      "description": "Parameter Group",
+      "arke_list": [
+        "string",
+        "integer",
+        "float",
+        "list",
+        "dict",
+        "date",
+        "datetime",
+        "time",
+        "boolean",
+        "dynamic",
+        "link",
+        "binary"
+      ]
+    },
+    {
+      "id": "arke_or_group",
+      "label": "Arke or Group",
+      "description": "Arke or Group",
+      "arke_list": [
+        "arke",
+        "group"
+      ]
+    }
+  ]
+}
\ No newline at end of file

From ff3e68939e869fe57370f7607f4ffe39ebcc1609 Mon Sep 17 00:00:00 2001
From: Ilyich Vismara <ilyich@arkemis.it>
Date: Mon, 3 Nov 2025 12:36:44 +0100
Subject: [PATCH 28/28] v0.4.0-bulk.1

---
 mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index 73f430a..7aabc1f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -1,7 +1,7 @@
 defmodule Arke.MixProject do
   use Mix.Project
 
-  @version "0.4.0-bulk.0"
+  @version "0.4.0-bulk.1"
   @scm_url "https://github.com/arkemishub/arke"
   @site_url "https://arkehub.com"