From ce945c04cabaa48d846aaf711a5f4feea9c6baa8 Mon Sep 17 00:00:00 2001 From: Robert Graff Date: Tue, 9 Dec 2025 18:11:13 -0800 Subject: [PATCH] improvement: add create_table_options for partitioned tables and more --- .formatter.exs | 1 + README.md | 1 + .../dsls/DSL-AshPostgres.DataLayer.md | 1 + .../topics/advanced/partitioned-tables.md | 232 ++++++++++++++++++ lib/data_layer.ex | 6 + lib/data_layer/info.ex | 5 + .../migration_generator.ex | 17 +- lib/migration_generator/operation.ex | 2 +- lib/migration_generator/phase.ex | 24 +- mix.exs | 3 +- test/migration_generator_test.exs | 83 +++++++ 11 files changed, 365 insertions(+), 10 deletions(-) create mode 100644 documentation/topics/advanced/partitioned-tables.md diff --git a/.formatter.exs b/.formatter.exs index 71cf7ae6..e5a29677 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -12,6 +12,7 @@ spark_locals_without_parens = [ code?: 1, concurrently: 1, create?: 1, + create_table_options: 1, deferrable: 1, down: 1, error_fields: 1, diff --git a/README.md b/README.md index 98ba97ee..73df94d0 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Minimum required PostgreSQL version: `13.0` - [Expressions](documentation/topics/advanced/expressions.md) - [Manual Relationships](documentation/topics/advanced/manual-relationships.md) +- [Partitioned Tables](documentation/topics/advanced/partitioned-tables.md) - [Schema Based Multitenancy](documentation/topics/advanced/schema-based-multitenancy.md) - [Read Replicas](documentation/topics/advanced/using-multiple-repos.md) diff --git a/documentation/dsls/DSL-AshPostgres.DataLayer.md b/documentation/dsls/DSL-AshPostgres.DataLayer.md index a3938dcf..f52955bf 100644 --- a/documentation/dsls/DSL-AshPostgres.DataLayer.md +++ b/documentation/dsls/DSL-AshPostgres.DataLayer.md @@ -56,6 +56,7 @@ end | [`table`](#postgres-table){: #postgres-table } | `String.t` | | The table to store and read the resource from. If this is changed, the migration generator will not remove the old table. | | [`schema`](#postgres-schema){: #postgres-schema } | `String.t` | | The schema that the table is located in. Schema-based multitenancy will supercede this option. If this is changed, the migration generator will not remove the old schema. | | [`polymorphic?`](#postgres-polymorphic?){: #postgres-polymorphic? } | `boolean` | `false` | Declares this resource as polymorphic. See the [polymorphic resources guide](/documentation/topics/resources/polymorphic-resources.md) for more. | +| [`create_table_options`](#postgres-create_table_options){: #postgres-create_table_options } | `String.t` | | Options passed to ecto's table/2 in the create migration. See the [Ecto.Migration.table/2](https://hexdocs.pm/ecto_sql/Ecto.Migration.html#table/2) documentation for more information. | ### postgres.custom_indexes diff --git a/documentation/topics/advanced/partitioned-tables.md b/documentation/topics/advanced/partitioned-tables.md new file mode 100644 index 00000000..bc6191e2 --- /dev/null +++ b/documentation/topics/advanced/partitioned-tables.md @@ -0,0 +1,232 @@ + + +# Partitioned Tables + +PostgreSQL supports table partitioning, which allows you to split a large table into smaller, more manageable pieces. Partitioning can improve query performance, simplify maintenance, and enable better data management strategies. + +For more information on PostgreSQL partitioning, see the [PostgreSQL partitioning documentation](https://www.postgresql.org/docs/current/ddl-partitioning.html). + +> ### Multitenancy and Partitioning {: .info} +> +> If you're interested in using partitions for multitenancy, start with AshPostgres's [Schema Based Multitenancy](schema-based-multitenancy.html) feature, which uses PostgreSQL schemas to separate tenant data. Schema-based multitenancy is generally the recommended approach for multitenancy in AshPostgres. + +## Setting Up a Partitioned Table + +To create a partitioned table in AshPostgres, you'll use the `create_table_options` DSL option to specify the partitioning strategy. This option passes configuration directly to Ecto's `create table/2` function. + +### Range Partitioning Example + +Here's an example of setting up a range-partitioned table by date: + +```elixir +defmodule MyApp.SensorReading do + use Ash.Resource, + domain: MyApp.Domain, + data_layer: AshPostgres.DataLayer + + attributes do + uuid_primary_key :id + attribute :sensor_id, :integer + attribute :reading_value, :float + create_timestamp :inserted_at + end + + postgres do + table "sensor_readings" + repo MyApp.Repo + + # Configure the table as a partitioned table + create_table_options "PARTITION BY RANGE (inserted_at)" + + # Create a default partition to catch any data that doesn't fit into specific partitions + custom_statements do + statement :default_partition do + up """ + CREATE TABLE IF NOT EXISTS sensor_readings_default + PARTITION OF sensor_readings DEFAULT; + """ + down """ + DROP TABLE IF EXISTS sensor_readings_default; + """ + end + end + end +end +``` + +### List Partitioning Example + +Here's an example of list partitioning by region: + +```elixir +defmodule MyApp.Order do + use Ash.Resource, + domain: MyApp.Domain, + data_layer: AshPostgres.DataLayer + + attributes do + uuid_primary_key :id + attribute :order_number, :string + attribute :region, :string + attribute :total, :decimal + create_timestamp :inserted_at + end + + postgres do + table "orders" + repo MyApp.Repo + + # Configure the table as a list-partitioned table + create_table_options "PARTITION BY LIST (region)" + + # Create a default partition + custom_statements do + statement :default_partition do + up """ + CREATE TABLE IF NOT EXISTS orders_default + PARTITION OF orders DEFAULT; + """ + down """ + DROP TABLE IF EXISTS orders_default; + """ + end + end + end +end +``` + +### Hash Partitioning Example + +Here's an example of hash partitioning: + +```elixir +defmodule MyApp.LogEntry do + use Ash.Resource, + domain: MyApp.Domain, + data_layer: AshPostgres.DataLayer + + attributes do + uuid_primary_key :id + attribute :user_id, :integer + attribute :message, :string + create_timestamp :inserted_at + end + + postgres do + table "log_entries" + repo MyApp.Repo + + # Configure the table as a hash-partitioned table + create_table_options "PARTITION BY HASH (user_id)" + + # Create a default partition + custom_statements do + statement :default_partition do + up """ + CREATE TABLE IF NOT EXISTS log_entries_default + PARTITION OF log_entries DEFAULT; + """ + down """ + DROP TABLE IF EXISTS log_entries_default; + """ + end + end + end +end +``` + +## Creating Additional Partitions + +After the initial migration, you can create additional partitions as needed using custom statements. For example, to create monthly partitions for a range-partitioned table: + +```elixir +postgres do + table "sensor_readings" + repo MyApp.Repo + + create_table_options "PARTITION BY RANGE (inserted_at)" + + custom_statements do + statement :default_partition do + up """ + CREATE TABLE IF NOT EXISTS sensor_readings_default + PARTITION OF sensor_readings DEFAULT; + """ + down """ + DROP TABLE IF EXISTS sensor_readings_default; + """ + end + + # Example: Create a partition for January 2024 + statement :january_2024_partition do + up """ + CREATE TABLE IF NOT EXISTS sensor_readings_2024_01 + PARTITION OF sensor_readings + FOR VALUES FROM ('2024-01-01') TO ('2024-02-01'); + """ + down """ + DROP TABLE IF EXISTS sensor_readings_2024_01; + """ + end + + # Example: Create a partition for February 2024 + statement :february_2024_partition do + up """ + CREATE TABLE IF NOT EXISTS sensor_readings_2024_02 + PARTITION OF sensor_readings + FOR VALUES FROM ('2024-02-01') TO ('2024-03-01'); + """ + down """ + DROP TABLE IF EXISTS sensor_readings_2024_02; + """ + end + end +end +``` + +## Dynamically Creating Partitions + +For list-partitioned tables, you may want to create partitions dynamically as part of a action. Here's an example helper function for creating partitions: + +```elixir +def create_partition(resource, partition_name, list_value) do + repo = AshPostgres.DataLayer.Info.repo(resource) + table_name = AshPostgres.DataLayer.Info.table(resource) + schema = AshPostgres.DataLayer.Info.schema(resource) || "public" + + sql = """ + CREATE TABLE IF NOT EXISTS "#{schema}"."#{partition_name}" + PARTITION OF "#{schema}"."#{table_name}" + FOR VALUES IN ('#{list_value}') + """ + + case Ecto.Adapters.SQL.query(repo, sql, []) do + {:ok, _} -> + :ok + + {:error, %{postgres: %{code: :duplicate_table}}} -> + :ok + + {:error, error} -> + {:error, "Failed to create partition for #{table_name}: #{inspect(error)}"} + end +end +``` + +Similarly, you'll want to dynamically drop partitions when they're no longer needed. + + + +> ### Partitioning is Complex {: .warning} +> +> Table partitioning is a complex topic with many considerations around performance, maintenance, foreign keys, and data management. This guide shows how to configure partitioned tables in AshPostgres, but it is not a comprehensive primer on PostgreSQL partitioning. For detailed information on partitioning strategies, best practices, and limitations, please refer to the [PostgreSQL partitioning documentation](https://www.postgresql.org/docs/current/ddl-partitioning.html). + +## See Also + +- [Ecto.Migration.table/2 documentation](https://hexdocs.pm/ecto_sql/Ecto.Migration.html#table/2) for more information on table options +- [PostgreSQL Partitioning documentation](https://www.postgresql.org/docs/current/ddl-partitioning.html) for detailed information on partitioning strategies +- [Custom Statements documentation](https://hexdocs.pm/ash_postgres/dsl-ashpostgres-datalayer.html#postgres-custom_statements) for more information on using custom statements in migrations diff --git a/lib/data_layer.ex b/lib/data_layer.ex index c2161ae2..3d0da15c 100644 --- a/lib/data_layer.ex +++ b/lib/data_layer.ex @@ -400,6 +400,12 @@ defmodule AshPostgres.DataLayer do doc: """ Declares this resource as polymorphic. See the [polymorphic resources guide](/documentation/topics/resources/polymorphic-resources.md) for more. """ + ], + create_table_options: [ + type: :string, + doc: """ + Options passed to ecto's table/2 in the create migration. See the [Ecto.Migration.table/2](https://hexdocs.pm/ecto_sql/Ecto.Migration.html#table/2) documentation for more information. + """ ] ] } diff --git a/lib/data_layer/info.ex b/lib/data_layer/info.ex index b36e9f86..922e68d4 100644 --- a/lib/data_layer/info.ex +++ b/lib/data_layer/info.ex @@ -226,4 +226,9 @@ defmodule AshPostgres.DataLayer.Info do def manage_tenant_update?(resource) do Extension.get_opt(resource, [:postgres, :manage_tenant], :update?, false) end + + @doc "String passed to table/2 in the create table migration for a given resource" + def create_table_options(resource) do + Extension.get_opt(resource, [:postgres], :create_table_options, nil) + end end diff --git a/lib/migration_generator/migration_generator.ex b/lib/migration_generator/migration_generator.ex index 425ccb53..984d245b 100644 --- a/lib/migration_generator/migration_generator.ex +++ b/lib/migration_generator/migration_generator.ex @@ -1366,7 +1366,8 @@ defmodule AshPostgres.MigrationGenerator do table: table, schema: schema, multitenancy: multitenancy, - repo: repo + repo: repo, + create_table_options: create_table_options } | rest ], @@ -1375,7 +1376,13 @@ defmodule AshPostgres.MigrationGenerator do ) do group_into_phases( rest, - %Phase.Create{table: table, schema: schema, multitenancy: multitenancy, repo: repo}, + %Phase.Create{ + table: table, + schema: schema, + multitenancy: multitenancy, + repo: repo, + create_table_options: create_table_options + }, acc ) end @@ -2031,7 +2038,8 @@ defmodule AshPostgres.MigrationGenerator do schema: snapshot.schema, repo: snapshot.repo, multitenancy: snapshot.multitenancy, - old_multitenancy: empty_snapshot.multitenancy + old_multitenancy: empty_snapshot.multitenancy, + create_table_options: snapshot.create_table_options } | acc ]) @@ -3103,7 +3111,8 @@ defmodule AshPostgres.MigrationGenerator do repo: AshPostgres.DataLayer.Info.repo(resource, :mutate), multitenancy: multitenancy(resource), base_filter: AshPostgres.DataLayer.Info.base_filter_sql(resource), - has_create_action: has_create_action?(resource) + has_create_action: has_create_action?(resource), + create_table_options: AshPostgres.DataLayer.Info.create_table_options(resource) } hash = diff --git a/lib/migration_generator/operation.ex b/lib/migration_generator/operation.ex index b7b09792..75adc85c 100644 --- a/lib/migration_generator/operation.ex +++ b/lib/migration_generator/operation.ex @@ -149,7 +149,7 @@ defmodule AshPostgres.MigrationGenerator.Operation do defmodule CreateTable do @moduledoc false - defstruct [:table, :schema, :multitenancy, :old_multitenancy, :repo] + defstruct [:table, :schema, :multitenancy, :old_multitenancy, :repo, :create_table_options] end defmodule AddAttribute do diff --git a/lib/migration_generator/phase.ex b/lib/migration_generator/phase.ex index 28e380d9..22c927ce 100644 --- a/lib/migration_generator/phase.ex +++ b/lib/migration_generator/phase.ex @@ -7,7 +7,15 @@ defmodule AshPostgres.MigrationGenerator.Phase do defmodule Create do @moduledoc false - defstruct [:table, :schema, :multitenancy, :repo, operations: [], commented?: false] + defstruct [ + :table, + :schema, + :multitenancy, + :repo, + :create_table_options, + operations: [], + commented?: false + ] import AshPostgres.MigrationGenerator.Operation.Helper, only: [as_atom: 1] @@ -16,10 +24,18 @@ defmodule AshPostgres.MigrationGenerator.Phase do table: table, operations: operations, multitenancy: multitenancy, - repo: repo + repo: repo, + create_table_options: create_table_options }) do + table_options_str = + if create_table_options do + ", options: #{inspect(create_table_options)}" + else + "" + end + if multitenancy.strategy == :context do - "create table(:#{as_atom(table)}, primary_key: false, prefix: prefix()) do\n" <> + "create table(:#{as_atom(table)}, primary_key: false, prefix: prefix()#{table_options_str}) do\n" <> Enum.map_join(operations, "\n", fn operation -> operation.__struct__.up(operation) end) <> "\nend" else @@ -38,7 +54,7 @@ defmodule AshPostgres.MigrationGenerator.Phase do end pre_create <> - "create table(:#{as_atom(table)}, primary_key: false#{opts}) do\n" <> + "create table(:#{as_atom(table)}, primary_key: false#{opts}#{table_options_str}) do\n" <> Enum.map_join(operations, "\n", fn operation -> operation.__struct__.up(operation) end) <> "\nend" end diff --git a/mix.exs b/mix.exs index 0b0af0d0..db5c090c 100644 --- a/mix.exs +++ b/mix.exs @@ -107,9 +107,10 @@ defmodule AshPostgres.MixProject do "documentation/topics/development/testing.md", "documentation/topics/development/upgrading-to-2.0.md", "documentation/topics/advanced/expressions.md", + "documentation/topics/advanced/manual-relationships.md", + "documentation/topics/advanced/partitioned-tables.md", "documentation/topics/advanced/schema-based-multitenancy.md", "documentation/topics/advanced/using-multiple-repos.md", - "documentation/topics/advanced/manual-relationships.md", {"documentation/dsls/DSL-AshPostgres.DataLayer.md", search_data: Spark.Docs.search_data_for(AshPostgres.DataLayer)}, "CHANGELOG.md" diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index b3d1fd17..4d17bcf4 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -3226,4 +3226,87 @@ defmodule AshPostgres.MigrationGeneratorTest do ~S[modify :category_id, references(:categories, column: :id, name: "products_category_id_fkey", type: :decimal, precision: 10, scale: 0] end end + + describe "create_table_options" do + setup do + on_exit(fn -> + File.rm_rf!("test_snapshots_path") + File.rm_rf!("test_migration_path") + File.rm_rf!("test_tenant_migration_path") + end) + end + + test "includes create_table_options in regular table migration" do + defposts do + postgres do + table "posts" + create_table_options("PARTITION BY RANGE (id)") + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true) + create_timestamp(:inserted_at) + end + end + + defdomain([Post]) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + quiet: true, + format: false, + auto_name: true + ) + + assert [file] = + Path.wildcard("test_migration_path/**/*_migrate_resources*.exs") + |> Enum.reject(&String.contains?(&1, "extensions")) + + file_contents = File.read!(file) + + assert file_contents =~ + ~S[create table(:posts, primary_key: false, options: "PARTITION BY RANGE (id)") do] + end + + test "includes create_table_options in context-based multitenancy migration" do + defposts do + postgres do + table "posts" + create_table_options("PARTITION BY RANGE (id)") + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true) + attribute(:user_id, :integer, public?: true) + end + + multitenancy do + strategy(:context) + end + end + + defdomain([Post]) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + tenant_migration_path: "test_tenant_migration_path", + quiet: true, + format: false, + auto_name: true + ) + + assert [file] = + Path.wildcard("test_tenant_migration_path/**/*_migrate_resources*.exs") + |> Enum.reject(&String.contains?(&1, "extensions")) + + file_contents = File.read!(file) + + assert file_contents =~ + ~S[create table(:posts, primary_key: false, prefix: prefix(), options: "PARTITION BY RANGE (id)") do] + end + end end