From 18b19a23a4640ee90081494b685e20f4f505e5ca Mon Sep 17 00:00:00 2001 From: alchemistmatt Date: Thu, 3 Oct 2019 16:05:32 -0700 Subject: [PATCH 1/3] Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns --- example_conf_file | 1 + sqlserver2pgsql.pl | 111 ++++++++++++++++++++++++++++++++------------- 2 files changed, 81 insertions(+), 31 deletions(-) diff --git a/example_conf_file b/example_conf_file index da617bb..7a36464 100644 --- a/example_conf_file +++ b/example_conf_file @@ -31,6 +31,7 @@ keep identifier case=1 # keep case of database objects; comment out to conv #camelcasetosnake=1 # Uncomment to convert to snake case; comment out to leave names unchanged (or lowercase) validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) #skip citext length check=1 # When defined, do not add a CHECK (char_length()) check for citext fields +#create generated always=1 # When defined, create CREATE GENERATED ALWAYS statements instead of CREATE SEQUENCE statements # Incremental job sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2e70cc9..0819032 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -52,7 +52,11 @@ our $use_pk_if_possible; our $pforce_ssl; our $stringtype_unspecified; +<<<<<<< HEAD our $skip_citext_length_check; +======= +our $create_generated_always; +>>>>>>> Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns # Will be set if we detect GIS objects our $requires_postgis=0; @@ -109,7 +113,11 @@ sub parse_conf_file 'ignore errors' => 'ignore_errors', 'postgresql force ssl' => 'pforce_ssl', 'stringtype unspecified' => 'stringtype_unspecified', +<<<<<<< HEAD 'skip citext length check' => 'skip_citext_length_check', +======= + 'create generated always' => 'create_generated_always', +>>>>>>> Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns ); # Open the conf file or die @@ -160,7 +168,11 @@ sub set_default_conf_values $sp=1433 unless (defined ($sp)); $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); +<<<<<<< HEAD $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); +======= + $create_generated_always=0 unless (defined ($create_generated_always)); +>>>>>>> Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -2584,35 +2596,63 @@ sub generate_schema foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); - if (defined $seqref->{STEP}) - { - print AFTER " INCREMENT BY ",$seqref->{STEP}; - } - if (defined $seqref->{MIN}) - { - print AFTER " MINVALUE ",$seqref->{MIN}; - } - if (defined $seqref->{MAX}) - { - print AFTER " MAXVALUE ",$seqref->{MAX}; - } - if (defined $seqref->{START}) - { - print AFTER " START WITH ",$seqref->{START}; - } - if (defined $seqref->{CACHE}) - { - print AFTER " CACHE ",$seqref->{CACHE}; - } - if (defined $seqref->{OWNERTABLE}) - { - print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), - '.',format_identifier($seqref->{OWNERTABLE}), - '.',format_identifier($seqref->{OWNERCOL}); - } - print AFTER ";\n"; - } + + if ($create_generated_always and defined $seqref->{OWNERTABLE}) + { + # Add a statement of the form + # ALTER TABLE "schema"."table_name" ALTER COLUMN "column_name" ADD GENERATED ALWAYS AS IDENTITY (start 1000); + + print AFTER "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($seqref->{OWNERTABLE}) . " "; + print AFTER "ALTER COLUMN " . format_identifier($seqref->{OWNERCOL}) . " ADD GENERATED ALWAYS AS IDENTITY"; + + if (defined $seqref->{START} or defined $seqref->{STEP}) + { + print AFTER " ("; + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + print AFTER ")"; + } + } + else + { + print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + if (defined $seqref->{MIN}) + { + print AFTER " MINVALUE ",$seqref->{MIN}; + } + if (defined $seqref->{MAX}) + { + print AFTER " MAXVALUE ",$seqref->{MAX}; + } + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + if (defined $seqref->{CACHE}) + { + print AFTER " CACHE ",$seqref->{CACHE}; + } + if (defined $seqref->{OWNERTABLE}) + { + print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), + '.',format_identifier($seqref->{OWNERTABLE}), + '.',format_identifier($seqref->{OWNERCOL}); + } + } + + print AFTER ";\n"; + } # Now PK. We have to go through all tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -2861,12 +2901,20 @@ sub generate_schema . " ALTER COLUMN " . format_identifier($col) . " SET DEFAULT " . $default_value . ";\n"; if ($colref->{DEFAULT}->{UNSURE}) - { + { print UNSURE $definition; } else { - print AFTER $definition; + if ($create_generated_always and ($definition =~ /nextval.+_seq/i)) + { + # Skip this set default item + } + else + { + print AFTER $definition; + } + } } } @@ -2880,6 +2928,7 @@ sub generate_schema my $seqref = $refschema->{SEQUENCES}->{$sequence}; # This may not be an identity. Skip it then next unless defined ($seqref->{OWNERCOL}); + next if defined ($create_generated_always); print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select " . ($seqref->{STEP} > 0 ? "max" : "min") . "(" . format_identifier($seqref->{OWNERCOL}) .") from " From 3ca0ec5b8469d45df6eb54bbc400cb5f911d0e9a Mon Sep 17 00:00:00 2001 From: madtibo Date: Thu, 29 Apr 2021 12:05:13 +0200 Subject: [PATCH 2/3] rename create_generated_always option to use_identity_column and activate it --- example_conf_file | 2 +- sqlserver2pgsql.pl | 32 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/example_conf_file b/example_conf_file index 7a36464..a93dea3 100644 --- a/example_conf_file +++ b/example_conf_file @@ -31,7 +31,7 @@ keep identifier case=1 # keep case of database objects; comment out to conv #camelcasetosnake=1 # Uncomment to convert to snake case; comment out to leave names unchanged (or lowercase) validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) #skip citext length check=1 # When defined, do not add a CHECK (char_length()) check for citext fields -#create generated always=1 # When defined, create CREATE GENERATED ALWAYS statements instead of CREATE SEQUENCE statements +use identity column=1 # if set, use identity columns statements ('CREATE GENERATED ALWAYS') instead of creating a dedicated sequence ('CREATE SEQUENCE') # Incremental job sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 0819032..5f6acf4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -52,11 +52,8 @@ our $use_pk_if_possible; our $pforce_ssl; our $stringtype_unspecified; -<<<<<<< HEAD our $skip_citext_length_check; -======= -our $create_generated_always; ->>>>>>> Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns +our $use_identity_column; # Will be set if we detect GIS objects our $requires_postgis=0; @@ -113,11 +110,8 @@ sub parse_conf_file 'ignore errors' => 'ignore_errors', 'postgresql force ssl' => 'pforce_ssl', 'stringtype unspecified' => 'stringtype_unspecified', -<<<<<<< HEAD - 'skip citext length check' => 'skip_citext_length_check', -======= - 'create generated always' => 'create_generated_always', ->>>>>>> Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns + 'skip citext length check' => 'skip_citext_length_check', + 'use identity column' => 'use_identity_column', ); # Open the conf file or die @@ -168,11 +162,8 @@ sub set_default_conf_values $sp=1433 unless (defined ($sp)); $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); -<<<<<<< HEAD $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); -======= - $create_generated_always=0 unless (defined ($create_generated_always)); ->>>>>>> Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns + $use_identity_column=0 unless (defined ($use_identity_column)); } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -804,6 +795,11 @@ sub usage '1' sort all tables. LIST_OF_TABLES gives a comma separated list of tables to sort in the form 'schema1.table1,schema2.table2'. Cases are compared insensitively. + -skip_citext_length_check (Default 0) + if set, do not add a CHECK (char_length()) check for citext fields + -use_identity_column (Default 1) + if set, use identity columns statements (GENERATED ALWAYS AS + IDENTITY) instead of creating a dedicated sequence (CREATE SEQUENCE) Kettle options: if you are generating for kettle, you must provide connection information. @@ -2597,7 +2593,7 @@ sub generate_schema { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - if ($create_generated_always and defined $seqref->{OWNERTABLE}) + if ($use_identity_column and defined $seqref->{OWNERTABLE}) { # Add a statement of the form # ALTER TABLE "schema"."table_name" ALTER COLUMN "column_name" ADD GENERATED ALWAYS AS IDENTITY (start 1000); @@ -2906,7 +2902,7 @@ sub generate_schema } else { - if ($create_generated_always and ($definition =~ /nextval.+_seq/i)) + if ($use_identity_column and ($definition =~ /nextval.+_seq/i)) { # Skip this set default item } @@ -2928,7 +2924,7 @@ sub generate_schema my $seqref = $refschema->{SEQUENCES}->{$sequence}; # This may not be an identity. Skip it then next unless defined ($seqref->{OWNERCOL}); - next if defined ($create_generated_always); + next if defined ($use_identity_column); print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select " . ($seqref->{STEP} > 0 ? "max" : "min") . "(" . format_identifier($seqref->{OWNERCOL}) .") from " @@ -3170,7 +3166,9 @@ sub resolve_name_conflicts "use_pk_if_possible=s" => \$use_pk_if_possible, "ignore_errors" => \$ignore_errors, "pforce_ssl" => \$pforce_ssl, - "stringtype_unspecified" => \$stringtype_unspecified + "stringtype_unspecified" => \$stringtype_unspecified, + "skip_citext_length_check" => \$skip_citext_length_check, + "use_identity_column" => \$use_identity_column ); # We don't understand command line or have been asked for usage From 70a733c34490b8784d1f7f6b87f64107ee5adb9a Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 4 May 2021 11:04:22 +0200 Subject: [PATCH 3/3] reindent --- sqlserver2pgsql.pl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5f6acf4..96b78fb 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -3157,18 +3157,18 @@ sub resolve_name_conflicts "i" => \$case_insensitive, "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int, - "drop_rowversion" => \$drop_rowversion, - "relabel_schemas=s" => \$relabel_schemas, - "keep_identifier_case" => \$keep_identifier_case, - "camel_to_snake" => \$camel_to_snake, - "validate_constraints=s" => \$validate_constraints, - "sort_size=i" => \$sort_size, - "use_pk_if_possible=s" => \$use_pk_if_possible, - "ignore_errors" => \$ignore_errors, - "pforce_ssl" => \$pforce_ssl, - "stringtype_unspecified" => \$stringtype_unspecified, + "drop_rowversion" => \$drop_rowversion, + "relabel_schemas=s" => \$relabel_schemas, + "keep_identifier_case" => \$keep_identifier_case, + "camel_to_snake" => \$camel_to_snake, + "validate_constraints=s" => \$validate_constraints, + "sort_size=i" => \$sort_size, + "use_pk_if_possible=s" => \$use_pk_if_possible, + "ignore_errors" => \$ignore_errors, + "pforce_ssl" => \$pforce_ssl, + "stringtype_unspecified" => \$stringtype_unspecified, "skip_citext_length_check" => \$skip_citext_length_check, - "use_identity_column" => \$use_identity_column + "use_identity_column" => \$use_identity_column ); # We don't understand command line or have been asked for usage