From 2f6d4c68dd790c1efea0ca5bae9297351d7249fd Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 27 Feb 2026 12:34:31 -0500 Subject: [PATCH 01/52] FECFILE-2913: If we delete the first_created_schedule_a, set it back to None to get updated. --- performance-testing/locust_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index 9105cee65..a81be9ea0 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -413,6 +413,8 @@ def delete_schedule_a_transaction(self): name="delete_schedule_a_transaction", ) if response.status_code == 204: + if transaction == self.first_created_schedule_a: + self.first_created_schedule_a = None return raise Exception("Failed to DELETE Schedule A transaction") From 229e7f6a451c5486ee6e9d5a70fe5757d137625a Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 27 Feb 2026 12:47:10 -0500 Subject: [PATCH 02/52] FECFILE-2913: Generalize which transaction is saved, add LONG_CHAINS bool. --- performance-testing/locust_run.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index a81be9ea0..73b54fcda 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -39,6 +39,9 @@ # Tracks state of Payload Contacts between test runs CREATED_PAYLOAD_CONTACTS = False +# Do we want long transaction chains? +LONG_CHAINS = False + # Lower the interval between log reports to prevent log queue overflow runners.WORKER_LOG_REPORT_INTERVAL = 2 @@ -92,9 +95,9 @@ class Tasks(TaskSet): "CAN": {}, "COM": {}, } - last_created_schedule_a = None - last_created_schedule_b = None - last_created_schedule_c = None + saved_schedule_a = None + saved_schedule_b = None + saved_schedule_c = None def on_start(self): logging.info("Logging in") @@ -235,7 +238,11 @@ def create_schedule_a_transaction(self): ) if response.status_code != 200: raise Exception("Failed to POST new Schedule A transaction") - self.last_created_schedule_a = response.json() + + # if LONG_CHAINS is true then we only want to save the first transaction, + # otherwise we always save the current one so we have the last transaction + if not self.saved_schedule_a or not LONG_CHAINS: + self.saved_schedule_a = response.json() @task(ceil(DATA_ENTRY_WEIGHT * SCHEDULE_B_MULTIPLIER / 2)) def create_schedule_b_transaction(self): @@ -269,7 +276,8 @@ def create_schedule_b_transaction(self): ) if response.status_code != 200: raise Exception("Failed to POST new Schedule B transaction") - self.last_created_schedule_b = response.json() + if not self.saved_schedule_b or not LONG_CHAINS: + self.saved_schedule_b = response.json() @task(ceil(DATA_ENTRY_WEIGHT * SCHEDULE_B_MULTIPLIER / 2)) def create_schedule_b_election_transaction(self): @@ -309,7 +317,8 @@ def create_schedule_b_election_transaction(self): ) if response.status_code != 200: raise Exception("Failed to POST new Schedule B transaction") - self.last_created_schedule_b = response.json() + if not self.saved_schedule_b or not LONG_CHAINS: + self.saved_schedule_b = response.json() @task(ceil(DATA_ENTRY_WEIGHT * SCHEDULE_C_MULTIPLIER)) def create_schedule_c_transaction(self): @@ -344,7 +353,8 @@ def create_schedule_c_transaction(self): ) if response.status_code != 200: raise Exception("Failed to POST new Schedule C transaction") - self.last_created_schedule_c = response.json() + if not self.saved_schedule_c or not LONG_CHAINS: + self.saved_schedule_c = response.json() @task(ceil(DATA_ENTRY_WEIGHT * SCHEDULE_D_MULTIPLIER)) def create_schedule_d_transaction(self): @@ -373,7 +383,7 @@ def create_schedule_d_transaction(self): ceil(DATA_ENTRY_WEIGHT * SCHEDULE_A_MULTIPLIER * UPDATE_TRANSACTION_MULTIPLIER) ) def update_schedule_a_transaction(self): - transaction_id = self.last_created_schedule_a + transaction_id = self.saved_schedule_a if transaction_id: response = self.client_get( f"/api/v1/transactions/{transaction_id}/", @@ -422,7 +432,7 @@ def delete_schedule_a_transaction(self): ceil(DATA_ENTRY_WEIGHT * SCHEDULE_B_MULTIPLIER * UPDATE_TRANSACTION_MULTIPLIER) ) def update_schedule_b_transaction(self): - transaction_id = self.last_created_schedule_b + transaction_id = self.saved_schedule_b if transaction_id: response = self.client_get( f"/api/v1/transactions/{transaction_id}/", @@ -458,7 +468,7 @@ def update_schedule_b_transaction(self): ceil(DATA_ENTRY_WEIGHT * SCHEDULE_C_MULTIPLIER * UPDATE_TRANSACTION_MULTIPLIER) ) def update_schedule_c_transaction(self): - transaction_id = self.last_created_schedule_c + transaction_id = self.saved_schedule_c if transaction_id: response = self.client_get( f"/api/v1/transactions/{transaction_id}/", From 567b284c57ad46708ebe1832a90768c3d7ce34a3 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 27 Feb 2026 12:49:06 -0500 Subject: [PATCH 03/52] FECFILE-2913: Missed one. --- performance-testing/locust_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index 73b54fcda..433e723ed 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -423,8 +423,8 @@ def delete_schedule_a_transaction(self): name="delete_schedule_a_transaction", ) if response.status_code == 204: - if transaction == self.first_created_schedule_a: - self.first_created_schedule_a = None + if transaction == self.saved_schedule_a: + self.saved_schedule_a = None return raise Exception("Failed to DELETE Schedule A transaction") From 11205bf29c589d08f0dfb5b3256fa90ec1e01551 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 27 Feb 2026 12:54:54 -0500 Subject: [PATCH 04/52] FECFILE-2913: Make LONG_CHAINS an env var. --- performance-testing/locust_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index 433e723ed..9a5cab32a 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -40,7 +40,7 @@ CREATED_PAYLOAD_CONTACTS = False # Do we want long transaction chains? -LONG_CHAINS = False +LONG_CHAINS = str(os.environ.get("LONG_CHAINS", "false")).lower() in ("true", "1") # Lower the interval between log reports to prevent log queue overflow runners.WORKER_LOG_REPORT_INTERVAL = 2 From e37ebd1a44139e4b32dd74be84c726af61755fc5 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 27 Feb 2026 12:56:06 -0500 Subject: [PATCH 05/52] FECFILE-2913: Comment LONG_CHAINS for clarity. --- performance-testing/locust_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index 9a5cab32a..3c5093c39 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -39,7 +39,7 @@ # Tracks state of Payload Contacts between test runs CREATED_PAYLOAD_CONTACTS = False -# Do we want long transaction chains? +# Do we want long transaction chains? Accepts "true"/"True"/"1", otherwise false LONG_CHAINS = str(os.environ.get("LONG_CHAINS", "false")).lower() in ("true", "1") # Lower the interval between log reports to prevent log queue overflow From 038ac19344028190ea2deb97a28dbf648e152dfd Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Mon, 9 Mar 2026 17:07:31 -0400 Subject: [PATCH 06/52] FECFILE-2734: Squashing All The Migrations. --- .../cash_on_hand/migrations/0001_initial.py | 51 -- ..._cashonhandyearly_cash_on_hand_and_more.py | 33 + ..._cashonhandyearly_cash_on_hand_and_more.py | 25 - .../migrations/0001_initial.py | 47 -- ...hed_0007_alter_committeeaccount_members.py | 196 ++++++ .../migrations/0002_membership.py | 83 --- ...ding_email_alter_membership_id_and_more.py | 93 --- .../0004_remove_duplicate_memberships.py | 43 -- .../migrations/0005_remove_pending_emails.py | 24 - ...r_membership_committee_account_and_more.py | 34 - .../0007_alter_committeeaccount_members.py | 24 - ...tial_squashed_0003_memotext_text_prefix.py | 27 + .../memo_text/migrations/0002_initial.py | 26 - .../migrations/0003_memotext_text_prefix.py | 18 - .../reports/migrations/00018_form24_name.py | 33 - .../migrations/00019_form24_name_fix.py | 21 - ..._deleted_squashed_00019_form24_name_fix.py | 363 +++++++++++ .../migrations/0007_remove_report_deleted.py | 17 - ...y_remove_form1m_committee_name_and_more.py | 193 ------ .../migrations/0009_report_can_delete.py | 180 ------ .../migrations/0010_report_can_unammend.py | 65 -- .../0011_remove_form3x_cash_on_hand_date.py | 16 - .../migrations/0012_alter_form99_text_code.py | 37 -- .../migrations/0013_form3_report_form_3.py | 478 -------------- .../migrations/0014_form99_swap_text_code.py | 17 - ...g_frequency_form3x_report_type_category.py | 23 - .../0016_determine_frequency_and_category.py | 37 -- ..._filing_frequency_form99_pdf_attachment.py | 23 - ...quashed_0026_alter_transaction_itemized.py | 396 ++++++++++++ ...03_alter_transaction_parent_transaction.py | 24 - .../0004_report_transactions_link_table.py | 31 - ...ulec_report_coverage_from_date_and_more.py | 65 -- ..._expenditure_memos_no_aggregation_group.py | 46 -- .../0007_schedulee_so_candidate_state.py | 17 - ...lendar_ytd_per_election_office_and_more.py | 261 -------- ...9_update_calculate_loan_payment_to_date.py | 260 -------- ...10_update_aggregate_trigger_performance.py | 205 ------ .../migrations/0011_transaction_can_delete.py | 107 ---- ...012_alter_transactions_blocking_reports.py | 27 - ...action_itemized_and_associated_triggers.py | 281 --------- .../migrations/0014_drop_transaction_view.py | 596 ------------------ .../0015_merge_transaction_triggers.py | 469 -------------- ...chedulef_transaction_contact_4_and_more.py | 81 --- ...17_schedulef_coordianted_to_coordinated.py | 19 - ...chedulef_general_election_year_and_more.py | 30 - .../0019_aggregate_committee_controls.py | 236 ------- .../0020_trigger_save_on_transactions.py | 76 --- .../0021_alter_transaction_reports.py | 24 - .../migrations/0022_schedule_f_aggregation.py | 54 -- ...optimize_calculate_loan_payment_to_date.py | 117 ---- ...024_scheduled_balance_at_close_and_more.py | 59 -- .../0025_drop_aggregate_triggers.py | 565 ----------------- .../0026_alter_transaction_itemized.py | 18 - .../migrations/0002_remove_user_cmtee_id.py | 27 - ...shed_0007_user_security_consent_version.py | 74 +++ .../0003_user_security_consent_date.py | 18 - .../migrations/0004_alter_user_managers.py | 20 - ...ent_date_user_security_consent_exp_date.py | 17 - .../0006_remove_old_login_accounts.py | 30 - .../0007_user_security_consent_version.py | 18 - ...ompleted_squashed_0003_polling_attempts.py | 54 ++ ...ploadsubmission_task_completed_and_more.py | 33 - ...ssion_fecfile_polling_attempts_and_more.py | 23 - 63 files changed, 1143 insertions(+), 5462 deletions(-) delete mode 100644 django-backend/fecfiler/cash_on_hand/migrations/0001_initial.py create mode 100644 django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py delete mode 100644 django-backend/fecfiler/cash_on_hand/migrations/0002_alter_cashonhandyearly_cash_on_hand_and_more.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0001_initial.py create mode 100644 django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0002_membership.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0005_remove_pending_emails.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0006_alter_membership_committee_account_and_more.py delete mode 100644 django-backend/fecfiler/committee_accounts/migrations/0007_alter_committeeaccount_members.py create mode 100644 django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py delete mode 100644 django-backend/fecfiler/memo_text/migrations/0002_initial.py delete mode 100644 django-backend/fecfiler/memo_text/migrations/0003_memotext_text_prefix.py delete mode 100644 django-backend/fecfiler/reports/migrations/00018_form24_name.py delete mode 100644 django-backend/fecfiler/reports/migrations/00019_form24_name_fix.py create mode 100644 django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py delete mode 100644 django-backend/fecfiler/reports/migrations/0007_remove_report_deleted.py delete mode 100644 django-backend/fecfiler/reports/migrations/0008_remove_form1m_city_remove_form1m_committee_name_and_more.py delete mode 100644 django-backend/fecfiler/reports/migrations/0009_report_can_delete.py delete mode 100644 django-backend/fecfiler/reports/migrations/0010_report_can_unammend.py delete mode 100644 django-backend/fecfiler/reports/migrations/0011_remove_form3x_cash_on_hand_date.py delete mode 100644 django-backend/fecfiler/reports/migrations/0012_alter_form99_text_code.py delete mode 100644 django-backend/fecfiler/reports/migrations/0013_form3_report_form_3.py delete mode 100644 django-backend/fecfiler/reports/migrations/0014_form99_swap_text_code.py delete mode 100644 django-backend/fecfiler/reports/migrations/0015_form3x_filing_frequency_form3x_report_type_category.py delete mode 100644 django-backend/fecfiler/reports/migrations/0016_determine_frequency_and_category.py delete mode 100644 django-backend/fecfiler/reports/migrations/0017_form99_filing_frequency_form99_pdf_attachment.py create mode 100644 django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0003_alter_transaction_parent_transaction.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0004_report_transactions_link_table.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0005_schedulec_report_coverage_from_date_and_more.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0006_independent_expenditure_memos_no_aggregation_group.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0007_schedulee_so_candidate_state.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0009_update_calculate_loan_payment_to_date.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0010_update_aggregate_trigger_performance.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0011_transaction_can_delete.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0012_alter_transactions_blocking_reports.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0013_transaction_itemized_and_associated_triggers.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0014_drop_transaction_view.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0015_merge_transaction_triggers.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0016_schedulef_transaction_contact_4_and_more.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0017_schedulef_coordianted_to_coordinated.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0018_schedulef_general_election_year_and_more.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0019_aggregate_committee_controls.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0020_trigger_save_on_transactions.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0021_alter_transaction_reports.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0022_schedule_f_aggregation.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0023_optimize_calculate_loan_payment_to_date.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0024_scheduled_balance_at_close_and_more.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0025_drop_aggregate_triggers.py delete mode 100644 django-backend/fecfiler/transactions/migrations/0026_alter_transaction_itemized.py delete mode 100644 django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id.py create mode 100644 django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py delete mode 100644 django-backend/fecfiler/user/migrations/0003_user_security_consent_date.py delete mode 100644 django-backend/fecfiler/user/migrations/0004_alter_user_managers.py delete mode 100644 django-backend/fecfiler/user/migrations/0005_rename_security_consent_date_user_security_consent_exp_date.py delete mode 100644 django-backend/fecfiler/user/migrations/0006_remove_old_login_accounts.py delete mode 100644 django-backend/fecfiler/user/migrations/0007_user_security_consent_version.py create mode 100644 django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py delete mode 100644 django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_and_more.py delete mode 100644 django-backend/fecfiler/web_services/migrations/0003_uploadsubmission_fecfile_polling_attempts_and_more.py diff --git a/django-backend/fecfiler/cash_on_hand/migrations/0001_initial.py b/django-backend/fecfiler/cash_on_hand/migrations/0001_initial.py deleted file mode 100644 index 407a0fb68..000000000 --- a/django-backend/fecfiler/cash_on_hand/migrations/0001_initial.py +++ /dev/null @@ -1,51 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-16 20:15 - -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("committee_accounts", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="CashOnHandYearly", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ( - "cash_on_hand", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ("year", models.TextField(blank=True, null=True)), - ("created", models.DateTimeField(auto_now_add=True)), - ("updated", models.DateTimeField(auto_now=True)), - ( - "committee_account", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="committee_accounts.committeeaccount", - ), - ), - ], - options={ - "db_table": "cash_on_hand_yearly", - }, - ), - ] diff --git a/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py b/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py new file mode 100644 index 000000000..25a2d2aed --- /dev/null +++ b/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.11 on 2026-03-06 21:37 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('cash_on_hand', '0001_initial'), ('cash_on_hand', '0002_alter_cashonhandyearly_cash_on_hand_and_more')] + + initial = True + + dependencies = [ + ('committee_accounts', '0001_squashed_0007_alter_committeeaccount_members'), + ] + + operations = [ + migrations.CreateModel( + name='CashOnHandYearly', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('cash_on_hand', models.DecimalField(decimal_places=2, max_digits=11)), + ('year', models.TextField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('committee_account', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='committee_accounts.committeeaccount')), + ], + options={ + 'db_table': 'cash_on_hand_yearly', + }, + ), + ] diff --git a/django-backend/fecfiler/cash_on_hand/migrations/0002_alter_cashonhandyearly_cash_on_hand_and_more.py b/django-backend/fecfiler/cash_on_hand/migrations/0002_alter_cashonhandyearly_cash_on_hand_and_more.py deleted file mode 100644 index 0eb4f21e1..000000000 --- a/django-backend/fecfiler/cash_on_hand/migrations/0002_alter_cashonhandyearly_cash_on_hand_and_more.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 5.1.1 on 2024-11-20 09:18 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('cash_on_hand', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='cashonhandyearly', - name='cash_on_hand', - field=models.DecimalField(decimal_places=2, default=0, max_digits=11), - preserve_default=False, - ), - migrations.AlterField( - model_name='cashonhandyearly', - name='year', - field=models.TextField(default=2020), - preserve_default=False, - ), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_initial.py b/django-backend/fecfiler/committee_accounts/migrations/0001_initial.py deleted file mode 100644 index d0af2c565..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_initial.py +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-16 20:15 - -import django.core.validators -from django.db import migrations, models -import uuid - - -class Migration(migrations.Migration): - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="CommitteeAccount", - fields=[ - ("deleted", models.DateTimeField(blank=True, null=True)), - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ( - "committee_id", - models.CharField( - max_length=9, - unique=True, - validators=[ - django.core.validators.RegexValidator( - "^C[0-9]{8}$", "invalid committee id format" - ) - ], - ), - ), - ("created", models.DateTimeField(auto_now_add=True)), - ("updated", models.DateTimeField(auto_now=True)), - ], - options={ - "db_table": "committee_accounts", - }, - ), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py new file mode 100644 index 000000000..3cd50758b --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -0,0 +1,196 @@ +# Generated by Django 5.2.11 on 2026-03-06 22:00 + +import django.core.validators +import django.db.migrations.operations.special +import django.db.models.deletion +import django.utils.timezone +import uuid +from django.conf import settings +from django.db import migrations, models + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# fecfiler.committee_accounts.migrations.0002_membership +# fecfiler.committee_accounts.migrations.0003_membership_pending_email_alter_membership_id_and_more +# fecfiler.committee_accounts.migrations.0004_remove_duplicate_memberships +# fecfiler.committee_accounts.migrations.0005_remove_pending_emails + + +def create_memberships(apps, schema_editor): + CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") + User = apps.get_model("user", "User") + Membership = apps.get_model("committee_accounts", "Membership") + db_alias = schema_editor.connection.alias + users = User.objects.using(db_alias).all() + for user in users: + committee = ( + CommitteeAccount.objects.using(db_alias) + .filter(committee_id=user.cmtee_id) + .first() + ) + if committee: + Membership.objects.create( + user=user, + committee_account=committee, + role="COMMITTEE_ADMINISTRATOR", + ) + + +def delete_pending_memberships(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") + Membership.objects.filter(user=None).delete() + + +def generate_new_uuid(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") + for membership in Membership.objects.all(): + membership.uuid = uuid.uuid4() + membership.save() + + +def delete_memberships_with_overlapping_emails(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") + Committee = apps.get_model("committee_accounts", "CommitteeAccount") + + for committee in Committee.objects.all(): + committee_memberships = Membership.objects.filter(committee_account=committee) + unique_pending_emails = set() + emails_to_prune = set() + + for membership in committee_memberships: + pending_email = str(membership.pending_email).lower() + if pending_email not in unique_pending_emails: + unique_pending_emails.add(pending_email) + else: + emails_to_prune.add(pending_email) + + for email in list(emails_to_prune): + overlapping_memberships = list( + committee_memberships.filter(pending_email__iexact=email).order_by("user") + ) + for membership_to_delete in overlapping_memberships[1:]: + membership_to_delete.delete() + + +def remove_pending_emails(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") + Membership.objects.filter(pending_email__isnull=False, user_id__isnull=False).update( + pending_email=None + ) + +class Migration(migrations.Migration): + + replaces = [('committee_accounts', '0001_initial'), ('committee_accounts', '0002_membership'), ('committee_accounts', '0003_membership_pending_email_alter_membership_id_and_more'), ('committee_accounts', '0004_remove_duplicate_memberships'), ('committee_accounts', '0005_remove_pending_emails'), ('committee_accounts', '0006_alter_membership_committee_account_and_more'), ('committee_accounts', '0007_alter_committeeaccount_members')] + + initial = True + + dependencies = [ + ('user', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='CommitteeAccount', + fields=[ + ('deleted', models.DateTimeField(blank=True, null=True)), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('committee_id', models.CharField(max_length=9, unique=True, validators=[django.core.validators.RegexValidator('^C[0-9]{8}$', 'invalid committee id format')])), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + options={ + 'db_table': 'committee_accounts', + }, + ), + migrations.CreateModel( + name='Membership', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('role', models.CharField(choices=[('COMMITTEE_ADMINISTRATOR', 'Committee Administrator'), ('REVIEWER', 'Reviewer')], max_length=25)), + ('committee_account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='committee_accounts.committeeaccount')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.AddField( + model_name='committeeaccount', + name='members', + field=models.ManyToManyField(through='committee_accounts.Membership', to=settings.AUTH_USER_MODEL), + ), + migrations.RunPython( + code=create_memberships, + ), + migrations.AddField( + model_name='membership', + name='pending_email', + field=models.EmailField(blank=True, max_length=254, null=True), + ), + migrations.AddField( + model_name='membership', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False, serialize=False), + ), + migrations.RunPython( + code=generate_new_uuid, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RemoveField( + model_name='membership', + name='id', + ), + migrations.RenameField( + model_name='membership', + old_name='uuid', + new_name='id', + ), + migrations.AlterField( + model_name='membership', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True), + ), + migrations.AlterField( + model_name='membership', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='membership', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='membership', + name='updated', + field=models.DateTimeField(auto_now=True), + ), + migrations.RunPython( + code=django.db.migrations.operations.special.RunPython.noop, + reverse_code=delete_pending_memberships, + ), + migrations.RunPython( + code=delete_memberships_with_overlapping_emails, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=remove_pending_emails, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterField( + model_name='membership', + name='committee_account', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='committee_accounts.committeeaccount'), + ), + migrations.AlterField( + model_name='membership', + name='role', + field=models.CharField(choices=[('COMMITTEE_ADMINISTRATOR', 'Committee Administrator'), ('MANAGER', 'Manager')], max_length=25), + ), + migrations.AlterField( + model_name='committeeaccount', + name='members', + field=models.ManyToManyField(through='committee_accounts.Membership', through_fields=('committee_account', 'user'), to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0002_membership.py b/django-backend/fecfiler/committee_accounts/migrations/0002_membership.py deleted file mode 100644 index 14a1fbb49..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0002_membership.py +++ /dev/null @@ -1,83 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-25 14:11 - -from django.db import migrations, models -import django.db.models.deletion -import django.contrib.auth.validators -import django.utils.timezone -from fecfiler.committee_accounts.models import Membership as MembershipModel - - -def create_memberships(apps, schema_editor): - CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") # noqa - User = apps.get_model("user", "User") # noqa - Membership = apps.get_model("committee_accounts", "Membership") # noqa - db_alias = schema_editor.connection.alias - users = User.objects.using(db_alias).all() - for user in users: - committee = ( - CommitteeAccount.objects.using(db_alias) - .filter(committee_id=user.cmtee_id) - .first() - ) - if committee: - membership = Membership( - user=user, - committee_account=committee, - role=MembershipModel.CommitteeRole.COMMITTEE_ADMINISTRATOR, - ) - membership.save() - - -class Migration(migrations.Migration): - dependencies = [ - ("user", "0001_initial"), - ("committee_accounts", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="Membership", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "role", - models.CharField( - choices=[ - ("COMMITTEE_ADMINISTRATOR", "Committee Administrator"), - ("REVIEWER", "Reviewer"), - ], - max_length=25, - ), - ), - ( - "committee_account", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="committee_accounts.committeeaccount", - ), - ), - ( - "user", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="user.user" - ), - ), - ], - ), - migrations.AddField( - model_name="committeeaccount", - name="members", - field=models.ManyToManyField( - through="committee_accounts.Membership", to="user.User" - ), - ), - migrations.RunPython(create_memberships), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py deleted file mode 100644 index 1e9809d26..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py +++ /dev/null @@ -1,93 +0,0 @@ -# Generated by Django 4.2.7 on 2024-02-16 20:43 - -from django.conf import settings -from django.db import migrations, models -from django.utils import timezone -import uuid - - -def delete_pending_memberships(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") # noqa - Membership.objects.filter(user=None).delete() - - -def generate_new_uuid(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") # noqa - for membership in Membership.objects.all(): - membership.uuid = uuid.uuid4() - membership.save() - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('committee_accounts', '0002_membership'), - ] - - operations = [ - migrations.AddField( - model_name='membership', - name='pending_email', - field=models.EmailField(blank=True, max_length=254, null=True), - ), - migrations.AddField( - model_name='membership', - name='uuid', - field=models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=False, - serialize=False, - unique=False - ) - ), - migrations.RunPython( - generate_new_uuid, - migrations.RunPython.noop, - ), - migrations.RemoveField( - model_name='membership', - name='id', - ), - migrations.RenameField( - model_name='membership', - old_name='uuid', - new_name='id' - ), - migrations.AlterField( - model_name='membership', - name='id', - field=models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True - ) - ), - migrations.AlterField( - model_name='membership', - name='user', - field=models.ForeignKey( - null=True, - on_delete=models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL - ), - ), - migrations.AddField( - model_name='membership', - name='created', - field=models.DateTimeField(auto_now_add=True, default=timezone.now), - preserve_default=False, - ), - migrations.AddField( - model_name='membership', - name='updated', - field=models.DateTimeField(auto_now=True), - ), - migrations.RunPython( - migrations.RunPython.noop, - delete_pending_memberships - ), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py b/django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py deleted file mode 100644 index 266317e05..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py +++ /dev/null @@ -1,43 +0,0 @@ -# Generated by Django 4.2.7 on 2024-02-16 20:43 - -from django.db import migrations - - -def delete_memberships_with_overlapping_emails(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") # noqa - Committee = apps.get_model("committee_accounts", "CommitteeAccount") # noqa - - for committee in Committee.objects.all(): - committee_memberships = Membership.objects.filter(committee_account=committee) - - unique_pending_emails = set() - emails_to_prune = set() - for membership in committee_memberships: - pending_email = str(membership.pending_email).lower() - if pending_email not in unique_pending_emails: - unique_pending_emails.add(pending_email) - else: - emails_to_prune.add(pending_email) - - for email in list(emails_to_prune): - # ordering by user places any memberships with a user first - overlapping_memberships = list( - committee_memberships.filter(pending_email__iexact=email).order_by('user') - ) - for membership_to_delete in overlapping_memberships[1:]: - membership_to_delete.delete() - - -class Migration(migrations.Migration): - - dependencies = [( - 'committee_accounts', - '0003_membership_pending_email_alter_membership_id_and_more' - )] - - operations = [ - migrations.RunPython( - delete_memberships_with_overlapping_emails, - migrations.RunPython.noop, - ), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0005_remove_pending_emails.py b/django-backend/fecfiler/committee_accounts/migrations/0005_remove_pending_emails.py deleted file mode 100644 index 2077e40d0..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0005_remove_pending_emails.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 5.1.4 on 2025-02-05 18:11 - -from django.db import migrations - - -def remove_pending_emails(apps, schema_editor): - # remove pending emails from memberships that have been redeemed - Membership = apps.get_model("committee_accounts", "Membership") # noqa - Membership.objects.filter(pending_email__isnull=False, user_id__isnull=False).update( - pending_email=None - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ("committee_accounts", "0004_remove_duplicate_memberships"), - ] - - operations = [ - migrations.RunPython( - remove_pending_emails, reverse_code=migrations.RunPython.noop - ), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0006_alter_membership_committee_account_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0006_alter_membership_committee_account_and_more.py deleted file mode 100644 index 8de0408b5..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0006_alter_membership_committee_account_and_more.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 5.1.5 on 2025-02-13 16:07 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("committee_accounts", "0005_remove_pending_emails"), - ] - - operations = [ - migrations.AlterField( - model_name="membership", - name="committee_account", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="committee_accounts.committeeaccount", - ), - ), - migrations.AlterField( - model_name="membership", - name="role", - field=models.CharField( - choices=[ - ("COMMITTEE_ADMINISTRATOR", "Committee Administrator"), - ("MANAGER", "Manager"), - ], - max_length=25, - ), - ), - ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0007_alter_committeeaccount_members.py deleted file mode 100644 index 532bc0a1d..000000000 --- a/django-backend/fecfiler/committee_accounts/migrations/0007_alter_committeeaccount_members.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 5.2 on 2025-04-30 17:10 - -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('committee_accounts', '0006_alter_membership_committee_account_and_more'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.AlterField( - model_name='committeeaccount', - name='members', - field=models.ManyToManyField( - through='committee_accounts.Membership', - through_fields=('committee_account', 'user'), - to=settings.AUTH_USER_MODEL - ), - ), - ] diff --git a/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py b/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py new file mode 100644 index 000000000..4e0bfc404 --- /dev/null +++ b/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py @@ -0,0 +1,27 @@ +# Generated by Django 5.2.11 on 2026-03-07 03:21 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('memo_text', '0002_initial'), ('memo_text', '0003_memotext_text_prefix')] + + dependencies = [ + ('memo_text', '0001_initial'), + ('reports', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='memotext', + name='report', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='reports.report'), + ), + migrations.AddField( + model_name='memotext', + name='text_prefix', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/django-backend/fecfiler/memo_text/migrations/0002_initial.py b/django-backend/fecfiler/memo_text/migrations/0002_initial.py deleted file mode 100644 index 66369ef09..000000000 --- a/django-backend/fecfiler/memo_text/migrations/0002_initial.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-16 20:15 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("reports", "0001_initial"), - ("memo_text", "0001_initial"), - ] - - operations = [ - migrations.AddField( - model_name="memotext", - name="report", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="reports.report", - ), - ), - ] diff --git a/django-backend/fecfiler/memo_text/migrations/0003_memotext_text_prefix.py b/django-backend/fecfiler/memo_text/migrations/0003_memotext_text_prefix.py deleted file mode 100644 index 68f88e025..000000000 --- a/django-backend/fecfiler/memo_text/migrations/0003_memotext_text_prefix.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-23 15:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('memo_text', '0002_initial'), - ] - - operations = [ - migrations.AddField( - model_name='memotext', - name='text_prefix', - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/00018_form24_name.py b/django-backend/fecfiler/reports/migrations/00018_form24_name.py deleted file mode 100644 index 77f529cd9..000000000 --- a/django-backend/fecfiler/reports/migrations/00018_form24_name.py +++ /dev/null @@ -1,33 +0,0 @@ -from django.db import migrations, models -from django_migration_linter import IgnoreMigration - - -def update_form24_names(apps, schema_editor): - form24 = apps.get_model("reports", "Form24") - form24_objects = list(form24.objects.all()) - for index, form in enumerate(form24_objects, start=1): - report_type = form.report_type_24_48 - form.name = f"{report_type}-HOUR Report: {index}" - form.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0017_form99_filing_frequency_form99_pdf_attachment"), - ] - - operations = [ - IgnoreMigration(), - migrations.AddField( - model_name="form24", - name="name", - field=models.TextField(null=True, blank=False), - ), - migrations.RunPython(update_form24_names), - migrations.AlterField( - model_name="form24", - name="name", - field=models.TextField(null=False, blank=False), - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/00019_form24_name_fix.py deleted file mode 100644 index 347f039b5..000000000 --- a/django-backend/fecfiler/reports/migrations/00019_form24_name_fix.py +++ /dev/null @@ -1,21 +0,0 @@ -from django.db import migrations - - -def update_form24_names(apps, schema_editor): - form24 = apps.get_model("reports", "Form24") - form24_objects = list(form24.objects.all()) - for form in form24_objects: - report_type = form.report_type_24_48 - form.name = f"{report_type}-HOUR: Report of Independent Expenditure" - form.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "00018_form24_name"), - ] - - operations = [ - migrations.RunPython(update_form24_names, migrations.RunPython.noop), - ] diff --git a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py new file mode 100644 index 000000000..efbcfb324 --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -0,0 +1,363 @@ +# Generated by Django 5.2.11 on 2026-03-07 03:24 + +import django.db.migrations.operations.special +import django.db.models.deletion +import uuid +from importlib import import_module +from django.db import migrations, models + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# fecfiler.reports.migrations.00018_form24_name +# fecfiler.reports.migrations.00019_form24_name_fix +# fecfiler.reports.migrations.0008_remove_form1m_city_remove_form1m_committee_name_and_more +# fecfiler.reports.migrations.0009_report_can_delete +# fecfiler.reports.migrations.0010_report_can_unammend + + +def _load_callable(module_name, function_name): + def _wrapper(apps, schema_editor): + try: + fn = getattr(import_module(module_name), function_name) + except Exception: + return django.db.migrations.operations.special.RunPython.noop(apps, schema_editor) + return fn(apps, schema_editor) + + return _wrapper + +class Migration(migrations.Migration): + + replaces = [('reports', '0007_remove_report_deleted'), ('reports', '0008_remove_form1m_city_remove_form1m_committee_name_and_more'), ('reports', '0009_report_can_delete'), ('reports', '0010_report_can_unammend'), ('reports', '0011_remove_form3x_cash_on_hand_date'), ('reports', '0012_alter_form99_text_code'), ('reports', '0013_form3_report_form_3'), ('reports', '0014_form99_swap_text_code'), ('reports', '0015_form3x_filing_frequency_form3x_report_type_category'), ('reports', '0016_determine_frequency_and_category'), ('reports', '0017_form99_filing_frequency_form99_pdf_attachment'), ('reports', '00018_form24_name'), ('reports', '00019_form24_name_fix')] + + dependencies = [ + ('reports', '0006_reporttransaction'), + ] + + operations = [ + migrations.RemoveField( + model_name='report', + name='deleted', + ), + migrations.AddField( + model_name='report', + name='city', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='report', + name='committee_name', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='report', + name='state', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='report', + name='street_1', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='report', + name='street_2', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='report', + name='zip', + field=models.TextField(blank=True, null=True), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.reports.migrations.0008_remove_form1m_city_remove_form1m_committee_name_and_more", + "migrate_committee_data", + ), + ), + migrations.RemoveField( + model_name='form1m', + name='city', + ), + migrations.RemoveField( + model_name='form1m', + name='committee_name', + ), + migrations.RemoveField( + model_name='form1m', + name='state', + ), + migrations.RemoveField( + model_name='form1m', + name='street_1', + ), + migrations.RemoveField( + model_name='form1m', + name='street_2', + ), + migrations.RemoveField( + model_name='form1m', + name='zip', + ), + migrations.RemoveField( + model_name='form24', + name='city', + ), + migrations.RemoveField( + model_name='form24', + name='state', + ), + migrations.RemoveField( + model_name='form24', + name='street_1', + ), + migrations.RemoveField( + model_name='form24', + name='street_2', + ), + migrations.RemoveField( + model_name='form24', + name='zip', + ), + migrations.RemoveField( + model_name='form3x', + name='city', + ), + migrations.RemoveField( + model_name='form3x', + name='state', + ), + migrations.RemoveField( + model_name='form3x', + name='street_1', + ), + migrations.RemoveField( + model_name='form3x', + name='street_2', + ), + migrations.RemoveField( + model_name='form3x', + name='zip', + ), + migrations.RemoveField( + model_name='form99', + name='city', + ), + migrations.RemoveField( + model_name='form99', + name='committee_name', + ), + migrations.RemoveField( + model_name='form99', + name='state', + ), + migrations.RemoveField( + model_name='form99', + name='street_1', + ), + migrations.RemoveField( + model_name='form99', + name='street_2', + ), + migrations.RemoveField( + model_name='form99', + name='zip', + ), + migrations.AddField( + model_name='report', + name='can_delete', + field=models.BooleanField(default=True), + ), + migrations.RunSQL( + sql='\n CREATE OR REPLACE FUNCTION update_report_can_delete(report RECORD)\n RETURNS VOID AS $$\n DECLARE\n r_can_delete boolean;\n BEGIN\n r_can_delete = report.upload_submission_id IS NULL\n AND (report.report_version IS NULL OR report.report_version = \'0\')\n AND (\n report.form_3x_id IS NULL OR\n (\n report.form_24_id IS NULL\n AND NOT EXISTS(\n SELECT DISTINCT rrt1.id\n FROM "reports_reporttransaction" rrt1\n JOIN "transactions_transaction" tt ON (\n rrt1."transaction_id" = tt."id"\n OR tt."reatt_redes_id" = rrt1."transaction_id"\n OR tt."parent_transaction_id" = rrt1."transaction_id"\n OR tt."debt_id" = rrt1."transaction_id"\n OR tt."loan_id" = rrt1."transaction_id"\n )\n INNER JOIN "reports_reporttransaction" rrt2 ON (\n rrt2."transaction_id" = tt."id"\n AND rrt2."report_id" <> report.id\n )\n WHERE rrt1."report_id" = report.id\n )\n )\n );\n UPDATE reports_report SET can_delete = r_can_delete\n WHERE id = report.id\n AND can_delete <> r_can_delete;\n END;\n $$ LANGUAGE plpgsql;\n ', + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.reports.migrations.0009_report_can_delete", + "create_trigger", + ), + reverse_code=_load_callable( + "fecfiler.reports.migrations.0009_report_can_delete", + "reverse_code", + ), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.reports.migrations.0009_report_can_delete", + "populate_existing_rows", + ), + ), + migrations.AddField( + model_name='report', + name='can_unamend', + field=models.BooleanField(default=False), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.reports.migrations.0010_report_can_unammend", + "create_trigger", + ), + ), + migrations.RemoveField( + model_name='form3x', + name='cash_on_hand_date', + ), + migrations.AddField( + model_name='form99', + name='text_code_2', + field=models.TextField(db_default=''), + ), + migrations.RunSQL( + sql="\n UPDATE reports_form99 SET text_code_2 = COALESCE(text_code, '');\n ALTER TABLE reports_form99 ALTER COLUMN text_code SET DEFAULT '';\n ALTER TABLE reports_form99 ALTER COLUMN text_code_2 SET NOT NULL;\n ", + reverse_sql='\n ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP DEFAULT;\n ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP NOT NULL;\n ', + state_operations=[migrations.AlterField( + model_name='form99', + name='text_code_2', + field=models.TextField(db_default='', default=''), + )], + ), + migrations.CreateModel( + name='Form3', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('change_of_address', models.BooleanField(blank=True, default=False, null=True)), + ('election_state', models.TextField(blank=True, null=True)), + ('election_district', models.TextField(blank=True, null=True)), + ('election_code', models.TextField(blank=True, null=True)), + ('date_of_election', models.DateField(blank=True, null=True)), + ('state_of_election', models.TextField(blank=True, null=True)), + ('L6a_total_contributions_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L6b_total_contribution_refunds_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L6c_net_contributions_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L7a_total_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L7b_total_offsets_to_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L7c_net_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L8_cash_on_hand_at_close_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L9_debts_owed_to_committee_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L10_debts_owed_by_committee_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11ai_individuals_itemized_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11aii_individuals_unitemized_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11aiii_total_individual_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11b_political_party_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11c_other_political_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11d_the_candidate_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11e_total_contributions_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L12_transfers_from_other_authorized_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L13a_loans_made_or_guaranteed_by_the_candidate_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L13b_all_other_loans_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L13c_total_loans_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L14_offsets_to_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L15_other_receipts_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L16_total_receipts_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L17_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L18_transfers_to_other_authorized_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_period', models.DecimalField(blank=True, db_column='l19a_period', decimal_places=2, max_digits=11, null=True)), + ('L19b_loan_repayments_of_all_other_loans_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L19c_total_loan_repayments_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20a_refunds_to_individuals_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20b_refunds_to_political_party_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20c_refunds_to_other_political_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20d_total_contribution_refunds_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L21_other_disbursements_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L22_total_disbursements_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L23_cash_on_hand_beginning_reporting_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L24_total_receipts_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L25_subtotals_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L26_total_disbursements_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L27_cash_on_hand_at_close_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L6a_total_contributions_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L6b_total_contribution_refunds_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L6c_net_contributions_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L7a_total_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L7b_total_offsets_to_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L7c_net_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11ai_individuals_itemized_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11aii_individuals_unitemized_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11aiii_total_individual_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11b_political_party_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11c_other_political_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11d_the_candidate_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L11e_total_contributions_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L12_transfers_from_other_authorized_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L13a_loans_made_or_guaranteed_by_the_candidate_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L13b_all_other_loans_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L13c_total_loans_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L14_offsets_to_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L15_other_receipts_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L16_total_receipts_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L17_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L18_transfers_to_other_authorized_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_ytd', models.DecimalField(blank=True, db_column='l19a_ytd', decimal_places=2, max_digits=11, null=True)), + ('L19b_loan_repayments_of_all_other_loans_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L19c_total_loan_repayments_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20a_refunds_to_individuals_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20b_refunds_to_political_party_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20c_refunds_to_other_political_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L20d_total_contribution_refunds_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L21_other_disbursements_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('L22_total_disbursements_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ], + ), + migrations.AddField( + model_name='report', + name='form_3', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='reports.form3'), + ), + migrations.RemoveField( + model_name='form99', + name='text_code', + ), + migrations.RenameField( + model_name='form99', + old_name='text_code_2', + new_name='text_code', + ), + migrations.AddField( + model_name='form3x', + name='filing_frequency', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='form3x', + name='report_type_category', + field=models.TextField(blank=True, null=True), + ), + migrations.RunSQL( + sql="\n UPDATE reports_form3x f\n SET filing_frequency = CASE\n WHEN r.report_code IN (\n 'M2', 'M3', 'M4', 'M5', 'M6', 'M7',\n 'M8', 'M9', 'M10', 'M11', 'M12'\n ) THEN 'M'\n ELSE 'Q'\n END,\n report_type_category = CASE\n WHEN r.report_code IN (\n 'M11', 'M12', 'MY'\n ) THEN 'Non-Election Year'\n ELSE 'Election Year'\n END\n FROM reports_report as r\n WHERE r.form_3x_id = f.id\n AND filing_frequency IS NULL AND report_type_category IS NULL;\n ", + reverse_sql='', + ), + migrations.AddField( + model_name='form99', + name='pdf_attachment', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AddField( + model_name='form99', + name='filing_frequency', + field=models.TextField(blank=True, max_length=1, null=True), + ), + migrations.AddField( + model_name='form24', + name='name', + field=models.TextField(null=True), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.reports.migrations.00018_form24_name", + "update_form24_names", + ), + ), + migrations.AlterField( + model_name='form24', + name='name', + field=models.TextField(), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.reports.migrations.00019_form24_name_fix", + "update_form24_names", + ), + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + ] diff --git a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted.py deleted file mode 100644 index e124f11a2..000000000 --- a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.2.10 on 2024-04-05 15:04 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('reports', '0006_reporttransaction'), - ] - - operations = [ - migrations.RemoveField( - model_name='report', - name='deleted', - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0008_remove_form1m_city_remove_form1m_committee_name_and_more.py b/django-backend/fecfiler/reports/migrations/0008_remove_form1m_city_remove_form1m_committee_name_and_more.py deleted file mode 100644 index 94fd45d4a..000000000 --- a/django-backend/fecfiler/reports/migrations/0008_remove_form1m_city_remove_form1m_committee_name_and_more.py +++ /dev/null @@ -1,193 +0,0 @@ -# Generated by Django 4.2.10 on 2024-04-19 19:10 - -from django.db import migrations, models -import structlog - -logger = structlog.get_logger(__name__) - - -def migrate_committee_data(apps, schema_editor): - Report = apps.get_model("reports", "Report") # noqa - Form24 = apps.get_model("reports", "Form24") # noqa - Form3x = apps.get_model("reports", "Form3X") # noqa - Form99 = apps.get_model("reports", "Form99") # noqa - Form1m = apps.get_model("reports", "Form1M") # noqa - - for form in Form24.objects.all(): - report = Report.objects.filter(form_24=form).first() - if report is not None: - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F24 Form has no corresponding report! {form}") - - for form in Form3x.objects.all(): - report = Report.objects.filter(form_3x=form).first() - if report is not None: - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F3X Form has no corresponding report! {form}") - - for form in Form99.objects.all(): - report = Report.objects.filter(form_99=form).first() - if report is not None: - report.committee_name = form.committee_name - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F99 Form has no corresponding report! {form}") - - for form in Form1m.objects.all(): - report = Report.objects.filter(form_1m=form).first() - if report is not None: - report.committee_name = form.committee_name - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F1M Form has no corresponding report! {form}") - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0007_remove_report_deleted"), - ] - - operations = [ - migrations.AddField( - model_name="report", - name="city", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="committee_name", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="state", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="street_1", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="street_2", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="zip", - field=models.TextField(blank=True, null=True), - ), - migrations.RunPython(migrate_committee_data), - migrations.RemoveField( - model_name="form1m", - name="city", - ), - migrations.RemoveField( - model_name="form1m", - name="committee_name", - ), - migrations.RemoveField( - model_name="form1m", - name="state", - ), - migrations.RemoveField( - model_name="form1m", - name="street_1", - ), - migrations.RemoveField( - model_name="form1m", - name="street_2", - ), - migrations.RemoveField( - model_name="form1m", - name="zip", - ), - migrations.RemoveField( - model_name="form24", - name="city", - ), - migrations.RemoveField( - model_name="form24", - name="state", - ), - migrations.RemoveField( - model_name="form24", - name="street_1", - ), - migrations.RemoveField( - model_name="form24", - name="street_2", - ), - migrations.RemoveField( - model_name="form24", - name="zip", - ), - migrations.RemoveField( - model_name="form3x", - name="city", - ), - migrations.RemoveField( - model_name="form3x", - name="state", - ), - migrations.RemoveField( - model_name="form3x", - name="street_1", - ), - migrations.RemoveField( - model_name="form3x", - name="street_2", - ), - migrations.RemoveField( - model_name="form3x", - name="zip", - ), - migrations.RemoveField( - model_name="form99", - name="city", - ), - migrations.RemoveField( - model_name="form99", - name="committee_name", - ), - migrations.RemoveField( - model_name="form99", - name="state", - ), - migrations.RemoveField( - model_name="form99", - name="street_1", - ), - migrations.RemoveField( - model_name="form99", - name="street_2", - ), - migrations.RemoveField( - model_name="form99", - name="zip", - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0009_report_can_delete.py b/django-backend/fecfiler/reports/migrations/0009_report_can_delete.py deleted file mode 100644 index 87cc66237..000000000 --- a/django-backend/fecfiler/reports/migrations/0009_report_can_delete.py +++ /dev/null @@ -1,180 +0,0 @@ -# Generated by Django 4.2.11 on 2024-07-02 17:21 - -from django.db import migrations, models -from django.db import connection - - -def populate_existing_rows(apps, schema_editor): - report_model = apps.get_model("reports", "Report") - for row in report_model.objects.all(): - row.can_delete = True - row.save() - - -def create_trigger(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION check_can_delete() - RETURNS TRIGGER AS $$ - BEGIN - PERFORM update_report_can_delete(NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION check_can_delete_previous() - RETURNS TRIGGER AS $$ - DECLARE - associated_report RECORD; - BEGIN - FOR associated_report IN ( - SELECT * FROM reports_report - WHERE committee_account_id = OLD.committee_account_id - AND can_delete = false - ) - LOOP - PERFORM update_report_can_delete(associated_report); - END LOOP; - - RETURN OLD; - END; - $$ LANGUAGE plpgsql; - - - CREATE OR REPLACE FUNCTION check_can_delete_transaction_update() - RETURNS TRIGGER AS $$ - DECLARE - associated_report RECORD; - BEGIN - SELECT * INTO associated_report FROM reports_report WHERE id IN ( - SELECT report_id FROM reports_reporttransaction - WHERE transaction_id = NEW.id LIMIT 1 - ); - PERFORM update_report_can_delete(associated_report); - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION check_can_delete_transaction_insert() - RETURNS TRIGGER AS $$ - DECLARE - associated_report RECORD; - BEGIN - FOR associated_report IN ( - SELECT r1.* - FROM reports_report r1 - JOIN reports_report r2 - ON r1.committee_account_id = r2.committee_account_id - AND EXTRACT(YEAR FROM r1.coverage_from_date) = ( - EXTRACT(YEAR FROM r2.coverage_from_date)) - WHERE r1.can_delete = true - AND r2.id = NEW.report_id - ) - LOOP - PERFORM update_report_can_delete(associated_report); - END LOOP; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER check_can_delete_report - AFTER INSERT OR UPDATE ON reports_report - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION check_can_delete(); - - CREATE TRIGGER check_can_delete_previous - AFTER DELETE ON reports_report - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION check_can_delete_previous(); - - CREATE TRIGGER check_can_delete_transaction_insert - AFTER INSERT ON reports_reporttransaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION check_can_delete_transaction_insert(); - """ - ) - - -def reverse_code(apps, schema_editor): - # Get models - report_model = apps.get_model("reports", "Report") - transaction_model = apps.get_model("transactions", "Transaction") - - # Drop triggers - triggers = [ - "check_can_delete_report", - "check_can_delete_previous", - "check_can_delete_transaction_update", - "check_can_delete_transaction_insert", - ] - - with schema_editor.atomic(): - for trigger in triggers: - schema_editor.execute( - "DROP TRIGGER IF EXISTS %s ON %s", (trigger, report_model._meta.db_table) - ) - schema_editor.execute( - "DROP TRIGGER IF EXISTS %s ON %s", - (trigger, transaction_model._meta.db_table), - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0008_remove_form1m_city_remove_form1m_committee_name_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="report", - name="can_delete", - field=models.BooleanField(default=True), - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION update_report_can_delete(report RECORD) - RETURNS VOID AS $$ - DECLARE - r_can_delete boolean; - BEGIN - r_can_delete = report.upload_submission_id IS NULL - AND (report.report_version IS NULL OR report.report_version = '0') - AND ( - report.form_3x_id IS NULL OR - ( - report.form_24_id IS NULL - AND NOT EXISTS( - SELECT DISTINCT rrt1.id - FROM "reports_reporttransaction" rrt1 - JOIN "transactions_transaction" tt ON ( - rrt1."transaction_id" = tt."id" - OR tt."reatt_redes_id" = rrt1."transaction_id" - OR tt."parent_transaction_id" = rrt1."transaction_id" - OR tt."debt_id" = rrt1."transaction_id" - OR tt."loan_id" = rrt1."transaction_id" - ) - INNER JOIN "reports_reporttransaction" rrt2 ON ( - rrt2."transaction_id" = tt."id" - AND rrt2."report_id" <> report.id - ) - WHERE rrt1."report_id" = report.id - ) - ) - ); - UPDATE reports_report SET can_delete = r_can_delete - WHERE id = report.id - AND can_delete <> r_can_delete; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunPython(create_trigger, reverse_code), - migrations.RunPython(populate_existing_rows), - ] diff --git a/django-backend/fecfiler/reports/migrations/0010_report_can_unammend.py b/django-backend/fecfiler/reports/migrations/0010_report_can_unammend.py deleted file mode 100644 index 90f64ed63..000000000 --- a/django-backend/fecfiler/reports/migrations/0010_report_can_unammend.py +++ /dev/null @@ -1,65 +0,0 @@ -# Generated by Django 5.0.8 on 2024-08-20 17:53 - -from django.db import migrations, models - - -def create_trigger(apps, schema_editor): - schema_editor.execute( - """ - CREATE OR REPLACE FUNCTION update_can_unamend() - RETURNS TRIGGER AS $$ - BEGIN - UPDATE reports_report - SET can_unamend = FALSE - WHERE id IN ( - SELECT report_id - FROM reports_reporttransaction - WHERE transaction_id = NEW.id - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER transaction_updated - AFTER UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION update_can_unamend(); - - CREATE OR REPLACE FUNCTION update_can_unamend_new_transaction() - RETURNS TRIGGER AS $$ - BEGIN - UPDATE reports_report - SET can_unamend = FALSE - WHERE id IN ( - SELECT report_id - FROM reports_reporttransaction - WHERE id = NEW.id - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER transaction_created - AFTER INSERT ON reports_reporttransaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION update_can_unamend_new_transaction(); - """ - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0009_report_can_delete"), - ] - - operations = [ - migrations.AddField( - model_name="report", - name="can_unamend", - field=models.BooleanField(default=False), - ), - migrations.RunPython(create_trigger), - ] diff --git a/django-backend/fecfiler/reports/migrations/0011_remove_form3x_cash_on_hand_date.py b/django-backend/fecfiler/reports/migrations/0011_remove_form3x_cash_on_hand_date.py deleted file mode 100644 index 3373a352c..000000000 --- a/django-backend/fecfiler/reports/migrations/0011_remove_form3x_cash_on_hand_date.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 4.2.11 on 2024-10-23 20:33 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("reports", "0010_report_can_unammend"), - ] - - operations = [ - migrations.RemoveField( - model_name="form3x", - name="cash_on_hand_date", - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0012_alter_form99_text_code.py b/django-backend/fecfiler/reports/migrations/0012_alter_form99_text_code.py deleted file mode 100644 index 2cbeb0f17..000000000 --- a/django-backend/fecfiler/reports/migrations/0012_alter_form99_text_code.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0011_remove_form3x_cash_on_hand_date"), - ] - - operations = [ - # Step 1: Add the column as NULLABLE (to prevent NOT NULL constraint errors) - migrations.AddField( - model_name="form99", - name="text_code_2", - field=models.TextField(db_default=""), - ), - migrations.RunSQL( - sql=""" - UPDATE reports_form99 SET text_code_2 = COALESCE(text_code, ''); - ALTER TABLE reports_form99 ALTER COLUMN text_code SET DEFAULT ''; - ALTER TABLE reports_form99 ALTER COLUMN text_code_2 SET NOT NULL; - """, - reverse_sql=""" - ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP DEFAULT; - ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP NOT NULL; - """, - state_operations=[ - migrations.AlterField( - model_name="form99", - name="text_code_2", - field=models.TextField( - null=False, blank=False, default="", db_default="" - ), - ), - ], - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0013_form3_report_form_3.py b/django-backend/fecfiler/reports/migrations/0013_form3_report_form_3.py deleted file mode 100644 index e236e08a3..000000000 --- a/django-backend/fecfiler/reports/migrations/0013_form3_report_form_3.py +++ /dev/null @@ -1,478 +0,0 @@ -# Generated by Django 5.1.5 on 2025-03-28 01:59 - -import django.db.models.deletion -import uuid -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0012_alter_form99_text_code"), - ] - - l19a_field = "L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_period" - operations = [ - migrations.CreateModel( - name="Form3", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ( - "change_of_address", - models.BooleanField(blank=True, default=False, null=True), - ), - ("election_state", models.TextField(blank=True, null=True)), - ("election_district", models.TextField(blank=True, null=True)), - ("election_code", models.TextField(blank=True, null=True)), - ("date_of_election", models.DateField(blank=True, null=True)), - ("state_of_election", models.TextField(blank=True, null=True)), - ( - "L6a_total_contributions_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L6b_total_contribution_refunds_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L6c_net_contributions_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L7a_total_operating_expenditures_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L7b_total_offsets_to_operating_expenditures_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L7c_net_operating_expenditures_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L8_cash_on_hand_at_close_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L9_debts_owed_to_committee_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L10_debts_owed_by_committee_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11ai_individuals_itemized_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11aii_individuals_unitemized_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11aiii_total_individual_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11b_political_party_committees_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11c_other_political_committees_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11d_the_candidate_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11e_total_contributions_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L12_transfers_from_other_authorized_committees_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L13a_loans_made_or_guaranteed_by_the_candidate_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L13b_all_other_loans_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L13c_total_loans_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L14_offsets_to_operating_expenditures_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L15_other_receipts_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L16_total_receipts_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L17_operating_expenditures_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L18_transfers_to_other_authorized_committees_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - l19a_field, - models.DecimalField( - blank=True, - db_column="l19a_period", - decimal_places=2, - max_digits=11, - null=True, - ), - ), - ( - "L19b_loan_repayments_of_all_other_loans_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L19c_total_loan_repayments_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20a_refunds_to_individuals_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20b_refunds_to_political_party_committees_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20c_refunds_to_other_political_committees_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20d_total_contribution_refunds_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L21_other_disbursements_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L22_total_disbursements_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L23_cash_on_hand_beginning_reporting_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L24_total_receipts_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L25_subtotals_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L26_total_disbursements_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L27_cash_on_hand_at_close_period", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L6a_total_contributions_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L6b_total_contribution_refunds_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L6c_net_contributions_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L7a_total_operating_expenditures_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L7b_total_offsets_to_operating_expenditures_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L7c_net_operating_expenditures_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11ai_individuals_itemized_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11aii_individuals_unitemized_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11aiii_total_individual_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11b_political_party_committees_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11c_other_political_committees_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11d_the_candidate_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L11e_total_contributions_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L12_transfers_from_other_authorized_committees_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L13a_loans_made_or_guaranteed_by_the_candidate_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L13b_all_other_loans_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L13c_total_loans_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L14_offsets_to_operating_expenditures_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L15_other_receipts_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L16_total_receipts_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L17_operating_expenditures_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L18_transfers_to_other_authorized_committees_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_ytd", - models.DecimalField( - blank=True, - db_column="l19a_ytd", - decimal_places=2, - max_digits=11, - null=True, - ), - ), - ( - "L19b_loan_repayments_of_all_other_loans_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L19c_total_loan_repayments_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20a_refunds_to_individuals_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20b_refunds_to_political_party_committees_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20c_refunds_to_other_political_committees_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L20d_total_contribution_refunds_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L21_other_disbursements_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "L22_total_disbursements_ytd", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ], - ), - migrations.AddField( - model_name="report", - name="form_3", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="reports.form3", - ), - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0014_form99_swap_text_code.py b/django-backend/fecfiler/reports/migrations/0014_form99_swap_text_code.py deleted file mode 100644 index 87dc826ec..000000000 --- a/django-backend/fecfiler/reports/migrations/0014_form99_swap_text_code.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.db import migrations -from django_migration_linter import IgnoreMigration - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0013_form3_report_form_3"), - ] - - operations = [ - IgnoreMigration(), - migrations.RemoveField(model_name="form99", name="text_code"), - migrations.RenameField( - model_name="form99", old_name="text_code_2", new_name="text_code" - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0015_form3x_filing_frequency_form3x_report_type_category.py b/django-backend/fecfiler/reports/migrations/0015_form3x_filing_frequency_form3x_report_type_category.py deleted file mode 100644 index db9f36da1..000000000 --- a/django-backend/fecfiler/reports/migrations/0015_form3x_filing_frequency_form3x_report_type_category.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.2 on 2025-05-20 19:59 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('reports', '0014_form99_swap_text_code'), - ] - - operations = [ - migrations.AddField( - model_name='form3x', - name='filing_frequency', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='form3x', - name='report_type_category', - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0016_determine_frequency_and_category.py b/django-backend/fecfiler/reports/migrations/0016_determine_frequency_and_category.py deleted file mode 100644 index 4f3b442f7..000000000 --- a/django-backend/fecfiler/reports/migrations/0016_determine_frequency_and_category.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 5.2 on 2025-05-20 19:59 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0015_form3x_filing_frequency_form3x_report_type_category"), - ] - - operations = [ - # for reports made before we added the filing_frequency - # and report_type_category fields - migrations.RunSQL( - """ - UPDATE reports_form3x f - SET filing_frequency = CASE - WHEN r.report_code IN ( - 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', - 'M8', 'M9', 'M10', 'M11', 'M12' - ) THEN 'M' - ELSE 'Q' - END, - report_type_category = CASE - WHEN r.report_code IN ( - 'M11', 'M12', 'MY' - ) THEN 'Non-Election Year' - ELSE 'Election Year' - END - FROM reports_report as r - WHERE r.form_3x_id = f.id - AND filing_frequency IS NULL AND report_type_category IS NULL; - """, - reverse_sql="", - ) - ] diff --git a/django-backend/fecfiler/reports/migrations/0017_form99_filing_frequency_form99_pdf_attachment.py b/django-backend/fecfiler/reports/migrations/0017_form99_filing_frequency_form99_pdf_attachment.py deleted file mode 100644 index dcb23d9a7..000000000 --- a/django-backend/fecfiler/reports/migrations/0017_form99_filing_frequency_form99_pdf_attachment.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.2 on 2025-06-04 21:11 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0016_determine_frequency_and_category"), - ] - - operations = [ - migrations.AddField( - model_name="form99", - name="filing_frequency", - field=models.TextField(blank=True, max_length=1, null=True), - ), - migrations.AddField( - model_name="form99", - name="pdf_attachment", - field=models.BooleanField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py new file mode 100644 index 000000000..d20eec2df --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -0,0 +1,396 @@ +# Generated by Django 5.2.11 on 2026-03-07 03:25 + +import django.contrib.postgres.fields +import django.db.migrations.operations.special +import django.db.models.deletion +import uuid +from django.db import migrations, models +from importlib import import_module + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# fecfiler.transactions.migrations.0004_report_transactions_link_table +# fecfiler.transactions.migrations.0005_schedulec_report_coverage_from_date_and_more +# fecfiler.transactions.migrations.0006_independent_expenditure_memos_no_aggregation_group +# fecfiler.transactions.migrations.0008_transaction__calendar_ytd_per_election_office_and_more +# fecfiler.transactions.migrations.0009_update_calculate_loan_payment_to_date +# fecfiler.transactions.migrations.0011_transaction_can_delete +# fecfiler.transactions.migrations.0012_alter_transactions_blocking_reports +# fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers +# fecfiler.transactions.migrations.0015_merge_transaction_triggers +# fecfiler.transactions.migrations.0020_trigger_save_on_transactions +# fecfiler.transactions.migrations.0022_schedule_f_aggregation +# fecfiler.transactions.migrations.0024_scheduled_balance_at_close_and_more + + +def _load_callable(module_name, function_name): + def _wrapper(apps, schema_editor): + try: + fn = getattr(import_module(module_name), function_name) + except Exception: + return django.db.migrations.operations.special.RunPython.noop(apps, schema_editor) + return fn(apps, schema_editor) + + return _wrapper + + +class Migration(migrations.Migration): + + replaces = [('transactions', '0003_alter_transaction_parent_transaction'), ('transactions', '0004_report_transactions_link_table'), ('transactions', '0005_schedulec_report_coverage_from_date_and_more'), ('transactions', '0006_independent_expenditure_memos_no_aggregation_group'), ('transactions', '0007_schedulee_so_candidate_state'), ('transactions', '0008_transaction__calendar_ytd_per_election_office_and_more'), ('transactions', '0009_update_calculate_loan_payment_to_date'), ('transactions', '0010_update_aggregate_trigger_performance'), ('transactions', '0011_transaction_can_delete'), ('transactions', '0012_alter_transactions_blocking_reports'), ('transactions', '0013_transaction_itemized_and_associated_triggers'), ('transactions', '0014_drop_transaction_view'), ('transactions', '0015_merge_transaction_triggers'), ('transactions', '0016_schedulef_transaction_contact_4_and_more'), ('transactions', '0017_schedulef_coordianted_to_coordinated'), ('transactions', '0018_schedulef_general_election_year_and_more'), ('transactions', '0019_aggregate_committee_controls'), ('transactions', '0020_trigger_save_on_transactions'), ('transactions', '0021_alter_transaction_reports'), ('transactions', '0022_schedule_f_aggregation'), ('transactions', '0023_optimize_calculate_loan_payment_to_date'), ('transactions', '0024_scheduled_balance_at_close_and_more'), ('transactions', '0025_drop_aggregate_triggers'), ('transactions', '0026_alter_transaction_itemized')] + + dependencies = [ + ('contacts', '0001_initial'), + ('reports', '0006_reporttransaction'), + ('reports', '0014_form99_swap_text_code'), + ('transactions', '0002_remove_schedulea_contributor_city_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='transaction', + name='parent_transaction', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='transactions.transaction'), + ), + migrations.AddField( + model_name='transaction', + name='reports', + field=models.ManyToManyField(through='reports.ReportTransaction', to='reports.report'), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0004_report_transactions_link_table", "add_link_table"), + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RemoveField( + model_name='transaction', + name='report', + ), + migrations.AddField( + model_name='schedulec', + name='report_coverage_through_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AddField( + model_name='scheduled', + name='report_coverage_from_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name='transaction', + name='parent_transaction', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='transactions.transaction'), + ), + migrations.AlterField( + model_name='transaction', + name='reports', + field=models.ManyToManyField(related_name='transactions', through='reports.ReportTransaction', to='reports.report'), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0005_schedulec_report_coverage_from_date_and_more", "set_coverage_date"), + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0006_independent_expenditure_memos_no_aggregation_group", "set_aggregation_group_to_none_for_ie_memos"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0006_independent_expenditure_memos_no_aggregation_group", "reverse_removing_aggregation_group_for_ie_memos"), + ), + migrations.AddField( + model_name='schedulee', + name='so_candidate_state', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='transaction', + name='_calendar_ytd_per_election_office', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.AddField( + model_name='transaction', + name='aggregate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.AddField( + model_name='transaction', + name='loan_payment_to_date', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.RunSQL( + sql='\n CREATE EXTENSION IF NOT EXISTS "uuid-ossp";\n ', + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n BEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT contribution_date\n INTO schedule_date\n FROM transactions_schedulea\n WHERE id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT expenditure_date\n INTO schedule_date\n FROM transactions_scheduleb\n WHERE id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET aggregate = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n v_election_code TEXT;\n v_candidate_office TEXT;\n v_candidate_state TEXT;\n v_candidate_district TEXT;\n BEGIN\n SELECT COALESCE(disbursement_date, dissemination_date)\n INTO schedule_date FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT election_code\n INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT candidate_office, candidate_state, candidate_district\n INTO v_candidate_office, v_candidate_state, v_candidate_district\n FROM contacts WHERE id = txn.contact_2_id;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n t.id,\n SUM(t.effective_amount) OVER\n (ORDER BY t.date, t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transaction_view__' || sql_committee_id || ' t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM t.date) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n BEGIN\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n loan_key,\n SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE loan_key LIKE (\n SELECT transaction_id FROM transactions_transaction\n WHERE id = $1\n ) || ''%%''; -- Match the loan_key with a transaction_id prefix\n\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id\n AND tt.loan_key LIKE ''%%LOAN'';\n '\n USING txn.id;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_aggregates()\n RETURNS TRIGGER AS $$\n DECLARE\n sql_committee_id TEXT;\n temp_table_name TEXT;\n BEGIN\n sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_');\n temp_table_name := 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_');\n RAISE NOTICE 'TESTING TRIGGER';\n\n -- If schedule_c2_id or schedule_d_id is not null, stop processing\n IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL\n THEN\n RETURN NEW;\n END IF;\n\n IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL\n THEN\n PERFORM calculate_entity_aggregates(\n NEW, sql_committee_id, temp_table_name || 'NEW');\n IF TG_OP = 'UPDATE'\n AND NEW.contact_1_id <> OLD.contact_1_id\n THEN\n PERFORM calculate_entity_aggregates(\n OLD, sql_committee_id, temp_table_name || 'OLD');\n END IF;\n\n ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL\n THEN\n PERFORM calculate_loan_payment_to_date(\n NEW, sql_committee_id, temp_table_name || 'NEW');\n\n ELSIF NEW.schedule_e_id IS NOT NULL\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n NEW, sql_committee_id, temp_table_name || 'NEW');\n IF TG_OP = 'UPDATE'\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n OLD, sql_committee_id, temp_table_name || 'OLD');\n END IF;\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE TRIGGER calculate_aggregates_trigger\n AFTER INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop\n EXECUTE FUNCTION calculate_aggregates();\n ", + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0008_transaction__calendar_ytd_per_election_office_and_more", "populate_existing_rows"), + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION get_temp_tablename()\n RETURNS TEXT AS $$\n BEGIN\n RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_');\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD,\n sql_committee_id TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n temp_table_name TEXT;\n BEGIN\n temp_table_name := get_temp_tablename();\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT contribution_date\n INTO schedule_date\n FROM transactions_schedulea\n WHERE id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT expenditure_date\n INTO schedule_date\n FROM transactions_scheduleb\n WHERE id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET aggregate = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD,\n sql_committee_id TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n v_election_code TEXT;\n v_candidate_office TEXT;\n v_candidate_state TEXT;\n v_candidate_district TEXT;\n temp_table_name TEXT;\n BEGIN\n temp_table_name := get_temp_tablename();\n SELECT COALESCE(disbursement_date, dissemination_date)\n INTO schedule_date FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT election_code\n INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT candidate_office, candidate_state, candidate_district\n INTO v_candidate_office, v_candidate_state, v_candidate_district\n FROM contacts WHERE id = txn.contact_2_id;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n t.id,\n SUM(t.effective_amount) OVER\n (ORDER BY t.date, t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transaction_view__' || sql_committee_id || ' t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM t.date) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD,\n sql_committee_id TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n temp_table_name TEXT;\n BEGIN\n temp_table_name := get_temp_tablename();\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n loan_key,\n SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE loan_key LIKE (\n SELECT\n CASE\n WHEN loan_id IS NULL THEN transaction_id\n ELSE (\n SELECT transaction_id\n FROM transactions_transaction\n WHERE id = t.loan_id\n )\n END\n FROM transactions_transaction t\n WHERE id = $1\n ) || ''%%''; -- Match the loan_key with a transaction_id prefix\n\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id\n AND tt.loan_key LIKE ''%%LOAN'';\n '\n USING txn.id;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_aggregates()\n RETURNS TRIGGER AS $$\n DECLARE\n sql_committee_id TEXT;\n BEGIN\n sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_');\n\n -- If schedule_c2_id or schedule_d_id is not null, stop processing\n IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL\n THEN\n RETURN NEW;\n END IF;\n\n IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL\n THEN\n PERFORM calculate_entity_aggregates(NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n AND NEW.contact_1_id <> OLD.contact_1_id\n THEN\n PERFORM calculate_entity_aggregates(OLD, sql_committee_id);\n END IF;\n END IF;\n\n IF NEW.schedule_c_id IS NOT NULL\n OR NEW.schedule_c1_id IS NOT NULL\n OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE'\n THEN\n PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id);\n END IF;\n\n IF NEW.schedule_e_id IS NOT NULL\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n OLD, sql_committee_id);\n END IF;\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\n -- Drop prior versions of these functions\n DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT);\n DROP FUNCTION calculate_calendar_ytd_per_election_office(RECORD, TEXT, TEXT);\n DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT);\n ', + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0009_update_calculate_loan_payment_to_date", "update_existing_rows"), + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date date;\n BEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT\n contribution_date INTO schedule_date\n FROM\n transactions_schedulea\n WHERE\n id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT\n expenditure_date INTO schedule_date\n FROM\n transactions_scheduleb\n WHERE\n id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET aggregate = tc.new_sum\n FROM (\n SELECT\n id,\n aggregate,\n date,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $4\n OR (\n tc.date = $4\n AND t.created >= $5\n )\n );\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\n END;\n $$\n LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID\n AS $$\n DECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\n BEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n t.date,\n SUM(t.effective_amount) OVER\n (ORDER BY t.date, t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transaction_view__' || sql_committee_id || ' t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM t.date) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\n END;\n $$\n LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\n CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID\n AS $$\n BEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n id,\n loan_key,\n SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE loan_key LIKE (\n SELECT\n CASE\n WHEN loan_id IS NULL THEN transaction_id\n ELSE (\n SELECT transaction_id\n FROM transactions_transaction\n WHERE id = t.loan_id\n )\n END\n FROM transactions_transaction t\n WHERE id = $1\n ) || ''%%'' -- Match the loan_key with a transaction_id prefix\n ) AS tc\n WHERE t.id = tc.id\n AND tc.loan_key LIKE ''%%LOAN''\n ;\n '\n USING txn.id;\n END;\n $$\n LANGUAGE plpgsql;\n ", + ), + migrations.AddField( + model_name='transaction', + name='blocking_reports', + field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=[], size=None), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "create_trigger_function"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "drop_trigger_function"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "create_trigger"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "drop_trigger"), + ), + migrations.AlterField( + model_name='transaction', + name='blocking_reports', + field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0012_alter_transactions_blocking_reports", "update_blocking_reports_default"), + ), + migrations.AddField( + model_name='transaction', + name='itemized', + field=models.BooleanField(db_default=True), + ), + migrations.CreateModel( + name='OverTwoHundredTypesScheduleA', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('type', models.TextField()), + ], + options={ + 'db_table': 'over_two_hundred_types_schedulea', + 'indexes': [models.Index(fields=['type'], name='over_two_hu_type_2c8314_idx')], + }, + ), + migrations.CreateModel( + name='OverTwoHundredTypesScheduleB', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('type', models.TextField()), + ], + options={ + 'db_table': 'over_two_hundred_types_scheduleb', + 'indexes': [models.Index(fields=['type'], name='over_two_hu_type_411a44_idx')], + }, + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "populate_over_two_hundred_types"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "drop_over_two_hundred_types"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "create_itemized_triggers_and_functions"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "drop_itemized_triggers_and_functions"), + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID AS $$\nDECLARE\n schedule_date date;\nBEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT\n contribution_date INTO schedule_date\n FROM\n transactions_schedulea\n WHERE\n id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT\n expenditure_date INTO schedule_date\n FROM\n transactions_scheduleb\n WHERE\n id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET aggregate = tc.new_sum\n FROM (\n SELECT\n t.id,\n COALESCE(\n sa.contribution_date,\n sb.expenditure_date,\n sc.loan_incurred_date,\n se.disbursement_date,\n se.dissemination_date\n ) AS date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id)\n ) OVER (\n ORDER BY\n COALESCE(\n sa.contribution_date,\n sb.expenditure_date,\n sc.loan_incurred_date,\n se.disbursement_date,\n se.dissemination_date\n ),\n t.created\n ) AS new_sum\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea AS sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb AS sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec AS sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 AS sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee AS se ON t.schedule_e_id = se.id\n LEFT JOIN transactions_scheduled AS sd ON t.schedule_d_id = sd.id\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM COALESCE(\n sa.contribution_date,\n sb.expenditure_date,\n sc.loan_incurred_date,\n se.disbursement_date,\n se.dissemination_date\n )) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE\n AND deleted IS NULL\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $4\n OR (\n tc.date = $4\n AND t.created >= $5\n )\n );\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID\nAS $$\nDECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\nBEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_effective_amount(\n transaction_type_identifier TEXT,\n amount NUMERIC,\n schedule_c_id UUID\n)\nRETURNS NUMERIC\nAS $$\nDECLARE\n effective_amount NUMERIC;\nBEGIN\n -- Case 1: transaction type is a refund\n IF transaction_type_identifier IN (\n 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'REFUND_PARTY_CONTRIBUTION',\n 'REFUND_PARTY_CONTRIBUTION_VOID',\n 'REFUND_PAC_CONTRIBUTION',\n 'REFUND_PAC_CONTRIBUTION_VOID',\n 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'REFUND_UNREGISTERED_CONTRIBUTION',\n 'REFUND_UNREGISTERED_CONTRIBUTION_VOID',\n 'REFUND_INDIVIDUAL_CONTRIBUTION',\n 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT'\n ) THEN\n effective_amount := amount * -1;\n\n -- Case 2: schedule_c exists (return NULL)\n ELSIF schedule_c_id IS NOT NULL THEN\n effective_amount := NULL;\n\n -- Default case: return the original amount\n ELSE\n effective_amount := amount;\n END IF;\n\n RETURN effective_amount;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_is_loan(\n loan UUID,\n transaction_type_identifier TEXT,\n schedule_c_id UUID\n)\nRETURNS TEXT\nAS $$\nDECLARE\n loan_key TEXT;\nBEGIN\n IF loan IS NOT NULL AND transaction_type_identifier\n IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE')\n THEN\n loan_key := 'F';\n\n ELSIF schedule_c_id IS NOT NULL THEN\n loan_key := 'T';\n\n ELSE\n loan_key := 'F';\n END IF;\n\n RETURN loan_key;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_original_loan_id(\n transaction_id text,\n loan UUID,\n transaction_type_identifier TEXT,\n schedule_c_id UUID,\n schedule_b_id UUID\n)\nRETURNS TEXT\nAS $$\nDECLARE\n loan_transaction_id text;\nBEGIN\n IF loan IS NOT NULL AND transaction_type_identifier\n IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE')\n THEN\n SELECT t.transaction_id\n INTO loan_transaction_id\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n WHERE t.id = loan;\n\n ELSIF schedule_c_id IS NOT NULL THEN\n loan_transaction_id := transaction_id;\n\n ELSE\n loan_transaction_id := NULL;\n END IF;\n\n RETURN loan_transaction_id;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_loan_date(\n trans_id TEXT,\n loan UUID,\n transaction_type_identifier TEXT,\n schedule_c_id UUID,\n schedule_b_id UUID\n)\nRETURNS DATE\nAS $$\nDECLARE\n date DATE;\nBEGIN\n IF loan IS NOT NULL AND transaction_type_identifier\n IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE')\n THEN\n SELECT sb.expenditure_date\n INTO date\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n WHERE t.transaction_id = trans_id;\n\n ELSIF schedule_c_id IS NOT NULL THEN\n SELECT s.report_coverage_through_date INTO date\n FROM transactions_schedulec s\n WHERE s.id = schedule_c_id;\n\n ELSE\n date := NULL;\n END IF;\n\n RETURN date;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\nCREATE OR REPLACE FUNCTION calculate_amount(\n schedule_a_contribution_amount NUMERIC,\n schedule_b_expenditure_amount NUMERIC,\n schedule_c_loan_amount NUMERIC,\n schedule_c2_guaranteed_amount NUMERIC,\n schedule_e_expenditure_amount NUMERIC,\n debt UUID, -- Reference to another transaction\n schedule_d_id UUID\n)\nRETURNS NUMERIC\nAS $$\nDECLARE\n debt_incurred_amount NUMERIC;\nBEGIN\n IF debt IS NOT NULL THEN\n SELECT sd.incurred_amount\n INTO debt_incurred_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id\n WHERE t.id = debt;\n ELSE\n debt_incurred_amount := NULL;\n END IF;\n\n RETURN COALESCE(\n schedule_a_contribution_amount,\n schedule_b_expenditure_amount,\n schedule_c_loan_amount,\n schedule_c2_guaranteed_amount,\n schedule_e_expenditure_amount,\n debt_incurred_amount,\n (SELECT incurred_amount FROM transactions_scheduled WHERE id = schedule_d_id)\n );\nEND;\n$$\nLANGUAGE plpgsql;\n', + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD, sql_committee_id TEXT\n)\nRETURNS VOID\nAS $$\nDECLARE\n pulled_forward_loans RECORD;\nBEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE data.original_loan_id = (\n SELECT calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n )\n AND data.date <= (\n SELECT calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n )\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING txn.id;\n\n -- Handle pulled-forward loans\n FOR pulled_forward_loans IN\n SELECT t.transaction_id\n FROM transactions_transaction t\n WHERE t.schedule_c_id IS NOT NULL\n AND t.loan_id = txn.loan_id\n LOOP\n -- Recalculate loan_payment_to_date for each pulled-forward loan\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE data.original_loan_id = $1\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING pulled_forward_loans.transaction_id;\n END LOOP;\nEND;\n$$\nLANGUAGE plpgsql;\n", + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "drop_old_triggers"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_drop_old_triggers"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "drop_old_functions"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_drop_old_functions"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "process_itemization"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_process_itemization"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "handle_parent_itemization"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_handle_parent_itemization"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "calculate_aggregates"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_calculate_aggregates"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "update_can_unamend"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_update_can_unamend"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "before_transactions_transaction"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "before_transactions_transaction"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "after_transactions_transaction"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_after_transactions_transaction"), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "create_triggers"), + reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_create_triggers"), + ), + migrations.CreateModel( + name='ScheduleF', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('filer_designated_to_make_coordinated_expenditures', models.BooleanField(blank=True, null=True)), + ('expenditure_date', models.DateField(blank=True, null=True)), + ('expenditure_amount', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('aggregate_general_elec_expended', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ('expenditure_purpose_descrip', models.TextField(blank=True)), + ('category_code', models.TextField(blank=True, null=True)), + ('memo_text_description', models.TextField(blank=True, null=True)), + ('general_election_year', models.TextField(blank=True)), + ], + ), + migrations.AddField( + model_name='transaction', + name='schedule_f', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='transactions.schedulef'), + ), + migrations.AddField( + model_name='transaction', + name='contact_4', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='contact_4_transaction_set', to='contacts.contact'), + ), + migrations.AddField( + model_name='transaction', + name='contact_5', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='contact_5_transaction_set', to='contacts.contact'), + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID\nAS $$\nDECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\nBEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n AND t.committee_account_id = $9\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created,\n txn.committee_account_id;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + reverse_sql="\nCREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID\nAS $$\nDECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\nBEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0020_trigger_save_on_transactions", "trigger_save_on_transactions"), + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterField( + model_name='transaction', + name='reports', + field=models.ManyToManyField(related_name='transactions', through='reports.ReportTransaction', through_fields=['transaction', 'report'], to='reports.report'), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0022_schedule_f_aggregation", "calculate_schedule_f_aggregates"), + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunSQL( + sql="\nCREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date(\n txn record, sql_committee_id text\n)\nRETURNS VOID AS $$\nBEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE (data.original_loan_id = (\n SELECT calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n )\n AND data.date <= (\n SELECT calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n ))\n OR data.original_loan_id IN (\n SELECT t.transaction_id\n FROM transactions_transaction t\n WHERE t.schedule_c_id IS NOT NULL\n AND t.loan_id = $2\n )\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING txn.id, txn.loan_id;\nEND;\n$$\nLANGUAGE plpgsql;\n", + ), + migrations.AddField( + model_name='scheduled', + name='balance_at_close', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.AddField( + model_name='scheduled', + name='beginning_balance', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.AddField( + model_name='scheduled', + name='incurred_prior', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.AddField( + model_name='scheduled', + name='payment_prior', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.AddField( + model_name='scheduled', + name='payment_amount', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + ), + migrations.RunPython( + code=_load_callable("fecfiler.transactions.migrations.0024_scheduled_balance_at_close_and_more", "run_aggregations_for_all_debts"), + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunSQL( + sql='\n -- Drop all aggregate-related triggers from\n -- transactions_transaction table\n DROP TRIGGER IF EXISTS calculate_aggregates_trigger\n ON transactions_transaction;\n DROP TRIGGER IF EXISTS after_transactions_transaction_trigger\n ON transactions_transaction;\n DROP TRIGGER IF EXISTS\n after_transactions_transaction_infinite_trigger\n ON transactions_transaction;\n DROP TRIGGER IF EXISTS before_transactions_transaction_trigger\n ON transactions_transaction;\n ', + reverse_sql='\n -- Recreate triggers for aggregate calculation\n CREATE TRIGGER before_transactions_transaction_trigger\n BEFORE INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n EXECUTE FUNCTION before_transactions_transaction();\n\n CREATE TRIGGER after_transactions_transaction_infinite_trigger\n AFTER INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n EXECUTE FUNCTION after_transactions_transaction_infinite();\n\n CREATE TRIGGER after_transactions_transaction_trigger\n AFTER INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n WHEN (pg_trigger_depth() = 0)\n EXECUTE FUNCTION after_transactions_transaction();\n ', + ), + migrations.RunSQL( + sql='\n -- Drop the calculate_entity_aggregates function\n DROP FUNCTION IF EXISTS calculate_entity_aggregates(\n txn RECORD, sql_committee_id TEXT, temp_table_name TEXT\n ) CASCADE;\n ', + reverse_sql="\n -- Recreate calculate_entity_aggregates function\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n BEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT contribution_date\n INTO schedule_date\n FROM transactions_schedulea\n WHERE id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT expenditure_date\n INTO schedule_date\n FROM transactions_scheduleb\n WHERE id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET aggregate = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\n -- Drop the calculate_calendar_ytd_per_election_office function\n DROP FUNCTION IF EXISTS calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id TEXT, temp_table_name TEXT\n ) CASCADE;\n ', + reverse_sql="\n -- Recreate calculate_calendar_ytd_per_election_office function\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\n BEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n AND t.committee_account_id = $9\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created,\n txn.committee_account_id;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\n -- Drop the calculate_effective_amount function\n DROP FUNCTION IF EXISTS calculate_effective_amount(\n transaction_type_identifier TEXT, amount NUMERIC,\n schedule_c_id UUID\n ) CASCADE;\n ', + reverse_sql="\n -- Recreate calculate_effective_amount function\n CREATE OR REPLACE FUNCTION calculate_effective_amount(\n transaction_type_identifier TEXT,\n amount NUMERIC,\n schedule_c_id UUID\n )\n RETURNS NUMERIC AS $$\n DECLARE\n effective_amount NUMERIC;\n BEGIN\n -- Case 1: transaction type is a refund\n IF transaction_type_identifier IN (\n 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'REFUND_PARTY_CONTRIBUTION',\n 'REFUND_PARTY_CONTRIBUTION_VOID',\n 'REFUND_PAC_CONTRIBUTION',\n 'REFUND_PAC_CONTRIBUTION_VOID',\n 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'REFUND_UNREGISTERED_CONTRIBUTION',\n 'REFUND_UNREGISTERED_CONTRIBUTION_VOID',\n 'REFUND_INDIVIDUAL_CONTRIBUTION',\n 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT'\n ) THEN\n effective_amount := amount * -1;\n\n -- Case 2: schedule_c exists (return NULL)\n ELSIF schedule_c_id IS NOT NULL THEN\n effective_amount := NULL;\n\n -- Default case: return the original amount\n ELSE\n effective_amount := amount;\n END IF;\n\n RETURN effective_amount;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\n -- Drop the calculate_amount function if it exists\n DROP FUNCTION IF EXISTS calculate_amount(\n contribution_amount NUMERIC,\n expenditure_amount NUMERIC,\n loan_amount NUMERIC,\n guaranteed_amount NUMERIC,\n schedule_e_expenditure_amount NUMERIC,\n debt_id UUID,\n schedule_d_id UUID\n ) CASCADE;\n ', + reverse_sql='\n -- Recreate calculate_amount function\n CREATE OR REPLACE FUNCTION calculate_amount(\n schedule_a_contribution_amount NUMERIC,\n schedule_b_expenditure_amount NUMERIC,\n schedule_c_loan_amount NUMERIC,\n schedule_c2_guaranteed_amount NUMERIC,\n schedule_e_expenditure_amount NUMERIC,\n debt UUID,\n schedule_d_id UUID\n )\n RETURNS NUMERIC AS $$\n DECLARE\n debt_incurred_amount NUMERIC;\n BEGIN\n IF debt IS NOT NULL THEN\n SELECT sd.incurred_amount\n INTO debt_incurred_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id\n WHERE t.id = debt;\n ELSE\n debt_incurred_amount := NULL;\n END IF;\n\n RETURN COALESCE(\n schedule_a_contribution_amount,\n schedule_b_expenditure_amount,\n schedule_c_loan_amount,\n schedule_c2_guaranteed_amount,\n schedule_e_expenditure_amount,\n debt_incurred_amount,\n (SELECT incurred_amount\n FROM transactions_scheduled\n WHERE id = schedule_d_id)\n );\n END;\n $$ LANGUAGE plpgsql;\n ', + ), + migrations.RunSQL( + sql='\n -- Drop the calculate_loan_payment_to_date function if it exists\n DROP FUNCTION IF EXISTS calculate_loan_payment_to_date(\n txn RECORD, sql_committee_id TEXT, temp_table_name TEXT\n ) CASCADE;\n ', + reverse_sql="\n -- Recreate calculate_loan_payment_to_date function\n CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date(\n txn record, sql_committee_id text\n )\n RETURNS VOID AS $$\n BEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa\n ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb\n ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc\n ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2\n ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se\n ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE (data.original_loan_id = (\n SELECT calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id\n FROM transactions_transaction\n WHERE id = $1),\n $1\n )\n )\n AND data.date <= (\n SELECT calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id\n FROM transactions_transaction\n WHERE id = $1),\n $1\n )\n ))\n OR data.original_loan_id IN (\n SELECT t.transaction_id\n FROM transactions_transaction t\n WHERE t.schedule_c_id IS NOT NULL\n AND t.loan_id = $2\n )\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING txn.id, txn.loan_id;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\n -- Drop the trigger handler functions\n DROP FUNCTION IF EXISTS after_transactions_transaction()\n CASCADE;\n DROP FUNCTION IF EXISTS\n after_transactions_transaction_infinite() CASCADE;\n DROP FUNCTION IF EXISTS before_transactions_transaction()\n CASCADE;\n DROP FUNCTION IF EXISTS\n before_transactions_transaction_insert_or_update()\n CASCADE;\n DROP FUNCTION IF EXISTS\n after_transactions_transaction_insert_or_update()\n CASCADE;\n ', + reverse_sql="\n -- Recreate trigger handler functions\n CREATE OR REPLACE FUNCTION before_transactions_transaction()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW := process_itemization(OLD, NEW);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE FUNCTION after_transactions_transaction()\n RETURNS TRIGGER AS $$\n BEGIN\n IF TG_OP = 'UPDATE'\n THEN\n NEW := calculate_aggregates(OLD, NEW, TG_OP);\n NEW := update_can_unamend(NEW);\n ELSE\n NEW := calculate_aggregates(OLD, NEW, TG_OP);\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW := handle_parent_itemization(OLD, NEW);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.RunSQL( + sql='\n -- Drop the main calculate_aggregates function that was\n -- called by triggers\n DROP FUNCTION IF EXISTS calculate_aggregates(\n old RECORD, new RECORD, tg_op TEXT\n ) CASCADE;\n DROP FUNCTION IF EXISTS calculate_aggregates() CASCADE;\n ', + reverse_sql="\n -- Recreate calculate_aggregates function\n CREATE OR REPLACE FUNCTION calculate_aggregates(\n OLD RECORD,\n NEW RECORD,\n TG_OP TEXT\n )\n RETURNS RECORD AS $$\n DECLARE\n sql_committee_id TEXT;\n BEGIN\n sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_');\n\n -- If schedule_c2_id or schedule_d_id is not null, stop processing\n IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL\n THEN\n RETURN NEW;\n END IF;\n\n IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL\n THEN\n PERFORM calculate_entity_aggregates(NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n AND NEW.contact_1_id <> OLD.contact_1_id\n THEN\n PERFORM calculate_entity_aggregates(OLD, sql_committee_id);\n END IF;\n END IF;\n\n IF NEW.schedule_c_id IS NOT NULL\n OR NEW.schedule_c1_id IS NOT NULL\n OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE'\n THEN\n PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id);\n END IF;\n\n IF NEW.schedule_e_id IS NOT NULL\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n OLD, sql_committee_id);\n END IF;\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n ", + ), + migrations.AlterField( + model_name='transaction', + name='itemized', + field=models.BooleanField(db_default=False), + ), + ] diff --git a/django-backend/fecfiler/transactions/migrations/0003_alter_transaction_parent_transaction.py b/django-backend/fecfiler/transactions/migrations/0003_alter_transaction_parent_transaction.py deleted file mode 100644 index b196f9c6a..000000000 --- a/django-backend/fecfiler/transactions/migrations/0003_alter_transaction_parent_transaction.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-05 03:55 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - dependencies = [ - ("transactions", "0002_remove_schedulea_contributor_city_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="transaction", - name="parent_transaction", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="children", - to="transactions.transaction", - ), - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0004_report_transactions_link_table.py b/django-backend/fecfiler/transactions/migrations/0004_report_transactions_link_table.py deleted file mode 100644 index 5f3e82cc4..000000000 --- a/django-backend/fecfiler/transactions/migrations/0004_report_transactions_link_table.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 4.2.7 on 2024-03-08 16:37 - -from django.db import migrations, models - - -def add_link_table(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - - for t in transaction.objects.all(): - t.reports.add(t.report) - t.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0006_reporttransaction"), - ("transactions", "0003_alter_transaction_parent_transaction"), - ] - - operations = [ - migrations.AddField( - model_name="transaction", - name="reports", - field=models.ManyToManyField( - through="reports.ReportTransaction", to="reports.report" - ), - ), - migrations.RunPython(add_link_table, migrations.RunPython.noop), - migrations.RemoveField(model_name="transaction", name="report"), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0005_schedulec_report_coverage_from_date_and_more.py b/django-backend/fecfiler/transactions/migrations/0005_schedulec_report_coverage_from_date_and_more.py deleted file mode 100644 index df255482f..000000000 --- a/django-backend/fecfiler/transactions/migrations/0005_schedulec_report_coverage_from_date_and_more.py +++ /dev/null @@ -1,65 +0,0 @@ -# Generated by Django 4.2.10 on 2024-03-25 16:45 - -from django.db import migrations -from django.db.models import Q, deletion, DateField, ForeignKey, ManyToManyField - - -def set_coverage_date(apps, schema_editor): - transaction_model = apps.get_model("transactions", "Transaction") - - for transaction in transaction_model.objects.filter( - Q(schedule_c__isnull=False) | Q(schedule_d__isnull=False) - ): - for report in transaction.reports.all(): - if report.coverage_from_date and transaction.schedule_d: - transaction.schedule_d.report_coverage_from_date = ( - report.coverage_from_date - ) - transaction.schedule_d.save() - if report.coverage_through_date and transaction.schedule_c: - transaction.schedule_c.report_coverage_through_date = ( - report.coverage_through_date - ) - transaction.schedule_c.save() - transaction.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("reports", "0006_reporttransaction"), - ("transactions", "0004_report_transactions_link_table"), - ] - - operations = [ - migrations.AddField( - model_name="schedulec", - name="report_coverage_through_date", - field=DateField(blank=True, null=True), - ), - migrations.AddField( - model_name="scheduled", - name="report_coverage_from_date", - field=DateField(blank=True, null=True), - ), - migrations.AlterField( - model_name="transaction", - name="parent_transaction", - field=ForeignKey( - blank=True, - null=True, - on_delete=deletion.CASCADE, - to="transactions.transaction", - ), - ), - migrations.AlterField( - model_name="transaction", - name="reports", - field=ManyToManyField( - related_name="transactions", - through="reports.ReportTransaction", - to="reports.report", - ), - ), - migrations.RunPython(set_coverage_date, migrations.RunPython.noop), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0006_independent_expenditure_memos_no_aggregation_group.py b/django-backend/fecfiler/transactions/migrations/0006_independent_expenditure_memos_no_aggregation_group.py deleted file mode 100644 index f984bba5e..000000000 --- a/django-backend/fecfiler/transactions/migrations/0006_independent_expenditure_memos_no_aggregation_group.py +++ /dev/null @@ -1,46 +0,0 @@ -# Generated by Django 4.2.10 on 2024-03-25 16:45 - -from django.db import migrations -from django.db.models import Q - - -def set_aggregation_group_to_none_for_ie_memos(apps, schema_editor): - transaction_model = apps.get_model("transactions", "Transaction") - - for transaction in transaction_model.objects.filter( - Q(transaction_type_identifier__in=[ - "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", - "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", - "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO" - ]) - ): - transaction.aggregation_group = None - transaction.save() - - -def reverse_removing_aggregation_group_for_ie_memos(apps, schema_editor): - transaction_model = apps.get_model("transactions", "Transaction") - - for transaction in transaction_model.objects.filter( - Q(transaction_type_identifier__in=[ - "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", - "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", - "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO" - ]) - ): - transaction.aggregation_group = "INDEPENDENT_EXPENDITURE" - transaction.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0005_schedulec_report_coverage_from_date_and_more"), - ] - - operations = [ - migrations.RunPython( - set_aggregation_group_to_none_for_ie_memos, - reverse_removing_aggregation_group_for_ie_memos, - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0007_schedulee_so_candidate_state.py b/django-backend/fecfiler/transactions/migrations/0007_schedulee_so_candidate_state.py deleted file mode 100644 index 3901f5c8f..000000000 --- a/django-backend/fecfiler/transactions/migrations/0007_schedulee_so_candidate_state.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.2.11 on 2024-06-04 14:02 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("transactions", "0006_independent_expenditure_memos_no_aggregation_group"), - ] - - operations = [ - migrations.AddField( - model_name="schedulee", - name="so_candidate_state", - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py b/django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py deleted file mode 100644 index 119c96d10..000000000 --- a/django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py +++ /dev/null @@ -1,261 +0,0 @@ -# Generated by Django 4.2.11 on 2024-05-21 20:49 - -from django.db import migrations, models - - -def populate_existing_rows(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - for row in transaction.objects.all(): - row.aggregate = 0.0 - row.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0007_schedulee_so_candidate_state"), - ] - - operations = [ - migrations.AddField( - model_name="transaction", - name="_calendar_ytd_per_election_office", - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.AddField( - model_name="transaction", - name="aggregate", - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.AddField( - model_name="transaction", - name="loan_payment_to_date", - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.RunSQL( - """ - CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - v_election_code TEXT; - v_candidate_office TEXT; - v_candidate_state TEXT; - v_candidate_district TEXT; - BEGIN - SELECT COALESCE(disbursement_date, dissemination_date) - INTO schedule_date FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT election_code - INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT candidate_office, candidate_state, candidate_district - INTO v_candidate_office, v_candidate_state, v_candidate_district - FROM contacts WHERE id = txn.contact_2_id; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - t.id, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - BEGIN - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT transaction_id FROM transactions_transaction - WHERE id = $1 - ) || ''%%''; -- Match the loan_key with a transaction_id prefix - - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id - AND tt.loan_key LIKE ''%%LOAN''; - ' - USING txn.id; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - temp_table_name TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - temp_table_name := 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); - RAISE NOTICE 'TESTING TRIGGER'; - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates( - NEW, sql_committee_id, temp_table_name || 'NEW'); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates( - OLD, sql_committee_id, temp_table_name || 'OLD'); - END IF; - - ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL - THEN - PERFORM calculate_loan_payment_to_date( - NEW, sql_committee_id, temp_table_name || 'NEW'); - - ELSIF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id, temp_table_name || 'NEW'); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id, temp_table_name || 'OLD'); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER calculate_aggregates_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION calculate_aggregates(); - """ - ), - migrations.RunPython(populate_existing_rows), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0009_update_calculate_loan_payment_to_date.py b/django-backend/fecfiler/transactions/migrations/0009_update_calculate_loan_payment_to_date.py deleted file mode 100644 index b4660cbb6..000000000 --- a/django-backend/fecfiler/transactions/migrations/0009_update_calculate_loan_payment_to_date.py +++ /dev/null @@ -1,260 +0,0 @@ -from django.db import migrations - - -def update_existing_rows(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - types = [ - "LOAN_RECEIVED_FROM_INDIVIDUAL", - "LOAN_RECEIVED_FROM BANK", - "LOAN_BY_COMMITTEE", - ] - for row in transaction.objects.filter(transaction_type_identifier__in=types): - row.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0008_transaction__calendar_ytd_per_election_office_and_more"), - ] - - operations = [ - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION get_temp_tablename() - RETURNS TEXT AS $$ - BEGIN - RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - v_election_code TEXT; - v_candidate_office TEXT; - v_candidate_state TEXT; - v_candidate_district TEXT; - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - SELECT COALESCE(disbursement_date, dissemination_date) - INTO schedule_date FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT election_code - INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT candidate_office, candidate_state, candidate_district - INTO v_candidate_office, v_candidate_state, v_candidate_district - FROM contacts WHERE id = txn.contact_2_id; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - t.id, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT - CASE - WHEN loan_id IS NULL THEN transaction_id - ELSE ( - SELECT transaction_id - FROM transactions_transaction - WHERE id = t.loan_id - ) - END - FROM transactions_transaction t - WHERE id = $1 - ) || ''%%''; -- Match the loan_key with a transaction_id prefix - - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id - AND tt.loan_key LIKE ''%%LOAN''; - ' - USING txn.id; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - -- Drop prior versions of these functions - DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT); - DROP FUNCTION calculate_calendar_ytd_per_election_office(RECORD, TEXT, TEXT); - DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT); - """ - ), - migrations.RunPython(update_existing_rows), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0010_update_aggregate_trigger_performance.py b/django-backend/fecfiler/transactions/migrations/0010_update_aggregate_trigger_performance.py deleted file mode 100644 index dba8033a4..000000000 --- a/django-backend/fecfiler/transactions/migrations/0010_update_aggregate_trigger_performance.py +++ /dev/null @@ -1,205 +0,0 @@ -# Generated by Django 4.2.11 on 2024-07-17 19:59 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0009_update_calculate_loan_payment_to_date"), - ] - - operations = [ - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT - contribution_date INTO schedule_date - FROM - transactions_schedulea - WHERE - id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT - expenditure_date INTO schedule_date - FROM - transactions_scheduleb - WHERE - id = txn.schedule_b_id; - END IF; - - EXECUTE ' - UPDATE transactions_transaction AS t - SET aggregate = tc.new_sum - FROM ( - SELECT - id, - aggregate, - date, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $4 - OR ( - tc.date = $4 - AND t.created >= $5 - ) - ); - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - t.date, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT - CASE - WHEN loan_id IS NULL THEN transaction_id - ELSE ( - SELECT transaction_id - FROM transactions_transaction - WHERE id = t.loan_id - ) - END - FROM transactions_transaction t - WHERE id = $1 - ) || ''%%'' -- Match the loan_key with a transaction_id prefix - ) AS tc - WHERE t.id = tc.id - AND tc.loan_key LIKE ''%%LOAN'' - ; - ' - USING txn.id; - END; - $$ - LANGUAGE plpgsql; - """ - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0011_transaction_can_delete.py b/django-backend/fecfiler/transactions/migrations/0011_transaction_can_delete.py deleted file mode 100644 index 502dcee6c..000000000 --- a/django-backend/fecfiler/transactions/migrations/0011_transaction_can_delete.py +++ /dev/null @@ -1,107 +0,0 @@ -# Generated by Django 5.0.8 on 2024-08-28 15:19 - -from django.db import connection, migrations, models -from django.contrib.postgres.fields import ArrayField - - -def create_trigger_function(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION update_transactions_can_delete() RETURNS TRIGGER AS $$ - BEGIN - UPDATE transactions_transaction - SET blocking_reports = CASE - WHEN NEW.upload_submission_id IS NOT NULL - THEN array_append(blocking_reports, NEW.id) - ELSE array_remove(blocking_reports, NEW.id) - END - -- all transactions in the submitted report - WHERE id IN ( - SELECT transaction_id - FROM reports_reporttransaction - WHERE report_id = NEW.id - ) - -- all transactions that are reattributed in the submtited report - OR id IN ( - SELECT reatt_redes_id - FROM reports_reporttransaction - JOIN transactions_transaction tt - ON reports_reporttransaction.transaction_id = tt.id - WHERE report_id = NEW.id - ) - -- all loans that are carried forward in the submitted report - OR id IN ( - SELECT loan_id - FROM reports_reporttransaction - JOIN transactions_transaction tt - ON reports_reporttransaction.transaction_id = tt.id - WHERE report_id = NEW.id - ) - -- all repayments to loans that are carried forward in the submitted report - OR loan_id IN ( - SELECT loan_id - FROM reports_reporttransaction - JOIN transactions_transaction tt - ON reports_reporttransaction.transaction_id = tt.id - WHERE report_id = NEW.id AND tt.schedule_c_id IS NOT NULL - ) - -- all debts that are carried forward in the submitted report - OR id IN ( - SELECT debt_id - FROM reports_reporttransaction - JOIN transactions_transaction tt - ON reports_reporttransaction.transaction_id = tt.id - WHERE report_id = NEW.id - ) - -- all repayments to debts that are carried forward in the submitted report - OR debt_id IN ( - SELECT debt_id - FROM reports_reporttransaction - JOIN transactions_transaction tt - ON reports_reporttransaction.transaction_id = tt.id - WHERE report_id = NEW.id AND tt.schedule_d_id IS NOT NULL - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def drop_trigger_function(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute("DROP FUNCTION IF EXISTS update_transactions_can_delete();") - - -def create_trigger(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE TRIGGER report_status_update - AFTER UPDATE OF upload_submission_id ON reports_report - FOR EACH ROW - EXECUTE FUNCTION update_transactions_can_delete(); - """ - ) - - -def drop_trigger(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute("DROP TRIGGER IF EXISTS report_status_update ON reports_report;") - - -class Migration(migrations.Migration): - dependencies = [ - ("transactions", "0010_update_aggregate_trigger_performance"), - ] - - operations = [ - migrations.AddField( - model_name="transaction", - name="blocking_reports", - field=ArrayField(models.UUIDField(), blank=False, default=list()), - ), - migrations.RunPython(create_trigger_function, reverse_code=drop_trigger_function), - migrations.RunPython(create_trigger, reverse_code=drop_trigger), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0012_alter_transactions_blocking_reports.py b/django-backend/fecfiler/transactions/migrations/0012_alter_transactions_blocking_reports.py deleted file mode 100644 index 2fc955acd..000000000 --- a/django-backend/fecfiler/transactions/migrations/0012_alter_transactions_blocking_reports.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.db import migrations, models -from django.contrib.postgres.fields import ArrayField - - -def update_blocking_reports_default(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - transaction._meta.get_field("blocking_reports").default = list - - -class Migration(migrations.Migration): - dependencies = [ - ("transactions", "0011_transaction_can_delete"), - ] - - operations = [ - migrations.AlterField( - model_name="transaction", - name="blocking_reports", - field=ArrayField( - base_field=models.UUIDField(), - blank=False, - default=list, - size=None, - ), - ), - migrations.RunPython(update_blocking_reports_default), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0013_transaction_itemized_and_associated_triggers.py b/django-backend/fecfiler/transactions/migrations/0013_transaction_itemized_and_associated_triggers.py deleted file mode 100644 index 98824f7d9..000000000 --- a/django-backend/fecfiler/transactions/migrations/0013_transaction_itemized_and_associated_triggers.py +++ /dev/null @@ -1,281 +0,0 @@ -from django.db import connection, migrations, models -from fecfiler.transactions.schedule_a.managers import ( - over_two_hundred_types as schedule_a_over_two_hundred_types, -) -from fecfiler.transactions.schedule_b.managers import ( - over_two_hundred_types as schedule_b_over_two_hundred_types, -) -import uuid - - -def populate_over_two_hundred_types(apps, schema_editor): - OverTwoHundredTypesScheduleA = apps.get_model( # noqa: N806 - "transactions", "OverTwoHundredTypesScheduleA" - ) - OverTwoHundredTypesScheduleB = apps.get_model( # noqa: N806 - "transactions", "OverTwoHundredTypesScheduleB" - ) - scha_types_to_create = [ - OverTwoHundredTypesScheduleA(type=type_to_create) - for type_to_create in schedule_a_over_two_hundred_types - ] - OverTwoHundredTypesScheduleA.objects.bulk_create(scha_types_to_create) - schb_types_to_create = [ - OverTwoHundredTypesScheduleB(type=type_to_create) - for type_to_create in schedule_b_over_two_hundred_types - ] - OverTwoHundredTypesScheduleB.objects.bulk_create(schb_types_to_create) - - -def drop_over_two_hundred_types(apps, schema_editor): - print("this reverses migration automatically.") - - -def create_itemized_triggers_and_functions(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION before_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - needs_itemized_set boolean; - itemization boolean; - BEGIN - needs_itemized_set := needs_itemized_set(OLD, NEW); - IF needs_itemized_set THEN - NEW.itemized := calculate_itemization(NEW); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION calculate_itemization( - txn RECORD - ) - RETURNS BOOLEAN AS $$ - DECLARE - itemized boolean; - BEGIN - itemized := TRUE; - IF txn.force_itemized IS NOT NULL THEN - itemized := txn.force_itemized; - ELSIF txn.aggregate < 0 THEN - itemized := TRUE; - ELSIF EXISTS ( - SELECT type from ( - SELECT type - FROM over_two_hundred_types_schedulea - UNION - SELECT type - FROM over_two_hundred_types_scheduleb - ) as scha_schb_types - WHERE type = txn.transaction_type_identifier - ) THEN - IF txn.aggregate > 200 THEN - itemized := TRUE; - ELSE - itemized := FALSE; - END IF; - END IF; - return itemized; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - parent_and_grandparent_ids uuid[]; - children_and_grandchildren_ids uuid[]; - BEGIN - IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN - IF NEW.itemized is TRUE THEN - parent_and_grandparent_ids := - get_parent_grandparent_transaction_ids(NEW); - PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); - ELSE - children_and_grandchildren_ids := - get_children_and_grandchildren_transaction_ids(NEW); - PERFORM set_itemization_for_ids( - FALSE,children_and_grandchildren_ids - ); - END IF; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION needs_itemized_set( - OLD RECORD, - NEW RECORD - ) - RETURNS BOOLEAN AS $$ - BEGIN - return OLD IS NULL OR ( - OLD.force_itemized IS DISTINCT FROM NEW.force_itemized - OR OLD.aggregate IS DISTINCT FROM NEW.aggregate - ); - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION set_itemization_for_ids( - itemization boolean, - ids uuid[] - ) - RETURNS VOID AS $$ - BEGIN - IF cardinality(ids) > 0 THEN - UPDATE transactions_transaction - SET - itemized = itemization - WHERE id = ANY (ids); - END IF; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION get_parent_grandparent_transaction_ids( - txn RECORD - ) - RETURNS uuid[] AS $$ - DECLARE - ids uuid[]; - BEGIN - SELECT array( - SELECT id - FROM transactions_transaction - WHERE id IN ( - txn.parent_transaction_id, - ( - SELECT parent_transaction_id - FROM transactions_transaction - WHERE id = txn.parent_transaction_id - ) - ) - ) into ids; - RETURN ids; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION get_children_and_grandchildren_transaction_ids( - txn RECORD - ) - RETURNS uuid[] AS $$ - DECLARE - ids uuid[]; - BEGIN - SELECT array( - SELECT id - FROM transactions_transaction - WHERE parent_transaction_id = ANY ( - array_prepend(txn.id, - array( - SELECT id - FROM transactions_transaction - WHERE parent_transaction_id = txn.id - ) - ) - ) - ) into ids; - RETURN ids; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER before_transactions_transaction_insert_or_update_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction_insert_or_update(); - - CREATE TRIGGER zafter_transactions_transaction_insert_or_update_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_insert_or_update(); - """ - ) - - -def drop_itemized_triggers_and_functions(apps, schema_editor): - schema_editor.execute( - """ - DROP TRIGGER - IF EXISTS zafter_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS before_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP FUNCTION IF EXISTS before_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS calculate_itemization; - DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS needs_itemized_set; - DROP FUNCTION IF EXISTS set_itemization_for_ids; - DROP FUNCTION IF EXISTS get_parent_grandparent_transaction_ids; - DROP FUNCTION IF EXISTS get_children_and_grandchildren_transaction_ids; - """ - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0012_alter_transactions_blocking_reports"), - ] - - operations = [ - migrations.AddField( - model_name="transaction", - name="itemized", - field=models.BooleanField(db_default=True), - ), - migrations.CreateModel( - name="OverTwoHundredTypesScheduleA", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ("type", models.TextField()), - ], - options={ - "db_table": "over_two_hundred_types_schedulea", - "indexes": [ - models.Index(fields=["type"], name="over_two_hu_type_2c8314_idx") - ], - }, - ), - migrations.CreateModel( - name="OverTwoHundredTypesScheduleB", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ("type", models.TextField()), - ], - options={ - "db_table": "over_two_hundred_types_scheduleb", - "indexes": [ - models.Index(fields=["type"], name="over_two_hu_type_411a44_idx") - ], - }, - ), - migrations.RunPython( - populate_over_two_hundred_types, - reverse_code=drop_over_two_hundred_types, - ), - migrations.RunPython( - create_itemized_triggers_and_functions, - reverse_code=drop_itemized_triggers_and_functions, - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0014_drop_transaction_view.py b/django-backend/fecfiler/transactions/migrations/0014_drop_transaction_view.py deleted file mode 100644 index 9b1cfd2b1..000000000 --- a/django-backend/fecfiler/transactions/migrations/0014_drop_transaction_view.py +++ /dev/null @@ -1,596 +0,0 @@ -# Generated by Django 4.2.11 on 2024-07-17 19:59 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0013_transaction_itemized_and_associated_triggers"), - ] - - operations = [ - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, sql_committee_id text -) -RETURNS VOID AS $$ -DECLARE - schedule_date date; -BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT - contribution_date INTO schedule_date - FROM - transactions_schedulea - WHERE - id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT - expenditure_date INTO schedule_date - FROM - transactions_scheduleb - WHERE - id = txn.schedule_b_id; - END IF; - - EXECUTE ' - UPDATE transactions_transaction AS t - SET aggregate = tc.new_sum - FROM ( - SELECT - t.id, - COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - ) AS date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id) - ) OVER ( - ORDER BY - COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - ), - t.created - ) AS new_sum - FROM transactions_transaction t - LEFT JOIN transactions_schedulea AS sa ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb AS sb ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec AS sc ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 AS sc2 ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee AS se ON t.schedule_e_id = se.id - LEFT JOIN transactions_scheduled AS sd ON t.schedule_d_id = sd.id - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - )) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE - AND deleted IS NULL - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $4 - OR ( - tc.date = $4 - AND t.created >= $5 - ) - ); - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; -END; -$$ -LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text -) -RETURNS VOID -AS $$ -DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; -BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; -END; -$$ -LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_effective_amount( - transaction_type_identifier TEXT, - amount NUMERIC, - schedule_c_id UUID -) -RETURNS NUMERIC -AS $$ -DECLARE - effective_amount NUMERIC; -BEGIN - -- Case 1: transaction type is a refund - IF transaction_type_identifier IN ( - 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', - 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', - 'REFUND_PARTY_CONTRIBUTION', - 'REFUND_PARTY_CONTRIBUTION_VOID', - 'REFUND_PAC_CONTRIBUTION', - 'REFUND_PAC_CONTRIBUTION_VOID', - 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'REFUND_UNREGISTERED_CONTRIBUTION', - 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', - 'REFUND_INDIVIDUAL_CONTRIBUTION', - 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' - ) THEN - effective_amount := amount * -1; - - -- Case 2: schedule_c exists (return NULL) - ELSIF schedule_c_id IS NOT NULL THEN - effective_amount := NULL; - - -- Default case: return the original amount - ELSE - effective_amount := amount; - END IF; - - RETURN effective_amount; -END; -$$ -LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_is_loan( - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID -) -RETURNS TEXT -AS $$ -DECLARE - loan_key TEXT; -BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - loan_key := 'F'; - - ELSIF schedule_c_id IS NOT NULL THEN - loan_key := 'T'; - - ELSE - loan_key := 'F'; - END IF; - - RETURN loan_key; -END; -$$ -LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_original_loan_id( - transaction_id text, - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID, - schedule_b_id UUID -) -RETURNS TEXT -AS $$ -DECLARE - loan_transaction_id text; -BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - SELECT t.transaction_id - INTO loan_transaction_id - FROM transactions_transaction t - LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id - WHERE t.id = loan; - - ELSIF schedule_c_id IS NOT NULL THEN - loan_transaction_id := transaction_id; - - ELSE - loan_transaction_id := NULL; - END IF; - - RETURN loan_transaction_id; -END; -$$ -LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_loan_date( - trans_id TEXT, - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID, - schedule_b_id UUID -) -RETURNS DATE -AS $$ -DECLARE - date DATE; -BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - SELECT sb.expenditure_date - INTO date - FROM transactions_transaction t - LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id - WHERE t.transaction_id = trans_id; - - ELSIF schedule_c_id IS NOT NULL THEN - SELECT s.report_coverage_through_date INTO date - FROM transactions_schedulec s - WHERE s.id = schedule_c_id; - - ELSE - date := NULL; - END IF; - - RETURN date; -END; -$$ -LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_amount( - schedule_a_contribution_amount NUMERIC, - schedule_b_expenditure_amount NUMERIC, - schedule_c_loan_amount NUMERIC, - schedule_c2_guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt UUID, -- Reference to another transaction - schedule_d_id UUID -) -RETURNS NUMERIC -AS $$ -DECLARE - debt_incurred_amount NUMERIC; -BEGIN - IF debt IS NOT NULL THEN - SELECT sd.incurred_amount - INTO debt_incurred_amount - FROM transactions_transaction t - LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id - WHERE t.id = debt; - ELSE - debt_incurred_amount := NULL; - END IF; - - RETURN COALESCE( - schedule_a_contribution_amount, - schedule_b_expenditure_amount, - schedule_c_loan_amount, - schedule_c2_guaranteed_amount, - schedule_e_expenditure_amount, - debt_incurred_amount, - (SELECT incurred_amount FROM transactions_scheduled WHERE id = schedule_d_id) - ); -END; -$$ -LANGUAGE plpgsql; -""" - ), - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, sql_committee_id TEXT -) -RETURNS VOID -AS $$ -DECLARE - pulled_forward_loans RECORD; -BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id FROM transactions_transaction WHERE id = $1), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id FROM transactions_transaction WHERE id = $1), - $1 - ) - ) - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING txn.id; - - -- Handle pulled-forward loans - FOR pulled_forward_loans IN - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = txn.loan_id - LOOP - -- Recalculate loan_payment_to_date for each pulled-forward loan - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE data.original_loan_id = $1 - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING pulled_forward_loans.transaction_id; - END LOOP; -END; -$$ -LANGUAGE plpgsql; -""" - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0015_merge_transaction_triggers.py b/django-backend/fecfiler/transactions/migrations/0015_merge_transaction_triggers.py deleted file mode 100644 index 8b87842cb..000000000 --- a/django-backend/fecfiler/transactions/migrations/0015_merge_transaction_triggers.py +++ /dev/null @@ -1,469 +0,0 @@ -from django.db import connection, migrations - - -def create_triggers(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE TRIGGER before_transactions_transaction_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction(); - - CREATE TRIGGER after_transactions_transaction_infinite_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_infinite(); - - CREATE TRIGGER after_transactions_transaction_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) - EXECUTE FUNCTION after_transactions_transaction(); - """ - ) - - -def reverse_create_triggers(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP TRIGGER - IF EXISTS before_transactions_transaction_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS after_transactions_transaction_infinite_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS after_transactions_transaction_trigger - ON transactions_transaction; - """ - ) - - -def before_transactions_transaction(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION before_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - NEW := process_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def after_transactions_transaction(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION after_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - IF TG_OP = 'UPDATE' - THEN - NEW := calculate_aggregates(OLD, NEW, TG_OP); - NEW := update_can_unamend(NEW); - ELSE - NEW := calculate_aggregates(OLD, NEW, TG_OP); - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite() - RETURNS TRIGGER AS $$ - BEGIN - NEW := handle_parent_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - """ - ) - - -def reverse_after_transactions_transaction(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS after_transactions_transaction - DROP FUNCTION IF EXISTS after_transactions_transaction_infinite - """ - ) - - -def drop_old_triggers(apps, schema_editor): - schema_editor.execute( - """ - DROP TRIGGER - IF EXISTS zafter_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS before_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS transaction_updated - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS calculate_aggregates_trigger - ON transactions_transaction; - """ - ) - - -def reverse_drop_old_triggers(apps, schema_editor): - schema_editor.execute( - """ - CREATE TRIGGER zafter_transactions_transaction_insert_or_update_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_insert_or_update(); - - CREATE TRIGGER before_transactions_transaction_insert_or_update_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction_insert_or_update(); - - CREATE TRIGGER transaction_updated - AFTER UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION update_can_unamend(); - - CREATE TRIGGER calculate_aggregates_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION calculate_aggregates(); - """ - ) - - -def drop_old_functions(apps, schema_editor): - schema_editor.execute( - """ - DROP FUNCTION IF EXISTS before_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS calculate_aggregates; - DROP FUNCTION IF EXISTS update_can_unamend; - """ - ) - - -def reverse_drop_old_functions(apps, schema_editor): - schema_editor.execute( - """ - CREATE OR REPLACE FUNCTION before_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - needs_itemized_set boolean; - itemization boolean; - BEGIN - needs_itemized_set := needs_itemized_set(OLD, NEW); - IF needs_itemized_set THEN - NEW.itemized := calculate_itemization(NEW); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - - CREATE OR REPLACE FUNCTION after_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - parent_and_grandparent_ids uuid[]; - children_and_grandchildren_ids uuid[]; - BEGIN - IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN - IF NEW.itemized is TRUE THEN - parent_and_grandparent_ids := - get_parent_grandparent_transaction_ids(NEW); - PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); - ELSE - children_and_grandchildren_ids := - get_children_and_grandchildren_transaction_ids(NEW); - PERFORM set_itemization_for_ids( - FALSE,children_and_grandchildren_ids - ); - END IF; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - - CREATE OR REPLACE FUNCTION update_can_unamend() - RETURNS TRIGGER AS $$ - BEGIN - UPDATE reports_report - SET can_unamend = FALSE - WHERE id IN ( - SELECT report_id - FROM reports_reporttransaction - WHERE transaction_id = NEW.id - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -# Replaces old before_transactions_transaction_insert_or_update() -def process_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION process_itemization( - OLD RECORD, - NEW RECORD - ) - RETURNS RECORD AS $$ - DECLARE - needs_itemized_set boolean; - itemization boolean; - BEGIN - needs_itemized_set := needs_itemized_set(OLD, NEW); - IF needs_itemized_set THEN - NEW.itemized := calculate_itemization(NEW); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_process_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS process_itemization - """ - ) - - -# Replaces old after_transactions_transaction_insert_or_update() -def handle_parent_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION handle_parent_itemization( - OLD RECORD, - NEW RECORD - ) - RETURNS RECORD AS $$ - DECLARE - parent_and_grandparent_ids uuid[]; - children_and_grandchildren_ids uuid[]; - BEGIN - IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN - IF NEW.itemized is TRUE THEN - parent_and_grandparent_ids := - get_parent_grandparent_transaction_ids(NEW); - PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); - ELSE - children_and_grandchildren_ids := - get_children_and_grandchildren_transaction_ids(NEW); - PERFORM set_itemization_for_ids( - FALSE,children_and_grandchildren_ids - ); - END IF; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_handle_parent_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS handle_parent_itemization - """ - ) - - -def calculate_aggregates(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates( - OLD RECORD, - NEW RECORD, - TG_OP TEXT - ) - RETURNS RECORD AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_calculate_aggregates(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS calculate_aggregates - """ - ) - - -def update_can_unamend(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION update_can_unamend( - NEW RECORD - ) - RETURNS RECORD AS $$ - BEGIN - UPDATE reports_report - SET can_unamend = FALSE - WHERE id IN ( - SELECT report_id - FROM reports_reporttransaction - WHERE transaction_id = NEW.id - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_update_can_unamend(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS update_can_unamend - """ - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0014_drop_transaction_view"), - ] - - operations = [ - migrations.RunPython(drop_old_triggers, reverse_code=reverse_drop_old_triggers), - migrations.RunPython(drop_old_functions, reverse_code=reverse_drop_old_functions), - migrations.RunPython( - process_itemization, reverse_code=reverse_process_itemization - ), - migrations.RunPython( - handle_parent_itemization, reverse_code=reverse_handle_parent_itemization - ), - migrations.RunPython( - calculate_aggregates, reverse_code=reverse_calculate_aggregates - ), - migrations.RunPython(update_can_unamend, reverse_code=reverse_update_can_unamend), - migrations.RunPython( - before_transactions_transaction, - reverse_code=before_transactions_transaction, - ), - migrations.RunPython( - after_transactions_transaction, - reverse_code=reverse_after_transactions_transaction, - ), - migrations.RunPython(create_triggers, reverse_code=reverse_create_triggers), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0016_schedulef_transaction_contact_4_and_more.py b/django-backend/fecfiler/transactions/migrations/0016_schedulef_transaction_contact_4_and_more.py deleted file mode 100644 index 62a09c018..000000000 --- a/django-backend/fecfiler/transactions/migrations/0016_schedulef_transaction_contact_4_and_more.py +++ /dev/null @@ -1,81 +0,0 @@ -# Generated by Django 5.1.5 on 2025-03-06 01:56 - -import django.db.models.deletion -import uuid -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("contacts", "0001_initial"), - ("transactions", "0015_merge_transaction_triggers"), - ] - - operations = [ - migrations.CreateModel( - name="ScheduleF", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ( - "filer_designated_to_make_coordianted_expenditures", - models.BooleanField(blank=True, null=True), - ), - ("expenditure_date", models.DateField(blank=True, null=True)), - ( - "expenditure_amount", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ( - "aggregate_general_elec_expended", - models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - ("expenditure_purpose_descrip", models.TextField(blank=True)), - ("category_code", models.TextField(blank=True)), - ("memo_text_description", models.TextField(blank=True)), - ], - ), - migrations.AddField( - model_name="transaction", - name="contact_4", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="contact_4_transaction_set", - to="contacts.contact", - ), - ), - migrations.AddField( - model_name="transaction", - name="contact_5", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="contact_5_transaction_set", - to="contacts.contact", - ), - ), - migrations.AddField( - model_name="transaction", - name="schedule_f", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="transactions.schedulef", - ), - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0017_schedulef_coordianted_to_coordinated.py b/django-backend/fecfiler/transactions/migrations/0017_schedulef_coordianted_to_coordinated.py deleted file mode 100644 index 9bfe5905e..000000000 --- a/django-backend/fecfiler/transactions/migrations/0017_schedulef_coordianted_to_coordinated.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Dan 1.0 on 2025-03-17 16:25ish - -from django.db import migrations -import django_migration_linter as linter - - -class Migration(migrations.Migration): - dependencies = [ - ("transactions", "0016_schedulef_transaction_contact_4_and_more"), - ] - - operations = [ - linter.IgnoreMigration(), - migrations.RenameField( - model_name="schedulef", - old_name="filer_designated_to_make_coordianted_expenditures", - new_name="filer_designated_to_make_coordinated_expenditures", - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0018_schedulef_general_election_year_and_more.py b/django-backend/fecfiler/transactions/migrations/0018_schedulef_general_election_year_and_more.py deleted file mode 100644 index 407ebff0a..000000000 --- a/django-backend/fecfiler/transactions/migrations/0018_schedulef_general_election_year_and_more.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 5.1.7 on 2025-03-26 14:54 - -from django.db import migrations, models -import django_migration_linter as linter - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0017_schedulef_coordianted_to_coordinated"), - ] - - operations = [ - linter.IgnoreMigration(), - migrations.AddField( - model_name="schedulef", - name="general_election_year", - field=models.TextField(blank=True), - ), - migrations.AlterField( - model_name="schedulef", - name="category_code", - field=models.TextField(blank=True, null=True), - ), - migrations.AlterField( - model_name="schedulef", - name="memo_text_description", - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0019_aggregate_committee_controls.py b/django-backend/fecfiler/transactions/migrations/0019_aggregate_committee_controls.py deleted file mode 100644 index 9a5f1c408..000000000 --- a/django-backend/fecfiler/transactions/migrations/0019_aggregate_committee_controls.py +++ /dev/null @@ -1,236 +0,0 @@ -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0018_schedulef_general_election_year_and_more"), - ] - - old_election_aggregate_function = """ -CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text -) -RETURNS VOID -AS $$ -DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; -BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; -END; -$$ -LANGUAGE plpgsql; - """ - - new_election_aggregate_function = """ -CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text -) -RETURNS VOID -AS $$ -DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; -BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - AND t.committee_account_id = $9 - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created, - txn.committee_account_id; -END; -$$ -LANGUAGE plpgsql; - """ - - operations = [ - migrations.RunSQL( - new_election_aggregate_function, reverse_sql=old_election_aggregate_function - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0020_trigger_save_on_transactions.py b/django-backend/fecfiler/transactions/migrations/0020_trigger_save_on_transactions.py deleted file mode 100644 index f0408c911..000000000 --- a/django-backend/fecfiler/transactions/migrations/0020_trigger_save_on_transactions.py +++ /dev/null @@ -1,76 +0,0 @@ -from django.db import migrations -import structlog - -logger = structlog.get_logger(__name__) - - -def trigger_save_on_transactions(apps, schema_editor): - transactions = apps.get_model("transactions", "transaction") - committees = apps.get_model("committee_accounts", "committeeaccount") - contacts = apps.get_model("contacts", "contact") - - # Update transactions for each committee - for committee in committees.objects.all(): - logger.info(f"Committee:{committee.committee_id}") - - # For each contact, update the first schedule A transaction - for contact in contacts.objects.filter(committee_account=committee): - logger.info(f"Contact: {contact.id}") - first_schedule_a = ( - transactions.objects.filter( - schedule_a__isnull=False, - contact_1=contact, - committee_account=committee, - ) - .order_by("schedule_a__contribution_date", "created") - .first() - ) - if first_schedule_a: - logger.info(f"Saving first Schedule A: {first_schedule_a.id}") - first_schedule_a.save() - - # Election Aggregates - elections = transactions.objects.filter( - schedule_e__isnull=False, - committee_account=committee, - ).values( - "contact_2__candidate_office", - "contact_2__candidate_state", - "contact_2__candidate_district", - "schedule_e__election_code", - ) - for election in elections: - logger.info("Finding first schedule E for election") - first_schedule_e = ( - transactions.objects.filter( - schedule_e__isnull=False, - contact_2__candidate_office=election[ - "contact_2__candidate_office" - ], - contact_2__candidate_state=election["contact_2__candidate_state"], - contact_2__candidate_district=election[ - "contact_2__candidate_district" - ], - schedule_e__election_code=election["schedule_e__election_code"], - committee_account=committee, - ) - .order_by( - "schedule_e__disbursement_date", - "created", - ) - .first() - ) - if first_schedule_e: - logger.info(f"Saving first Schedule E: {first_schedule_e.id}") - first_schedule_e.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0019_aggregate_committee_controls"), - ] - - operations = [ - migrations.RunPython(trigger_save_on_transactions, migrations.RunPython.noop), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0021_alter_transaction_reports.py b/django-backend/fecfiler/transactions/migrations/0021_alter_transaction_reports.py deleted file mode 100644 index 02685bde8..000000000 --- a/django-backend/fecfiler/transactions/migrations/0021_alter_transaction_reports.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 5.2 on 2025-04-30 17:10 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('reports', '0014_form99_swap_text_code'), - ('transactions', '0020_trigger_save_on_transactions'), - ] - - operations = [ - migrations.AlterField( - model_name='transaction', - name='reports', - field=models.ManyToManyField( - related_name='transactions', - through='reports.ReportTransaction', - through_fields=['transaction', 'report'], - to='reports.report' - ), - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0022_schedule_f_aggregation.py b/django-backend/fecfiler/transactions/migrations/0022_schedule_f_aggregation.py deleted file mode 100644 index 0ba3dde8f..000000000 --- a/django-backend/fecfiler/transactions/migrations/0022_schedule_f_aggregation.py +++ /dev/null @@ -1,54 +0,0 @@ -# Generated by Django 5.2 on 2025-05-07 18:18 - -from django.db import migrations -from django.db.models import F -from fecfiler.transactions.utils_aggregation_queries import filter_queryset_for_previous_transactions_in_aggregation # noqa: E501 - - -def calculate_schedule_f_aggregates(apps, schema_editor): - CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") # noqa - Transaction = apps.get_model("transactions", "Transaction") # noqa - - for committee in CommitteeAccount.objects.all(): - schedule_f_transactions = Transaction.objects.all().filter( - committee_account=committee, - schedule_f__isnull=False, - ).annotate( - date=F("schedule_f__expenditure_date"), - amount=F("schedule_f__expenditure_amount") - ).order_by("date") - - for trans in schedule_f_transactions: - previous_transactions = filter_queryset_for_previous_transactions_in_aggregation( # noqa: E501 - schedule_f_transactions, - trans.date, - trans.aggregation_group, - trans.id, - None, - trans.contact_2.id, - None, - trans.schedule_f.general_election_year - ) - - previous_transaction = previous_transactions.first() - previous_aggregate = 0 - if previous_transaction: - previous_aggregate = ( - previous_transaction.schedule_f.aggregate_general_elec_expended - ) - - trans.schedule_f.aggregate_general_elec_expended = ( - trans.amount + previous_aggregate - ) - trans.schedule_f.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ('transactions', '0021_alter_transaction_reports'), - ] - - operations = [ - migrations.RunPython(calculate_schedule_f_aggregates, migrations.RunPython.noop), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0023_optimize_calculate_loan_payment_to_date.py b/django-backend/fecfiler/transactions/migrations/0023_optimize_calculate_loan_payment_to_date.py deleted file mode 100644 index af27ba807..000000000 --- a/django-backend/fecfiler/transactions/migrations/0023_optimize_calculate_loan_payment_to_date.py +++ /dev/null @@ -1,117 +0,0 @@ -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0022_schedule_f_aggregation"), - ] - - operations = [ - migrations.RunSQL( - """ -CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( - txn record, sql_committee_id text -) -RETURNS VOID AS $$ -BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE (data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id FROM transactions_transaction WHERE id = $1), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id FROM transactions_transaction WHERE id = $1), - $1 - ) - )) - OR data.original_loan_id IN ( - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = $2 - ) - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING txn.id, txn.loan_id; -END; -$$ -LANGUAGE plpgsql; -""" - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0024_scheduled_balance_at_close_and_more.py b/django-backend/fecfiler/transactions/migrations/0024_scheduled_balance_at_close_and_more.py deleted file mode 100644 index ceef9321b..000000000 --- a/django-backend/fecfiler/transactions/migrations/0024_scheduled_balance_at_close_and_more.py +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by Django 5.2.7 on 2025-11-25 03:23 - -from django.db import migrations, models -from fecfiler.transactions.aggregation import process_aggregation_for_debts - - -def run_aggregations_for_all_debts(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - all_root_debts = transaction.objects.filter( - schedule_d__isnull=False, debt__isnull=True - ) - for debt in all_root_debts: - process_aggregation_for_debts(debt) - - -class Migration(migrations.Migration): - - dependencies = [ - ('transactions', '0023_optimize_calculate_loan_payment_to_date'), - ] - - operations = [ - migrations.AddField( - model_name='scheduled', - name='balance_at_close', - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.AddField( - model_name='scheduled', - name='beginning_balance', - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.AddField( - model_name='scheduled', - name='incurred_prior', - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.AddField( - model_name='scheduled', - name='payment_prior', - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.AddField( - model_name='scheduled', - name='payment_amount', - field=models.DecimalField( - blank=True, decimal_places=2, max_digits=11, null=True - ), - ), - migrations.RunPython(run_aggregations_for_all_debts, migrations.RunPython.noop) - ] diff --git a/django-backend/fecfiler/transactions/migrations/0025_drop_aggregate_triggers.py b/django-backend/fecfiler/transactions/migrations/0025_drop_aggregate_triggers.py deleted file mode 100644 index da0351adc..000000000 --- a/django-backend/fecfiler/transactions/migrations/0025_drop_aggregate_triggers.py +++ /dev/null @@ -1,565 +0,0 @@ -# Generated migration to drop database trigger functions -# This migration moves aggregate calculation from database triggers to Django code - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0024_scheduled_balance_at_close_and_more"), - ] - - operations = [ - migrations.RunSQL( - """ - -- Drop all aggregate-related triggers from - -- transactions_transaction table - DROP TRIGGER IF EXISTS calculate_aggregates_trigger - ON transactions_transaction; - DROP TRIGGER IF EXISTS after_transactions_transaction_trigger - ON transactions_transaction; - DROP TRIGGER IF EXISTS - after_transactions_transaction_infinite_trigger - ON transactions_transaction; - DROP TRIGGER IF EXISTS before_transactions_transaction_trigger - ON transactions_transaction; - """, - reverse_sql=""" - -- Recreate triggers for aggregate calculation - CREATE TRIGGER before_transactions_transaction_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction(); - - CREATE TRIGGER after_transactions_transaction_infinite_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_infinite(); - - CREATE TRIGGER after_transactions_transaction_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) - EXECUTE FUNCTION after_transactions_transaction(); - """, - ), - migrations.RunSQL( - """ - -- Drop the calculate_entity_aggregates function - DROP FUNCTION IF EXISTS calculate_entity_aggregates( - txn RECORD, sql_committee_id TEXT, temp_table_name TEXT - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_entity_aggregates function - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - """ - -- Drop the calculate_calendar_ytd_per_election_office function - DROP FUNCTION IF EXISTS calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id TEXT, temp_table_name TEXT - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_calendar_ytd_per_election_office function - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - AND t.committee_account_id = $9 - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created, - txn.committee_account_id; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - """ - -- Drop the calculate_effective_amount function - DROP FUNCTION IF EXISTS calculate_effective_amount( - transaction_type_identifier TEXT, amount NUMERIC, - schedule_c_id UUID - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_effective_amount function - CREATE OR REPLACE FUNCTION calculate_effective_amount( - transaction_type_identifier TEXT, - amount NUMERIC, - schedule_c_id UUID - ) - RETURNS NUMERIC AS $$ - DECLARE - effective_amount NUMERIC; - BEGIN - -- Case 1: transaction type is a refund - IF transaction_type_identifier IN ( - 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', - 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', - 'REFUND_PARTY_CONTRIBUTION', - 'REFUND_PARTY_CONTRIBUTION_VOID', - 'REFUND_PAC_CONTRIBUTION', - 'REFUND_PAC_CONTRIBUTION_VOID', - 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'REFUND_UNREGISTERED_CONTRIBUTION', - 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', - 'REFUND_INDIVIDUAL_CONTRIBUTION', - 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' - ) THEN - effective_amount := amount * -1; - - -- Case 2: schedule_c exists (return NULL) - ELSIF schedule_c_id IS NOT NULL THEN - effective_amount := NULL; - - -- Default case: return the original amount - ELSE - effective_amount := amount; - END IF; - - RETURN effective_amount; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - """ - -- Drop the calculate_amount function if it exists - DROP FUNCTION IF EXISTS calculate_amount( - contribution_amount NUMERIC, - expenditure_amount NUMERIC, - loan_amount NUMERIC, - guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt_id UUID, - schedule_d_id UUID - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_amount function - CREATE OR REPLACE FUNCTION calculate_amount( - schedule_a_contribution_amount NUMERIC, - schedule_b_expenditure_amount NUMERIC, - schedule_c_loan_amount NUMERIC, - schedule_c2_guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt UUID, - schedule_d_id UUID - ) - RETURNS NUMERIC AS $$ - DECLARE - debt_incurred_amount NUMERIC; - BEGIN - IF debt IS NOT NULL THEN - SELECT sd.incurred_amount - INTO debt_incurred_amount - FROM transactions_transaction t - LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id - WHERE t.id = debt; - ELSE - debt_incurred_amount := NULL; - END IF; - - RETURN COALESCE( - schedule_a_contribution_amount, - schedule_b_expenditure_amount, - schedule_c_loan_amount, - schedule_c2_guaranteed_amount, - schedule_e_expenditure_amount, - debt_incurred_amount, - (SELECT incurred_amount - FROM transactions_scheduled - WHERE id = schedule_d_id) - ); - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - """ - -- Drop the calculate_loan_payment_to_date function if it exists - DROP FUNCTION IF EXISTS calculate_loan_payment_to_date( - txn RECORD, sql_committee_id TEXT, temp_table_name TEXT - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_loan_payment_to_date function - CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( - txn record, sql_committee_id text - ) - RETURNS VOID AS $$ - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se - ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE (data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id - FROM transactions_transaction - WHERE id = $1), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id - FROM transactions_transaction - WHERE id = $1), - $1 - ) - )) - OR data.original_loan_id IN ( - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = $2 - ) - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING txn.id, txn.loan_id; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - """ - -- Drop the trigger handler functions - DROP FUNCTION IF EXISTS after_transactions_transaction() - CASCADE; - DROP FUNCTION IF EXISTS - after_transactions_transaction_infinite() CASCADE; - DROP FUNCTION IF EXISTS before_transactions_transaction() - CASCADE; - DROP FUNCTION IF EXISTS - before_transactions_transaction_insert_or_update() - CASCADE; - DROP FUNCTION IF EXISTS - after_transactions_transaction_insert_or_update() - CASCADE; - """, - reverse_sql=""" - -- Recreate trigger handler functions - CREATE OR REPLACE FUNCTION before_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - NEW := process_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - IF TG_OP = 'UPDATE' - THEN - NEW := calculate_aggregates(OLD, NEW, TG_OP); - NEW := update_can_unamend(NEW); - ELSE - NEW := calculate_aggregates(OLD, NEW, TG_OP); - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite() - RETURNS TRIGGER AS $$ - BEGIN - NEW := handle_parent_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - """ - -- Drop the main calculate_aggregates function that was - -- called by triggers - DROP FUNCTION IF EXISTS calculate_aggregates( - old RECORD, new RECORD, tg_op TEXT - ) CASCADE; - DROP FUNCTION IF EXISTS calculate_aggregates() CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_aggregates function - CREATE OR REPLACE FUNCTION calculate_aggregates( - OLD RECORD, - NEW RECORD, - TG_OP TEXT - ) - RETURNS RECORD AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """, - ), - ] diff --git a/django-backend/fecfiler/transactions/migrations/0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0026_alter_transaction_itemized.py deleted file mode 100644 index 5f27857b8..000000000 --- a/django-backend/fecfiler/transactions/migrations/0026_alter_transaction_itemized.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.2.9 on 2025-12-31 19:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("transactions", "0025_drop_aggregate_triggers"), - ] - - operations = [ - migrations.AlterField( - model_name="transaction", - name="itemized", - field=models.BooleanField(db_default=False), - ), - ] diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id.py deleted file mode 100644 index bf1ee44fc..000000000 --- a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-25 14:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("user", "0001_initial"), - ("committee_accounts", "0002_membership"), - ] - - operations = [ - migrations.RemoveField( - model_name="user", - name="cmtee_id", - ), - migrations.AlterField( - model_name="user", - name="first_name", - field=models.CharField(blank=True, max_length=150, null=True), - ), - migrations.AlterField( - model_name="user", - name="last_name", - field=models.CharField(blank=True, max_length=150, null=True), - ), - ] diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py new file mode 100644 index 000000000..b26f34516 --- /dev/null +++ b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py @@ -0,0 +1,74 @@ +# Generated by Django 5.2.11 on 2026-03-06 22:06 + +import django.db.migrations.operations.special +import fecfiler.user.managers +from django.db import migrations, models +from django.db.models import Q + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# fecfiler.user.migrations.0006_remove_old_login_accounts + + +def remove_old_login_accounts(apps, schema_editor): + User = apps.get_model("user", "User") + + users_to_delete = User.objects.filter( + Q(username__contains="@") | Q(username="adminnxg") | Q(username="tt") + ) + for user in users_to_delete: + user.membership_set.all().delete() + users_to_delete.delete() + +class Migration(migrations.Migration): + + replaces = [('user', '0002_remove_user_cmtee_id'), ('user', '0003_user_security_consent_date'), ('user', '0004_alter_user_managers'), ('user', '0005_rename_security_consent_date_user_security_consent_exp_date'), ('user', '0006_remove_old_login_accounts'), ('user', '0007_user_security_consent_version')] + + dependencies = [ + ('committee_accounts', '0002_membership'), + ('user', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='user', + name='cmtee_id', + ), + migrations.AlterField( + model_name='user', + name='first_name', + field=models.CharField(blank=True, max_length=150, null=True), + ), + migrations.AlterField( + model_name='user', + name='last_name', + field=models.CharField(blank=True, max_length=150, null=True), + ), + migrations.AddField( + model_name='user', + name='security_consent_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AlterModelManagers( + name='user', + managers=[ + ('objects', fecfiler.user.managers.UserManager()), + ], + ), + migrations.RenameField( + model_name='user', + old_name='security_consent_date', + new_name='security_consent_exp_date', + ), + migrations.RunPython( + code=remove_old_login_accounts, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AddField( + model_name='user', + name='security_consent_version', + field=models.CharField(blank=True, null=True), + ), + ] diff --git a/django-backend/fecfiler/user/migrations/0003_user_security_consent_date.py b/django-backend/fecfiler/user/migrations/0003_user_security_consent_date.py deleted file mode 100644 index 33d45b9a9..000000000 --- a/django-backend/fecfiler/user/migrations/0003_user_security_consent_date.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.7 on 2024-02-01 22:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('user', '0002_remove_user_cmtee_id'), - ] - - operations = [ - migrations.AddField( - model_name='user', - name='security_consent_date', - field=models.DateField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py b/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py deleted file mode 100644 index dd7809c98..000000000 --- a/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 4.2.7 on 2024-02-16 20:43 - -from django.db import migrations -import fecfiler.user.managers - - -class Migration(migrations.Migration): - - dependencies = [ - ('user', '0003_user_security_consent_date'), - ] - - operations = [ - migrations.AlterModelManagers( - name='user', - managers=[ - ('objects', fecfiler.user.managers.UserManager()), - ], - ), - ] diff --git a/django-backend/fecfiler/user/migrations/0005_rename_security_consent_date_user_security_consent_exp_date.py b/django-backend/fecfiler/user/migrations/0005_rename_security_consent_date_user_security_consent_exp_date.py deleted file mode 100644 index 671e09b09..000000000 --- a/django-backend/fecfiler/user/migrations/0005_rename_security_consent_date_user_security_consent_exp_date.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.2.7 on 2024-03-11 14:58 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("user", "0004_alter_user_managers"), - ] - - operations = [ - migrations.RenameField( - model_name="user", - old_name="security_consent_date", - new_name="security_consent_exp_date", - ), - ] diff --git a/django-backend/fecfiler/user/migrations/0006_remove_old_login_accounts.py b/django-backend/fecfiler/user/migrations/0006_remove_old_login_accounts.py deleted file mode 100644 index f281f6bbd..000000000 --- a/django-backend/fecfiler/user/migrations/0006_remove_old_login_accounts.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.db import migrations -from django.db.models import Q - - -def remove_old_login_accounts(apps, schema_editor): - User = apps.get_model("user", "User") # noqa - - users_to_delete = User.objects.filter( - Q(username__contains="@") | Q(username="adminnxg") | Q(username="tt") - ) - for user in users_to_delete: - user.membership_set.all().delete() - users_to_delete.delete() - - -class Migration(migrations.Migration): - - dependencies = [ - ( - "user", - "0005_rename_security_consent_date_user_security_consent_exp_date", - ) - ] - - operations = [ - migrations.RunPython( - remove_old_login_accounts, - migrations.RunPython.noop, - ), - ] diff --git a/django-backend/fecfiler/user/migrations/0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0007_user_security_consent_version.py deleted file mode 100644 index eb25f6d77..000000000 --- a/django-backend/fecfiler/user/migrations/0007_user_security_consent_version.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.2.2 on 2025-10-03 16:39 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('user', '0006_remove_old_login_accounts'), - ] - - operations = [ - migrations.AddField( - model_name='user', - name='security_consent_version', - field=models.CharField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py b/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py new file mode 100644 index 000000000..5499b71e8 --- /dev/null +++ b/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py @@ -0,0 +1,54 @@ +# Generated by Django 5.2.11 on 2026-03-07 03:21 + +import django.db.migrations.operations.special +from django.db import migrations, models +from django.db.models import F + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# fecfiler.web_services.migrations.0002_uploadsubmission_task_completed_and_more + + +def set_default_task_completed_times(apps, schema_editor): + uploads = apps.get_model("web_services", "UploadSubmission") + web_prints = apps.get_model("web_services", "WebPrintSubmission") + + uploads.objects.all().update(task_completed=F("updated")) + web_prints.objects.all().update(task_completed=F("updated")) + +class Migration(migrations.Migration): + + replaces = [('web_services', '0002_uploadsubmission_task_completed_and_more'), ('web_services', '0003_uploadsubmission_fecfile_polling_attempts_and_more')] + + dependencies = [ + ('web_services', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='uploadsubmission', + name='task_completed', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='webprintsubmission', + name='task_completed', + field=models.DateTimeField(null=True), + ), + migrations.RunPython( + code=set_default_task_completed_times, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AddField( + model_name='uploadsubmission', + name='fecfile_polling_attempts', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='webprintsubmission', + name='fecfile_polling_attempts', + field=models.IntegerField(default=0), + ), + ] diff --git a/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_and_more.py b/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_and_more.py deleted file mode 100644 index 1ee32610c..000000000 --- a/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_and_more.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 5.0.8 on 2024-08-22 19:42 - -from django.db import migrations, models -from django.db.models import F - - -def set_default_task_completed_times(apps, schema_editor): - uploads = apps.get_model("web_services", "UploadSubmission") - web_prints = apps.get_model("web_services", "WebPrintSubmission") - - uploads.objects.all().update(task_completed=F("updated")) - web_prints.objects.all().update(task_completed=F("updated")) - - -class Migration(migrations.Migration): - - dependencies = [ - ('web_services', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='uploadsubmission', - name='task_completed', - field=models.DateTimeField(null=True), - ), - migrations.AddField( - model_name='webprintsubmission', - name='task_completed', - field=models.DateTimeField(null=True), - ), - migrations.RunPython(set_default_task_completed_times, migrations.RunPython.noop) - ] diff --git a/django-backend/fecfiler/web_services/migrations/0003_uploadsubmission_fecfile_polling_attempts_and_more.py b/django-backend/fecfiler/web_services/migrations/0003_uploadsubmission_fecfile_polling_attempts_and_more.py deleted file mode 100644 index 86f41f0dd..000000000 --- a/django-backend/fecfiler/web_services/migrations/0003_uploadsubmission_fecfile_polling_attempts_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.0.8 on 2024-09-11 12:22 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('web_services', '0002_uploadsubmission_task_completed_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='uploadsubmission', - name='fecfile_polling_attempts', - field=models.IntegerField(default=0), - ), - migrations.AddField( - model_name='webprintsubmission', - name='fecfile_polling_attempts', - field=models.IntegerField(default=0), - ), - ] From 99e68cf7d1f714e62b7c750abd0d43d62f95c1cc Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Mon, 9 Mar 2026 17:47:37 -0400 Subject: [PATCH 07/52] FECFILE-2734: Linting and SQL clarity. --- ..._cashonhandyearly_cash_on_hand_and_more.py | 39 +- ...hed_0007_alter_committeeaccount_members.py | 181 +- ...tial_squashed_0003_memotext_text_prefix.py | 21 +- ..._deleted_squashed_00019_form24_name_fix.py | 834 +++-- ...quashed_0026_alter_transaction_itemized.py | 2734 +++++++++++++++-- ...shed_0007_user_security_consent_version.py | 44 +- ...ompleted_squashed_0003_polling_attempts.py | 24 +- 7 files changed, 3434 insertions(+), 443 deletions(-) diff --git a/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py b/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py index 25a2d2aed..e53712657 100644 --- a/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py +++ b/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py @@ -7,27 +7,46 @@ class Migration(migrations.Migration): - replaces = [('cash_on_hand', '0001_initial'), ('cash_on_hand', '0002_alter_cashonhandyearly_cash_on_hand_and_more')] + replaces = [ + ("cash_on_hand", "0001_initial"), + ("cash_on_hand", "0002_alter_cashonhandyearly_cash_on_hand_and_more"), + ] initial = True dependencies = [ - ('committee_accounts', '0001_squashed_0007_alter_committeeaccount_members'), + ("committee_accounts", "0001_squashed_0007_alter_committeeaccount_members"), ] operations = [ migrations.CreateModel( - name='CashOnHandYearly', + name="CashOnHandYearly", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('cash_on_hand', models.DecimalField(decimal_places=2, max_digits=11)), - ('year', models.TextField()), - ('created', models.DateTimeField(auto_now_add=True)), - ('updated', models.DateTimeField(auto_now=True)), - ('committee_account', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='committee_accounts.committeeaccount')), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ("cash_on_hand", models.DecimalField(decimal_places=2, max_digits=11)), + ("year", models.TextField()), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ( + "committee_account", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="committee_accounts.committeeaccount", + ), + ), ], options={ - 'db_table': 'cash_on_hand_yearly', + "db_table": "cash_on_hand_yearly", }, ), ] diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py index 3cd50758b..754f71ea4 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -80,56 +80,119 @@ def remove_pending_emails(apps, schema_editor): pending_email=None ) + class Migration(migrations.Migration): - replaces = [('committee_accounts', '0001_initial'), ('committee_accounts', '0002_membership'), ('committee_accounts', '0003_membership_pending_email_alter_membership_id_and_more'), ('committee_accounts', '0004_remove_duplicate_memberships'), ('committee_accounts', '0005_remove_pending_emails'), ('committee_accounts', '0006_alter_membership_committee_account_and_more'), ('committee_accounts', '0007_alter_committeeaccount_members')] + replaces = [ + ("committee_accounts", "0001_initial"), + ("committee_accounts", "0002_membership"), + ( + "committee_accounts", + "0003_membership_pending_email_alter_membership_id_and_more", + ), + ("committee_accounts", "0004_remove_duplicate_memberships"), + ("committee_accounts", "0005_remove_pending_emails"), + ("committee_accounts", "0006_alter_membership_committee_account_and_more"), + ("committee_accounts", "0007_alter_committeeaccount_members"), + ] initial = True dependencies = [ - ('user', '0001_initial'), + ("user", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.CreateModel( - name='CommitteeAccount', + name="CommitteeAccount", fields=[ - ('deleted', models.DateTimeField(blank=True, null=True)), - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('committee_id', models.CharField(max_length=9, unique=True, validators=[django.core.validators.RegexValidator('^C[0-9]{8}$', 'invalid committee id format')])), - ('created', models.DateTimeField(auto_now_add=True)), - ('updated', models.DateTimeField(auto_now=True)), + ("deleted", models.DateTimeField(blank=True, null=True)), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ( + "committee_id", + models.CharField( + max_length=9, + unique=True, + validators=[ + django.core.validators.RegexValidator( + "^C[0-9]{8}$", "invalid committee id format" + ) + ], + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), ], options={ - 'db_table': 'committee_accounts', + "db_table": "committee_accounts", }, ), migrations.CreateModel( - name='Membership', + name="Membership", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('role', models.CharField(choices=[('COMMITTEE_ADMINISTRATOR', 'Committee Administrator'), ('REVIEWER', 'Reviewer')], max_length=25)), - ('committee_account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='committee_accounts.committeeaccount')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "role", + models.CharField( + choices=[ + ("COMMITTEE_ADMINISTRATOR", "Committee Administrator"), + ("REVIEWER", "Reviewer"), + ], + max_length=25, + ), + ), + ( + "committee_account", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="committee_accounts.committeeaccount", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], ), migrations.AddField( - model_name='committeeaccount', - name='members', - field=models.ManyToManyField(through='committee_accounts.Membership', to=settings.AUTH_USER_MODEL), + model_name="committeeaccount", + name="members", + field=models.ManyToManyField( + through="committee_accounts.Membership", to=settings.AUTH_USER_MODEL + ), ), migrations.RunPython( code=create_memberships, ), migrations.AddField( - model_name='membership', - name='pending_email', + model_name="membership", + name="pending_email", field=models.EmailField(blank=True, max_length=254, null=True), ), migrations.AddField( - model_name='membership', - name='uuid', + model_name="membership", + name="uuid", field=models.UUIDField(default=uuid.uuid4, editable=False, serialize=False), ), migrations.RunPython( @@ -137,33 +200,45 @@ class Migration(migrations.Migration): reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RemoveField( - model_name='membership', - name='id', + model_name="membership", + name="id", ), migrations.RenameField( - model_name='membership', - old_name='uuid', - new_name='id', + model_name="membership", + old_name="uuid", + new_name="id", ), migrations.AlterField( - model_name='membership', - name='id', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True), + model_name="membership", + name="id", + field=models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), ), migrations.AlterField( - model_name='membership', - name='user', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + model_name="membership", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), ), migrations.AddField( - model_name='membership', - name='created', - field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + model_name="membership", + name="created", + field=models.DateTimeField( + auto_now_add=True, default=django.utils.timezone.now + ), preserve_default=False, ), migrations.AddField( - model_name='membership', - name='updated', + model_name="membership", + name="updated", field=models.DateTimeField(auto_now=True), ), migrations.RunPython( @@ -179,18 +254,32 @@ class Migration(migrations.Migration): reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.AlterField( - model_name='membership', - name='committee_account', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='committee_accounts.committeeaccount'), + model_name="membership", + name="committee_account", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="committee_accounts.committeeaccount", + ), ), migrations.AlterField( - model_name='membership', - name='role', - field=models.CharField(choices=[('COMMITTEE_ADMINISTRATOR', 'Committee Administrator'), ('MANAGER', 'Manager')], max_length=25), + model_name="membership", + name="role", + field=models.CharField( + choices=[ + ("COMMITTEE_ADMINISTRATOR", "Committee Administrator"), + ("MANAGER", "Manager"), + ], + max_length=25, + ), ), migrations.AlterField( - model_name='committeeaccount', - name='members', - field=models.ManyToManyField(through='committee_accounts.Membership', through_fields=('committee_account', 'user'), to=settings.AUTH_USER_MODEL), + model_name="committeeaccount", + name="members", + field=models.ManyToManyField( + through="committee_accounts.Membership", + through_fields=("committee_account", "user"), + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py b/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py index 4e0bfc404..69bcd9ebc 100644 --- a/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py +++ b/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py @@ -6,22 +6,27 @@ class Migration(migrations.Migration): - replaces = [('memo_text', '0002_initial'), ('memo_text', '0003_memotext_text_prefix')] + replaces = [("memo_text", "0002_initial"), ("memo_text", "0003_memotext_text_prefix")] dependencies = [ - ('memo_text', '0001_initial'), - ('reports', '0001_initial'), + ("memo_text", "0001_initial"), + ("reports", "0001_initial"), ] operations = [ migrations.AddField( - model_name='memotext', - name='report', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='reports.report'), + model_name="memotext", + name="report", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="reports.report", + ), ), migrations.AddField( - model_name='memotext', - name='text_prefix', + model_name="memotext", + name="text_prefix", field=models.TextField(blank=True, null=True), ), ] diff --git a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py index efbcfb324..9b6919b9e 100644 --- a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -4,6 +4,7 @@ import django.db.models.deletion import uuid from importlib import import_module +from textwrap import dedent from django.db import migrations, models @@ -12,165 +13,226 @@ # RunPython operations to refer to the local versions: # fecfiler.reports.migrations.00018_form24_name # fecfiler.reports.migrations.00019_form24_name_fix -# fecfiler.reports.migrations.0008_remove_form1m_city_remove_form1m_committee_name_and_more +# fecfiler.reports.migrations. +# 0008_remove_form1m_city_remove_form1m_committee_name_and_more # fecfiler.reports.migrations.0009_report_can_delete # fecfiler.reports.migrations.0010_report_can_unammend +def _strip_sql(sql): + return dedent(sql).strip() + + def _load_callable(module_name, function_name): def _wrapper(apps, schema_editor): try: fn = getattr(import_module(module_name), function_name) except Exception: - return django.db.migrations.operations.special.RunPython.noop(apps, schema_editor) + return django.db.migrations.operations.special.RunPython.noop( + apps, schema_editor + ) return fn(apps, schema_editor) return _wrapper + class Migration(migrations.Migration): - replaces = [('reports', '0007_remove_report_deleted'), ('reports', '0008_remove_form1m_city_remove_form1m_committee_name_and_more'), ('reports', '0009_report_can_delete'), ('reports', '0010_report_can_unammend'), ('reports', '0011_remove_form3x_cash_on_hand_date'), ('reports', '0012_alter_form99_text_code'), ('reports', '0013_form3_report_form_3'), ('reports', '0014_form99_swap_text_code'), ('reports', '0015_form3x_filing_frequency_form3x_report_type_category'), ('reports', '0016_determine_frequency_and_category'), ('reports', '0017_form99_filing_frequency_form99_pdf_attachment'), ('reports', '00018_form24_name'), ('reports', '00019_form24_name_fix')] + replaces = [ + ("reports", "0007_remove_report_deleted"), + ("reports", "0008_remove_form1m_city_remove_form1m_committee_name_and_more"), + ("reports", "0009_report_can_delete"), + ("reports", "0010_report_can_unammend"), + ("reports", "0011_remove_form3x_cash_on_hand_date"), + ("reports", "0012_alter_form99_text_code"), + ("reports", "0013_form3_report_form_3"), + ("reports", "0014_form99_swap_text_code"), + ("reports", "0015_form3x_filing_frequency_form3x_report_type_category"), + ("reports", "0016_determine_frequency_and_category"), + ("reports", "0017_form99_filing_frequency_form99_pdf_attachment"), + ("reports", "00018_form24_name"), + ("reports", "00019_form24_name_fix"), + ] dependencies = [ - ('reports', '0006_reporttransaction'), + ("reports", "0006_reporttransaction"), ] operations = [ migrations.RemoveField( - model_name='report', - name='deleted', + model_name="report", + name="deleted", ), migrations.AddField( - model_name='report', - name='city', + model_name="report", + name="city", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='report', - name='committee_name', + model_name="report", + name="committee_name", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='report', - name='state', + model_name="report", + name="state", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='report', - name='street_1', + model_name="report", + name="street_1", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='report', - name='street_2', + model_name="report", + name="street_2", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='report', - name='zip', + model_name="report", + name="zip", field=models.TextField(blank=True, null=True), ), migrations.RunPython( code=_load_callable( - "fecfiler.reports.migrations.0008_remove_form1m_city_remove_form1m_committee_name_and_more", + "fecfiler.reports.migrations." + "0008_remove_form1m_city_remove_form1m_committee_name_and_more", "migrate_committee_data", ), ), migrations.RemoveField( - model_name='form1m', - name='city', + model_name="form1m", + name="city", ), migrations.RemoveField( - model_name='form1m', - name='committee_name', + model_name="form1m", + name="committee_name", ), migrations.RemoveField( - model_name='form1m', - name='state', + model_name="form1m", + name="state", ), migrations.RemoveField( - model_name='form1m', - name='street_1', + model_name="form1m", + name="street_1", ), migrations.RemoveField( - model_name='form1m', - name='street_2', + model_name="form1m", + name="street_2", ), migrations.RemoveField( - model_name='form1m', - name='zip', + model_name="form1m", + name="zip", ), migrations.RemoveField( - model_name='form24', - name='city', + model_name="form24", + name="city", ), migrations.RemoveField( - model_name='form24', - name='state', + model_name="form24", + name="state", ), migrations.RemoveField( - model_name='form24', - name='street_1', + model_name="form24", + name="street_1", ), migrations.RemoveField( - model_name='form24', - name='street_2', + model_name="form24", + name="street_2", ), migrations.RemoveField( - model_name='form24', - name='zip', + model_name="form24", + name="zip", ), migrations.RemoveField( - model_name='form3x', - name='city', + model_name="form3x", + name="city", ), migrations.RemoveField( - model_name='form3x', - name='state', + model_name="form3x", + name="state", ), migrations.RemoveField( - model_name='form3x', - name='street_1', + model_name="form3x", + name="street_1", ), migrations.RemoveField( - model_name='form3x', - name='street_2', + model_name="form3x", + name="street_2", ), migrations.RemoveField( - model_name='form3x', - name='zip', + model_name="form3x", + name="zip", ), migrations.RemoveField( - model_name='form99', - name='city', + model_name="form99", + name="city", ), migrations.RemoveField( - model_name='form99', - name='committee_name', + model_name="form99", + name="committee_name", ), migrations.RemoveField( - model_name='form99', - name='state', + model_name="form99", + name="state", ), migrations.RemoveField( - model_name='form99', - name='street_1', + model_name="form99", + name="street_1", ), migrations.RemoveField( - model_name='form99', - name='street_2', + model_name="form99", + name="street_2", ), migrations.RemoveField( - model_name='form99', - name='zip', + model_name="form99", + name="zip", ), migrations.AddField( - model_name='report', - name='can_delete', + model_name="report", + name="can_delete", field=models.BooleanField(default=True), ), migrations.RunSQL( - sql='\n CREATE OR REPLACE FUNCTION update_report_can_delete(report RECORD)\n RETURNS VOID AS $$\n DECLARE\n r_can_delete boolean;\n BEGIN\n r_can_delete = report.upload_submission_id IS NULL\n AND (report.report_version IS NULL OR report.report_version = \'0\')\n AND (\n report.form_3x_id IS NULL OR\n (\n report.form_24_id IS NULL\n AND NOT EXISTS(\n SELECT DISTINCT rrt1.id\n FROM "reports_reporttransaction" rrt1\n JOIN "transactions_transaction" tt ON (\n rrt1."transaction_id" = tt."id"\n OR tt."reatt_redes_id" = rrt1."transaction_id"\n OR tt."parent_transaction_id" = rrt1."transaction_id"\n OR tt."debt_id" = rrt1."transaction_id"\n OR tt."loan_id" = rrt1."transaction_id"\n )\n INNER JOIN "reports_reporttransaction" rrt2 ON (\n rrt2."transaction_id" = tt."id"\n AND rrt2."report_id" <> report.id\n )\n WHERE rrt1."report_id" = report.id\n )\n )\n );\n UPDATE reports_report SET can_delete = r_can_delete\n WHERE id = report.id\n AND can_delete <> r_can_delete;\n END;\n $$ LANGUAGE plpgsql;\n ', + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION update_report_can_delete(report RECORD) + RETURNS VOID AS $$ + DECLARE + r_can_delete boolean; + BEGIN + r_can_delete = report.upload_submission_id IS NULL + AND (report.report_version IS NULL OR report.report_version = '0') + AND ( + report.form_3x_id IS NULL OR + ( + report.form_24_id IS NULL + AND NOT EXISTS( + SELECT DISTINCT rrt1.id + FROM "reports_reporttransaction" rrt1 + JOIN "transactions_transaction" tt ON ( + rrt1."transaction_id" = tt."id" + OR tt."reatt_redes_id" = rrt1."transaction_id" + OR tt."parent_transaction_id" = + rrt1."transaction_id" + OR tt."debt_id" = rrt1."transaction_id" + OR tt."loan_id" = rrt1."transaction_id" + ) + INNER JOIN "reports_reporttransaction" rrt2 ON ( + rrt2."transaction_id" = tt."id" + AND rrt2."report_id" <> report.id + ) + WHERE rrt1."report_id" = report.id + ) + ) + ); + UPDATE reports_report SET can_delete = r_can_delete + WHERE id = report.id + AND can_delete <> r_can_delete; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunPython( code=_load_callable( @@ -189,8 +251,8 @@ class Migration(migrations.Migration): ), ), migrations.AddField( - model_name='report', - name='can_unamend', + model_name="report", + name="can_unamend", field=models.BooleanField(default=False), ), migrations.RunPython( @@ -200,146 +262,558 @@ class Migration(migrations.Migration): ), ), migrations.RemoveField( - model_name='form3x', - name='cash_on_hand_date', + model_name="form3x", + name="cash_on_hand_date", ), migrations.AddField( - model_name='form99', - name='text_code_2', - field=models.TextField(db_default=''), + model_name="form99", + name="text_code_2", + field=models.TextField(db_default=""), ), migrations.RunSQL( - sql="\n UPDATE reports_form99 SET text_code_2 = COALESCE(text_code, '');\n ALTER TABLE reports_form99 ALTER COLUMN text_code SET DEFAULT '';\n ALTER TABLE reports_form99 ALTER COLUMN text_code_2 SET NOT NULL;\n ", - reverse_sql='\n ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP DEFAULT;\n ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP NOT NULL;\n ', - state_operations=[migrations.AlterField( - model_name='form99', - name='text_code_2', - field=models.TextField(db_default='', default=''), - )], + sql=_strip_sql( + """ + UPDATE reports_form99 SET text_code_2 = COALESCE(text_code, ''); + ALTER TABLE reports_form99 ALTER COLUMN text_code SET DEFAULT ''; + ALTER TABLE reports_form99 ALTER COLUMN text_code_2 SET NOT NULL; + """ + ), + reverse_sql=_strip_sql( + """ + ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP DEFAULT; + ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP NOT NULL; + """ + ), + state_operations=[ + migrations.AlterField( + model_name="form99", + name="text_code_2", + field=models.TextField(db_default="", default=""), + ) + ], ), migrations.CreateModel( - name='Form3', + name="Form3", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('change_of_address', models.BooleanField(blank=True, default=False, null=True)), - ('election_state', models.TextField(blank=True, null=True)), - ('election_district', models.TextField(blank=True, null=True)), - ('election_code', models.TextField(blank=True, null=True)), - ('date_of_election', models.DateField(blank=True, null=True)), - ('state_of_election', models.TextField(blank=True, null=True)), - ('L6a_total_contributions_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L6b_total_contribution_refunds_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L6c_net_contributions_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L7a_total_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L7b_total_offsets_to_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L7c_net_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L8_cash_on_hand_at_close_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L9_debts_owed_to_committee_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L10_debts_owed_by_committee_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11ai_individuals_itemized_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11aii_individuals_unitemized_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11aiii_total_individual_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11b_political_party_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11c_other_political_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11d_the_candidate_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11e_total_contributions_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L12_transfers_from_other_authorized_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L13a_loans_made_or_guaranteed_by_the_candidate_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L13b_all_other_loans_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L13c_total_loans_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L14_offsets_to_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L15_other_receipts_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L16_total_receipts_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L17_operating_expenditures_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L18_transfers_to_other_authorized_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_period', models.DecimalField(blank=True, db_column='l19a_period', decimal_places=2, max_digits=11, null=True)), - ('L19b_loan_repayments_of_all_other_loans_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L19c_total_loan_repayments_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20a_refunds_to_individuals_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20b_refunds_to_political_party_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20c_refunds_to_other_political_committees_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20d_total_contribution_refunds_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L21_other_disbursements_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L22_total_disbursements_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L23_cash_on_hand_beginning_reporting_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L24_total_receipts_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L25_subtotals_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L26_total_disbursements_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L27_cash_on_hand_at_close_period', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L6a_total_contributions_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L6b_total_contribution_refunds_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L6c_net_contributions_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L7a_total_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L7b_total_offsets_to_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L7c_net_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11ai_individuals_itemized_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11aii_individuals_unitemized_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11aiii_total_individual_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11b_political_party_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11c_other_political_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11d_the_candidate_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L11e_total_contributions_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L12_transfers_from_other_authorized_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L13a_loans_made_or_guaranteed_by_the_candidate_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L13b_all_other_loans_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L13c_total_loans_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L14_offsets_to_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L15_other_receipts_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L16_total_receipts_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L17_operating_expenditures_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L18_transfers_to_other_authorized_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_ytd', models.DecimalField(blank=True, db_column='l19a_ytd', decimal_places=2, max_digits=11, null=True)), - ('L19b_loan_repayments_of_all_other_loans_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L19c_total_loan_repayments_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20a_refunds_to_individuals_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20b_refunds_to_political_party_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20c_refunds_to_other_political_committees_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L20d_total_contribution_refunds_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L21_other_disbursements_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('L22_total_disbursements_ytd', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ( + "change_of_address", + models.BooleanField(blank=True, default=False, null=True), + ), + ("election_state", models.TextField(blank=True, null=True)), + ("election_district", models.TextField(blank=True, null=True)), + ("election_code", models.TextField(blank=True, null=True)), + ("date_of_election", models.DateField(blank=True, null=True)), + ("state_of_election", models.TextField(blank=True, null=True)), + ( + "L6a_total_contributions_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L6b_total_contribution_refunds_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L6c_net_contributions_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L7a_total_operating_expenditures_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L7b_total_offsets_to_operating_expenditures_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L7c_net_operating_expenditures_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L8_cash_on_hand_at_close_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L9_debts_owed_to_committee_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L10_debts_owed_by_committee_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11ai_individuals_itemized_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11aii_individuals_unitemized_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11aiii_total_individual_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11b_political_party_committees_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11c_other_political_committees_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11d_the_candidate_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11e_total_contributions_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L12_transfers_from_other_authorized_committees_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L13a_loans_made_or_guaranteed_by_the_candidate_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L13b_all_other_loans_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L13c_total_loans_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L14_offsets_to_operating_expenditures_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L15_other_receipts_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L16_total_receipts_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L17_operating_expenditures_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L18_transfers_to_other_authorized_committees_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L19a_loan_repayments_of_loans_made_or_" + "guaranteed_by_candidate_period", + models.DecimalField( + blank=True, + db_column="l19a_period", + decimal_places=2, + max_digits=11, + null=True, + ), + ), + ( + "L19b_loan_repayments_of_all_other_loans_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L19c_total_loan_repayments_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20a_refunds_to_individuals_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20b_refunds_to_political_party_committees_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20c_refunds_to_other_political_committees_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20d_total_contribution_refunds_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L21_other_disbursements_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L22_total_disbursements_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L23_cash_on_hand_beginning_reporting_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L24_total_receipts_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L25_subtotals_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L26_total_disbursements_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L27_cash_on_hand_at_close_period", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L6a_total_contributions_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L6b_total_contribution_refunds_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L6c_net_contributions_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L7a_total_operating_expenditures_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L7b_total_offsets_to_operating_expenditures_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L7c_net_operating_expenditures_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11ai_individuals_itemized_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11aii_individuals_unitemized_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11aiii_total_individual_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11b_political_party_committees_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11c_other_political_committees_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11d_the_candidate_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L11e_total_contributions_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L12_transfers_from_other_authorized_committees_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L13a_loans_made_or_guaranteed_by_the_candidate_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L13b_all_other_loans_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L13c_total_loans_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L14_offsets_to_operating_expenditures_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L15_other_receipts_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L16_total_receipts_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L17_operating_expenditures_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L18_transfers_to_other_authorized_committees_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L19a_loan_repayments_of_loans_made_or_guaranteed_by_candidate_ytd", + models.DecimalField( + blank=True, + db_column="l19a_ytd", + decimal_places=2, + max_digits=11, + null=True, + ), + ), + ( + "L19b_loan_repayments_of_all_other_loans_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L19c_total_loan_repayments_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20a_refunds_to_individuals_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20b_refunds_to_political_party_committees_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20c_refunds_to_other_political_committees_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L20d_total_contribution_refunds_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L21_other_disbursements_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "L22_total_disbursements_ytd", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), ], ), migrations.AddField( - model_name='report', - name='form_3', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='reports.form3'), + model_name="report", + name="form_3", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="reports.form3", + ), ), migrations.RemoveField( - model_name='form99', - name='text_code', + model_name="form99", + name="text_code", ), migrations.RenameField( - model_name='form99', - old_name='text_code_2', - new_name='text_code', + model_name="form99", + old_name="text_code_2", + new_name="text_code", ), migrations.AddField( - model_name='form3x', - name='filing_frequency', + model_name="form3x", + name="filing_frequency", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='form3x', - name='report_type_category', + model_name="form3x", + name="report_type_category", field=models.TextField(blank=True, null=True), ), migrations.RunSQL( - sql="\n UPDATE reports_form3x f\n SET filing_frequency = CASE\n WHEN r.report_code IN (\n 'M2', 'M3', 'M4', 'M5', 'M6', 'M7',\n 'M8', 'M9', 'M10', 'M11', 'M12'\n ) THEN 'M'\n ELSE 'Q'\n END,\n report_type_category = CASE\n WHEN r.report_code IN (\n 'M11', 'M12', 'MY'\n ) THEN 'Non-Election Year'\n ELSE 'Election Year'\n END\n FROM reports_report as r\n WHERE r.form_3x_id = f.id\n AND filing_frequency IS NULL AND report_type_category IS NULL;\n ", - reverse_sql='', + sql=_strip_sql( + """ + UPDATE reports_form3x f + SET filing_frequency = CASE + WHEN r.report_code IN ( + 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', + 'M8', 'M9', 'M10', 'M11', 'M12' + ) THEN 'M' + ELSE 'Q' + END, + report_type_category = CASE + WHEN r.report_code IN ( + 'M11', 'M12', 'MY' + ) THEN 'Non-Election Year' + ELSE 'Election Year' + END + FROM reports_report as r + WHERE r.form_3x_id = f.id + AND filing_frequency IS NULL AND report_type_category IS NULL; + """ + ), + reverse_sql=_strip_sql( + """ + """ + ), ), migrations.AddField( - model_name='form99', - name='pdf_attachment', + model_name="form99", + name="pdf_attachment", field=models.BooleanField(blank=True, null=True), ), migrations.AddField( - model_name='form99', - name='filing_frequency', + model_name="form99", + name="filing_frequency", field=models.TextField(blank=True, max_length=1, null=True), ), migrations.AddField( - model_name='form24', - name='name', + model_name="form24", + name="name", field=models.TextField(null=True), ), migrations.RunPython( @@ -349,8 +823,8 @@ class Migration(migrations.Migration): ), ), migrations.AlterField( - model_name='form24', - name='name', + model_name="form24", + name="name", field=models.TextField(), ), migrations.RunPython( diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py index d20eec2df..e7a518bc3 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -6,6 +6,7 @@ import uuid from django.db import migrations, models from importlib import import_module +from textwrap import dedent # Functions from the following migrations need manual copying. @@ -13,8 +14,10 @@ # RunPython operations to refer to the local versions: # fecfiler.transactions.migrations.0004_report_transactions_link_table # fecfiler.transactions.migrations.0005_schedulec_report_coverage_from_date_and_more -# fecfiler.transactions.migrations.0006_independent_expenditure_memos_no_aggregation_group -# fecfiler.transactions.migrations.0008_transaction__calendar_ytd_per_election_office_and_more +# fecfiler.transactions.migrations. +# 0006_independent_expenditure_memos_no_aggregation_group +# fecfiler.transactions.migrations. +# 0008_transaction__calendar_ytd_per_election_office_and_more # fecfiler.transactions.migrations.0009_update_calculate_loan_payment_to_date # fecfiler.transactions.migrations.0011_transaction_can_delete # fecfiler.transactions.migrations.0012_alter_transactions_blocking_reports @@ -25,12 +28,18 @@ # fecfiler.transactions.migrations.0024_scheduled_balance_at_close_and_more +def _strip_sql(sql): + return dedent(sql).strip() + + def _load_callable(module_name, function_name): def _wrapper(apps, schema_editor): try: fn = getattr(import_module(module_name), function_name) except Exception: - return django.db.migrations.operations.special.RunPython.noop(apps, schema_editor) + return django.db.migrations.operations.special.RunPython.noop( + apps, schema_editor + ) return fn(apps, schema_editor) return _wrapper @@ -38,359 +47,2742 @@ def _wrapper(apps, schema_editor): class Migration(migrations.Migration): - replaces = [('transactions', '0003_alter_transaction_parent_transaction'), ('transactions', '0004_report_transactions_link_table'), ('transactions', '0005_schedulec_report_coverage_from_date_and_more'), ('transactions', '0006_independent_expenditure_memos_no_aggregation_group'), ('transactions', '0007_schedulee_so_candidate_state'), ('transactions', '0008_transaction__calendar_ytd_per_election_office_and_more'), ('transactions', '0009_update_calculate_loan_payment_to_date'), ('transactions', '0010_update_aggregate_trigger_performance'), ('transactions', '0011_transaction_can_delete'), ('transactions', '0012_alter_transactions_blocking_reports'), ('transactions', '0013_transaction_itemized_and_associated_triggers'), ('transactions', '0014_drop_transaction_view'), ('transactions', '0015_merge_transaction_triggers'), ('transactions', '0016_schedulef_transaction_contact_4_and_more'), ('transactions', '0017_schedulef_coordianted_to_coordinated'), ('transactions', '0018_schedulef_general_election_year_and_more'), ('transactions', '0019_aggregate_committee_controls'), ('transactions', '0020_trigger_save_on_transactions'), ('transactions', '0021_alter_transaction_reports'), ('transactions', '0022_schedule_f_aggregation'), ('transactions', '0023_optimize_calculate_loan_payment_to_date'), ('transactions', '0024_scheduled_balance_at_close_and_more'), ('transactions', '0025_drop_aggregate_triggers'), ('transactions', '0026_alter_transaction_itemized')] + replaces = [ + ("transactions", "0003_alter_transaction_parent_transaction"), + ("transactions", "0004_report_transactions_link_table"), + ("transactions", "0005_schedulec_report_coverage_from_date_and_more"), + ("transactions", "0006_independent_expenditure_memos_no_aggregation_group"), + ("transactions", "0007_schedulee_so_candidate_state"), + ("transactions", "0008_transaction__calendar_ytd_per_election_office_and_more"), + ("transactions", "0009_update_calculate_loan_payment_to_date"), + ("transactions", "0010_update_aggregate_trigger_performance"), + ("transactions", "0011_transaction_can_delete"), + ("transactions", "0012_alter_transactions_blocking_reports"), + ("transactions", "0013_transaction_itemized_and_associated_triggers"), + ("transactions", "0014_drop_transaction_view"), + ("transactions", "0015_merge_transaction_triggers"), + ("transactions", "0016_schedulef_transaction_contact_4_and_more"), + ("transactions", "0017_schedulef_coordianted_to_coordinated"), + ("transactions", "0018_schedulef_general_election_year_and_more"), + ("transactions", "0019_aggregate_committee_controls"), + ("transactions", "0020_trigger_save_on_transactions"), + ("transactions", "0021_alter_transaction_reports"), + ("transactions", "0022_schedule_f_aggregation"), + ("transactions", "0023_optimize_calculate_loan_payment_to_date"), + ("transactions", "0024_scheduled_balance_at_close_and_more"), + ("transactions", "0025_drop_aggregate_triggers"), + ("transactions", "0026_alter_transaction_itemized"), + ] dependencies = [ - ('contacts', '0001_initial'), - ('reports', '0006_reporttransaction'), - ('reports', '0014_form99_swap_text_code'), - ('transactions', '0002_remove_schedulea_contributor_city_and_more'), + ("contacts", "0001_initial"), + ("reports", "0006_reporttransaction"), + ("reports", "0014_form99_swap_text_code"), + ("transactions", "0002_remove_schedulea_contributor_city_and_more"), ] operations = [ migrations.AlterField( - model_name='transaction', - name='parent_transaction', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='transactions.transaction'), + model_name="transaction", + name="parent_transaction", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="transactions.transaction", + ), ), migrations.AddField( - model_name='transaction', - name='reports', - field=models.ManyToManyField(through='reports.ReportTransaction', to='reports.report'), + model_name="transaction", + name="reports", + field=models.ManyToManyField( + through="reports.ReportTransaction", to="reports.report" + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0004_report_transactions_link_table", "add_link_table"), + code=_load_callable( + "fecfiler.transactions.migrations.0004_report_transactions_link_table", + "add_link_table", + ), reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RemoveField( - model_name='transaction', - name='report', + model_name="transaction", + name="report", ), migrations.AddField( - model_name='schedulec', - name='report_coverage_through_date', + model_name="schedulec", + name="report_coverage_through_date", field=models.DateField(blank=True, null=True), ), migrations.AddField( - model_name='scheduled', - name='report_coverage_from_date', + model_name="scheduled", + name="report_coverage_from_date", field=models.DateField(blank=True, null=True), ), migrations.AlterField( - model_name='transaction', - name='parent_transaction', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='transactions.transaction'), + model_name="transaction", + name="parent_transaction", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="transactions.transaction", + ), ), migrations.AlterField( - model_name='transaction', - name='reports', - field=models.ManyToManyField(related_name='transactions', through='reports.ReportTransaction', to='reports.report'), + model_name="transaction", + name="reports", + field=models.ManyToManyField( + related_name="transactions", + through="reports.ReportTransaction", + to="reports.report", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0005_schedulec_report_coverage_from_date_and_more", "set_coverage_date"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0005_schedulec_report_coverage_from_date_and_more", + "set_coverage_date", + ), reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0006_independent_expenditure_memos_no_aggregation_group", "set_aggregation_group_to_none_for_ie_memos"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0006_independent_expenditure_memos_no_aggregation_group", "reverse_removing_aggregation_group_for_ie_memos"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0006_independent_expenditure_memos_no_aggregation_group", + "set_aggregation_group_to_none_for_ie_memos", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations." + "0006_independent_expenditure_memos_no_aggregation_group", + "reverse_removing_aggregation_group_for_ie_memos", + ), ), migrations.AddField( - model_name='schedulee', - name='so_candidate_state', + model_name="schedulee", + name="so_candidate_state", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='transaction', - name='_calendar_ytd_per_election_office', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="transaction", + name="_calendar_ytd_per_election_office", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.AddField( - model_name='transaction', - name='aggregate', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="transaction", + name="aggregate", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.AddField( - model_name='transaction', - name='loan_payment_to_date', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="transaction", + name="loan_payment_to_date", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.RunSQL( - sql='\n CREATE EXTENSION IF NOT EXISTS "uuid-ossp";\n ', + sql=_strip_sql( + """ + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n BEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT contribution_date\n INTO schedule_date\n FROM transactions_schedulea\n WHERE id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT expenditure_date\n INTO schedule_date\n FROM transactions_scheduleb\n WHERE id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET aggregate = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT contribution_date + INTO schedule_date + FROM transactions_schedulea + WHERE id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT expenditure_date + INTO schedule_date + FROM transactions_scheduleb + WHERE id = txn.schedule_b_id; + END IF; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET aggregate = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n v_election_code TEXT;\n v_candidate_office TEXT;\n v_candidate_state TEXT;\n v_candidate_district TEXT;\n BEGIN\n SELECT COALESCE(disbursement_date, dissemination_date)\n INTO schedule_date FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT election_code\n INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT candidate_office, candidate_state, candidate_district\n INTO v_candidate_office, v_candidate_state, v_candidate_district\n FROM contacts WHERE id = txn.contact_2_id;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n t.id,\n SUM(t.effective_amount) OVER\n (ORDER BY t.date, t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transaction_view__' || sql_committee_id || ' t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM t.date) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + v_election_code TEXT; + v_candidate_office TEXT; + v_candidate_state TEXT; + v_candidate_district TEXT; + BEGIN + SELECT COALESCE(disbursement_date, dissemination_date) + INTO schedule_date FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT election_code + INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT candidate_office, candidate_state, candidate_district + INTO v_candidate_office, v_candidate_state, v_candidate_district + FROM contacts WHERE id = txn.contact_2_id; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + t.id, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum + FROM transactions_schedulee e + JOIN transaction_view__' || sql_committee_id || ' t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM t.date) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n BEGIN\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n loan_key,\n SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE loan_key LIKE (\n SELECT transaction_id FROM transactions_transaction\n WHERE id = $1\n ) || ''%%''; -- Match the loan_key with a transaction_id prefix\n\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id\n AND tt.loan_key LIKE ''%%LOAN'';\n '\n USING txn.id;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + BEGIN + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT transaction_id FROM transactions_transaction + WHERE id = $1 + ) || ''%%''; -- Match the loan_key with a transaction_id prefix + + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id + AND tt.loan_key LIKE ''%%LOAN''; + ' + USING txn.id; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_aggregates()\n RETURNS TRIGGER AS $$\n DECLARE\n sql_committee_id TEXT;\n temp_table_name TEXT;\n BEGIN\n sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_');\n temp_table_name := 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_');\n RAISE NOTICE 'TESTING TRIGGER';\n\n -- If schedule_c2_id or schedule_d_id is not null, stop processing\n IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL\n THEN\n RETURN NEW;\n END IF;\n\n IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL\n THEN\n PERFORM calculate_entity_aggregates(\n NEW, sql_committee_id, temp_table_name || 'NEW');\n IF TG_OP = 'UPDATE'\n AND NEW.contact_1_id <> OLD.contact_1_id\n THEN\n PERFORM calculate_entity_aggregates(\n OLD, sql_committee_id, temp_table_name || 'OLD');\n END IF;\n\n ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL\n THEN\n PERFORM calculate_loan_payment_to_date(\n NEW, sql_committee_id, temp_table_name || 'NEW');\n\n ELSIF NEW.schedule_e_id IS NOT NULL\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n NEW, sql_committee_id, temp_table_name || 'NEW');\n IF TG_OP = 'UPDATE'\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n OLD, sql_committee_id, temp_table_name || 'OLD');\n END IF;\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE TRIGGER calculate_aggregates_trigger\n AFTER INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop\n EXECUTE FUNCTION calculate_aggregates();\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_aggregates() + RETURNS TRIGGER AS $$ + DECLARE + sql_committee_id TEXT; + temp_table_name TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + temp_table_name := 'temp_' || + REPLACE(uuid_generate_v4()::TEXT, '-', '_'); + RAISE NOTICE 'TESTING TRIGGER'; + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates( + NEW, sql_committee_id, temp_table_name || 'NEW'); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates( + OLD, sql_committee_id, temp_table_name || 'OLD'); + END IF; + + ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL + THEN + PERFORM calculate_loan_payment_to_date( + NEW, sql_committee_id, temp_table_name || 'NEW'); + + ELSIF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id, temp_table_name || 'NEW'); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id, temp_table_name || 'OLD'); + END IF; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER calculate_aggregates_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION calculate_aggregates(); + """ + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0008_transaction__calendar_ytd_per_election_office_and_more", "populate_existing_rows"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0008_transaction__calendar_ytd_per_election_office_and_more", + "populate_existing_rows", + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION get_temp_tablename()\n RETURNS TEXT AS $$\n BEGIN\n RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_');\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION get_temp_tablename() + RETURNS TEXT AS $$ + BEGIN + RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD,\n sql_committee_id TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n temp_table_name TEXT;\n BEGIN\n temp_table_name := get_temp_tablename();\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT contribution_date\n INTO schedule_date\n FROM transactions_schedulea\n WHERE id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT expenditure_date\n INTO schedule_date\n FROM transactions_scheduleb\n WHERE id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET aggregate = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, + sql_committee_id TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + temp_table_name TEXT; + BEGIN + temp_table_name := get_temp_tablename(); + IF txn.schedule_a_id IS NOT NULL THEN + SELECT contribution_date + INTO schedule_date + FROM transactions_schedulea + WHERE id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT expenditure_date + INTO schedule_date + FROM transactions_scheduleb + WHERE id = txn.schedule_b_id; + END IF; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET aggregate = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD,\n sql_committee_id TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n v_election_code TEXT;\n v_candidate_office TEXT;\n v_candidate_state TEXT;\n v_candidate_district TEXT;\n temp_table_name TEXT;\n BEGIN\n temp_table_name := get_temp_tablename();\n SELECT COALESCE(disbursement_date, dissemination_date)\n INTO schedule_date FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT election_code\n INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n SELECT candidate_office, candidate_state, candidate_district\n INTO v_candidate_office, v_candidate_state, v_candidate_district\n FROM contacts WHERE id = txn.contact_2_id;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n t.id,\n SUM(t.effective_amount) OVER\n (ORDER BY t.date, t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transaction_view__' || sql_committee_id || ' t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM t.date) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, + sql_committee_id TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + v_election_code TEXT; + v_candidate_office TEXT; + v_candidate_state TEXT; + v_candidate_district TEXT; + temp_table_name TEXT; + BEGIN + temp_table_name := get_temp_tablename(); + SELECT COALESCE(disbursement_date, dissemination_date) + INTO schedule_date FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT election_code + INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT candidate_office, candidate_state, candidate_district + INTO v_candidate_office, v_candidate_state, v_candidate_district + FROM contacts WHERE id = txn.contact_2_id; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + t.id, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum + FROM transactions_schedulee e + JOIN transaction_view__' || sql_committee_id || ' t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM t.date) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD,\n sql_committee_id TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n temp_table_name TEXT;\n BEGIN\n temp_table_name := get_temp_tablename();\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n loan_key,\n SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE loan_key LIKE (\n SELECT\n CASE\n WHEN loan_id IS NULL THEN transaction_id\n ELSE (\n SELECT transaction_id\n FROM transactions_transaction\n WHERE id = t.loan_id\n )\n END\n FROM transactions_transaction t\n WHERE id = $1\n ) || ''%%''; -- Match the loan_key with a transaction_id prefix\n\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id\n AND tt.loan_key LIKE ''%%LOAN'';\n '\n USING txn.id;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, + sql_committee_id TEXT + ) + RETURNS VOID AS $$ + DECLARE + temp_table_name TEXT; + BEGIN + temp_table_name := get_temp_tablename(); + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT + CASE + WHEN loan_id IS NULL THEN transaction_id + ELSE ( + SELECT transaction_id + FROM transactions_transaction + WHERE id = t.loan_id + ) + END + FROM transactions_transaction t + WHERE id = $1 + ) || ''%%''; -- Match the loan_key with a transaction_id prefix + + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id + AND tt.loan_key LIKE ''%%LOAN''; + ' + USING txn.id; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_aggregates()\n RETURNS TRIGGER AS $$\n DECLARE\n sql_committee_id TEXT;\n BEGIN\n sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_');\n\n -- If schedule_c2_id or schedule_d_id is not null, stop processing\n IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL\n THEN\n RETURN NEW;\n END IF;\n\n IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL\n THEN\n PERFORM calculate_entity_aggregates(NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n AND NEW.contact_1_id <> OLD.contact_1_id\n THEN\n PERFORM calculate_entity_aggregates(OLD, sql_committee_id);\n END IF;\n END IF;\n\n IF NEW.schedule_c_id IS NOT NULL\n OR NEW.schedule_c1_id IS NOT NULL\n OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE'\n THEN\n PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id);\n END IF;\n\n IF NEW.schedule_e_id IS NOT NULL\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n OLD, sql_committee_id);\n END IF;\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_aggregates() + RETURNS TRIGGER AS $$ + DECLARE + sql_committee_id TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates(NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates(OLD, sql_committee_id); + END IF; + END IF; + + IF NEW.schedule_c_id IS NOT NULL + OR NEW.schedule_c1_id IS NOT NULL + OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' + THEN + PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); + END IF; + + IF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id); + END IF; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop prior versions of these functions\n DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT);\n DROP FUNCTION calculate_calendar_ytd_per_election_office(RECORD, TEXT, TEXT);\n DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT);\n ', + sql=_strip_sql( + """ + -- Drop prior versions of these functions + DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT); + DROP FUNCTION calculate_calendar_ytd_per_election_office( + RECORD, TEXT, TEXT + ); + DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT); + """ + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0009_update_calculate_loan_payment_to_date", "update_existing_rows"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0009_update_calculate_loan_payment_to_date", + "update_existing_rows", + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date date;\n BEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT\n contribution_date INTO schedule_date\n FROM\n transactions_schedulea\n WHERE\n id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT\n expenditure_date INTO schedule_date\n FROM\n transactions_scheduleb\n WHERE\n id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET aggregate = tc.new_sum\n FROM (\n SELECT\n id,\n aggregate,\n date,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $4\n OR (\n tc.date = $4\n AND t.created >= $5\n )\n );\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\n END;\n $$\n LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, sql_committee_id text + ) + RETURNS VOID AS $$ + DECLARE + schedule_date date; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT + contribution_date INTO schedule_date + FROM + transactions_schedulea + WHERE + id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT + expenditure_date INTO schedule_date + FROM + transactions_scheduleb + WHERE + id = txn.schedule_b_id; + END IF; + + EXECUTE ' + UPDATE transactions_transaction AS t + SET aggregate = tc.new_sum + FROM ( + SELECT + id, + aggregate, + date, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $4 + OR ( + tc.date = $4 + AND t.created >= $5 + ) + ); + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID\n AS $$\n DECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\n BEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n t.date,\n SUM(t.effective_amount) OVER\n (ORDER BY t.date, t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transaction_view__' || sql_committee_id || ' t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM t.date) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\n END;\n $$\n LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id text + ) + RETURNS VOID + AS $$ + DECLARE + schedule_date date; + v_election_code text; + v_candidate_office text; + v_candidate_state text; + v_candidate_district text; + BEGIN + SELECT + COALESCE(disbursement_date, dissemination_date) + INTO schedule_date + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT election_code INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT + candidate_office, + candidate_state, + candidate_district INTO v_candidate_office, + v_candidate_state, + v_candidate_district + FROM contacts + WHERE id = txn.contact_2_id; + EXECUTE ' + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tc.new_sum + FROM ( + SELECT + t.id, + t.date, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum + FROM transactions_schedulee e + JOIN transaction_view__' || sql_committee_id || ' t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM t.date) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $7 + OR ( + tc.date = $7 + AND t.created >= $8 + ) + ); + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\n CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID\n AS $$\n BEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n id,\n loan_key,\n SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE loan_key LIKE (\n SELECT\n CASE\n WHEN loan_id IS NULL THEN transaction_id\n ELSE (\n SELECT transaction_id\n FROM transactions_transaction\n WHERE id = t.loan_id\n )\n END\n FROM transactions_transaction t\n WHERE id = $1\n ) || ''%%'' -- Match the loan_key with a transaction_id prefix\n ) AS tc\n WHERE t.id = tc.id\n AND tc.loan_key LIKE ''%%LOAN''\n ;\n '\n USING txn.id;\n END;\n $$\n LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, sql_committee_id text + ) + RETURNS VOID + AS $$ + BEGIN + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT + CASE + WHEN loan_id IS NULL THEN transaction_id + ELSE ( + SELECT transaction_id + FROM transactions_transaction + WHERE id = t.loan_id + ) + END + FROM transactions_transaction t + WHERE id = $1 + ) || ''%%'' -- Match the loan_key with a transaction_id prefix + ) AS tc + WHERE t.id = tc.id + AND tc.loan_key LIKE ''%%LOAN'' + ; + ' + USING txn.id; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.AddField( - model_name='transaction', - name='blocking_reports', - field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=[], size=None), + model_name="transaction", + name="blocking_reports", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), default=[], size=None + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "create_trigger_function"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "drop_trigger_function"), + code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "create_trigger_function", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "drop_trigger_function", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "create_trigger"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0011_transaction_can_delete", "drop_trigger"), + code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "create_trigger", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "drop_trigger", + ), ), migrations.AlterField( - model_name='transaction', - name='blocking_reports', - field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None), + model_name="transaction", + name="blocking_reports", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), default=list, size=None + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0012_alter_transactions_blocking_reports", "update_blocking_reports_default"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0012_alter_transactions_blocking_reports", + "update_blocking_reports_default", + ), ), migrations.AddField( - model_name='transaction', - name='itemized', + model_name="transaction", + name="itemized", field=models.BooleanField(db_default=True), ), migrations.CreateModel( - name='OverTwoHundredTypesScheduleA', + name="OverTwoHundredTypesScheduleA", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('type', models.TextField()), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ("type", models.TextField()), ], options={ - 'db_table': 'over_two_hundred_types_schedulea', - 'indexes': [models.Index(fields=['type'], name='over_two_hu_type_2c8314_idx')], + "db_table": "over_two_hundred_types_schedulea", + "indexes": [ + models.Index(fields=["type"], name="over_two_hu_type_2c8314_idx") + ], }, ), migrations.CreateModel( - name='OverTwoHundredTypesScheduleB', + name="OverTwoHundredTypesScheduleB", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('type', models.TextField()), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ("type", models.TextField()), ], options={ - 'db_table': 'over_two_hundred_types_scheduleb', - 'indexes': [models.Index(fields=['type'], name='over_two_hu_type_411a44_idx')], + "db_table": "over_two_hundred_types_scheduleb", + "indexes": [ + models.Index(fields=["type"], name="over_two_hu_type_411a44_idx") + ], }, ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "populate_over_two_hundred_types"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "drop_over_two_hundred_types"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "populate_over_two_hundred_types", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "drop_over_two_hundred_types", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "create_itemized_triggers_and_functions"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers", "drop_itemized_triggers_and_functions"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "create_itemized_triggers_and_functions", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "drop_itemized_triggers_and_functions", + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID AS $$\nDECLARE\n schedule_date date;\nBEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT\n contribution_date INTO schedule_date\n FROM\n transactions_schedulea\n WHERE\n id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT\n expenditure_date INTO schedule_date\n FROM\n transactions_scheduleb\n WHERE\n id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET aggregate = tc.new_sum\n FROM (\n SELECT\n t.id,\n COALESCE(\n sa.contribution_date,\n sb.expenditure_date,\n sc.loan_incurred_date,\n se.disbursement_date,\n se.dissemination_date\n ) AS date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id)\n ) OVER (\n ORDER BY\n COALESCE(\n sa.contribution_date,\n sb.expenditure_date,\n sc.loan_incurred_date,\n se.disbursement_date,\n se.dissemination_date\n ),\n t.created\n ) AS new_sum\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea AS sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb AS sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec AS sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 AS sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee AS se ON t.schedule_e_id = se.id\n LEFT JOIN transactions_scheduled AS sd ON t.schedule_d_id = sd.id\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM COALESCE(\n sa.contribution_date,\n sb.expenditure_date,\n sc.loan_incurred_date,\n se.disbursement_date,\n se.dissemination_date\n )) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE\n AND deleted IS NULL\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $4\n OR (\n tc.date = $4\n AND t.created >= $5\n )\n );\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, sql_committee_id text + ) + RETURNS VOID AS $$ + DECLARE + schedule_date date; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT + contribution_date INTO schedule_date + FROM + transactions_schedulea + WHERE + id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT + expenditure_date INTO schedule_date + FROM + transactions_scheduleb + WHERE + id = txn.schedule_b_id; + END IF; + + EXECUTE ' + UPDATE transactions_transaction AS t + SET aggregate = tc.new_sum + FROM ( + SELECT + t.id, + COALESCE( + sa.contribution_date, + sb.expenditure_date, + sc.loan_incurred_date, + se.disbursement_date, + se.dissemination_date + ) AS date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id) + ) OVER ( + ORDER BY + COALESCE( + sa.contribution_date, + sb.expenditure_date, + sc.loan_incurred_date, + se.disbursement_date, + se.dissemination_date + ), + t.created + ) AS new_sum + FROM transactions_transaction t + LEFT JOIN transactions_schedulea AS sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb AS sb + ON t.schedule_b_id = sb.id + LEFT JOIN transactions_schedulec AS sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 AS sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee AS se + ON t.schedule_e_id = se.id + LEFT JOIN transactions_scheduled AS sd + ON t.schedule_d_id = sd.id + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM COALESCE( + sa.contribution_date, + sb.expenditure_date, + sc.loan_incurred_date, + se.disbursement_date, + se.dissemination_date + )) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE + AND deleted IS NULL + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $4 + OR ( + tc.date = $4 + AND t.created >= $5 + ) + ); + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID\nAS $$\nDECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\nBEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id text + ) + RETURNS VOID + AS $$ + DECLARE + schedule_date date; + v_election_code text; + v_candidate_office text; + v_candidate_state text; + v_candidate_district text; + BEGIN + SELECT + COALESCE(disbursement_date, dissemination_date) + INTO schedule_date + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT election_code INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT + candidate_office, + candidate_state, + candidate_district INTO v_candidate_office, + v_candidate_state, + v_candidate_district + FROM contacts + WHERE id = txn.contact_2_id; + EXECUTE ' + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tc.new_sum + FROM ( + SELECT + t.id, + Coalesce( + e.disbursement_date, + e.dissemination_date + ) as date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + NULL, + NULL, + NULL, + NULL, + e.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) + ) OVER (ORDER BY Coalesce( + e.disbursement_date, + e.dissemination_date + ), t.created) AS new_sum + FROM transactions_schedulee e + JOIN transactions_transaction t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM Coalesce( + e.disbursement_date, + e.dissemination_date + )) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $7 + OR ( + tc.date = $7 + AND t.created >= $8 + ) + ); + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_effective_amount(\n transaction_type_identifier TEXT,\n amount NUMERIC,\n schedule_c_id UUID\n)\nRETURNS NUMERIC\nAS $$\nDECLARE\n effective_amount NUMERIC;\nBEGIN\n -- Case 1: transaction type is a refund\n IF transaction_type_identifier IN (\n 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'REFUND_PARTY_CONTRIBUTION',\n 'REFUND_PARTY_CONTRIBUTION_VOID',\n 'REFUND_PAC_CONTRIBUTION',\n 'REFUND_PAC_CONTRIBUTION_VOID',\n 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'REFUND_UNREGISTERED_CONTRIBUTION',\n 'REFUND_UNREGISTERED_CONTRIBUTION_VOID',\n 'REFUND_INDIVIDUAL_CONTRIBUTION',\n 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT'\n ) THEN\n effective_amount := amount * -1;\n\n -- Case 2: schedule_c exists (return NULL)\n ELSIF schedule_c_id IS NOT NULL THEN\n effective_amount := NULL;\n\n -- Default case: return the original amount\n ELSE\n effective_amount := amount;\n END IF;\n\n RETURN effective_amount;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_effective_amount( + transaction_type_identifier TEXT, + amount NUMERIC, + schedule_c_id UUID + ) + RETURNS NUMERIC + AS $$ + DECLARE + effective_amount NUMERIC; + BEGIN + -- Case 1: transaction type is a refund + IF transaction_type_identifier IN ( + 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', + 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', + 'REFUND_PARTY_CONTRIBUTION', + 'REFUND_PARTY_CONTRIBUTION_VOID', + 'REFUND_PAC_CONTRIBUTION', + 'REFUND_PAC_CONTRIBUTION_VOID', + 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'REFUND_UNREGISTERED_CONTRIBUTION', + 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', + 'REFUND_INDIVIDUAL_CONTRIBUTION', + 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' + ) THEN + effective_amount := amount * -1; + + -- Case 2: schedule_c exists (return NULL) + ELSIF schedule_c_id IS NOT NULL THEN + effective_amount := NULL; + + -- Default case: return the original amount + ELSE + effective_amount := amount; + END IF; + + RETURN effective_amount; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_is_loan(\n loan UUID,\n transaction_type_identifier TEXT,\n schedule_c_id UUID\n)\nRETURNS TEXT\nAS $$\nDECLARE\n loan_key TEXT;\nBEGIN\n IF loan IS NOT NULL AND transaction_type_identifier\n IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE')\n THEN\n loan_key := 'F';\n\n ELSIF schedule_c_id IS NOT NULL THEN\n loan_key := 'T';\n\n ELSE\n loan_key := 'F';\n END IF;\n\n RETURN loan_key;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_is_loan( + loan UUID, + transaction_type_identifier TEXT, + schedule_c_id UUID + ) + RETURNS TEXT + AS $$ + DECLARE + loan_key TEXT; + BEGIN + IF loan IS NOT NULL AND transaction_type_identifier + IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') + THEN + loan_key := 'F'; + + ELSIF schedule_c_id IS NOT NULL THEN + loan_key := 'T'; + + ELSE + loan_key := 'F'; + END IF; + + RETURN loan_key; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_original_loan_id(\n transaction_id text,\n loan UUID,\n transaction_type_identifier TEXT,\n schedule_c_id UUID,\n schedule_b_id UUID\n)\nRETURNS TEXT\nAS $$\nDECLARE\n loan_transaction_id text;\nBEGIN\n IF loan IS NOT NULL AND transaction_type_identifier\n IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE')\n THEN\n SELECT t.transaction_id\n INTO loan_transaction_id\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n WHERE t.id = loan;\n\n ELSIF schedule_c_id IS NOT NULL THEN\n loan_transaction_id := transaction_id;\n\n ELSE\n loan_transaction_id := NULL;\n END IF;\n\n RETURN loan_transaction_id;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_original_loan_id( + transaction_id text, + loan UUID, + transaction_type_identifier TEXT, + schedule_c_id UUID, + schedule_b_id UUID + ) + RETURNS TEXT + AS $$ + DECLARE + loan_transaction_id text; + BEGIN + IF loan IS NOT NULL AND transaction_type_identifier + IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') + THEN + SELECT t.transaction_id + INTO loan_transaction_id + FROM transactions_transaction t + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + WHERE t.id = loan; + + ELSIF schedule_c_id IS NOT NULL THEN + loan_transaction_id := transaction_id; + + ELSE + loan_transaction_id := NULL; + END IF; + + RETURN loan_transaction_id; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_loan_date(\n trans_id TEXT,\n loan UUID,\n transaction_type_identifier TEXT,\n schedule_c_id UUID,\n schedule_b_id UUID\n)\nRETURNS DATE\nAS $$\nDECLARE\n date DATE;\nBEGIN\n IF loan IS NOT NULL AND transaction_type_identifier\n IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE')\n THEN\n SELECT sb.expenditure_date\n INTO date\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n WHERE t.transaction_id = trans_id;\n\n ELSIF schedule_c_id IS NOT NULL THEN\n SELECT s.report_coverage_through_date INTO date\n FROM transactions_schedulec s\n WHERE s.id = schedule_c_id;\n\n ELSE\n date := NULL;\n END IF;\n\n RETURN date;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_loan_date( + trans_id TEXT, + loan UUID, + transaction_type_identifier TEXT, + schedule_c_id UUID, + schedule_b_id UUID + ) + RETURNS DATE + AS $$ + DECLARE + date DATE; + BEGIN + IF loan IS NOT NULL AND transaction_type_identifier + IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') + THEN + SELECT sb.expenditure_date + INTO date + FROM transactions_transaction t + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + WHERE t.transaction_id = trans_id; + + ELSIF schedule_c_id IS NOT NULL THEN + SELECT s.report_coverage_through_date INTO date + FROM transactions_schedulec s + WHERE s.id = schedule_c_id; + + ELSE + date := NULL; + END IF; + + RETURN date; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\nCREATE OR REPLACE FUNCTION calculate_amount(\n schedule_a_contribution_amount NUMERIC,\n schedule_b_expenditure_amount NUMERIC,\n schedule_c_loan_amount NUMERIC,\n schedule_c2_guaranteed_amount NUMERIC,\n schedule_e_expenditure_amount NUMERIC,\n debt UUID, -- Reference to another transaction\n schedule_d_id UUID\n)\nRETURNS NUMERIC\nAS $$\nDECLARE\n debt_incurred_amount NUMERIC;\nBEGIN\n IF debt IS NOT NULL THEN\n SELECT sd.incurred_amount\n INTO debt_incurred_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id\n WHERE t.id = debt;\n ELSE\n debt_incurred_amount := NULL;\n END IF;\n\n RETURN COALESCE(\n schedule_a_contribution_amount,\n schedule_b_expenditure_amount,\n schedule_c_loan_amount,\n schedule_c2_guaranteed_amount,\n schedule_e_expenditure_amount,\n debt_incurred_amount,\n (SELECT incurred_amount FROM transactions_scheduled WHERE id = schedule_d_id)\n );\nEND;\n$$\nLANGUAGE plpgsql;\n', + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_amount( + schedule_a_contribution_amount NUMERIC, + schedule_b_expenditure_amount NUMERIC, + schedule_c_loan_amount NUMERIC, + schedule_c2_guaranteed_amount NUMERIC, + schedule_e_expenditure_amount NUMERIC, + debt UUID, -- Reference to another transaction + schedule_d_id UUID + ) + RETURNS NUMERIC + AS $$ + DECLARE + debt_incurred_amount NUMERIC; + BEGIN + IF debt IS NOT NULL THEN + SELECT sd.incurred_amount + INTO debt_incurred_amount + FROM transactions_transaction t + LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id + WHERE t.id = debt; + ELSE + debt_incurred_amount := NULL; + END IF; + + RETURN COALESCE( + schedule_a_contribution_amount, + schedule_b_expenditure_amount, + schedule_c_loan_amount, + schedule_c2_guaranteed_amount, + schedule_e_expenditure_amount, + debt_incurred_amount, + ( + SELECT incurred_amount + FROM transactions_scheduled + WHERE id = schedule_d_id + ) + ); + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_loan_payment_to_date(\n txn RECORD, sql_committee_id TEXT\n)\nRETURNS VOID\nAS $$\nDECLARE\n pulled_forward_loans RECORD;\nBEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE data.original_loan_id = (\n SELECT calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n )\n AND data.date <= (\n SELECT calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n )\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING txn.id;\n\n -- Handle pulled-forward loans\n FOR pulled_forward_loans IN\n SELECT t.transaction_id\n FROM transactions_transaction t\n WHERE t.schedule_c_id IS NOT NULL\n AND t.loan_id = txn.loan_id\n LOOP\n -- Recalculate loan_payment_to_date for each pulled-forward loan\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE data.original_loan_id = $1\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING pulled_forward_loans.transaction_id;\n END LOOP;\nEND;\n$$\nLANGUAGE plpgsql;\n", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, sql_committee_id TEXT + ) + RETURNS VOID + AS $$ + DECLARE + pulled_forward_loans RECORD; + BEGIN + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + data.id, + data.original_loan_id, + data.is_loan, + SUM(data.effective_amount) OVER ( + PARTITION BY data.original_loan_id + ORDER BY data.date + ) AS new_sum + FROM ( + SELECT + t.id, + calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS date, + calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS original_loan_id, + calculate_is_loan( + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id + ) AS is_loan, + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) AS effective_amount + FROM transactions_transaction t + LEFT JOIN transactions_schedulea sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + LEFT JOIN transactions_schedulec sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee se + ON t.schedule_e_id = se.id + WHERE t.deleted IS NULL + ) AS data + WHERE data.original_loan_id = ( + SELECT calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + ( + SELECT loan_id + FROM transactions_transaction + WHERE id = $1 + ), + $1 + ) + ) + AND data.date <= ( + SELECT calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + ( + SELECT loan_id + FROM transactions_transaction + WHERE id = $1 + ), + $1 + ) + ) + ) AS tc + WHERE t.id = tc.id + AND tc.is_loan = ''T''; + ' + USING txn.id; + + -- Handle pulled-forward loans + FOR pulled_forward_loans IN + SELECT t.transaction_id + FROM transactions_transaction t + WHERE t.schedule_c_id IS NOT NULL + AND t.loan_id = txn.loan_id + LOOP + -- Recalculate loan_payment_to_date for each pulled-forward loan + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + data.id, + data.original_loan_id, + data.is_loan, + SUM(data.effective_amount) OVER ( + PARTITION BY data.original_loan_id + ORDER BY data.date + ) AS new_sum + FROM ( + SELECT + t.id, + calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS date, + calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS original_loan_id, + calculate_is_loan( + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id + ) AS is_loan, + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) AS effective_amount + FROM transactions_transaction t + LEFT JOIN transactions_schedulea sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + LEFT JOIN transactions_schedulec sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee se + ON t.schedule_e_id = se.id + WHERE t.deleted IS NULL + ) AS data + WHERE data.original_loan_id = $1 + ) AS tc + WHERE t.id = tc.id + AND tc.is_loan = ''T''; + ' + USING pulled_forward_loans.transaction_id; + END LOOP; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "drop_old_triggers"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_drop_old_triggers"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "drop_old_triggers", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_drop_old_triggers", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "drop_old_functions"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_drop_old_functions"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "drop_old_functions", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_drop_old_functions", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "process_itemization"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_process_itemization"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "process_itemization", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_process_itemization", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "handle_parent_itemization"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_handle_parent_itemization"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "handle_parent_itemization", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_handle_parent_itemization", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "calculate_aggregates"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_calculate_aggregates"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "calculate_aggregates", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_calculate_aggregates", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "update_can_unamend"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_update_can_unamend"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "update_can_unamend", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_update_can_unamend", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "before_transactions_transaction"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "before_transactions_transaction"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "before_transactions_transaction", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "before_transactions_transaction", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "after_transactions_transaction"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_after_transactions_transaction"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "after_transactions_transaction", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_after_transactions_transaction", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "create_triggers"), - reverse_code=_load_callable("fecfiler.transactions.migrations.0015_merge_transaction_triggers", "reverse_create_triggers"), + code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "create_triggers", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0015_merge_transaction_triggers", + "reverse_create_triggers", + ), ), migrations.CreateModel( - name='ScheduleF', + name="ScheduleF", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('filer_designated_to_make_coordinated_expenditures', models.BooleanField(blank=True, null=True)), - ('expenditure_date', models.DateField(blank=True, null=True)), - ('expenditure_amount', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('aggregate_general_elec_expended', models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True)), - ('expenditure_purpose_descrip', models.TextField(blank=True)), - ('category_code', models.TextField(blank=True, null=True)), - ('memo_text_description', models.TextField(blank=True, null=True)), - ('general_election_year', models.TextField(blank=True)), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ( + "filer_designated_to_make_coordinated_expenditures", + models.BooleanField(blank=True, null=True), + ), + ("expenditure_date", models.DateField(blank=True, null=True)), + ( + "expenditure_amount", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ( + "aggregate_general_elec_expended", + models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + ("expenditure_purpose_descrip", models.TextField(blank=True)), + ("category_code", models.TextField(blank=True, null=True)), + ("memo_text_description", models.TextField(blank=True, null=True)), + ("general_election_year", models.TextField(blank=True)), ], ), migrations.AddField( - model_name='transaction', - name='schedule_f', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='transactions.schedulef'), + model_name="transaction", + name="schedule_f", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="transactions.schedulef", + ), ), migrations.AddField( - model_name='transaction', - name='contact_4', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='contact_4_transaction_set', to='contacts.contact'), + model_name="transaction", + name="contact_4", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="contact_4_transaction_set", + to="contacts.contact", + ), ), migrations.AddField( - model_name='transaction', - name='contact_5', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='contact_5_transaction_set', to='contacts.contact'), + model_name="transaction", + name="contact_5", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="contact_5_transaction_set", + to="contacts.contact", + ), ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID\nAS $$\nDECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\nBEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n AND t.committee_account_id = $9\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created,\n txn.committee_account_id;\nEND;\n$$\nLANGUAGE plpgsql;\n ", - reverse_sql="\nCREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n)\nRETURNS VOID\nAS $$\nDECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\nBEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created;\nEND;\n$$\nLANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id text + ) + RETURNS VOID + AS $$ + DECLARE + schedule_date date; + v_election_code text; + v_candidate_office text; + v_candidate_state text; + v_candidate_district text; + BEGIN + SELECT + COALESCE(disbursement_date, dissemination_date) + INTO schedule_date + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT election_code INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT + candidate_office, + candidate_state, + candidate_district INTO v_candidate_office, + v_candidate_state, + v_candidate_district + FROM contacts + WHERE id = txn.contact_2_id; + EXECUTE ' + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tc.new_sum + FROM ( + SELECT + t.id, + Coalesce( + e.disbursement_date, + e.dissemination_date + ) as date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + NULL, + NULL, + NULL, + NULL, + e.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) + ) OVER (ORDER BY Coalesce( + e.disbursement_date, + e.dissemination_date + ), t.created) AS new_sum + FROM transactions_schedulee e + JOIN transactions_transaction t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM Coalesce( + e.disbursement_date, + e.dissemination_date + )) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE + AND t.committee_account_id = $9 + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $7 + OR ( + tc.date = $7 + AND t.created >= $8 + ) + ); + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created, + txn.committee_account_id; + END; + $$ + LANGUAGE plpgsql; + """ + ), + reverse_sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id text + ) + RETURNS VOID + AS $$ + DECLARE + schedule_date date; + v_election_code text; + v_candidate_office text; + v_candidate_state text; + v_candidate_district text; + BEGIN + SELECT + COALESCE(disbursement_date, dissemination_date) + INTO schedule_date + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT election_code INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT + candidate_office, + candidate_state, + candidate_district INTO v_candidate_office, + v_candidate_state, + v_candidate_district + FROM contacts + WHERE id = txn.contact_2_id; + EXECUTE ' + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tc.new_sum + FROM ( + SELECT + t.id, + Coalesce( + e.disbursement_date, + e.dissemination_date + ) as date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + NULL, + NULL, + NULL, + NULL, + e.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) + ) OVER (ORDER BY Coalesce( + e.disbursement_date, + e.dissemination_date + ), t.created) AS new_sum + FROM transactions_schedulee e + JOIN transactions_transaction t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM Coalesce( + e.disbursement_date, + e.dissemination_date + )) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $7 + OR ( + tc.date = $7 + AND t.created >= $8 + ) + ); + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0020_trigger_save_on_transactions", "trigger_save_on_transactions"), + code=_load_callable( + "fecfiler.transactions.migrations.0020_trigger_save_on_transactions", + "trigger_save_on_transactions", + ), reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.AlterField( - model_name='transaction', - name='reports', - field=models.ManyToManyField(related_name='transactions', through='reports.ReportTransaction', through_fields=['transaction', 'report'], to='reports.report'), + model_name="transaction", + name="reports", + field=models.ManyToManyField( + related_name="transactions", + through="reports.ReportTransaction", + through_fields=["transaction", "report"], + to="reports.report", + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0022_schedule_f_aggregation", "calculate_schedule_f_aggregates"), + code=_load_callable( + "fecfiler.transactions.migrations.0022_schedule_f_aggregation", + "calculate_schedule_f_aggregates", + ), reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunSQL( - sql="\nCREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date(\n txn record, sql_committee_id text\n)\nRETURNS VOID AS $$\nBEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2 ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE (data.original_loan_id = (\n SELECT calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n )\n AND data.date <= (\n SELECT calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id FROM transactions_transaction WHERE id = $1),\n $1\n )\n ))\n OR data.original_loan_id IN (\n SELECT t.transaction_id\n FROM transactions_transaction t\n WHERE t.schedule_c_id IS NOT NULL\n AND t.loan_id = $2\n )\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING txn.id, txn.loan_id;\nEND;\n$$\nLANGUAGE plpgsql;\n", + sql=_strip_sql( + """ + CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( + txn record, sql_committee_id text + ) + RETURNS VOID AS $$ + BEGIN + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + data.id, + data.original_loan_id, + data.is_loan, + SUM(data.effective_amount) OVER ( + PARTITION BY data.original_loan_id + ORDER BY data.date + ) AS new_sum + FROM ( + SELECT + t.id, + calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS date, + calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS original_loan_id, + calculate_is_loan( + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id + ) AS is_loan, + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) AS effective_amount + FROM transactions_transaction t + LEFT JOIN transactions_schedulea sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + LEFT JOIN transactions_schedulec sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee se + ON t.schedule_e_id = se.id + WHERE t.deleted IS NULL + ) AS data + WHERE (data.original_loan_id = ( + SELECT calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + ( + SELECT loan_id + FROM transactions_transaction + WHERE id = $1 + ), + $1 + ) + ) + AND data.date <= ( + SELECT calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + ( + SELECT loan_id + FROM transactions_transaction + WHERE id = $1 + ), + $1 + ) + )) + OR data.original_loan_id IN ( + SELECT t.transaction_id + FROM transactions_transaction t + WHERE t.schedule_c_id IS NOT NULL + AND t.loan_id = $2 + ) + ) AS tc + WHERE t.id = tc.id + AND tc.is_loan = ''T''; + ' + USING txn.id, txn.loan_id; + END; + $$ + LANGUAGE plpgsql; + """ + ), ), migrations.AddField( - model_name='scheduled', - name='balance_at_close', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="scheduled", + name="balance_at_close", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.AddField( - model_name='scheduled', - name='beginning_balance', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="scheduled", + name="beginning_balance", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.AddField( - model_name='scheduled', - name='incurred_prior', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="scheduled", + name="incurred_prior", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.AddField( - model_name='scheduled', - name='payment_prior', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="scheduled", + name="payment_prior", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.AddField( - model_name='scheduled', - name='payment_amount', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=11, null=True), + model_name="scheduled", + name="payment_amount", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), ), migrations.RunPython( - code=_load_callable("fecfiler.transactions.migrations.0024_scheduled_balance_at_close_and_more", "run_aggregations_for_all_debts"), + code=_load_callable( + "fecfiler.transactions.migrations." + "0024_scheduled_balance_at_close_and_more", + "run_aggregations_for_all_debts", + ), reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunSQL( - sql='\n -- Drop all aggregate-related triggers from\n -- transactions_transaction table\n DROP TRIGGER IF EXISTS calculate_aggregates_trigger\n ON transactions_transaction;\n DROP TRIGGER IF EXISTS after_transactions_transaction_trigger\n ON transactions_transaction;\n DROP TRIGGER IF EXISTS\n after_transactions_transaction_infinite_trigger\n ON transactions_transaction;\n DROP TRIGGER IF EXISTS before_transactions_transaction_trigger\n ON transactions_transaction;\n ', - reverse_sql='\n -- Recreate triggers for aggregate calculation\n CREATE TRIGGER before_transactions_transaction_trigger\n BEFORE INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n EXECUTE FUNCTION before_transactions_transaction();\n\n CREATE TRIGGER after_transactions_transaction_infinite_trigger\n AFTER INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n EXECUTE FUNCTION after_transactions_transaction_infinite();\n\n CREATE TRIGGER after_transactions_transaction_trigger\n AFTER INSERT OR UPDATE ON transactions_transaction\n FOR EACH ROW\n WHEN (pg_trigger_depth() = 0)\n EXECUTE FUNCTION after_transactions_transaction();\n ', + sql=_strip_sql( + """ + -- Drop all aggregate-related triggers from + -- transactions_transaction table + DROP TRIGGER IF EXISTS calculate_aggregates_trigger + ON transactions_transaction; + DROP TRIGGER IF EXISTS after_transactions_transaction_trigger + ON transactions_transaction; + DROP TRIGGER IF EXISTS + after_transactions_transaction_infinite_trigger + ON transactions_transaction; + DROP TRIGGER IF EXISTS before_transactions_transaction_trigger + ON transactions_transaction; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate triggers for aggregate calculation + CREATE TRIGGER before_transactions_transaction_trigger + BEFORE INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION before_transactions_transaction(); + + CREATE TRIGGER after_transactions_transaction_infinite_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION after_transactions_transaction_infinite(); + + CREATE TRIGGER after_transactions_transaction_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION after_transactions_transaction(); + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the calculate_entity_aggregates function\n DROP FUNCTION IF EXISTS calculate_entity_aggregates(\n txn RECORD, sql_committee_id TEXT, temp_table_name TEXT\n ) CASCADE;\n ', - reverse_sql="\n -- Recreate calculate_entity_aggregates function\n CREATE OR REPLACE FUNCTION calculate_entity_aggregates(\n txn RECORD,\n sql_committee_id TEXT,\n temp_table_name TEXT\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date DATE;\n BEGIN\n IF txn.schedule_a_id IS NOT NULL THEN\n SELECT contribution_date\n INTO schedule_date\n FROM transactions_schedulea\n WHERE id = txn.schedule_a_id;\n ELSIF txn.schedule_b_id IS NOT NULL THEN\n SELECT expenditure_date\n INTO schedule_date\n FROM transactions_scheduleb\n WHERE id = txn.schedule_b_id;\n END IF;\n\n EXECUTE '\n CREATE TEMPORARY TABLE ' || temp_table_name || '\n ON COMMIT DROP AS\n SELECT\n id,\n SUM(effective_amount) OVER (ORDER BY date, created)\n AS new_sum\n FROM transaction_view__' || sql_committee_id || '\n WHERE\n contact_1_id = $1\n AND EXTRACT(YEAR FROM date) = $2\n AND aggregation_group = $3\n AND force_unaggregated IS NOT TRUE;\n\n UPDATE transactions_transaction AS t\n SET aggregate = tt.new_sum\n FROM ' || temp_table_name || ' AS tt\n WHERE t.id = tt.id;\n '\n USING\n txn.contact_1_id,\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + -- Drop the calculate_entity_aggregates function + DROP FUNCTION IF EXISTS calculate_entity_aggregates( + txn RECORD, sql_committee_id TEXT, temp_table_name TEXT + ) CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate calculate_entity_aggregates function + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT contribution_date + INTO schedule_date + FROM transactions_schedulea + WHERE id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT expenditure_date + INTO schedule_date + FROM transactions_scheduleb + WHERE id = txn.schedule_b_id; + END IF; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET aggregate = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the calculate_calendar_ytd_per_election_office function\n DROP FUNCTION IF EXISTS calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id TEXT, temp_table_name TEXT\n ) CASCADE;\n ', - reverse_sql="\n -- Recreate calculate_calendar_ytd_per_election_office function\n CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office(\n txn RECORD, sql_committee_id text\n )\n RETURNS VOID AS $$\n DECLARE\n schedule_date date;\n v_election_code text;\n v_candidate_office text;\n v_candidate_state text;\n v_candidate_district text;\n BEGIN\n SELECT\n COALESCE(disbursement_date, dissemination_date) INTO schedule_date\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT election_code INTO v_election_code\n FROM transactions_schedulee\n WHERE id = txn.schedule_e_id;\n\n SELECT\n candidate_office,\n candidate_state,\n candidate_district INTO v_candidate_office,\n v_candidate_state,\n v_candidate_district\n FROM contacts\n WHERE id = txn.contact_2_id;\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET _calendar_ytd_per_election_office = tc.new_sum\n FROM (\n SELECT\n t.id,\n Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ) as date,\n SUM(\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n NULL,\n NULL,\n NULL,\n NULL,\n e.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n )\n ) OVER (ORDER BY Coalesce(\n e.disbursement_date,\n e.dissemination_date\n ), t.created) AS new_sum\n FROM transactions_schedulee e\n JOIN transactions_transaction t\n ON e.id = t.schedule_e_id\n JOIN contacts c\n ON t.contact_2_id = c.id\n WHERE\n e.election_code = $1\n AND c.candidate_office = $2\n AND (\n c.candidate_state = $3\n OR (\n c.candidate_state IS NULL\n AND $3 = ''''\n )\n )\n AND (\n c.candidate_district = $4\n OR (\n c.candidate_district IS NULL\n AND $4 = ''''\n )\n )\n AND EXTRACT(YEAR FROM Coalesce(\n e.disbursement_date,\n e.dissemination_date\n )) = $5\n AND aggregation_group = $6\n AND force_unaggregated IS NOT TRUE\n AND t.committee_account_id = $9\n ) AS tc\n WHERE t.id = tc.id\n AND (\n tc.date > $7\n OR (\n tc.date = $7\n AND t.created >= $8\n )\n );\n '\n USING\n v_election_code,\n v_candidate_office,\n COALESCE(v_candidate_state, ''),\n COALESCE(v_candidate_district, ''),\n EXTRACT(YEAR FROM schedule_date),\n txn.aggregation_group,\n schedule_date,\n txn.created,\n txn.committee_account_id;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + -- Drop the calculate_calendar_ytd_per_election_office function + DROP FUNCTION IF EXISTS calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id TEXT, temp_table_name TEXT + ) CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate calculate_calendar_ytd_per_election_office function + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id text + ) + RETURNS VOID AS $$ + DECLARE + schedule_date date; + v_election_code text; + v_candidate_office text; + v_candidate_state text; + v_candidate_district text; + BEGIN + SELECT + COALESCE(disbursement_date, dissemination_date) + INTO schedule_date + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT election_code INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT + candidate_office, + candidate_state, + candidate_district INTO v_candidate_office, + v_candidate_state, + v_candidate_district + FROM contacts + WHERE id = txn.contact_2_id; + EXECUTE ' + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tc.new_sum + FROM ( + SELECT + t.id, + Coalesce( + e.disbursement_date, + e.dissemination_date + ) as date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + NULL, + NULL, + NULL, + NULL, + e.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) + ) OVER (ORDER BY Coalesce( + e.disbursement_date, + e.dissemination_date + ), t.created) AS new_sum + FROM transactions_schedulee e + JOIN transactions_transaction t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM Coalesce( + e.disbursement_date, + e.dissemination_date + )) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE + AND t.committee_account_id = $9 + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $7 + OR ( + tc.date = $7 + AND t.created >= $8 + ) + ); + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created, + txn.committee_account_id; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the calculate_effective_amount function\n DROP FUNCTION IF EXISTS calculate_effective_amount(\n transaction_type_identifier TEXT, amount NUMERIC,\n schedule_c_id UUID\n ) CASCADE;\n ', - reverse_sql="\n -- Recreate calculate_effective_amount function\n CREATE OR REPLACE FUNCTION calculate_effective_amount(\n transaction_type_identifier TEXT,\n amount NUMERIC,\n schedule_c_id UUID\n )\n RETURNS NUMERIC AS $$\n DECLARE\n effective_amount NUMERIC;\n BEGIN\n -- Case 1: transaction type is a refund\n IF transaction_type_identifier IN (\n 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT',\n 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT',\n 'REFUND_PARTY_CONTRIBUTION',\n 'REFUND_PARTY_CONTRIBUTION_VOID',\n 'REFUND_PAC_CONTRIBUTION',\n 'REFUND_PAC_CONTRIBUTION_VOID',\n 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT',\n 'REFUND_UNREGISTERED_CONTRIBUTION',\n 'REFUND_UNREGISTERED_CONTRIBUTION_VOID',\n 'REFUND_INDIVIDUAL_CONTRIBUTION',\n 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT',\n 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT'\n ) THEN\n effective_amount := amount * -1;\n\n -- Case 2: schedule_c exists (return NULL)\n ELSIF schedule_c_id IS NOT NULL THEN\n effective_amount := NULL;\n\n -- Default case: return the original amount\n ELSE\n effective_amount := amount;\n END IF;\n\n RETURN effective_amount;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + -- Drop the calculate_effective_amount function + DROP FUNCTION IF EXISTS calculate_effective_amount( + transaction_type_identifier TEXT, amount NUMERIC, + schedule_c_id UUID + ) CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate calculate_effective_amount function + CREATE OR REPLACE FUNCTION calculate_effective_amount( + transaction_type_identifier TEXT, + amount NUMERIC, + schedule_c_id UUID + ) + RETURNS NUMERIC AS $$ + DECLARE + effective_amount NUMERIC; + BEGIN + -- Case 1: transaction type is a refund + IF transaction_type_identifier IN ( + 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', + 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', + 'REFUND_PARTY_CONTRIBUTION', + 'REFUND_PARTY_CONTRIBUTION_VOID', + 'REFUND_PAC_CONTRIBUTION', + 'REFUND_PAC_CONTRIBUTION_VOID', + 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'REFUND_UNREGISTERED_CONTRIBUTION', + 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', + 'REFUND_INDIVIDUAL_CONTRIBUTION', + 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' + ) THEN + effective_amount := amount * -1; + + -- Case 2: schedule_c exists (return NULL) + ELSIF schedule_c_id IS NOT NULL THEN + effective_amount := NULL; + + -- Default case: return the original amount + ELSE + effective_amount := amount; + END IF; + + RETURN effective_amount; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the calculate_amount function if it exists\n DROP FUNCTION IF EXISTS calculate_amount(\n contribution_amount NUMERIC,\n expenditure_amount NUMERIC,\n loan_amount NUMERIC,\n guaranteed_amount NUMERIC,\n schedule_e_expenditure_amount NUMERIC,\n debt_id UUID,\n schedule_d_id UUID\n ) CASCADE;\n ', - reverse_sql='\n -- Recreate calculate_amount function\n CREATE OR REPLACE FUNCTION calculate_amount(\n schedule_a_contribution_amount NUMERIC,\n schedule_b_expenditure_amount NUMERIC,\n schedule_c_loan_amount NUMERIC,\n schedule_c2_guaranteed_amount NUMERIC,\n schedule_e_expenditure_amount NUMERIC,\n debt UUID,\n schedule_d_id UUID\n )\n RETURNS NUMERIC AS $$\n DECLARE\n debt_incurred_amount NUMERIC;\n BEGIN\n IF debt IS NOT NULL THEN\n SELECT sd.incurred_amount\n INTO debt_incurred_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id\n WHERE t.id = debt;\n ELSE\n debt_incurred_amount := NULL;\n END IF;\n\n RETURN COALESCE(\n schedule_a_contribution_amount,\n schedule_b_expenditure_amount,\n schedule_c_loan_amount,\n schedule_c2_guaranteed_amount,\n schedule_e_expenditure_amount,\n debt_incurred_amount,\n (SELECT incurred_amount\n FROM transactions_scheduled\n WHERE id = schedule_d_id)\n );\n END;\n $$ LANGUAGE plpgsql;\n ', + sql=_strip_sql( + """ + -- Drop the calculate_amount function if it exists + DROP FUNCTION IF EXISTS calculate_amount( + contribution_amount NUMERIC, + expenditure_amount NUMERIC, + loan_amount NUMERIC, + guaranteed_amount NUMERIC, + schedule_e_expenditure_amount NUMERIC, + debt_id UUID, + schedule_d_id UUID + ) CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate calculate_amount function + CREATE OR REPLACE FUNCTION calculate_amount( + schedule_a_contribution_amount NUMERIC, + schedule_b_expenditure_amount NUMERIC, + schedule_c_loan_amount NUMERIC, + schedule_c2_guaranteed_amount NUMERIC, + schedule_e_expenditure_amount NUMERIC, + debt UUID, + schedule_d_id UUID + ) + RETURNS NUMERIC AS $$ + DECLARE + debt_incurred_amount NUMERIC; + BEGIN + IF debt IS NOT NULL THEN + SELECT sd.incurred_amount + INTO debt_incurred_amount + FROM transactions_transaction t + LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id + WHERE t.id = debt; + ELSE + debt_incurred_amount := NULL; + END IF; + + RETURN COALESCE( + schedule_a_contribution_amount, + schedule_b_expenditure_amount, + schedule_c_loan_amount, + schedule_c2_guaranteed_amount, + schedule_e_expenditure_amount, + debt_incurred_amount, + (SELECT incurred_amount + FROM transactions_scheduled + WHERE id = schedule_d_id) + ); + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the calculate_loan_payment_to_date function if it exists\n DROP FUNCTION IF EXISTS calculate_loan_payment_to_date(\n txn RECORD, sql_committee_id TEXT, temp_table_name TEXT\n ) CASCADE;\n ', - reverse_sql="\n -- Recreate calculate_loan_payment_to_date function\n CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date(\n txn record, sql_committee_id text\n )\n RETURNS VOID AS $$\n BEGIN\n EXECUTE '\n UPDATE transactions_transaction AS t\n SET loan_payment_to_date = tc.new_sum\n FROM (\n SELECT\n data.id,\n data.original_loan_id,\n data.is_loan,\n SUM(data.effective_amount) OVER (\n PARTITION BY data.original_loan_id\n ORDER BY data.date\n ) AS new_sum\n FROM (\n SELECT\n t.id,\n calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS date,\n calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n ) AS original_loan_id,\n calculate_is_loan(\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id\n ) AS is_loan,\n calculate_effective_amount(\n t.transaction_type_identifier,\n calculate_amount(\n sa.contribution_amount,\n sb.expenditure_amount,\n sc.loan_amount,\n sc2.guaranteed_amount,\n se.expenditure_amount,\n t.debt_id,\n t.schedule_d_id\n ),\n t.schedule_c_id\n ) AS effective_amount\n FROM transactions_transaction t\n LEFT JOIN transactions_schedulea sa\n ON t.schedule_a_id = sa.id\n LEFT JOIN transactions_scheduleb sb\n ON t.schedule_b_id = sb.id\n LEFT JOIN transactions_schedulec sc\n ON t.schedule_c_id = sc.id\n LEFT JOIN transactions_schedulec2 sc2\n ON t.schedule_c2_id = sc2.id\n LEFT JOIN transactions_schedulee se\n ON t.schedule_e_id = se.id\n WHERE t.deleted IS NULL\n ) AS data\n WHERE (data.original_loan_id = (\n SELECT calculate_original_loan_id(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id\n FROM transactions_transaction\n WHERE id = $1),\n $1\n )\n )\n AND data.date <= (\n SELECT calculate_loan_date(\n t.transaction_id,\n t.loan_id,\n t.transaction_type_identifier,\n t.schedule_c_id,\n t.schedule_b_id\n )\n FROM transactions_transaction t\n WHERE t.id = COALESCE(\n (SELECT loan_id\n FROM transactions_transaction\n WHERE id = $1),\n $1\n )\n ))\n OR data.original_loan_id IN (\n SELECT t.transaction_id\n FROM transactions_transaction t\n WHERE t.schedule_c_id IS NOT NULL\n AND t.loan_id = $2\n )\n ) AS tc\n WHERE t.id = tc.id\n AND tc.is_loan = ''T'';\n '\n USING txn.id, txn.loan_id;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + -- Drop the calculate_loan_payment_to_date function if it exists + DROP FUNCTION IF EXISTS calculate_loan_payment_to_date( + txn RECORD, sql_committee_id TEXT, temp_table_name TEXT + ) CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate calculate_loan_payment_to_date function + CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( + txn record, sql_committee_id text + ) + RETURNS VOID AS $$ + BEGIN + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + data.id, + data.original_loan_id, + data.is_loan, + SUM(data.effective_amount) OVER ( + PARTITION BY data.original_loan_id + ORDER BY data.date + ) AS new_sum + FROM ( + SELECT + t.id, + calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS date, + calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS original_loan_id, + calculate_is_loan( + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id + ) AS is_loan, + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) AS effective_amount + FROM transactions_transaction t + LEFT JOIN transactions_schedulea sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + LEFT JOIN transactions_schedulec sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee se + ON t.schedule_e_id = se.id + WHERE t.deleted IS NULL + ) AS data + WHERE (data.original_loan_id = ( + SELECT calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + (SELECT loan_id + FROM transactions_transaction + WHERE id = $1), + $1 + ) + ) + AND data.date <= ( + SELECT calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + (SELECT loan_id + FROM transactions_transaction + WHERE id = $1), + $1 + ) + )) + OR data.original_loan_id IN ( + SELECT t.transaction_id + FROM transactions_transaction t + WHERE t.schedule_c_id IS NOT NULL + AND t.loan_id = $2 + ) + ) AS tc + WHERE t.id = tc.id + AND tc.is_loan = ''T''; + ' + USING txn.id, txn.loan_id; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the trigger handler functions\n DROP FUNCTION IF EXISTS after_transactions_transaction()\n CASCADE;\n DROP FUNCTION IF EXISTS\n after_transactions_transaction_infinite() CASCADE;\n DROP FUNCTION IF EXISTS before_transactions_transaction()\n CASCADE;\n DROP FUNCTION IF EXISTS\n before_transactions_transaction_insert_or_update()\n CASCADE;\n DROP FUNCTION IF EXISTS\n after_transactions_transaction_insert_or_update()\n CASCADE;\n ', - reverse_sql="\n -- Recreate trigger handler functions\n CREATE OR REPLACE FUNCTION before_transactions_transaction()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW := process_itemization(OLD, NEW);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE FUNCTION after_transactions_transaction()\n RETURNS TRIGGER AS $$\n BEGIN\n IF TG_OP = 'UPDATE'\n THEN\n NEW := calculate_aggregates(OLD, NEW, TG_OP);\n NEW := update_can_unamend(NEW);\n ELSE\n NEW := calculate_aggregates(OLD, NEW, TG_OP);\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW := handle_parent_itemization(OLD, NEW);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + -- Drop the trigger handler functions + DROP FUNCTION IF EXISTS after_transactions_transaction() + CASCADE; + DROP FUNCTION IF EXISTS + after_transactions_transaction_infinite() CASCADE; + DROP FUNCTION IF EXISTS before_transactions_transaction() + CASCADE; + DROP FUNCTION IF EXISTS + before_transactions_transaction_insert_or_update() + CASCADE; + DROP FUNCTION IF EXISTS + after_transactions_transaction_insert_or_update() + CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate trigger handler functions + CREATE OR REPLACE FUNCTION before_transactions_transaction() + RETURNS TRIGGER AS $$ + BEGIN + NEW := process_itemization(OLD, NEW); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION after_transactions_transaction() + RETURNS TRIGGER AS $$ + BEGIN + IF TG_OP = 'UPDATE' + THEN + NEW := calculate_aggregates(OLD, NEW, TG_OP); + NEW := update_can_unamend(NEW); + ELSE + NEW := calculate_aggregates(OLD, NEW, TG_OP); + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite() + RETURNS TRIGGER AS $$ + BEGIN + NEW := handle_parent_itemization(OLD, NEW); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.RunSQL( - sql='\n -- Drop the main calculate_aggregates function that was\n -- called by triggers\n DROP FUNCTION IF EXISTS calculate_aggregates(\n old RECORD, new RECORD, tg_op TEXT\n ) CASCADE;\n DROP FUNCTION IF EXISTS calculate_aggregates() CASCADE;\n ', - reverse_sql="\n -- Recreate calculate_aggregates function\n CREATE OR REPLACE FUNCTION calculate_aggregates(\n OLD RECORD,\n NEW RECORD,\n TG_OP TEXT\n )\n RETURNS RECORD AS $$\n DECLARE\n sql_committee_id TEXT;\n BEGIN\n sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_');\n\n -- If schedule_c2_id or schedule_d_id is not null, stop processing\n IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL\n THEN\n RETURN NEW;\n END IF;\n\n IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL\n THEN\n PERFORM calculate_entity_aggregates(NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n AND NEW.contact_1_id <> OLD.contact_1_id\n THEN\n PERFORM calculate_entity_aggregates(OLD, sql_committee_id);\n END IF;\n END IF;\n\n IF NEW.schedule_c_id IS NOT NULL\n OR NEW.schedule_c1_id IS NOT NULL\n OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE'\n THEN\n PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id);\n END IF;\n\n IF NEW.schedule_e_id IS NOT NULL\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n NEW, sql_committee_id);\n IF TG_OP = 'UPDATE'\n THEN\n PERFORM calculate_calendar_ytd_per_election_office(\n OLD, sql_committee_id);\n END IF;\n END IF;\n\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n ", + sql=_strip_sql( + """ + -- Drop the main calculate_aggregates function that was + -- called by triggers + DROP FUNCTION IF EXISTS calculate_aggregates( + old RECORD, new RECORD, tg_op TEXT + ) CASCADE; + DROP FUNCTION IF EXISTS calculate_aggregates() CASCADE; + """ + ), + reverse_sql=_strip_sql( + """ + -- Recreate calculate_aggregates function + CREATE OR REPLACE FUNCTION calculate_aggregates( + OLD RECORD, + NEW RECORD, + TG_OP TEXT + ) + RETURNS RECORD AS $$ + DECLARE + sql_committee_id TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates(NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates(OLD, sql_committee_id); + END IF; + END IF; + + IF NEW.schedule_c_id IS NOT NULL + OR NEW.schedule_c1_id IS NOT NULL + OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' + THEN + PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); + END IF; + + IF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id); + END IF; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ), ), migrations.AlterField( - model_name='transaction', - name='itemized', + model_name="transaction", + name="itemized", field=models.BooleanField(db_default=False), ), ] diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py index b26f34516..99a4f5861 100644 --- a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py +++ b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py @@ -22,53 +22,61 @@ def remove_old_login_accounts(apps, schema_editor): user.membership_set.all().delete() users_to_delete.delete() + class Migration(migrations.Migration): - replaces = [('user', '0002_remove_user_cmtee_id'), ('user', '0003_user_security_consent_date'), ('user', '0004_alter_user_managers'), ('user', '0005_rename_security_consent_date_user_security_consent_exp_date'), ('user', '0006_remove_old_login_accounts'), ('user', '0007_user_security_consent_version')] + replaces = [ + ("user", "0002_remove_user_cmtee_id"), + ("user", "0003_user_security_consent_date"), + ("user", "0004_alter_user_managers"), + ("user", "0005_rename_security_consent_date_user_security_consent_exp_date"), + ("user", "0006_remove_old_login_accounts"), + ("user", "0007_user_security_consent_version"), + ] dependencies = [ - ('committee_accounts', '0002_membership'), - ('user', '0001_initial'), + ("committee_accounts", "0002_membership"), + ("user", "0001_initial"), ] operations = [ migrations.RemoveField( - model_name='user', - name='cmtee_id', + model_name="user", + name="cmtee_id", ), migrations.AlterField( - model_name='user', - name='first_name', + model_name="user", + name="first_name", field=models.CharField(blank=True, max_length=150, null=True), ), migrations.AlterField( - model_name='user', - name='last_name', + model_name="user", + name="last_name", field=models.CharField(blank=True, max_length=150, null=True), ), migrations.AddField( - model_name='user', - name='security_consent_date', + model_name="user", + name="security_consent_date", field=models.DateField(blank=True, null=True), ), migrations.AlterModelManagers( - name='user', + name="user", managers=[ - ('objects', fecfiler.user.managers.UserManager()), + ("objects", fecfiler.user.managers.UserManager()), ], ), migrations.RenameField( - model_name='user', - old_name='security_consent_date', - new_name='security_consent_exp_date', + model_name="user", + old_name="security_consent_date", + new_name="security_consent_exp_date", ), migrations.RunPython( code=remove_old_login_accounts, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.AddField( - model_name='user', - name='security_consent_version', + model_name="user", + name="security_consent_version", field=models.CharField(blank=True, null=True), ), ] diff --git a/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py b/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py index 5499b71e8..5da677458 100644 --- a/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py +++ b/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py @@ -18,23 +18,27 @@ def set_default_task_completed_times(apps, schema_editor): uploads.objects.all().update(task_completed=F("updated")) web_prints.objects.all().update(task_completed=F("updated")) + class Migration(migrations.Migration): - replaces = [('web_services', '0002_uploadsubmission_task_completed_and_more'), ('web_services', '0003_uploadsubmission_fecfile_polling_attempts_and_more')] + replaces = [ + ("web_services", "0002_uploadsubmission_task_completed_and_more"), + ("web_services", "0003_uploadsubmission_fecfile_polling_attempts_and_more"), + ] dependencies = [ - ('web_services', '0001_initial'), + ("web_services", "0001_initial"), ] operations = [ migrations.AddField( - model_name='uploadsubmission', - name='task_completed', + model_name="uploadsubmission", + name="task_completed", field=models.DateTimeField(null=True), ), migrations.AddField( - model_name='webprintsubmission', - name='task_completed', + model_name="webprintsubmission", + name="task_completed", field=models.DateTimeField(null=True), ), migrations.RunPython( @@ -42,13 +46,13 @@ class Migration(migrations.Migration): reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.AddField( - model_name='uploadsubmission', - name='fecfile_polling_attempts', + model_name="uploadsubmission", + name="fecfile_polling_attempts", field=models.IntegerField(default=0), ), migrations.AddField( - model_name='webprintsubmission', - name='fecfile_polling_attempts', + model_name="webprintsubmission", + name="fecfile_polling_attempts", field=models.IntegerField(default=0), ), ] From 096cd56bd2043d68a867597cbb1cb5029f1a6998 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 07:38:14 -0400 Subject: [PATCH 08/52] FECFILE-2734: Fixed committee_accounts dependecy post-squash. --- django-backend/fecfiler/contacts/migrations/0001_initial.py | 5 ++++- django-backend/fecfiler/memo_text/migrations/0001_initial.py | 5 ++++- django-backend/fecfiler/reports/migrations/0001_initial.py | 5 ++++- .../fecfiler/transactions/migrations/0001_initial.py | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/contacts/migrations/0001_initial.py b/django-backend/fecfiler/contacts/migrations/0001_initial.py index f933ee380..38ddb40d1 100644 --- a/django-backend/fecfiler/contacts/migrations/0001_initial.py +++ b/django-backend/fecfiler/contacts/migrations/0001_initial.py @@ -9,7 +9,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("committee_accounts", "0001_initial"), + ( + "committee_accounts", + "0001_squashed_0007_alter_committeeaccount_members", + ), ] operations = [ diff --git a/django-backend/fecfiler/memo_text/migrations/0001_initial.py b/django-backend/fecfiler/memo_text/migrations/0001_initial.py index 8d99be1ea..0a493cc37 100644 --- a/django-backend/fecfiler/memo_text/migrations/0001_initial.py +++ b/django-backend/fecfiler/memo_text/migrations/0001_initial.py @@ -9,7 +9,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("committee_accounts", "0001_initial"), + ( + "committee_accounts", + "0001_squashed_0007_alter_committeeaccount_members", + ), ] operations = [ diff --git a/django-backend/fecfiler/reports/migrations/0001_initial.py b/django-backend/fecfiler/reports/migrations/0001_initial.py index 2b38af6ca..41ca47370 100644 --- a/django-backend/fecfiler/reports/migrations/0001_initial.py +++ b/django-backend/fecfiler/reports/migrations/0001_initial.py @@ -9,7 +9,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("committee_accounts", "0001_initial"), + ( + "committee_accounts", + "0001_squashed_0007_alter_committeeaccount_members", + ), ] operations = [ diff --git a/django-backend/fecfiler/transactions/migrations/0001_initial.py b/django-backend/fecfiler/transactions/migrations/0001_initial.py index 0c5a8fd5b..c3c2a2f8f 100644 --- a/django-backend/fecfiler/transactions/migrations/0001_initial.py +++ b/django-backend/fecfiler/transactions/migrations/0001_initial.py @@ -10,7 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("committee_accounts", "0001_initial"), + ( + "committee_accounts", + "0001_squashed_0007_alter_committeeaccount_members", + ), ("contacts", "0001_initial"), ("reports", "0001_initial"), ("memo_text", "0001_initial"), From b44085bec3b5f31775e1ac34572a310f959fafa1 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 08:15:55 -0400 Subject: [PATCH 09/52] FECFILE-2734: Fixed other committee_accounts dependecy post-squash. --- .../fecfiler/reports/migrations/0006_reporttransaction.py | 6 ++++-- ..._cmtee_id_squashed_0007_user_security_consent_version.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/reports/migrations/0006_reporttransaction.py b/django-backend/fecfiler/reports/migrations/0006_reporttransaction.py index 146665aa2..b632cad43 100644 --- a/django-backend/fecfiler/reports/migrations/0006_reporttransaction.py +++ b/django-backend/fecfiler/reports/migrations/0006_reporttransaction.py @@ -8,8 +8,10 @@ class Migration(migrations.Migration): dependencies = [ - ('committee_accounts', - '0003_membership_pending_email_alter_membership_id_and_more'), + ( + "committee_accounts", + "0001_squashed_0007_alter_committeeaccount_members", + ), ('transactions', '0002_remove_schedulea_contributor_city_and_more'), ('reports', '0005_remove_form1m_iii_candidate_district_and_more'), ] diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py index 99a4f5861..e1dc05ffd 100644 --- a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py +++ b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py @@ -35,7 +35,10 @@ class Migration(migrations.Migration): ] dependencies = [ - ("committee_accounts", "0002_membership"), + ( + "committee_accounts", + "0001_squashed_0007_alter_committeeaccount_members", + ), ("user", "0001_initial"), ] From b51b881cd0e9886953b7cb7d3696c62c1fe8130c Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 08:23:32 -0400 Subject: [PATCH 10/52] FECFILE-2734: Surely the last broken dependency. --- ...003_alter_parent_squashed_0026_alter_transaction_itemized.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py index e7a518bc3..a78bdab22 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -77,7 +77,7 @@ class Migration(migrations.Migration): dependencies = [ ("contacts", "0001_initial"), ("reports", "0006_reporttransaction"), - ("reports", "0014_form99_swap_text_code"), + ("reports", "0007_0007_remove_report_deleted_squashed_00019_form24_name_fix"), ("transactions", "0002_remove_schedulea_contributor_city_and_more"), ] From d0830ae3191ed7978027ca9c33158a689e7fc39f Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 14:20:02 -0400 Subject: [PATCH 11/52] FECFILE-2734: Simplify big RunSQL blocks. --- ..._deleted_squashed_00019_form24_name_fix.py | 33 +- ...quashed_0026_alter_transaction_itemized.py | 2544 ++++++++--------- 2 files changed, 1238 insertions(+), 1339 deletions(-) diff --git a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py index 9b6919b9e..5bf953ba0 100644 --- a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -19,10 +19,6 @@ # fecfiler.reports.migrations.0010_report_can_unammend -def _strip_sql(sql): - return dedent(sql).strip() - - def _load_callable(module_name, function_name): def _wrapper(apps, schema_editor): try: @@ -194,8 +190,7 @@ class Migration(migrations.Migration): field=models.BooleanField(default=True), ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" CREATE OR REPLACE FUNCTION update_report_can_delete(report RECORD) RETURNS VOID AS $$ DECLARE @@ -231,8 +226,7 @@ class Migration(migrations.Migration): AND can_delete <> r_can_delete; END; $$ LANGUAGE plpgsql; - """ - ), + """ ), migrations.RunPython( code=_load_callable( @@ -271,19 +265,15 @@ class Migration(migrations.Migration): field=models.TextField(db_default=""), ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" UPDATE reports_form99 SET text_code_2 = COALESCE(text_code, ''); ALTER TABLE reports_form99 ALTER COLUMN text_code SET DEFAULT ''; ALTER TABLE reports_form99 ALTER COLUMN text_code_2 SET NOT NULL; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP DEFAULT; ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP NOT NULL; - """ - ), + """, state_operations=[ migrations.AlterField( model_name="form99", @@ -775,8 +765,7 @@ class Migration(migrations.Migration): field=models.TextField(blank=True, null=True), ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" UPDATE reports_form3x f SET filing_frequency = CASE WHEN r.report_code IN ( @@ -794,12 +783,8 @@ class Migration(migrations.Migration): FROM reports_report as r WHERE r.form_3x_id = f.id AND filing_frequency IS NULL AND report_type_category IS NULL; - """ - ), - reverse_sql=_strip_sql( - """ - """ - ), + """, + reverse_sql="", ), migrations.AddField( model_name="form99", diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py index a78bdab22..2b8fe8c9c 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -28,10 +28,6 @@ # fecfiler.transactions.migrations.0024_scheduled_balance_at_close_and_more -def _strip_sql(sql): - return dedent(sql).strip() - - def _load_callable(module_name, function_name): def _wrapper(apps, schema_editor): try: @@ -187,234 +183,224 @@ class Migration(migrations.Migration): ), ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - """ - ), + """ + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT contribution_date + INTO schedule_date + FROM transactions_schedulea + WHERE id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT expenditure_date + INTO schedule_date + FROM transactions_scheduleb + WHERE id = txn.schedule_b_id; + END IF; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET aggregate = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - v_election_code TEXT; - v_candidate_office TEXT; - v_candidate_state TEXT; - v_candidate_district TEXT; - BEGIN - SELECT COALESCE(disbursement_date, dissemination_date) - INTO schedule_date FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT election_code - INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT candidate_office, candidate_state, candidate_district - INTO v_candidate_office, v_candidate_state, v_candidate_district - FROM contacts WHERE id = txn.contact_2_id; + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + v_election_code TEXT; + v_candidate_office TEXT; + v_candidate_state TEXT; + v_candidate_district TEXT; + BEGIN + SELECT COALESCE(disbursement_date, dissemination_date) + INTO schedule_date FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT election_code + INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT candidate_office, candidate_state, candidate_district + INTO v_candidate_office, v_candidate_state, v_candidate_district + FROM contacts WHERE id = txn.contact_2_id; - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - t.id, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + t.id, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum + FROM transactions_schedulee e + JOIN transaction_view__' || sql_committee_id || ' t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), + ) + AND EXTRACT(YEAR FROM t.date) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - BEGIN - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT transaction_id FROM transactions_transaction - WHERE id = $1 - ) || ''%%''; -- Match the loan_key with a transaction_id prefix - - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id - AND tt.loan_key LIKE ''%%LOAN''; - ' - USING txn.id; - END; - $$ LANGUAGE plpgsql; - """ - ), + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + BEGIN + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT transaction_id FROM transactions_transaction + WHERE id = $1 + ) || ''%%''; -- Match the loan_key with a transaction_id prefix + + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id + AND tt.loan_key LIKE ''%%LOAN''; + ' + USING txn.id; + END; + $$ LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - temp_table_name TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - temp_table_name := 'temp_' || - REPLACE(uuid_generate_v4()::TEXT, '-', '_'); - RAISE NOTICE 'TESTING TRIGGER'; - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; + """ + CREATE OR REPLACE FUNCTION calculate_aggregates() + RETURNS TRIGGER AS $$ + DECLARE + sql_committee_id TEXT; + temp_table_name TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + temp_table_name := 'temp_' || + REPLACE(uuid_generate_v4()::TEXT, '-', '_'); + RAISE NOTICE 'TESTING TRIGGER'; + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates( + NEW, sql_committee_id, temp_table_name || 'NEW'); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id THEN PERFORM calculate_entity_aggregates( - NEW, sql_committee_id, temp_table_name || 'NEW'); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates( - OLD, sql_committee_id, temp_table_name || 'OLD'); - END IF; + OLD, sql_committee_id, temp_table_name || 'OLD'); + END IF; - ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL - THEN - PERFORM calculate_loan_payment_to_date( - NEW, sql_committee_id, temp_table_name || 'NEW'); + ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL + THEN + PERFORM calculate_loan_payment_to_date( + NEW, sql_committee_id, temp_table_name || 'NEW'); - ELSIF NEW.schedule_e_id IS NOT NULL + ELSIF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id, temp_table_name || 'NEW'); + IF TG_OP = 'UPDATE' THEN PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id, temp_table_name || 'NEW'); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id, temp_table_name || 'OLD'); - END IF; + OLD, sql_committee_id, temp_table_name || 'OLD'); END IF; + END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; - CREATE TRIGGER calculate_aggregates_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION calculate_aggregates(); - """ - ), + CREATE TRIGGER calculate_aggregates_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION calculate_aggregates(); + """ ), migrations.RunPython( code=_load_callable( @@ -424,694 +410,320 @@ class Migration(migrations.Migration): ), ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION get_temp_tablename() - RETURNS TEXT AS $$ - BEGIN - RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); - END; - $$ LANGUAGE plpgsql; - """ - ), + """ + CREATE OR REPLACE FUNCTION get_temp_tablename() + RETURNS TEXT AS $$ + BEGIN + RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); + END; + $$ LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, + sql_committee_id TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + temp_table_name TEXT; + BEGIN + temp_table_name := get_temp_tablename(); + IF txn.schedule_a_id IS NOT NULL THEN + SELECT contribution_date + INTO schedule_date + FROM transactions_schedulea + WHERE id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT expenditure_date + INTO schedule_date + FROM transactions_scheduleb + WHERE id = txn.schedule_b_id; + END IF; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET aggregate = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, + sql_committee_id TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + v_election_code TEXT; + v_candidate_office TEXT; + v_candidate_state TEXT; + v_candidate_district TEXT; + temp_table_name TEXT; + BEGIN + temp_table_name := get_temp_tablename(); + SELECT COALESCE(disbursement_date, dissemination_date) + INTO schedule_date FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT election_code + INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT candidate_office, candidate_state, candidate_district + INTO v_candidate_office, v_candidate_state, v_candidate_district + FROM contacts WHERE id = txn.contact_2_id; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + t.id, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum + FROM transactions_schedulee e + JOIN transaction_view__' || sql_committee_id || ' t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM t.date) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, + sql_committee_id TEXT + ) + RETURNS VOID AS $$ + DECLARE + temp_table_name TEXT; + BEGIN + temp_table_name := get_temp_tablename(); + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT + CASE + WHEN loan_id IS NULL THEN transaction_id + ELSE ( + SELECT transaction_id + FROM transactions_transaction + WHERE id = t.loan_id + ) + END + FROM transactions_transaction t + WHERE id = $1 + ) || ''%%''; -- Match the loan_key with a transaction_id prefix + + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id + AND tt.loan_key LIKE ''%%LOAN''; + ' + USING txn.id; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_aggregates() + RETURNS TRIGGER AS $$ + DECLARE + sql_committee_id TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates(NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates(OLD, sql_committee_id); + END IF; + END IF; + + IF NEW.schedule_c_id IS NOT NULL + OR NEW.schedule_c1_id IS NOT NULL + OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' + THEN + PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); + END IF; + + IF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id); END IF; + END IF; - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + -- Drop prior versions of these functions + DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT); + DROP FUNCTION calculate_calendar_ytd_per_election_office( + RECORD, TEXT, TEXT + ); + DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT); + """ + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.transactions.migrations." + "0009_update_calculate_loan_payment_to_date", + "update_existing_rows", + ), + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, sql_committee_id text + ) + RETURNS VOID AS $$ + DECLARE + schedule_date date; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT + contribution_date INTO schedule_date + FROM + transactions_schedulea + WHERE + id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT + expenditure_date INTO schedule_date + FROM + transactions_scheduleb + WHERE + id = txn.schedule_b_id; + END IF; + + EXECUTE ' + UPDATE transactions_transaction AS t + SET aggregate = tc.new_sum + FROM ( SELECT id, + aggregate, + date, SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum + AS new_sum FROM transaction_view__' || sql_committee_id || ' WHERE contact_1_id = $1 AND EXTRACT(YEAR FROM date) = $2 AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; + AND force_unaggregated IS NOT TRUE + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $4 + OR ( + tc.date = $4 + AND t.created >= $5 + ) + ); ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - v_election_code TEXT; - v_candidate_office TEXT; - v_candidate_state TEXT; - v_candidate_district TEXT; - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - SELECT COALESCE(disbursement_date, dissemination_date) - INTO schedule_date FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT election_code - INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT candidate_office, candidate_state, candidate_district - INTO v_candidate_office, v_candidate_state, v_candidate_district - FROM contacts WHERE id = txn.contact_2_id; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - t.id, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT - CASE - WHEN loan_id IS NULL THEN transaction_id - ELSE ( - SELECT transaction_id - FROM transactions_transaction - WHERE id = t.loan_id - ) - END - FROM transactions_transaction t - WHERE id = $1 - ) || ''%%''; -- Match the loan_key with a transaction_id prefix - - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id - AND tt.loan_key LIKE ''%%LOAN''; - ' - USING txn.id; - END; - $$ LANGUAGE plpgsql; - """ - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - -- Drop prior versions of these functions - DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT); - DROP FUNCTION calculate_calendar_ytd_per_election_office( - RECORD, TEXT, TEXT - ); - DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT); - """ - ), - ), - migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0009_update_calculate_loan_payment_to_date", - "update_existing_rows", - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT - contribution_date INTO schedule_date - FROM - transactions_schedulea - WHERE - id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT - expenditure_date INTO schedule_date - FROM - transactions_scheduleb - WHERE - id = txn.schedule_b_id; - END IF; - - EXECUTE ' - UPDATE transactions_transaction AS t - SET aggregate = tc.new_sum - FROM ( - SELECT - id, - aggregate, - date, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $4 - OR ( - tc.date = $4 - AND t.created >= $5 - ) - ); - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) - INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - t.date, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT - CASE - WHEN loan_id IS NULL THEN transaction_id - ELSE ( - SELECT transaction_id - FROM transactions_transaction - WHERE id = t.loan_id - ) - END - FROM transactions_transaction t - WHERE id = $1 - ) || ''%%'' -- Match the loan_key with a transaction_id prefix - ) AS tc - WHERE t.id = tc.id - AND tc.loan_key LIKE ''%%LOAN'' - ; - ' - USING txn.id; - END; - $$ - LANGUAGE plpgsql; - """ - ), - ), - migrations.AddField( - model_name="transaction", - name="blocking_reports", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.UUIDField(), default=[], size=None - ), - ), - migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "create_trigger_function", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "drop_trigger_function", - ), - ), - migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "create_trigger", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "drop_trigger", - ), - ), - migrations.AlterField( - model_name="transaction", - name="blocking_reports", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.UUIDField(), default=list, size=None - ), - ), - migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0012_alter_transactions_blocking_reports", - "update_blocking_reports_default", - ), - ), - migrations.AddField( - model_name="transaction", - name="itemized", - field=models.BooleanField(db_default=True), - ), - migrations.CreateModel( - name="OverTwoHundredTypesScheduleA", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ("type", models.TextField()), - ], - options={ - "db_table": "over_two_hundred_types_schedulea", - "indexes": [ - models.Index(fields=["type"], name="over_two_hu_type_2c8314_idx") - ], - }, - ), - migrations.CreateModel( - name="OverTwoHundredTypesScheduleB", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - ("type", models.TextField()), - ], - options={ - "db_table": "over_two_hundred_types_scheduleb", - "indexes": [ - models.Index(fields=["type"], name="over_two_hu_type_411a44_idx") - ], - }, - ), - migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "populate_over_two_hundred_types", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "drop_over_two_hundred_types", - ), - ), - migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "create_itemized_triggers_and_functions", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "drop_itemized_triggers_and_functions", - ), - ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT - contribution_date INTO schedule_date - FROM - transactions_schedulea - WHERE - id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT - expenditure_date INTO schedule_date - FROM - transactions_scheduleb - WHERE - id = txn.schedule_b_id; - END IF; - - EXECUTE ' - UPDATE transactions_transaction AS t - SET aggregate = tc.new_sum - FROM ( - SELECT - t.id, - COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - ) AS date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id) - ) OVER ( - ORDER BY - COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - ), - t.created - ) AS new_sum - FROM transactions_transaction t - LEFT JOIN transactions_schedulea AS sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb AS sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec AS sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 AS sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee AS se - ON t.schedule_e_id = se.id - LEFT JOIN transactions_scheduled AS sd - ON t.schedule_d_id = sd.id - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - )) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE - AND deleted IS NULL - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $4 - OR ( - tc.date = $4 - AND t.created >= $5 - ) - ); - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ + ) + RETURNS VOID + AS $$ DECLARE schedule_date date; v_election_code text; @@ -1143,30 +755,11 @@ class Migration(migrations.Migration): FROM ( SELECT t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum + t.date, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum FROM transactions_schedulee e - JOIN transactions_transaction t + JOIN transaction_view__' || sql_committee_id || ' t ON e.id = t.schedule_e_id JOIN contacts c ON t.contact_2_id = c.id @@ -1187,10 +780,7 @@ class Migration(migrations.Migration): AND $4 = '''' ) ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 + AND EXTRACT(YEAR FROM t.date) = $5 AND aggregation_group = $6 AND force_unaggregated IS NOT TRUE ) AS tc @@ -1212,235 +802,708 @@ class Migration(migrations.Migration): txn.aggregation_group, schedule_date, txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), + END; + $$ + LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_effective_amount( - transaction_type_identifier TEXT, - amount NUMERIC, - schedule_c_id UUID - ) - RETURNS NUMERIC + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, sql_committee_id text + ) + RETURNS VOID AS $$ - DECLARE - effective_amount NUMERIC; - BEGIN - -- Case 1: transaction type is a refund - IF transaction_type_identifier IN ( - 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', - 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', - 'REFUND_PARTY_CONTRIBUTION', - 'REFUND_PARTY_CONTRIBUTION_VOID', - 'REFUND_PAC_CONTRIBUTION', - 'REFUND_PAC_CONTRIBUTION_VOID', - 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'REFUND_UNREGISTERED_CONTRIBUTION', - 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', - 'REFUND_INDIVIDUAL_CONTRIBUTION', - 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' - ) THEN - effective_amount := amount * -1; - - -- Case 2: schedule_c exists (return NULL) - ELSIF schedule_c_id IS NOT NULL THEN - effective_amount := NULL; - - -- Default case: return the original amount - ELSE - effective_amount := amount; - END IF; - - RETURN effective_amount; - END; - $$ - LANGUAGE plpgsql; - """ + BEGIN + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT + CASE + WHEN loan_id IS NULL THEN transaction_id + ELSE ( + SELECT transaction_id + FROM transactions_transaction + WHERE id = t.loan_id + ) + END + FROM transactions_transaction t + WHERE id = $1 + ) || ''%%'' -- Match the loan_key with a transaction_id prefix + ) AS tc + WHERE t.id = tc.id + AND tc.loan_key LIKE ''%%LOAN'' + ; + ' + USING txn.id; + END; + $$ + LANGUAGE plpgsql; + """ + ), + migrations.AddField( + model_name="transaction", + name="blocking_reports", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), default=[], size=None ), ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_is_loan( - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID - ) - RETURNS TEXT - AS $$ - DECLARE - loan_key TEXT; - BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - loan_key := 'F'; - - ELSIF schedule_c_id IS NOT NULL THEN - loan_key := 'T'; - - ELSE - loan_key := 'F'; - END IF; - - RETURN loan_key; - END; - $$ - LANGUAGE plpgsql; - """ + migrations.RunPython( + code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "create_trigger_function", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "drop_trigger_function", + ), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "create_trigger", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations.0011_transaction_can_delete", + "drop_trigger", + ), + ), + migrations.AlterField( + model_name="transaction", + name="blocking_reports", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), default=list, size=None + ), + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.transactions.migrations." + "0012_alter_transactions_blocking_reports", + "update_blocking_reports_default", + ), + ), + migrations.AddField( + model_name="transaction", + name="itemized", + field=models.BooleanField(db_default=True), + ), + migrations.CreateModel( + name="OverTwoHundredTypesScheduleA", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ("type", models.TextField()), + ], + options={ + "db_table": "over_two_hundred_types_schedulea", + "indexes": [ + models.Index(fields=["type"], name="over_two_hu_type_2c8314_idx") + ], + }, + ), + migrations.CreateModel( + name="OverTwoHundredTypesScheduleB", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ("type", models.TextField()), + ], + options={ + "db_table": "over_two_hundred_types_scheduleb", + "indexes": [ + models.Index(fields=["type"], name="over_two_hu_type_411a44_idx") + ], + }, + ), + migrations.RunPython( + code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "populate_over_two_hundred_types", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "drop_over_two_hundred_types", ), ), - migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_original_loan_id( - transaction_id text, - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID, - schedule_b_id UUID - ) - RETURNS TEXT - AS $$ - DECLARE - loan_transaction_id text; - BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - SELECT t.transaction_id - INTO loan_transaction_id - FROM transactions_transaction t - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - WHERE t.id = loan; - - ELSIF schedule_c_id IS NOT NULL THEN - loan_transaction_id := transaction_id; - - ELSE - loan_transaction_id := NULL; - END IF; - - RETURN loan_transaction_id; - END; - $$ - LANGUAGE plpgsql; - """ + migrations.RunPython( + code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "create_itemized_triggers_and_functions", + ), + reverse_code=_load_callable( + "fecfiler.transactions.migrations." + "0013_transaction_itemized_and_associated_triggers", + "drop_itemized_triggers_and_functions", ), ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_loan_date( - trans_id TEXT, - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID, - schedule_b_id UUID - ) - RETURNS DATE - AS $$ - DECLARE - date DATE; - BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - SELECT sb.expenditure_date - INTO date + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, sql_committee_id text + ) + RETURNS VOID AS $$ + DECLARE + schedule_date date; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT + contribution_date INTO schedule_date + FROM + transactions_schedulea + WHERE + id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT + expenditure_date INTO schedule_date + FROM + transactions_scheduleb + WHERE + id = txn.schedule_b_id; + END IF; + + EXECUTE ' + UPDATE transactions_transaction AS t + SET aggregate = tc.new_sum + FROM ( + SELECT + t.id, + COALESCE( + sa.contribution_date, + sb.expenditure_date, + sc.loan_incurred_date, + se.disbursement_date, + se.dissemination_date + ) AS date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id) + ) OVER ( + ORDER BY + COALESCE( + sa.contribution_date, + sb.expenditure_date, + sc.loan_incurred_date, + se.disbursement_date, + se.dissemination_date + ), + t.created + ) AS new_sum FROM transactions_transaction t - LEFT JOIN transactions_scheduleb sb + LEFT JOIN transactions_schedulea AS sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb AS sb ON t.schedule_b_id = sb.id - WHERE t.transaction_id = trans_id; - - ELSIF schedule_c_id IS NOT NULL THEN - SELECT s.report_coverage_through_date INTO date - FROM transactions_schedulec s - WHERE s.id = schedule_c_id; - - ELSE - date := NULL; - END IF; - - RETURN date; - END; - $$ - LANGUAGE plpgsql; - """ - ), + LEFT JOIN transactions_schedulec AS sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 AS sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee AS se + ON t.schedule_e_id = se.id + LEFT JOIN transactions_scheduled AS sd + ON t.schedule_d_id = sd.id + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM COALESCE( + sa.contribution_date, + sb.expenditure_date, + sc.loan_incurred_date, + se.disbursement_date, + se.dissemination_date + )) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE + AND deleted IS NULL + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $4 + OR ( + tc.date = $4 + AND t.created >= $5 + ) + ); + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_amount( - schedule_a_contribution_amount NUMERIC, - schedule_b_expenditure_amount NUMERIC, - schedule_c_loan_amount NUMERIC, - schedule_c2_guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt UUID, -- Reference to another transaction - schedule_d_id UUID - ) - RETURNS NUMERIC - AS $$ - DECLARE - debt_incurred_amount NUMERIC; - BEGIN - IF debt IS NOT NULL THEN - SELECT sd.incurred_amount - INTO debt_incurred_amount - FROM transactions_transaction t - LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id - WHERE t.id = debt; - ELSE - debt_incurred_amount := NULL; - END IF; - - RETURN COALESCE( - schedule_a_contribution_amount, - schedule_b_expenditure_amount, - schedule_c_loan_amount, - schedule_c2_guaranteed_amount, - schedule_e_expenditure_amount, - debt_incurred_amount, - ( - SELECT incurred_amount - FROM transactions_scheduled - WHERE id = schedule_d_id + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, sql_committee_id text + ) + RETURNS VOID + AS $$ + DECLARE + schedule_date date; + v_election_code text; + v_candidate_office text; + v_candidate_state text; + v_candidate_district text; + BEGIN + SELECT + COALESCE(disbursement_date, dissemination_date) + INTO schedule_date + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT election_code INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + + SELECT + candidate_office, + candidate_state, + candidate_district INTO v_candidate_office, + v_candidate_state, + v_candidate_district + FROM contacts + WHERE id = txn.contact_2_id; + EXECUTE ' + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tc.new_sum + FROM ( + SELECT + t.id, + Coalesce( + e.disbursement_date, + e.dissemination_date + ) as date, + SUM( + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + NULL, + NULL, + NULL, + NULL, + e.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) + ) OVER (ORDER BY Coalesce( + e.disbursement_date, + e.dissemination_date + ), t.created) AS new_sum + FROM transactions_schedulee e + JOIN transactions_transaction t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM Coalesce( + e.disbursement_date, + e.dissemination_date + )) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE + ) AS tc + WHERE t.id = tc.id + AND ( + tc.date > $7 + OR ( + tc.date = $7 + AND t.created >= $8 ) ); - END; - $$ - LANGUAGE plpgsql; - """ - ), + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group, + schedule_date, + txn.created; + END; + $$ + LANGUAGE plpgsql; + """ ), migrations.RunSQL( - sql=_strip_sql( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, sql_committee_id TEXT - ) - RETURNS VOID - AS $$ - DECLARE - pulled_forward_loans RECORD; - BEGIN + """ + CREATE OR REPLACE FUNCTION calculate_effective_amount( + transaction_type_identifier TEXT, + amount NUMERIC, + schedule_c_id UUID + ) + RETURNS NUMERIC + AS $$ + DECLARE + effective_amount NUMERIC; + BEGIN + -- Case 1: transaction type is a refund + IF transaction_type_identifier IN ( + 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', + 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', + 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', + 'REFUND_PARTY_CONTRIBUTION', + 'REFUND_PARTY_CONTRIBUTION_VOID', + 'REFUND_PAC_CONTRIBUTION', + 'REFUND_PAC_CONTRIBUTION_VOID', + 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', + 'REFUND_UNREGISTERED_CONTRIBUTION', + 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', + 'REFUND_INDIVIDUAL_CONTRIBUTION', + 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', + 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' + ) THEN + effective_amount := amount * -1; + + -- Case 2: schedule_c exists (return NULL) + ELSIF schedule_c_id IS NOT NULL THEN + effective_amount := NULL; + + -- Default case: return the original amount + ELSE + effective_amount := amount; + END IF; + + RETURN effective_amount; + END; + $$ + LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_is_loan( + loan UUID, + transaction_type_identifier TEXT, + schedule_c_id UUID + ) + RETURNS TEXT + AS $$ + DECLARE + loan_key TEXT; + BEGIN + IF loan IS NOT NULL AND transaction_type_identifier + IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') + THEN + loan_key := 'F'; + + ELSIF schedule_c_id IS NOT NULL THEN + loan_key := 'T'; + + ELSE + loan_key := 'F'; + END IF; + + RETURN loan_key; + END; + $$ + LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_original_loan_id( + transaction_id text, + loan UUID, + transaction_type_identifier TEXT, + schedule_c_id UUID, + schedule_b_id UUID + ) + RETURNS TEXT + AS $$ + DECLARE + loan_transaction_id text; + BEGIN + IF loan IS NOT NULL AND transaction_type_identifier + IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') + THEN + SELECT t.transaction_id + INTO loan_transaction_id + FROM transactions_transaction t + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + WHERE t.id = loan; + + ELSIF schedule_c_id IS NOT NULL THEN + loan_transaction_id := transaction_id; + + ELSE + loan_transaction_id := NULL; + END IF; + + RETURN loan_transaction_id; + END; + $$ + LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_loan_date( + trans_id TEXT, + loan UUID, + transaction_type_identifier TEXT, + schedule_c_id UUID, + schedule_b_id UUID + ) + RETURNS DATE + AS $$ + DECLARE + date DATE; + BEGIN + IF loan IS NOT NULL AND transaction_type_identifier + IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') + THEN + SELECT sb.expenditure_date + INTO date + FROM transactions_transaction t + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + WHERE t.transaction_id = trans_id; + + ELSIF schedule_c_id IS NOT NULL THEN + SELECT s.report_coverage_through_date INTO date + FROM transactions_schedulec s + WHERE s.id = schedule_c_id; + + ELSE + date := NULL; + END IF; + + RETURN date; + END; + $$ + LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_amount( + schedule_a_contribution_amount NUMERIC, + schedule_b_expenditure_amount NUMERIC, + schedule_c_loan_amount NUMERIC, + schedule_c2_guaranteed_amount NUMERIC, + schedule_e_expenditure_amount NUMERIC, + debt UUID, -- Reference to another transaction + schedule_d_id UUID + ) + RETURNS NUMERIC + AS $$ + DECLARE + debt_incurred_amount NUMERIC; + BEGIN + IF debt IS NOT NULL THEN + SELECT sd.incurred_amount + INTO debt_incurred_amount + FROM transactions_transaction t + LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id + WHERE t.id = debt; + ELSE + debt_incurred_amount := NULL; + END IF; + + RETURN COALESCE( + schedule_a_contribution_amount, + schedule_b_expenditure_amount, + schedule_c_loan_amount, + schedule_c2_guaranteed_amount, + schedule_e_expenditure_amount, + debt_incurred_amount, + ( + SELECT incurred_amount + FROM transactions_scheduled + WHERE id = schedule_d_id + ) + ); + END; + $$ + LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, sql_committee_id TEXT + ) + RETURNS VOID + AS $$ + DECLARE + pulled_forward_loans RECORD; + BEGIN + EXECUTE ' + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tc.new_sum + FROM ( + SELECT + data.id, + data.original_loan_id, + data.is_loan, + SUM(data.effective_amount) OVER ( + PARTITION BY data.original_loan_id + ORDER BY data.date + ) AS new_sum + FROM ( + SELECT + t.id, + calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS date, + calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) AS original_loan_id, + calculate_is_loan( + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id + ) AS is_loan, + calculate_effective_amount( + t.transaction_type_identifier, + calculate_amount( + sa.contribution_amount, + sb.expenditure_amount, + sc.loan_amount, + sc2.guaranteed_amount, + se.expenditure_amount, + t.debt_id, + t.schedule_d_id + ), + t.schedule_c_id + ) AS effective_amount + FROM transactions_transaction t + LEFT JOIN transactions_schedulea sa + ON t.schedule_a_id = sa.id + LEFT JOIN transactions_scheduleb sb + ON t.schedule_b_id = sb.id + LEFT JOIN transactions_schedulec sc + ON t.schedule_c_id = sc.id + LEFT JOIN transactions_schedulec2 sc2 + ON t.schedule_c2_id = sc2.id + LEFT JOIN transactions_schedulee se + ON t.schedule_e_id = se.id + WHERE t.deleted IS NULL + ) AS data + WHERE data.original_loan_id = ( + SELECT calculate_original_loan_id( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + ( + SELECT loan_id + FROM transactions_transaction + WHERE id = $1 + ), + $1 + ) + ) + AND data.date <= ( + SELECT calculate_loan_date( + t.transaction_id, + t.loan_id, + t.transaction_type_identifier, + t.schedule_c_id, + t.schedule_b_id + ) + FROM transactions_transaction t + WHERE t.id = COALESCE( + ( + SELECT loan_id + FROM transactions_transaction + WHERE id = $1 + ), + $1 + ) + ) + ) AS tc + WHERE t.id = tc.id + AND tc.is_loan = ''T''; + ' + USING txn.id; + + -- Handle pulled-forward loans + FOR pulled_forward_loans IN + SELECT t.transaction_id + FROM transactions_transaction t + WHERE t.schedule_c_id IS NOT NULL + AND t.loan_id = txn.loan_id + LOOP + -- Recalculate loan_payment_to_date for each pulled-forward loan EXECUTE ' UPDATE transactions_transaction AS t SET loan_payment_to_date = tc.new_sum @@ -1501,128 +1564,17 @@ class Migration(migrations.Migration): ON t.schedule_e_id = se.id WHERE t.deleted IS NULL ) AS data - WHERE data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - ( - SELECT loan_id - FROM transactions_transaction - WHERE id = $1 - ), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - ( - SELECT loan_id - FROM transactions_transaction - WHERE id = $1 - ), - $1 - ) - ) + WHERE data.original_loan_id = $1 ) AS tc WHERE t.id = tc.id AND tc.is_loan = ''T''; ' - USING txn.id; - - -- Handle pulled-forward loans - FOR pulled_forward_loans IN - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = txn.loan_id - LOOP - -- Recalculate loan_payment_to_date for each pulled-forward loan - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se - ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE data.original_loan_id = $1 - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING pulled_forward_loans.transaction_id; - END LOOP; - END; - $$ - LANGUAGE plpgsql; - """ - ), + USING pulled_forward_loans.transaction_id; + END LOOP; + END; + $$ + LANGUAGE plpgsql; + """ ), migrations.RunPython( code=_load_callable( @@ -1781,8 +1733,7 @@ class Migration(migrations.Migration): ), ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( txn RECORD, sql_committee_id text ) @@ -1893,10 +1844,8 @@ class Migration(migrations.Migration): END; $$ LANGUAGE plpgsql; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( txn RECORD, sql_committee_id text ) @@ -2005,8 +1954,7 @@ class Migration(migrations.Migration): END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunPython( code=_load_callable( @@ -2033,8 +1981,7 @@ class Migration(migrations.Migration): reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( txn record, sql_committee_id text ) @@ -2150,8 +2097,7 @@ class Migration(migrations.Migration): END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.AddField( model_name="scheduled", @@ -2197,8 +2143,7 @@ class Migration(migrations.Migration): reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop all aggregate-related triggers from -- transactions_transaction table DROP TRIGGER IF EXISTS calculate_aggregates_trigger @@ -2210,10 +2155,8 @@ class Migration(migrations.Migration): ON transactions_transaction; DROP TRIGGER IF EXISTS before_transactions_transaction_trigger ON transactions_transaction; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate triggers for aggregate calculation CREATE TRIGGER before_transactions_transaction_trigger BEFORE INSERT OR UPDATE ON transactions_transaction @@ -2230,20 +2173,16 @@ class Migration(migrations.Migration): FOR EACH ROW WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION after_transactions_transaction(); - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the calculate_entity_aggregates function DROP FUNCTION IF EXISTS calculate_entity_aggregates( txn RECORD, sql_committee_id TEXT, temp_table_name TEXT ) CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate calculate_entity_aggregates function CREATE OR REPLACE FUNCTION calculate_entity_aggregates( txn RECORD, @@ -2291,20 +2230,16 @@ class Migration(migrations.Migration): txn.aggregation_group; END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the calculate_calendar_ytd_per_election_office function DROP FUNCTION IF EXISTS calculate_calendar_ytd_per_election_office( txn RECORD, sql_committee_id TEXT, temp_table_name TEXT ) CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate calculate_calendar_ytd_per_election_office function CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( txn RECORD, sql_committee_id text @@ -2414,21 +2349,17 @@ class Migration(migrations.Migration): txn.committee_account_id; END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the calculate_effective_amount function DROP FUNCTION IF EXISTS calculate_effective_amount( transaction_type_identifier TEXT, amount NUMERIC, schedule_c_id UUID ) CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate calculate_effective_amount function CREATE OR REPLACE FUNCTION calculate_effective_amount( transaction_type_identifier TEXT, @@ -2476,12 +2407,10 @@ class Migration(migrations.Migration): RETURN effective_amount; END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the calculate_amount function if it exists DROP FUNCTION IF EXISTS calculate_amount( contribution_amount NUMERIC, @@ -2492,10 +2421,8 @@ class Migration(migrations.Migration): debt_id UUID, schedule_d_id UUID ) CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate calculate_amount function CREATE OR REPLACE FUNCTION calculate_amount( schedule_a_contribution_amount NUMERIC, @@ -2533,20 +2460,16 @@ class Migration(migrations.Migration): ); END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the calculate_loan_payment_to_date function if it exists DROP FUNCTION IF EXISTS calculate_loan_payment_to_date( txn RECORD, sql_committee_id TEXT, temp_table_name TEXT ) CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate calculate_loan_payment_to_date function CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( txn record, sql_committee_id text @@ -2658,12 +2581,10 @@ class Migration(migrations.Migration): USING txn.id, txn.loan_id; END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the trigger handler functions DROP FUNCTION IF EXISTS after_transactions_transaction() CASCADE; @@ -2677,10 +2598,8 @@ class Migration(migrations.Migration): DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update() CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate trigger handler functions CREATE OR REPLACE FUNCTION before_transactions_transaction() RETURNS TRIGGER AS $$ @@ -2712,22 +2631,18 @@ class Migration(migrations.Migration): RETURN NEW; END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.RunSQL( - sql=_strip_sql( - """ + sql=""" -- Drop the main calculate_aggregates function that was -- called by triggers DROP FUNCTION IF EXISTS calculate_aggregates( old RECORD, new RECORD, tg_op TEXT ) CASCADE; DROP FUNCTION IF EXISTS calculate_aggregates() CASCADE; - """ - ), - reverse_sql=_strip_sql( - """ + """, + reverse_sql=""" -- Recreate calculate_aggregates function CREATE OR REPLACE FUNCTION calculate_aggregates( OLD RECORD, @@ -2777,8 +2692,7 @@ class Migration(migrations.Migration): RETURN NEW; END; $$ LANGUAGE plpgsql; - """ - ), + """, ), migrations.AlterField( model_name="transaction", From d824dac9d61506b80f3fa3cb43b1c77b23c4e9f5 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 15:03:39 -0400 Subject: [PATCH 12/52] FECFILE-2734: Inline functions from squashed migrations. --- ...hed_0007_alter_committeeaccount_members.py | 1 + ..._deleted_squashed_00019_form24_name_fix.py | 305 ++++- ...quashed_0026_alter_transaction_itemized.py | 1174 ++++++++++++++--- 3 files changed, 1238 insertions(+), 242 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py index 754f71ea4..b7553ccad 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -184,6 +184,7 @@ class Migration(migrations.Migration): ), migrations.RunPython( code=create_memberships, + reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.AddField( model_name="membership", diff --git a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py index 5bf953ba0..63fca1faf 100644 --- a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -3,33 +3,252 @@ import django.db.migrations.operations.special import django.db.models.deletion import uuid -from importlib import import_module -from textwrap import dedent -from django.db import migrations, models - - -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# fecfiler.reports.migrations.00018_form24_name -# fecfiler.reports.migrations.00019_form24_name_fix -# fecfiler.reports.migrations. -# 0008_remove_form1m_city_remove_form1m_committee_name_and_more -# fecfiler.reports.migrations.0009_report_can_delete -# fecfiler.reports.migrations.0010_report_can_unammend - - -def _load_callable(module_name, function_name): - def _wrapper(apps, schema_editor): - try: - fn = getattr(import_module(module_name), function_name) - except Exception: - return django.db.migrations.operations.special.RunPython.noop( - apps, schema_editor +import structlog +from django.db import connection, migrations, models + +logger = structlog.get_logger(__name__) + + +def _migrate_committee_data(apps, schema_editor): + Report = apps.get_model("reports", "Report") # noqa + Form24 = apps.get_model("reports", "Form24") # noqa + Form3x = apps.get_model("reports", "Form3X") # noqa + Form99 = apps.get_model("reports", "Form99") # noqa + Form1m = apps.get_model("reports", "Form1M") # noqa + + for form in Form24.objects.all(): + report = Report.objects.filter(form_24=form).first() + if report is not None: + report.street_1 = form.street_1 + report.street_2 = form.street_2 + report.city = form.city + report.state = form.state + report.zip = form.zip + report.save() + else: + logger.error(f"F24 Form has no corresponding report! {form}") + + for form in Form3x.objects.all(): + report = Report.objects.filter(form_3x=form).first() + if report is not None: + report.street_1 = form.street_1 + report.street_2 = form.street_2 + report.city = form.city + report.state = form.state + report.zip = form.zip + report.save() + else: + logger.error(f"F3X Form has no corresponding report! {form}") + + for form in Form99.objects.all(): + report = Report.objects.filter(form_99=form).first() + if report is not None: + report.committee_name = form.committee_name + report.street_1 = form.street_1 + report.street_2 = form.street_2 + report.city = form.city + report.state = form.state + report.zip = form.zip + report.save() + else: + logger.error(f"F99 Form has no corresponding report! {form}") + + for form in Form1m.objects.all(): + report = Report.objects.filter(form_1m=form).first() + if report is not None: + report.committee_name = form.committee_name + report.street_1 = form.street_1 + report.street_2 = form.street_2 + report.city = form.city + report.state = form.state + report.zip = form.zip + report.save() + else: + logger.error(f"F1M Form has no corresponding report! {form}") + + +def _create_can_delete_trigger(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION check_can_delete() + RETURNS TRIGGER AS $$ + BEGIN + PERFORM update_report_can_delete(NEW); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION check_can_delete_previous() + RETURNS TRIGGER AS $$ + DECLARE + associated_report RECORD; + BEGIN + FOR associated_report IN ( + SELECT * FROM reports_report + WHERE committee_account_id = OLD.committee_account_id + AND can_delete = false ) - return fn(apps, schema_editor) + LOOP + PERFORM update_report_can_delete(associated_report); + END LOOP; + + RETURN OLD; + END; + $$ LANGUAGE plpgsql; - return _wrapper + + CREATE OR REPLACE FUNCTION check_can_delete_transaction_update() + RETURNS TRIGGER AS $$ + DECLARE + associated_report RECORD; + BEGIN + SELECT * INTO associated_report FROM reports_report WHERE id IN ( + SELECT report_id FROM reports_reporttransaction + WHERE transaction_id = NEW.id LIMIT 1 + ); + PERFORM update_report_can_delete(associated_report); + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION check_can_delete_transaction_insert() + RETURNS TRIGGER AS $$ + DECLARE + associated_report RECORD; + BEGIN + FOR associated_report IN ( + SELECT r1.* + FROM reports_report r1 + JOIN reports_report r2 + ON r1.committee_account_id = r2.committee_account_id + AND EXTRACT(YEAR FROM r1.coverage_from_date) = ( + EXTRACT(YEAR FROM r2.coverage_from_date)) + WHERE r1.can_delete = true + AND r2.id = NEW.report_id + ) + LOOP + PERFORM update_report_can_delete(associated_report); + END LOOP; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER check_can_delete_report + AFTER INSERT OR UPDATE ON reports_report + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION check_can_delete(); + + CREATE TRIGGER check_can_delete_previous + AFTER DELETE ON reports_report + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION check_can_delete_previous(); + + CREATE TRIGGER check_can_delete_transaction_insert + AFTER INSERT ON reports_reporttransaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION check_can_delete_transaction_insert(); + """ + ) + + +def _reverse_can_delete_trigger(apps, schema_editor): + report_model = apps.get_model("reports", "Report") + transaction_model = apps.get_model("transactions", "Transaction") + + triggers = [ + "check_can_delete_report", + "check_can_delete_previous", + "check_can_delete_transaction_update", + "check_can_delete_transaction_insert", + ] + + with schema_editor.atomic(): + for trigger in triggers: + schema_editor.execute( + "DROP TRIGGER IF EXISTS %s ON %s", + (trigger, report_model._meta.db_table), + ) + schema_editor.execute( + "DROP TRIGGER IF EXISTS %s ON %s", + (trigger, transaction_model._meta.db_table), + ) + + +def _populate_can_delete(apps, schema_editor): + report_model = apps.get_model("reports", "Report") + for row in report_model.objects.all(): + row.can_delete = True + row.save() + + +def _create_can_unamend_trigger(apps, schema_editor): + schema_editor.execute( + """ + CREATE OR REPLACE FUNCTION update_can_unamend() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE reports_report + SET can_unamend = FALSE + WHERE id IN ( + SELECT report_id + FROM reports_reporttransaction + WHERE transaction_id = NEW.id + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER transaction_updated + AFTER UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION update_can_unamend(); + + CREATE OR REPLACE FUNCTION update_can_unamend_new_transaction() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE reports_report + SET can_unamend = FALSE + WHERE id IN ( + SELECT report_id + FROM reports_reporttransaction + WHERE id = NEW.id + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER transaction_created + AFTER INSERT ON reports_reporttransaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION update_can_unamend_new_transaction(); + """ + ) + + +def _update_form24_names_v1(apps, schema_editor): + form24 = apps.get_model("reports", "Form24") + form24_objects = list(form24.objects.all()) + for index, form in enumerate(form24_objects, start=1): + report_type = form.report_type_24_48 + form.name = f"{report_type}-HOUR Report: {index}" + form.save() + + +def _update_form24_names_v2(apps, schema_editor): + form24 = apps.get_model("reports", "Form24") + form24_objects = list(form24.objects.all()) + for form in form24_objects: + report_type = form.report_type_24_48 + form.name = f"{report_type}-HOUR: Report of Independent Expenditure" + form.save() class Migration(migrations.Migration): @@ -90,11 +309,7 @@ class Migration(migrations.Migration): field=models.TextField(blank=True, null=True), ), migrations.RunPython( - code=_load_callable( - "fecfiler.reports.migrations." - "0008_remove_form1m_city_remove_form1m_committee_name_and_more", - "migrate_committee_data", - ), + code=_migrate_committee_data, ), migrations.RemoveField( model_name="form1m", @@ -229,20 +444,11 @@ class Migration(migrations.Migration): """ ), migrations.RunPython( - code=_load_callable( - "fecfiler.reports.migrations.0009_report_can_delete", - "create_trigger", - ), - reverse_code=_load_callable( - "fecfiler.reports.migrations.0009_report_can_delete", - "reverse_code", - ), + code=_create_can_delete_trigger, + reverse_code=_reverse_can_delete_trigger, ), migrations.RunPython( - code=_load_callable( - "fecfiler.reports.migrations.0009_report_can_delete", - "populate_existing_rows", - ), + code=_populate_can_delete, ), migrations.AddField( model_name="report", @@ -250,10 +456,7 @@ class Migration(migrations.Migration): field=models.BooleanField(default=False), ), migrations.RunPython( - code=_load_callable( - "fecfiler.reports.migrations.0010_report_can_unammend", - "create_trigger", - ), + code=_create_can_unamend_trigger, ), migrations.RemoveField( model_name="form3x", @@ -802,10 +1005,7 @@ class Migration(migrations.Migration): field=models.TextField(null=True), ), migrations.RunPython( - code=_load_callable( - "fecfiler.reports.migrations.00018_form24_name", - "update_form24_names", - ), + code=_update_form24_names_v1, ), migrations.AlterField( model_name="form24", @@ -813,10 +1013,7 @@ class Migration(migrations.Migration): field=models.TextField(), ), migrations.RunPython( - code=_load_callable( - "fecfiler.reports.migrations.00019_form24_name_fix", - "update_form24_names", - ), + code=_update_form24_names_v2, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), ] diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py index 2b8fe8c9c..527ccad41 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -3,42 +3,959 @@ import django.contrib.postgres.fields import django.db.migrations.operations.special import django.db.models.deletion +import structlog import uuid -from django.db import migrations, models -from importlib import import_module -from textwrap import dedent - - -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# fecfiler.transactions.migrations.0004_report_transactions_link_table -# fecfiler.transactions.migrations.0005_schedulec_report_coverage_from_date_and_more -# fecfiler.transactions.migrations. -# 0006_independent_expenditure_memos_no_aggregation_group -# fecfiler.transactions.migrations. -# 0008_transaction__calendar_ytd_per_election_office_and_more -# fecfiler.transactions.migrations.0009_update_calculate_loan_payment_to_date -# fecfiler.transactions.migrations.0011_transaction_can_delete -# fecfiler.transactions.migrations.0012_alter_transactions_blocking_reports -# fecfiler.transactions.migrations.0013_transaction_itemized_and_associated_triggers -# fecfiler.transactions.migrations.0015_merge_transaction_triggers -# fecfiler.transactions.migrations.0020_trigger_save_on_transactions -# fecfiler.transactions.migrations.0022_schedule_f_aggregation -# fecfiler.transactions.migrations.0024_scheduled_balance_at_close_and_more - - -def _load_callable(module_name, function_name): - def _wrapper(apps, schema_editor): - try: - fn = getattr(import_module(module_name), function_name) - except Exception: - return django.db.migrations.operations.special.RunPython.noop( - apps, schema_editor +from django.db import connection, migrations, models +from django.db.models import F, Q +from fecfiler.transactions.aggregation import process_aggregation_for_debts +from fecfiler.transactions.schedule_a.managers import ( + over_two_hundred_types as schedule_a_over_two_hundred_types, +) +from fecfiler.transactions.schedule_b.managers import ( + over_two_hundred_types as schedule_b_over_two_hundred_types, +) +from fecfiler.transactions.utils_aggregation_queries import ( + filter_queryset_for_previous_transactions_in_aggregation, +) + + +logger = structlog.get_logger(__name__) + + +# Inlined from 0004_report_transactions_link_table +def add_link_table(apps, schema_editor): + transaction = apps.get_model("transactions", "Transaction") + + for t in transaction.objects.all(): + t.reports.add(t.report) + t.save() + + +# Inlined from 0005_schedulec_report_coverage_from_date_and_more +def set_coverage_date(apps, schema_editor): + transaction_model = apps.get_model("transactions", "Transaction") + + for transaction in transaction_model.objects.filter( + Q(schedule_c__isnull=False) | Q(schedule_d__isnull=False) + ): + for report in transaction.reports.all(): + if report.coverage_from_date and transaction.schedule_d: + transaction.schedule_d.report_coverage_from_date = ( + report.coverage_from_date + ) + transaction.schedule_d.save() + if report.coverage_through_date and transaction.schedule_c: + transaction.schedule_c.report_coverage_through_date = ( + report.coverage_through_date + ) + transaction.schedule_c.save() + transaction.save() + + +# Inlined from 0006_independent_expenditure_memos_no_aggregation_group +def set_aggregation_group_to_none_for_ie_memos(apps, schema_editor): + transaction_model = apps.get_model("transactions", "Transaction") + + for transaction in transaction_model.objects.filter( + Q( + transaction_type_identifier__in=[ + "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", + "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", + "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO", + ] + ) + ): + transaction.aggregation_group = None + transaction.save() + + +def reverse_removing_aggregation_group_for_ie_memos(apps, schema_editor): + transaction_model = apps.get_model("transactions", "Transaction") + + for transaction in transaction_model.objects.filter( + Q( + transaction_type_identifier__in=[ + "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", + "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", + "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO", + ] + ) + ): + transaction.aggregation_group = "INDEPENDENT_EXPENDITURE" + transaction.save() + + +# Inlined from 0008_transaction__calendar_ytd_per_election_office_and_more +def populate_existing_rows(apps, schema_editor): + transaction = apps.get_model("transactions", "Transaction") + for row in transaction.objects.all(): + row.aggregate = 0.0 + row.save() + + +# Inlined from 0009_update_calculate_loan_payment_to_date +def update_existing_rows(apps, schema_editor): + transaction = apps.get_model("transactions", "Transaction") + types = [ + "LOAN_RECEIVED_FROM_INDIVIDUAL", + "LOAN_RECEIVED_FROM BANK", + "LOAN_BY_COMMITTEE", + ] + for row in transaction.objects.filter(transaction_type_identifier__in=types): + row.save() + + +# Inlined from 0011_transaction_can_delete +def create_trigger_function(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION update_transactions_can_delete() RETURNS TRIGGER AS $$ + BEGIN + UPDATE transactions_transaction + SET blocking_reports = CASE + WHEN NEW.upload_submission_id IS NOT NULL + THEN array_append(blocking_reports, NEW.id) + ELSE array_remove(blocking_reports, NEW.id) + END + -- all transactions in the submitted report + WHERE id IN ( + SELECT transaction_id + FROM reports_reporttransaction + WHERE report_id = NEW.id + ) + -- all transactions that are reattributed in the submtited report + OR id IN ( + SELECT reatt_redes_id + FROM reports_reporttransaction + JOIN transactions_transaction tt + ON reports_reporttransaction.transaction_id = tt.id + WHERE report_id = NEW.id + ) + -- all loans that are carried forward in the submitted report + OR id IN ( + SELECT loan_id + FROM reports_reporttransaction + JOIN transactions_transaction tt + ON reports_reporttransaction.transaction_id = tt.id + WHERE report_id = NEW.id + ) + -- all repayments to loans that are carried forward in the submitted report + OR loan_id IN ( + SELECT loan_id + FROM reports_reporttransaction + JOIN transactions_transaction tt + ON reports_reporttransaction.transaction_id = tt.id + WHERE report_id = NEW.id AND tt.schedule_c_id IS NOT NULL + ) + -- all debts that are carried forward in the submitted report + OR id IN ( + SELECT debt_id + FROM reports_reporttransaction + JOIN transactions_transaction tt + ON reports_reporttransaction.transaction_id = tt.id + WHERE report_id = NEW.id + ) + -- all repayments to debts that are carried forward in the submitted report + OR debt_id IN ( + SELECT debt_id + FROM reports_reporttransaction + JOIN transactions_transaction tt + ON reports_reporttransaction.transaction_id = tt.id + WHERE report_id = NEW.id AND tt.schedule_d_id IS NOT NULL + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) + + +def drop_trigger_function(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute("DROP FUNCTION IF EXISTS update_transactions_can_delete();") + + +def create_trigger(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE TRIGGER report_status_update + AFTER UPDATE OF upload_submission_id ON reports_report + FOR EACH ROW + EXECUTE FUNCTION update_transactions_can_delete(); + """ + ) + + +def drop_trigger(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute("DROP TRIGGER IF EXISTS report_status_update ON reports_report;") + + +# Inlined from 0012_alter_transactions_blocking_reports +def update_blocking_reports_default(apps, schema_editor): + transaction = apps.get_model("transactions", "Transaction") + transaction._meta.get_field("blocking_reports").default = list + + +# Inlined from 0013_transaction_itemized_and_associated_triggers +def populate_over_two_hundred_types(apps, schema_editor): + OverTwoHundredTypesScheduleA = apps.get_model( # noqa: N806 + "transactions", "OverTwoHundredTypesScheduleA" + ) + OverTwoHundredTypesScheduleB = apps.get_model( # noqa: N806 + "transactions", "OverTwoHundredTypesScheduleB" + ) + scha_types_to_create = [ + OverTwoHundredTypesScheduleA(type=type_to_create) + for type_to_create in schedule_a_over_two_hundred_types + ] + OverTwoHundredTypesScheduleA.objects.bulk_create(scha_types_to_create) + schb_types_to_create = [ + OverTwoHundredTypesScheduleB(type=type_to_create) + for type_to_create in schedule_b_over_two_hundred_types + ] + OverTwoHundredTypesScheduleB.objects.bulk_create(schb_types_to_create) + + +def drop_over_two_hundred_types(apps, schema_editor): + print("this reverses migration automatically.") + + +def create_itemized_triggers_and_functions(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION before_transactions_transaction_insert_or_update() + RETURNS TRIGGER AS $$ + DECLARE + needs_itemized_set boolean; + itemization boolean; + BEGIN + needs_itemized_set := needs_itemized_set(OLD, NEW); + IF needs_itemized_set THEN + NEW.itemized := calculate_itemization(NEW); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION calculate_itemization( + txn RECORD + ) + RETURNS BOOLEAN AS $$ + DECLARE + itemized boolean; + BEGIN + itemized := TRUE; + IF txn.force_itemized IS NOT NULL THEN + itemized := txn.force_itemized; + ELSIF txn.aggregate < 0 THEN + itemized := TRUE; + ELSIF EXISTS ( + SELECT type from ( + SELECT type + FROM over_two_hundred_types_schedulea + UNION + SELECT type + FROM over_two_hundred_types_scheduleb + ) as scha_schb_types + WHERE type = txn.transaction_type_identifier + ) THEN + IF txn.aggregate > 200 THEN + itemized := TRUE; + ELSE + itemized := FALSE; + END IF; + END IF; + return itemized; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION after_transactions_transaction_insert_or_update() + RETURNS TRIGGER AS $$ + DECLARE + parent_and_grandparent_ids uuid[]; + children_and_grandchildren_ids uuid[]; + BEGIN + IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN + IF NEW.itemized is TRUE THEN + parent_and_grandparent_ids := + get_parent_grandparent_transaction_ids(NEW); + PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); + ELSE + children_and_grandchildren_ids := + get_children_and_grandchildren_transaction_ids(NEW); + PERFORM set_itemization_for_ids( + FALSE,children_and_grandchildren_ids + ); + END IF; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION needs_itemized_set( + OLD RECORD, + NEW RECORD + ) + RETURNS BOOLEAN AS $$ + BEGIN + return OLD IS NULL OR ( + OLD.force_itemized IS DISTINCT FROM NEW.force_itemized + OR OLD.aggregate IS DISTINCT FROM NEW.aggregate + ); + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION set_itemization_for_ids( + itemization boolean, + ids uuid[] + ) + RETURNS VOID AS $$ + BEGIN + IF cardinality(ids) > 0 THEN + UPDATE transactions_transaction + SET + itemized = itemization + WHERE id = ANY (ids); + END IF; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION get_parent_grandparent_transaction_ids( + txn RECORD + ) + RETURNS uuid[] AS $$ + DECLARE + ids uuid[]; + BEGIN + SELECT array( + SELECT id + FROM transactions_transaction + WHERE id IN ( + txn.parent_transaction_id, + ( + SELECT parent_transaction_id + FROM transactions_transaction + WHERE id = txn.parent_transaction_id + ) + ) + ) into ids; + RETURN ids; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION get_children_and_grandchildren_transaction_ids( + txn RECORD + ) + RETURNS uuid[] AS $$ + DECLARE + ids uuid[]; + BEGIN + SELECT array( + SELECT id + FROM transactions_transaction + WHERE parent_transaction_id = ANY ( + array_prepend(txn.id, + array( + SELECT id + FROM transactions_transaction + WHERE parent_transaction_id = txn.id + ) + ) + ) + ) into ids; + RETURN ids; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER before_transactions_transaction_insert_or_update_trigger + BEFORE INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION before_transactions_transaction_insert_or_update(); + + CREATE TRIGGER zafter_transactions_transaction_insert_or_update_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION after_transactions_transaction_insert_or_update(); + """ + ) + + +def drop_itemized_triggers_and_functions(apps, schema_editor): + schema_editor.execute( + """ + DROP TRIGGER + IF EXISTS zafter_transactions_transaction_insert_or_update_trigger + ON transactions_transaction; + + DROP TRIGGER + IF EXISTS before_transactions_transaction_insert_or_update_trigger + ON transactions_transaction; + + DROP FUNCTION IF EXISTS before_transactions_transaction_insert_or_update; + DROP FUNCTION IF EXISTS calculate_itemization; + DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update; + DROP FUNCTION IF EXISTS needs_itemized_set; + DROP FUNCTION IF EXISTS set_itemization_for_ids; + DROP FUNCTION IF EXISTS get_parent_grandparent_transaction_ids; + DROP FUNCTION IF EXISTS get_children_and_grandchildren_transaction_ids; + """ + ) + + +# Inlined from 0015_merge_transaction_triggers +def create_triggers(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE TRIGGER before_transactions_transaction_trigger + BEFORE INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION before_transactions_transaction(); + + CREATE TRIGGER after_transactions_transaction_infinite_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION after_transactions_transaction_infinite(); + + CREATE TRIGGER after_transactions_transaction_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION after_transactions_transaction(); + """ + ) + + +def reverse_create_triggers(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + DROP TRIGGER + IF EXISTS before_transactions_transaction_trigger + ON transactions_transaction; + + DROP TRIGGER + IF EXISTS after_transactions_transaction_infinite_trigger + ON transactions_transaction; + + DROP TRIGGER + IF EXISTS after_transactions_transaction_trigger + ON transactions_transaction; + """ + ) + + +def before_transactions_transaction(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION before_transactions_transaction() + RETURNS TRIGGER AS $$ + BEGIN + NEW := process_itemization(OLD, NEW); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) + + +def after_transactions_transaction(apps, schema_editor): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION after_transactions_transaction() + RETURNS TRIGGER AS $$ + BEGIN + IF TG_OP = 'UPDATE' + THEN + NEW := calculate_aggregates(OLD, NEW, TG_OP); + NEW := update_can_unamend(NEW); + ELSE + NEW := calculate_aggregates(OLD, NEW, TG_OP); + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite() + RETURNS TRIGGER AS $$ + BEGIN + NEW := handle_parent_itemization(OLD, NEW); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + """ + ) + + +def reverse_after_transactions_transaction(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + DROP FUNCTION IF EXISTS after_transactions_transaction + DROP FUNCTION IF EXISTS after_transactions_transaction_infinite + """ + ) + + +def drop_old_triggers(apps, schema_editor): + schema_editor.execute( + """ + DROP TRIGGER + IF EXISTS zafter_transactions_transaction_insert_or_update_trigger + ON transactions_transaction; + + DROP TRIGGER + IF EXISTS before_transactions_transaction_insert_or_update_trigger + ON transactions_transaction; + + DROP TRIGGER + IF EXISTS transaction_updated + ON transactions_transaction; + + DROP TRIGGER + IF EXISTS calculate_aggregates_trigger + ON transactions_transaction; + """ + ) + + +def reverse_drop_old_triggers(apps, schema_editor): + schema_editor.execute( + """ + CREATE TRIGGER zafter_transactions_transaction_insert_or_update_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION after_transactions_transaction_insert_or_update(); + + CREATE TRIGGER before_transactions_transaction_insert_or_update_trigger + BEFORE INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + EXECUTE FUNCTION before_transactions_transaction_insert_or_update(); + + CREATE TRIGGER transaction_updated + AFTER UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION update_can_unamend(); + + CREATE TRIGGER calculate_aggregates_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION calculate_aggregates(); + """ + ) + + +def drop_old_functions(apps, schema_editor): + schema_editor.execute( + """ + DROP FUNCTION IF EXISTS before_transactions_transaction_insert_or_update; + DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update; + DROP FUNCTION IF EXISTS calculate_aggregates; + DROP FUNCTION IF EXISTS update_can_unamend; + """ + ) + + +def reverse_drop_old_functions(apps, schema_editor): + schema_editor.execute( + """ + CREATE OR REPLACE FUNCTION before_transactions_transaction_insert_or_update() + RETURNS TRIGGER AS $$ + DECLARE + needs_itemized_set boolean; + itemization boolean; + BEGIN + needs_itemized_set := needs_itemized_set(OLD, NEW); + IF needs_itemized_set THEN + NEW.itemized := calculate_itemization(NEW); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + + CREATE OR REPLACE FUNCTION after_transactions_transaction_insert_or_update() + RETURNS TRIGGER AS $$ + DECLARE + parent_and_grandparent_ids uuid[]; + children_and_grandchildren_ids uuid[]; + BEGIN + IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN + IF NEW.itemized is TRUE THEN + parent_and_grandparent_ids := + get_parent_grandparent_transaction_ids(NEW); + PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); + ELSE + children_and_grandchildren_ids := + get_children_and_grandchildren_transaction_ids(NEW); + PERFORM set_itemization_for_ids( + FALSE,children_and_grandchildren_ids + ); + END IF; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION calculate_aggregates() + RETURNS TRIGGER AS $$ + DECLARE + sql_committee_id TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates(NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates(OLD, sql_committee_id); + END IF; + END IF; + + IF NEW.schedule_c_id IS NOT NULL + OR NEW.schedule_c1_id IS NOT NULL + OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' + THEN + PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); + END IF; + + IF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id); + END IF; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + + CREATE OR REPLACE FUNCTION update_can_unamend() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE reports_report + SET can_unamend = FALSE + WHERE id IN ( + SELECT report_id + FROM reports_reporttransaction + WHERE transaction_id = NEW.id + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) + + +def process_itemization(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION process_itemization( + OLD RECORD, + NEW RECORD + ) + RETURNS RECORD AS $$ + DECLARE + needs_itemized_set boolean; + itemization boolean; + BEGIN + needs_itemized_set := needs_itemized_set(OLD, NEW); + IF needs_itemized_set THEN + NEW.itemized := calculate_itemization(NEW); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) + + +def reverse_process_itemization(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + DROP FUNCTION IF EXISTS process_itemization + """ + ) + + +def handle_parent_itemization(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION handle_parent_itemization( + OLD RECORD, + NEW RECORD + ) + RETURNS RECORD AS $$ + DECLARE + parent_and_grandparent_ids uuid[]; + children_and_grandchildren_ids uuid[]; + BEGIN + IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN + IF NEW.itemized is TRUE THEN + parent_and_grandparent_ids := + get_parent_grandparent_transaction_ids(NEW); + PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); + ELSE + children_and_grandchildren_ids := + get_children_and_grandchildren_transaction_ids(NEW); + PERFORM set_itemization_for_ids( + FALSE,children_and_grandchildren_ids + ); + END IF; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) + + +def reverse_handle_parent_itemization(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + DROP FUNCTION IF EXISTS handle_parent_itemization + """ + ) + + +def calculate_aggregates(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION calculate_aggregates( + OLD RECORD, + NEW RECORD, + TG_OP TEXT + ) + RETURNS RECORD AS $$ + DECLARE + sql_committee_id TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates(NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates(OLD, sql_committee_id); + END IF; + END IF; + + IF NEW.schedule_c_id IS NOT NULL + OR NEW.schedule_c1_id IS NOT NULL + OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' + THEN + PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); + END IF; + + IF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id); + END IF; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) + + +def reverse_calculate_aggregates(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + DROP FUNCTION IF EXISTS calculate_aggregates + """ + ) + + +def update_can_unamend(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE OR REPLACE FUNCTION update_can_unamend( + NEW RECORD ) - return fn(apps, schema_editor) + RETURNS RECORD AS $$ + BEGIN + UPDATE reports_report + SET can_unamend = FALSE + WHERE id IN ( + SELECT report_id + FROM reports_reporttransaction + WHERE transaction_id = NEW.id + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + ) - return _wrapper + +def reverse_update_can_unamend(apps, schema): + with connection.cursor() as cursor: + cursor.execute( + """ + DROP FUNCTION IF EXISTS update_can_unamend + """ + ) + + +# Inlined from 0020_trigger_save_on_transactions +def trigger_save_on_transactions(apps, schema_editor): + transactions = apps.get_model("transactions", "transaction") + committees = apps.get_model("committee_accounts", "committeeaccount") + contacts = apps.get_model("contacts", "contact") + + # Update transactions for each committee + for committee in committees.objects.all(): + logger.info(f"Committee:{committee.committee_id}") + + # For each contact, update the first schedule A transaction + for contact in contacts.objects.filter(committee_account=committee): + logger.info(f"Contact: {contact.id}") + first_schedule_a = ( + transactions.objects.filter( + schedule_a__isnull=False, + contact_1=contact, + committee_account=committee, + ) + .order_by("schedule_a__contribution_date", "created") + .first() + ) + if first_schedule_a: + logger.info(f"Saving first Schedule A: {first_schedule_a.id}") + first_schedule_a.save() + + # Election Aggregates + elections = transactions.objects.filter( + schedule_e__isnull=False, + committee_account=committee, + ).values( + "contact_2__candidate_office", + "contact_2__candidate_state", + "contact_2__candidate_district", + "schedule_e__election_code", + ) + for election in elections: + logger.info("Finding first schedule E for election") + first_schedule_e = ( + transactions.objects.filter( + schedule_e__isnull=False, + contact_2__candidate_office=election["contact_2__candidate_office"], + contact_2__candidate_state=election["contact_2__candidate_state"], + contact_2__candidate_district=election[ + "contact_2__candidate_district" + ], + schedule_e__election_code=election["schedule_e__election_code"], + committee_account=committee, + ) + .order_by( + "schedule_e__disbursement_date", + "created", + ) + .first() + ) + if first_schedule_e: + logger.info(f"Saving first Schedule E: {first_schedule_e.id}") + first_schedule_e.save() + + +# Inlined from 0022_schedule_f_aggregation +def calculate_schedule_f_aggregates(apps, schema_editor): + CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") # noqa + Transaction = apps.get_model("transactions", "Transaction") # noqa + + for committee in CommitteeAccount.objects.all(): + schedule_f_transactions = ( + Transaction.objects.all() + .filter( + committee_account=committee, + schedule_f__isnull=False, + ) + .annotate( + date=F("schedule_f__expenditure_date"), + amount=F("schedule_f__expenditure_amount"), + ) + .order_by("date") + ) + + for trans in schedule_f_transactions: + previous_transactions = ( + filter_queryset_for_previous_transactions_in_aggregation( # noqa: E501 + schedule_f_transactions, + trans.date, + trans.aggregation_group, + trans.id, + None, + trans.contact_2.id, + None, + trans.schedule_f.general_election_year, + ) + ) + + previous_transaction = previous_transactions.first() + previous_aggregate = 0 + if previous_transaction: + previous_aggregate = ( + previous_transaction.schedule_f.aggregate_general_elec_expended + ) + + trans.schedule_f.aggregate_general_elec_expended = ( + trans.amount + previous_aggregate + ) + trans.schedule_f.save() + + +# Inlined from 0024_scheduled_balance_at_close_and_more +def run_aggregations_for_all_debts(apps, schema_editor): + transaction = apps.get_model("transactions", "Transaction") + all_root_debts = transaction.objects.filter( + schedule_d__isnull=False, debt__isnull=True + ) + for debt in all_root_debts: + process_aggregation_for_debts(debt) class Migration(migrations.Migration): @@ -97,10 +1014,7 @@ class Migration(migrations.Migration): ), ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0004_report_transactions_link_table", - "add_link_table", - ), + code=add_link_table, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RemoveField( @@ -137,24 +1051,12 @@ class Migration(migrations.Migration): ), ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0005_schedulec_report_coverage_from_date_and_more", - "set_coverage_date", - ), + code=set_coverage_date, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0006_independent_expenditure_memos_no_aggregation_group", - "set_aggregation_group_to_none_for_ie_memos", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations." - "0006_independent_expenditure_memos_no_aggregation_group", - "reverse_removing_aggregation_group_for_ie_memos", - ), + code=set_aggregation_group_to_none_for_ie_memos, + reverse_code=reverse_removing_aggregation_group_for_ie_memos, ), migrations.AddField( model_name="schedulee", @@ -403,11 +1305,7 @@ class Migration(migrations.Migration): """ ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0008_transaction__calendar_ytd_per_election_office_and_more", - "populate_existing_rows", - ), + code=populate_existing_rows, ), migrations.RunSQL( """ @@ -649,11 +1547,7 @@ class Migration(migrations.Migration): """ ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0009_update_calculate_loan_payment_to_date", - "update_existing_rows", - ), + code=update_existing_rows, ), migrations.RunSQL( """ @@ -856,24 +1750,12 @@ class Migration(migrations.Migration): ), ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "create_trigger_function", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "drop_trigger_function", - ), + code=create_trigger_function, + reverse_code=drop_trigger_function, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "create_trigger", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0011_transaction_can_delete", - "drop_trigger", - ), + code=create_trigger, + reverse_code=drop_trigger, ), migrations.AlterField( model_name="transaction", @@ -883,11 +1765,7 @@ class Migration(migrations.Migration): ), ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0012_alter_transactions_blocking_reports", - "update_blocking_reports_default", - ), + code=update_blocking_reports_default, ), migrations.AddField( model_name="transaction", @@ -939,28 +1817,12 @@ class Migration(migrations.Migration): }, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "populate_over_two_hundred_types", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "drop_over_two_hundred_types", - ), + code=populate_over_two_hundred_types, + reverse_code=drop_over_two_hundred_types, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "create_itemized_triggers_and_functions", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations." - "0013_transaction_itemized_and_associated_triggers", - "drop_itemized_triggers_and_functions", - ), + code=create_itemized_triggers_and_functions, + reverse_code=drop_itemized_triggers_and_functions, ), migrations.RunSQL( """ @@ -1577,94 +2439,40 @@ class Migration(migrations.Migration): """ ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "drop_old_triggers", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_drop_old_triggers", - ), + code=drop_old_triggers, + reverse_code=reverse_drop_old_triggers, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "drop_old_functions", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_drop_old_functions", - ), + code=drop_old_functions, + reverse_code=reverse_drop_old_functions, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "process_itemization", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_process_itemization", - ), + code=process_itemization, + reverse_code=reverse_process_itemization, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "handle_parent_itemization", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_handle_parent_itemization", - ), + code=handle_parent_itemization, + reverse_code=reverse_handle_parent_itemization, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "calculate_aggregates", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_calculate_aggregates", - ), + code=calculate_aggregates, + reverse_code=reverse_calculate_aggregates, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "update_can_unamend", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_update_can_unamend", - ), + code=update_can_unamend, + reverse_code=reverse_update_can_unamend, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "before_transactions_transaction", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "before_transactions_transaction", - ), + code=before_transactions_transaction, + reverse_code=before_transactions_transaction, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "after_transactions_transaction", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_after_transactions_transaction", - ), + code=after_transactions_transaction, + reverse_code=reverse_after_transactions_transaction, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "create_triggers", - ), - reverse_code=_load_callable( - "fecfiler.transactions.migrations.0015_merge_transaction_triggers", - "reverse_create_triggers", - ), + code=create_triggers, + reverse_code=reverse_create_triggers, ), migrations.CreateModel( name="ScheduleF", @@ -1957,10 +2765,7 @@ class Migration(migrations.Migration): """, ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0020_trigger_save_on_transactions", - "trigger_save_on_transactions", - ), + code=trigger_save_on_transactions, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.AlterField( @@ -1974,10 +2779,7 @@ class Migration(migrations.Migration): ), ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations.0022_schedule_f_aggregation", - "calculate_schedule_f_aggregates", - ), + code=calculate_schedule_f_aggregates, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunSQL( @@ -2135,11 +2937,7 @@ class Migration(migrations.Migration): ), ), migrations.RunPython( - code=_load_callable( - "fecfiler.transactions.migrations." - "0024_scheduled_balance_at_close_and_more", - "run_aggregations_for_all_debts", - ), + code=run_aggregations_for_all_debts, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.RunSQL( From dff1b67cf06e8cca5c475b373a3ff2feeb30bf4b Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 15:36:23 -0400 Subject: [PATCH 13/52] FECFILE-2734: Adding IgnoreMigrations() as needed with great reluctance. --- .../0001_squashed_0007_alter_committeeaccount_members.py | 2 ++ .../0005_remove_form1m_iii_candidate_district_and_more.py | 2 ++ ...emove_report_deleted_squashed_00019_form24_name_fix.py | 2 ++ .../0002_remove_schedulea_contributor_city_and_more.py | 2 ++ ...ter_parent_squashed_0026_alter_transaction_itemized.py | 2 ++ ...mtee_id_squashed_0007_user_security_consent_version.py | 8 ++------ ...ssion_task_completed_squashed_0003_polling_attempts.py | 8 ++------ 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py index b7553ccad..4973836cb 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -7,6 +7,7 @@ import uuid from django.conf import settings from django.db import migrations, models +from django_migration_linter import IgnoreMigration # Functions from the following migrations need manual copying. @@ -104,6 +105,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.CreateModel( name="CommitteeAccount", fields=[ diff --git a/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py index 6b6e81462..493bed56a 100644 --- a/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py +++ b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py @@ -1,6 +1,7 @@ # Generated by Django 4.2.7 on 2024-01-24 09:51 from django.db import migrations +from django_migration_linter import IgnoreMigration class Migration(migrations.Migration): @@ -10,6 +11,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.RemoveField( model_name='form1m', name='III_candidate_district', diff --git a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py index 63fca1faf..36ed1136d 100644 --- a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -5,6 +5,7 @@ import uuid import structlog from django.db import connection, migrations, models +from django_migration_linter import IgnoreMigration logger = structlog.get_logger(__name__) @@ -274,6 +275,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.RemoveField( model_name="report", name="deleted", diff --git a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py index b1f92a36d..efb36d049 100644 --- a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py +++ b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py @@ -1,6 +1,7 @@ # Generated by Django 4.2.7 on 2024-01-24 09:51 from django.db import migrations +from django_migration_linter import IgnoreMigration class Migration(migrations.Migration): @@ -10,6 +11,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.RemoveField( model_name='schedulea', name='contributor_city', diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py index 527ccad41..7aec3341a 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -17,6 +17,7 @@ from fecfiler.transactions.utils_aggregation_queries import ( filter_queryset_for_previous_transactions_in_aggregation, ) +from django_migration_linter import IgnoreMigration logger = structlog.get_logger(__name__) @@ -995,6 +996,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.AlterField( model_name="transaction", name="parent_transaction", diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py index e1dc05ffd..23d044d9d 100644 --- a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py +++ b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py @@ -4,12 +4,7 @@ import fecfiler.user.managers from django.db import migrations, models from django.db.models import Q - - -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# fecfiler.user.migrations.0006_remove_old_login_accounts +from django_migration_linter import IgnoreMigration def remove_old_login_accounts(apps, schema_editor): @@ -43,6 +38,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.RemoveField( model_name="user", name="cmtee_id", diff --git a/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py b/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py index 5da677458..e2ad18be6 100644 --- a/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py +++ b/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py @@ -3,12 +3,7 @@ import django.db.migrations.operations.special from django.db import migrations, models from django.db.models import F - - -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# fecfiler.web_services.migrations.0002_uploadsubmission_task_completed_and_more +from django_migration_linter import IgnoreMigration def set_default_task_completed_times(apps, schema_editor): @@ -31,6 +26,7 @@ class Migration(migrations.Migration): ] operations = [ + IgnoreMigration(), migrations.AddField( model_name="uploadsubmission", name="task_completed", From ebbce8b5e9143e757f7f1b4c76002425d9af0c60 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 15:56:24 -0400 Subject: [PATCH 14/52] FECFILE-2734: Removed the last old comment block. --- .../0001_squashed_0007_alter_committeeaccount_members.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py index 4973836cb..03b65978f 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -10,15 +10,6 @@ from django_migration_linter import IgnoreMigration -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# fecfiler.committee_accounts.migrations.0002_membership -# fecfiler.committee_accounts.migrations.0003_membership_pending_email_alter_membership_id_and_more -# fecfiler.committee_accounts.migrations.0004_remove_duplicate_memberships -# fecfiler.committee_accounts.migrations.0005_remove_pending_emails - - def create_memberships(apps, schema_editor): CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") User = apps.get_model("user", "User") From fadd65aca688f1faea94f79e8fa44f4e1acdb4ce Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 16:13:11 -0400 Subject: [PATCH 15/52] FECFILE-2734: Renamed some migrations for sanity. --- ...ix.py => 0002_initial_squashed_0003_memotext_text_prefix.py} | 0 ...007_remove_report_deleted_squashed_00019_form24_name_fix.py} | 0 ...03_alter_parent_squashed_0026_alter_transaction_itemized.py} | 2 +- ...submission_task_completed_squashed_0003_polling_attempts.py} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename django-backend/fecfiler/memo_text/migrations/{0002_0002_initial_squashed_0003_memotext_text_prefix.py => 0002_initial_squashed_0003_memotext_text_prefix.py} (100%) rename django-backend/fecfiler/reports/migrations/{0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py => 0007_remove_report_deleted_squashed_00019_form24_name_fix.py} (100%) rename django-backend/fecfiler/transactions/migrations/{0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py => 0003_alter_parent_squashed_0026_alter_transaction_itemized.py} (99%) rename django-backend/fecfiler/web_services/migrations/{0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py => 0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py} (100%) diff --git a/django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py b/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py similarity index 100% rename from django-backend/fecfiler/memo_text/migrations/0002_0002_initial_squashed_0003_memotext_text_prefix.py rename to django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py diff --git a/django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py similarity index 100% rename from django-backend/fecfiler/reports/migrations/0007_0007_remove_report_deleted_squashed_00019_form24_name_fix.py rename to django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py diff --git a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py similarity index 99% rename from django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py rename to django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py index 7aec3341a..24bea6716 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -991,7 +991,7 @@ class Migration(migrations.Migration): dependencies = [ ("contacts", "0001_initial"), ("reports", "0006_reporttransaction"), - ("reports", "0007_0007_remove_report_deleted_squashed_00019_form24_name_fix"), + ("reports", "0007_remove_report_deleted_squashed_00019_form24_name_fix"), ("transactions", "0002_remove_schedulea_contributor_city_and_more"), ] diff --git a/django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py b/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py similarity index 100% rename from django-backend/fecfiler/web_services/migrations/0002_0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py rename to django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py From bc5ec72911bcc16115f03237f3a62e067e068a69 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 18:34:06 -0400 Subject: [PATCH 16/52] FECFILE-2734: Refactoring away changes into initial state. --- ...hed_0007_alter_committeeaccount_members.py | 113 +---- .../reports/migrations/0001_initial.py | 103 +--- ..._form1m_iii_candidate_district_and_more.py | 193 +------ ..._deleted_squashed_00019_form24_name_fix.py | 267 ---------- .../transactions/migrations/0001_initial.py | 167 ------- ...ove_schedulea_contributor_city_and_more.py | 469 +----------------- ...quashed_0026_alter_transaction_itemized.py | 19 - .../fecfiler/user/migrations/0001_initial.py | 1 - ...shed_0007_user_security_consent_version.py | 13 +- .../web_services/migrations/0001_initial.py | 4 + ...ompleted_squashed_0003_polling_attempts.py | 24 +- 11 files changed, 37 insertions(+), 1336 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py index 03b65978f..c25d66720 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -3,11 +3,9 @@ import django.core.validators import django.db.migrations.operations.special import django.db.models.deletion -import django.utils.timezone import uuid from django.conf import settings from django.db import migrations, models -from django_migration_linter import IgnoreMigration def create_memberships(apps, schema_editor): @@ -96,7 +94,6 @@ class Migration(migrations.Migration): ] operations = [ - IgnoreMigration(), migrations.CreateModel( name="CommitteeAccount", fields=[ @@ -135,11 +132,12 @@ class Migration(migrations.Migration): fields=[ ( "id", - models.AutoField( - auto_created=True, + models.UUIDField( + default=uuid.uuid4, + editable=False, primary_key=True, serialize=False, - verbose_name="ID", + unique=True, ), ), ( @@ -147,14 +145,21 @@ class Migration(migrations.Migration): models.CharField( choices=[ ("COMMITTEE_ADMINISTRATOR", "Committee Administrator"), - ("REVIEWER", "Reviewer"), + ("MANAGER", "Manager"), ], max_length=25, ), ), + ( + "pending_email", + models.EmailField(blank=True, max_length=254, null=True), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), ( "committee_account", models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="committee_accounts.committeeaccount", ), @@ -162,6 +167,7 @@ class Migration(migrations.Migration): ( "user", models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, ), @@ -172,73 +178,15 @@ class Migration(migrations.Migration): model_name="committeeaccount", name="members", field=models.ManyToManyField( - through="committee_accounts.Membership", to=settings.AUTH_USER_MODEL + through="committee_accounts.Membership", + through_fields=("committee_account", "user"), + to=settings.AUTH_USER_MODEL, ), ), migrations.RunPython( code=create_memberships, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), - migrations.AddField( - model_name="membership", - name="pending_email", - field=models.EmailField(blank=True, max_length=254, null=True), - ), - migrations.AddField( - model_name="membership", - name="uuid", - field=models.UUIDField(default=uuid.uuid4, editable=False, serialize=False), - ), - migrations.RunPython( - code=generate_new_uuid, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RemoveField( - model_name="membership", - name="id", - ), - migrations.RenameField( - model_name="membership", - old_name="uuid", - new_name="id", - ), - migrations.AlterField( - model_name="membership", - name="id", - field=models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - unique=True, - ), - ), - migrations.AlterField( - model_name="membership", - name="user", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL, - ), - ), - migrations.AddField( - model_name="membership", - name="created", - field=models.DateTimeField( - auto_now_add=True, default=django.utils.timezone.now - ), - preserve_default=False, - ), - migrations.AddField( - model_name="membership", - name="updated", - field=models.DateTimeField(auto_now=True), - ), - migrations.RunPython( - code=django.db.migrations.operations.special.RunPython.noop, - reverse_code=delete_pending_memberships, - ), migrations.RunPython( code=delete_memberships_with_overlapping_emails, reverse_code=django.db.migrations.operations.special.RunPython.noop, @@ -247,33 +195,4 @@ class Migration(migrations.Migration): code=remove_pending_emails, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), - migrations.AlterField( - model_name="membership", - name="committee_account", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="committee_accounts.committeeaccount", - ), - ), - migrations.AlterField( - model_name="membership", - name="role", - field=models.CharField( - choices=[ - ("COMMITTEE_ADMINISTRATOR", "Committee Administrator"), - ("MANAGER", "Manager"), - ], - max_length=25, - ), - ), - migrations.AlterField( - model_name="committeeaccount", - name="members", - field=models.ManyToManyField( - through="committee_accounts.Membership", - through_fields=("committee_account", "user"), - to=settings.AUTH_USER_MODEL, - ), - ), ] diff --git a/django-backend/fecfiler/reports/migrations/0001_initial.py b/django-backend/fecfiler/reports/migrations/0001_initial.py index 41ca47370..5e43031fa 100644 --- a/django-backend/fecfiler/reports/migrations/0001_initial.py +++ b/django-backend/fecfiler/reports/migrations/0001_initial.py @@ -29,12 +29,6 @@ class Migration(migrations.Migration): unique=True, ), ), - ("committee_name", models.TextField(blank=True, null=True)), - ("street_1", models.TextField(blank=True, null=True)), - ("street_2", models.TextField(blank=True, null=True)), - ("city", models.TextField(blank=True, null=True)), - ("state", models.TextField(blank=True, null=True)), - ("zip", models.TextField(blank=True, null=True)), ( "committee_type", models.CharField(blank=True, max_length=1, null=True), @@ -43,75 +37,10 @@ class Migration(migrations.Migration): "affiliated_date_form_f1_filed", models.DateField(blank=True, null=True), ), - ( - "affiliated_committee_fec_id", - models.TextField(blank=True, null=True), - ), - ("affiliated_committee_name", models.TextField(blank=True, null=True)), - ("I_candidate_id_number", models.TextField(blank=True, null=True)), - ("I_candidate_last_name", models.TextField(blank=True, null=True)), - ("I_candidate_first_name", models.TextField(blank=True, null=True)), - ("I_candidate_middle_name", models.TextField(blank=True, null=True)), - ("I_candidate_prefix", models.TextField(blank=True, null=True)), - ("I_candidate_suffix", models.TextField(blank=True, null=True)), - ( - "I_candidate_office", - models.CharField(blank=True, max_length=1, null=True), - ), - ("I_candidate_state", models.TextField(blank=True, null=True)), - ("I_candidate_district", models.TextField(blank=True, null=True)), ("I_date_of_contribution", models.DateField(blank=True, null=True)), - ("II_candidate_id_number", models.TextField(blank=True, null=True)), - ("II_candidate_last_name", models.TextField(blank=True, null=True)), - ("II_candidate_first_name", models.TextField(blank=True, null=True)), - ("II_candidate_middle_name", models.TextField(blank=True, null=True)), - ("II_candidate_prefix", models.TextField(blank=True, null=True)), - ("II_candidate_suffix", models.TextField(blank=True, null=True)), - ( - "II_candidate_office", - models.CharField(blank=True, max_length=1, null=True), - ), - ("II_candidate_state", models.TextField(blank=True, null=True)), - ("II_candidate_district", models.TextField(blank=True, null=True)), ("II_date_of_contribution", models.DateField(blank=True, null=True)), - ("III_candidate_id_number", models.TextField(blank=True, null=True)), - ("III_candidate_last_name", models.TextField(blank=True, null=True)), - ("III_candidate_first_name", models.TextField(blank=True, null=True)), - ("III_candidate_middle_name", models.TextField(blank=True, null=True)), - ("III_candidate_prefix", models.TextField(blank=True, null=True)), - ("III_candidate_suffix", models.TextField(blank=True, null=True)), - ( - "III_candidate_office", - models.CharField(blank=True, max_length=1, null=True), - ), - ("III_candidate_state", models.TextField(blank=True, null=True)), - ("III_candidate_district", models.TextField(blank=True, null=True)), ("III_date_of_contribution", models.DateField(blank=True, null=True)), - ("IV_candidate_id_number", models.TextField(blank=True, null=True)), - ("IV_candidate_last_name", models.TextField(blank=True, null=True)), - ("IV_candidate_first_name", models.TextField(blank=True, null=True)), - ("IV_candidate_middle_name", models.TextField(blank=True, null=True)), - ("IV_candidate_prefix", models.TextField(blank=True, null=True)), - ("IV_candidate_suffix", models.TextField(blank=True, null=True)), - ( - "IV_candidate_office", - models.CharField(blank=True, max_length=1, null=True), - ), - ("IV_candidate_state", models.TextField(blank=True, null=True)), - ("IV_candidate_district", models.TextField(blank=True, null=True)), ("IV_date_of_contribution", models.DateField(blank=True, null=True)), - ("V_candidate_id_number", models.TextField(blank=True, null=True)), - ("V_candidate_last_name", models.TextField(blank=True, null=True)), - ("V_candidate_first_name", models.TextField(blank=True, null=True)), - ("V_candidate_middle_name", models.TextField(blank=True, null=True)), - ("V_candidate_prefix", models.TextField(blank=True, null=True)), - ("V_candidate_suffix", models.TextField(blank=True, null=True)), - ( - "V_candidate_office", - models.CharField(blank=True, max_length=1, null=True), - ), - ("V_candidate_state", models.TextField(blank=True, null=True)), - ("V_candidate_district", models.TextField(blank=True, null=True)), ("V_date_of_contribution", models.DateField(blank=True, null=True)), ], ), @@ -130,11 +59,7 @@ class Migration(migrations.Migration): ), ("report_type_24_48", models.TextField(blank=True, null=True)), ("original_amendment_date", models.DateField(blank=True, null=True)), - ("street_1", models.TextField(blank=True, null=True)), - ("street_2", models.TextField(blank=True, null=True)), - ("city", models.TextField(blank=True, null=True)), - ("state", models.TextField(blank=True, null=True)), - ("zip", models.TextField(blank=True, null=True)), + ("name", models.TextField()), ], ), migrations.CreateModel( @@ -154,11 +79,6 @@ class Migration(migrations.Migration): "change_of_address", models.BooleanField(blank=True, default=False, null=True), ), - ("street_1", models.TextField(blank=True, null=True)), - ("street_2", models.TextField(blank=True, null=True)), - ("city", models.TextField(blank=True, null=True)), - ("state", models.TextField(blank=True, null=True)), - ("zip", models.TextField(blank=True, null=True)), ("election_code", models.TextField(blank=True, null=True)), ("date_of_election", models.DateField(blank=True, null=True)), ("state_of_election", models.TextField(blank=True, null=True)), @@ -166,7 +86,6 @@ class Migration(migrations.Migration): "qualified_committee", models.BooleanField(blank=True, default=False, null=True), ), - ("cash_on_hand_date", models.DateField(blank=True, null=True)), ( "L6b_cash_on_hand_beginning_period", models.DecimalField( @@ -783,20 +702,16 @@ class Migration(migrations.Migration): unique=True, ), ), - ("committee_name", models.TextField(blank=True, null=True)), - ("street_1", models.TextField(blank=True, null=True)), - ("street_2", models.TextField(blank=True, null=True)), - ("city", models.TextField(blank=True, null=True)), - ("state", models.TextField(blank=True, null=True)), - ("zip", models.TextField(blank=True, null=True)), - ("text_code", models.TextField(blank=True, null=True)), + ( + "text_code", + models.TextField(blank=False, db_default="", default=""), + ), ("message_text", models.TextField(blank=True, null=True)), ], ), migrations.CreateModel( name="Report", fields=[ - ("deleted", models.DateTimeField(blank=True, null=True)), ( "id", models.UUIDField( @@ -813,6 +728,12 @@ class Migration(migrations.Migration): ("report_code", models.TextField(blank=True, null=True)), ("coverage_from_date", models.DateField(blank=True, null=True)), ("coverage_through_date", models.DateField(blank=True, null=True)), + ("committee_name", models.TextField(blank=True, null=True)), + ("street_1", models.TextField(blank=True, null=True)), + ("street_2", models.TextField(blank=True, null=True)), + ("city", models.TextField(blank=True, null=True)), + ("state", models.TextField(blank=True, null=True)), + ("zip", models.TextField(blank=True, null=True)), ("treasurer_last_name", models.TextField(blank=True, null=True)), ("treasurer_first_name", models.TextField(blank=True, null=True)), ("treasurer_middle_name", models.TextField(blank=True, null=True)), @@ -839,6 +760,8 @@ class Migration(migrations.Migration): blank=True, default=None, max_length=44, null=True ), ), + ("can_delete", models.BooleanField(default=True)), + ("can_unamend", models.BooleanField(default=False)), ("created", models.DateTimeField(auto_now_add=True)), ("updated", models.DateTimeField(auto_now=True)), ( diff --git a/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py index 493bed56a..8c35add58 100644 --- a/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py +++ b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py @@ -1,7 +1,6 @@ # Generated by Django 4.2.7 on 2024-01-24 09:51 from django.db import migrations -from django_migration_linter import IgnoreMigration class Migration(migrations.Migration): @@ -10,194 +9,4 @@ class Migration(migrations.Migration): ('reports', '0004_form1m_date_committee_met_requirements_and_more'), ] - operations = [ - IgnoreMigration(), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_district', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_first_name', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_id_number', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_last_name', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_middle_name', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_office', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_prefix', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_state', - ), - migrations.RemoveField( - model_name='form1m', - name='III_candidate_suffix', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_district', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_first_name', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_id_number', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_last_name', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_middle_name', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_office', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_prefix', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_state', - ), - migrations.RemoveField( - model_name='form1m', - name='II_candidate_suffix', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_district', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_first_name', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_id_number', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_last_name', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_middle_name', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_office', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_prefix', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_state', - ), - migrations.RemoveField( - model_name='form1m', - name='IV_candidate_suffix', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_district', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_first_name', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_id_number', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_last_name', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_middle_name', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_office', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_prefix', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_state', - ), - migrations.RemoveField( - model_name='form1m', - name='I_candidate_suffix', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_district', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_first_name', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_id_number', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_last_name', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_middle_name', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_office', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_prefix', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_state', - ), - migrations.RemoveField( - model_name='form1m', - name='V_candidate_suffix', - ), - migrations.RemoveField( - model_name='form1m', - name='affiliated_committee_fec_id', - ), - migrations.RemoveField( - model_name='form1m', - name='affiliated_committee_name', - ), - ] + operations = [] diff --git a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py index 36ed1136d..332daee34 100644 --- a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -1,73 +1,13 @@ # Generated by Django 5.2.11 on 2026-03-07 03:24 -import django.db.migrations.operations.special import django.db.models.deletion import uuid import structlog from django.db import connection, migrations, models -from django_migration_linter import IgnoreMigration logger = structlog.get_logger(__name__) -def _migrate_committee_data(apps, schema_editor): - Report = apps.get_model("reports", "Report") # noqa - Form24 = apps.get_model("reports", "Form24") # noqa - Form3x = apps.get_model("reports", "Form3X") # noqa - Form99 = apps.get_model("reports", "Form99") # noqa - Form1m = apps.get_model("reports", "Form1M") # noqa - - for form in Form24.objects.all(): - report = Report.objects.filter(form_24=form).first() - if report is not None: - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F24 Form has no corresponding report! {form}") - - for form in Form3x.objects.all(): - report = Report.objects.filter(form_3x=form).first() - if report is not None: - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F3X Form has no corresponding report! {form}") - - for form in Form99.objects.all(): - report = Report.objects.filter(form_99=form).first() - if report is not None: - report.committee_name = form.committee_name - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F99 Form has no corresponding report! {form}") - - for form in Form1m.objects.all(): - report = Report.objects.filter(form_1m=form).first() - if report is not None: - report.committee_name = form.committee_name - report.street_1 = form.street_1 - report.street_2 = form.street_2 - report.city = form.city - report.state = form.state - report.zip = form.zip - report.save() - else: - logger.error(f"F1M Form has no corresponding report! {form}") - - def _create_can_delete_trigger(apps, schema_editor): with connection.cursor() as cursor: cursor.execute( @@ -234,24 +174,6 @@ def _create_can_unamend_trigger(apps, schema_editor): ) -def _update_form24_names_v1(apps, schema_editor): - form24 = apps.get_model("reports", "Form24") - form24_objects = list(form24.objects.all()) - for index, form in enumerate(form24_objects, start=1): - report_type = form.report_type_24_48 - form.name = f"{report_type}-HOUR Report: {index}" - form.save() - - -def _update_form24_names_v2(apps, schema_editor): - form24 = apps.get_model("reports", "Form24") - form24_objects = list(form24.objects.all()) - for form in form24_objects: - report_type = form.report_type_24_48 - form.name = f"{report_type}-HOUR: Report of Independent Expenditure" - form.save() - - class Migration(migrations.Migration): replaces = [ @@ -275,137 +197,6 @@ class Migration(migrations.Migration): ] operations = [ - IgnoreMigration(), - migrations.RemoveField( - model_name="report", - name="deleted", - ), - migrations.AddField( - model_name="report", - name="city", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="committee_name", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="state", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="street_1", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="street_2", - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name="report", - name="zip", - field=models.TextField(blank=True, null=True), - ), - migrations.RunPython( - code=_migrate_committee_data, - ), - migrations.RemoveField( - model_name="form1m", - name="city", - ), - migrations.RemoveField( - model_name="form1m", - name="committee_name", - ), - migrations.RemoveField( - model_name="form1m", - name="state", - ), - migrations.RemoveField( - model_name="form1m", - name="street_1", - ), - migrations.RemoveField( - model_name="form1m", - name="street_2", - ), - migrations.RemoveField( - model_name="form1m", - name="zip", - ), - migrations.RemoveField( - model_name="form24", - name="city", - ), - migrations.RemoveField( - model_name="form24", - name="state", - ), - migrations.RemoveField( - model_name="form24", - name="street_1", - ), - migrations.RemoveField( - model_name="form24", - name="street_2", - ), - migrations.RemoveField( - model_name="form24", - name="zip", - ), - migrations.RemoveField( - model_name="form3x", - name="city", - ), - migrations.RemoveField( - model_name="form3x", - name="state", - ), - migrations.RemoveField( - model_name="form3x", - name="street_1", - ), - migrations.RemoveField( - model_name="form3x", - name="street_2", - ), - migrations.RemoveField( - model_name="form3x", - name="zip", - ), - migrations.RemoveField( - model_name="form99", - name="city", - ), - migrations.RemoveField( - model_name="form99", - name="committee_name", - ), - migrations.RemoveField( - model_name="form99", - name="state", - ), - migrations.RemoveField( - model_name="form99", - name="street_1", - ), - migrations.RemoveField( - model_name="form99", - name="street_2", - ), - migrations.RemoveField( - model_name="form99", - name="zip", - ), - migrations.AddField( - model_name="report", - name="can_delete", - field=models.BooleanField(default=True), - ), migrations.RunSQL( sql=""" CREATE OR REPLACE FUNCTION update_report_can_delete(report RECORD) @@ -452,41 +243,9 @@ class Migration(migrations.Migration): migrations.RunPython( code=_populate_can_delete, ), - migrations.AddField( - model_name="report", - name="can_unamend", - field=models.BooleanField(default=False), - ), migrations.RunPython( code=_create_can_unamend_trigger, ), - migrations.RemoveField( - model_name="form3x", - name="cash_on_hand_date", - ), - migrations.AddField( - model_name="form99", - name="text_code_2", - field=models.TextField(db_default=""), - ), - migrations.RunSQL( - sql=""" - UPDATE reports_form99 SET text_code_2 = COALESCE(text_code, ''); - ALTER TABLE reports_form99 ALTER COLUMN text_code SET DEFAULT ''; - ALTER TABLE reports_form99 ALTER COLUMN text_code_2 SET NOT NULL; - """, - reverse_sql=""" - ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP DEFAULT; - ALTER TABLE reports_form99 ALTER COLUMN text_code_2 DROP NOT NULL; - """, - state_operations=[ - migrations.AlterField( - model_name="form99", - name="text_code_2", - field=models.TextField(db_default="", default=""), - ) - ], - ), migrations.CreateModel( name="Form3", fields=[ @@ -950,15 +709,6 @@ class Migration(migrations.Migration): to="reports.form3", ), ), - migrations.RemoveField( - model_name="form99", - name="text_code", - ), - migrations.RenameField( - model_name="form99", - old_name="text_code_2", - new_name="text_code", - ), migrations.AddField( model_name="form3x", name="filing_frequency", @@ -1001,21 +751,4 @@ class Migration(migrations.Migration): name="filing_frequency", field=models.TextField(blank=True, max_length=1, null=True), ), - migrations.AddField( - model_name="form24", - name="name", - field=models.TextField(null=True), - ), - migrations.RunPython( - code=_update_form24_names_v1, - ), - migrations.AlterField( - model_name="form24", - name="name", - field=models.TextField(), - ), - migrations.RunPython( - code=_update_form24_names_v2, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), ] diff --git a/django-backend/fecfiler/transactions/migrations/0001_initial.py b/django-backend/fecfiler/transactions/migrations/0001_initial.py index c3c2a2f8f..84f130f89 100644 --- a/django-backend/fecfiler/transactions/migrations/0001_initial.py +++ b/django-backend/fecfiler/transactions/migrations/0001_initial.py @@ -33,20 +33,6 @@ class Migration(migrations.Migration): unique=True, ), ), - ( - "contributor_organization_name", - models.TextField(blank=True, null=True), - ), - ("contributor_last_name", models.TextField(blank=True, null=True)), - ("contributor_first_name", models.TextField(blank=True, null=True)), - ("contributor_middle_name", models.TextField(blank=True, null=True)), - ("contributor_prefix", models.TextField(blank=True, null=True)), - ("contributor_suffix", models.TextField(blank=True, null=True)), - ("contributor_street_1", models.TextField(blank=True, null=True)), - ("contributor_street_2", models.TextField(blank=True, null=True)), - ("contributor_city", models.TextField(blank=True, null=True)), - ("contributor_state", models.TextField(blank=True, null=True)), - ("contributor_zip", models.TextField(blank=True, null=True)), ("contribution_date", models.DateField(blank=True, null=True)), ( "contribution_amount", @@ -58,22 +44,6 @@ class Migration(migrations.Migration): "contribution_purpose_descrip", models.TextField(blank=True, null=True), ), - ("contributor_employer", models.TextField(blank=True, null=True)), - ("contributor_occupation", models.TextField(blank=True, null=True)), - ("donor_committee_fec_id", models.TextField(blank=True, null=True)), - ("donor_committee_name", models.TextField(blank=True, null=True)), - ("donor_candidate_fec_id", models.TextField(blank=True, null=True)), - ("donor_candidate_last_name", models.TextField(blank=True, null=True)), - ("donor_candidate_first_name", models.TextField(blank=True, null=True)), - ( - "donor_candidate_middle_name", - models.TextField(blank=True, null=True), - ), - ("donor_candidate_prefix", models.TextField(blank=True, null=True)), - ("donor_candidate_suffix", models.TextField(blank=True, null=True)), - ("donor_candidate_office", models.TextField(blank=True, null=True)), - ("donor_candidate_state", models.TextField(blank=True, null=True)), - ("donor_candidate_district", models.TextField(blank=True, null=True)), ("election_code", models.TextField(blank=True, null=True)), ("election_other_description", models.TextField(blank=True, null=True)), ("conduit_name", models.TextField(blank=True, null=True)), @@ -106,17 +76,6 @@ class Migration(migrations.Migration): unique=True, ), ), - ("payee_organization_name", models.TextField(blank=True, null=True)), - ("payee_last_name", models.TextField(blank=True, null=True)), - ("payee_first_name", models.TextField(blank=True, null=True)), - ("payee_middle_name", models.TextField(blank=True, null=True)), - ("payee_prefix", models.TextField(blank=True, null=True)), - ("payee_suffix", models.TextField(blank=True, null=True)), - ("payee_street_1", models.TextField(blank=True, null=True)), - ("payee_street_2", models.TextField(blank=True, null=True)), - ("payee_city", models.TextField(blank=True, null=True)), - ("payee_state", models.TextField(blank=True, null=True)), - ("payee_zip", models.TextField(blank=True, null=True)), ("expenditure_date", models.DateField(blank=True, null=True)), ( "expenditure_amount", @@ -137,47 +96,6 @@ class Migration(migrations.Migration): ("conduit_state", models.TextField(blank=True, null=True)), ("conduit_zip", models.TextField(blank=True, null=True)), ("category_code", models.TextField(blank=True, null=True)), - ( - "beneficiary_committee_fec_id", - models.TextField(blank=True, null=True), - ), - ("beneficiary_committee_name", models.TextField(blank=True, null=True)), - ( - "beneficiary_candidate_fec_id", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_last_name", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_first_name", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_middle_name", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_prefix", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_suffix", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_office", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_state", - models.TextField(blank=True, null=True), - ), - ( - "beneficiary_candidate_district", - models.TextField(blank=True, null=True), - ), ("memo_text_description", models.TextField(blank=True, null=True)), ( "reference_to_si_or_sl_system_code_that_identifies_the_account", @@ -203,17 +121,6 @@ class Migration(migrations.Migration): ), ), ("receipt_line_number", models.TextField(blank=True, null=True)), - ("lender_organization_name", models.TextField(blank=True, null=True)), - ("lender_last_name", models.TextField(blank=True, null=True)), - ("lender_first_name", models.TextField(blank=True, null=True)), - ("lender_middle_name", models.TextField(blank=True, null=True)), - ("lender_prefix", models.TextField(blank=True, null=True)), - ("lender_suffix", models.TextField(blank=True, null=True)), - ("lender_street_1", models.TextField(blank=True, null=True)), - ("lender_street_2", models.TextField(blank=True, null=True)), - ("lender_city", models.TextField(blank=True, null=True)), - ("lender_state", models.TextField(blank=True, null=True)), - ("lender_zip", models.TextField(blank=True, null=True)), ("election_code", models.TextField(blank=True, null=True)), ("election_other_description", models.TextField(blank=True, null=True)), ( @@ -230,22 +137,6 @@ class Migration(migrations.Migration): "personal_funds", models.BooleanField(blank=True, default=False, null=True), ), - ("lender_committee_id_number", models.TextField(blank=True, null=True)), - ("lender_candidate_id_number", models.TextField(blank=True, null=True)), - ("lender_candidate_last_name", models.TextField(blank=True, null=True)), - ( - "lender_candidate_first_name", - models.TextField(blank=True, null=True), - ), - ( - "lender_candidate_middle_name", - models.TextField(blank=True, null=True), - ), - ("lender_candidate_prefix", models.TextField(blank=True, null=True)), - ("lender_candidate_suffix", models.TextField(blank=True, null=True)), - ("lender_candidate_office", models.TextField(blank=True, null=True)), - ("lender_candidate_state", models.TextField(blank=True, null=True)), - ("lender_candidate_district", models.TextField(blank=True, null=True)), ("memo_text_description", models.TextField(blank=True, null=True)), ], ), @@ -262,12 +153,6 @@ class Migration(migrations.Migration): unique=True, ), ), - ("lender_organization_name", models.TextField(blank=True, null=True)), - ("lender_street_1", models.TextField(blank=True, null=True)), - ("lender_street_2", models.TextField(blank=True, null=True)), - ("lender_city", models.TextField(blank=True, null=True)), - ("lender_state", models.TextField(blank=True, null=True)), - ("lender_zip", models.TextField(blank=True, null=True)), ( "loan_amount", models.DecimalField( @@ -377,18 +262,6 @@ class Migration(migrations.Migration): unique=True, ), ), - ("guarantor_last_name", models.TextField(blank=True, null=True)), - ("guarantor_first_name", models.TextField(blank=True, null=True)), - ("guarantor_middle_name", models.TextField(blank=True, null=True)), - ("guarantor_prefix", models.TextField(blank=True, null=True)), - ("guarantor_suffix", models.TextField(blank=True, null=True)), - ("guarantor_street_1", models.TextField(blank=True, null=True)), - ("guarantor_street_2", models.TextField(blank=True, null=True)), - ("guarantor_city", models.TextField(blank=True, null=True)), - ("guarantor_state", models.TextField(blank=True, null=True)), - ("guarantor_zip", models.TextField(blank=True, null=True)), - ("guarantor_employer", models.TextField(blank=True, null=True)), - ("guarantor_occupation", models.TextField(blank=True, null=True)), ( "guaranteed_amount", models.DecimalField( @@ -411,17 +284,6 @@ class Migration(migrations.Migration): ), ), ("receipt_line_number", models.TextField(blank=True, null=True)), - ("creditor_organization_name", models.TextField(blank=True, null=True)), - ("creditor_last_name", models.TextField(blank=True, null=True)), - ("creditor_first_name", models.TextField(blank=True, null=True)), - ("creditor_middle_name", models.TextField(blank=True, null=True)), - ("creditor_prefix", models.TextField(blank=True, null=True)), - ("creditor_suffix", models.TextField(blank=True, null=True)), - ("creditor_street_1", models.TextField(blank=True, null=True)), - ("creditor_street_2", models.TextField(blank=True, null=True)), - ("creditor_city", models.TextField(blank=True, null=True)), - ("creditor_state", models.TextField(blank=True, null=True)), - ("creditor_zip", models.TextField(blank=True, null=True)), ( "purpose_of_debt_or_obligation", models.TextField(blank=True, null=True), @@ -447,17 +309,6 @@ class Migration(migrations.Migration): unique=True, ), ), - ("payee_organization_name", models.TextField(blank=True, null=True)), - ("payee_last_name", models.TextField(blank=True, null=True)), - ("payee_first_name", models.TextField(blank=True, null=True)), - ("payee_middle_name", models.TextField(blank=True, null=True)), - ("payee_prefix", models.TextField(blank=True, null=True)), - ("payee_suffix", models.TextField(blank=True, null=True)), - ("payee_street_1", models.TextField(blank=True, null=True)), - ("payee_street_2", models.TextField(blank=True, null=True)), - ("payee_city", models.TextField(blank=True, null=True)), - ("payee_state", models.TextField(blank=True, null=True)), - ("payee_zip", models.TextField(blank=True, null=True)), ("election_code", models.TextField(blank=True, null=True)), ("election_other_description", models.TextField(blank=True, null=True)), ("dissemination_date", models.DateField(blank=True, null=True)), @@ -475,15 +326,6 @@ class Migration(migrations.Migration): ("category_code", models.TextField(blank=True, null=True)), ("payee_cmtte_fec_id_number", models.TextField(blank=True, null=True)), ("support_oppose_code", models.TextField(blank=True, null=True)), - ("so_candidate_id_number", models.TextField(blank=True, null=True)), - ("so_candidate_last_name", models.TextField(blank=True, null=True)), - ("so_candidate_first_name", models.TextField(blank=True, null=True)), - ("so_candidate_middle_name", models.TextField(blank=True, null=True)), - ("so_candidate_prefix", models.TextField(blank=True, null=True)), - ("so_candidate_suffix", models.TextField(blank=True, null=True)), - ("so_candidate_office", models.TextField(blank=True, null=True)), - ("so_candidate_district", models.TextField(blank=True, null=True)), - ("so_candidate_state", models.TextField(blank=True, null=True)), ("completing_last_name", models.TextField(blank=True, null=True)), ("completing_first_name", models.TextField(blank=True, null=True)), ("completing_middle_name", models.TextField(blank=True, null=True)), @@ -610,15 +452,6 @@ class Migration(migrations.Migration): to="transactions.transaction", ), ), - ( - "report", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="reports.report", - ), - ), ( "schedule_a", models.ForeignKey( diff --git a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py index efb36d049..f17af0076 100644 --- a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py +++ b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py @@ -1,7 +1,6 @@ # Generated by Django 4.2.7 on 2024-01-24 09:51 from django.db import migrations -from django_migration_linter import IgnoreMigration class Migration(migrations.Migration): @@ -10,470 +9,4 @@ class Migration(migrations.Migration): ('transactions', '0001_initial'), ] - operations = [ - IgnoreMigration(), - migrations.RemoveField( - model_name='schedulea', - name='contributor_city', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_employer', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_first_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_last_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_middle_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_occupation', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_organization_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_prefix', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_state', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_street_1', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_street_2', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_suffix', - ), - migrations.RemoveField( - model_name='schedulea', - name='contributor_zip', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_district', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_fec_id', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_first_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_last_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_middle_name', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_office', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_prefix', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_state', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_candidate_suffix', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_committee_fec_id', - ), - migrations.RemoveField( - model_name='schedulea', - name='donor_committee_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_district', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_fec_id', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_first_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_last_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_middle_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_office', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_prefix', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_state', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_candidate_suffix', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_committee_fec_id', - ), - migrations.RemoveField( - model_name='scheduleb', - name='beneficiary_committee_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_city', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_first_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_last_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_middle_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_organization_name', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_prefix', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_state', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_street_1', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_street_2', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_suffix', - ), - migrations.RemoveField( - model_name='scheduleb', - name='payee_zip', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_district', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_first_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_id_number', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_last_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_middle_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_office', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_prefix', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_state', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_candidate_suffix', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_city', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_committee_id_number', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_first_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_last_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_middle_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_organization_name', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_prefix', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_state', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_street_1', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_street_2', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_suffix', - ), - migrations.RemoveField( - model_name='schedulec', - name='lender_zip', - ), - migrations.RemoveField( - model_name='schedulec1', - name='lender_city', - ), - migrations.RemoveField( - model_name='schedulec1', - name='lender_organization_name', - ), - migrations.RemoveField( - model_name='schedulec1', - name='lender_state', - ), - migrations.RemoveField( - model_name='schedulec1', - name='lender_street_1', - ), - migrations.RemoveField( - model_name='schedulec1', - name='lender_street_2', - ), - migrations.RemoveField( - model_name='schedulec1', - name='lender_zip', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_city', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_employer', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_first_name', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_last_name', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_middle_name', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_occupation', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_prefix', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_state', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_street_1', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_street_2', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_suffix', - ), - migrations.RemoveField( - model_name='schedulec2', - name='guarantor_zip', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_city', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_first_name', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_last_name', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_middle_name', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_organization_name', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_prefix', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_state', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_street_1', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_street_2', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_suffix', - ), - migrations.RemoveField( - model_name='scheduled', - name='creditor_zip', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_city', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_first_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_last_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_middle_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_organization_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_prefix', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_state', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_street_1', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_street_2', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_suffix', - ), - migrations.RemoveField( - model_name='schedulee', - name='payee_zip', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_district', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_first_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_id_number', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_last_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_middle_name', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_office', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_prefix', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_state', - ), - migrations.RemoveField( - model_name='schedulee', - name='so_candidate_suffix', - ), - ] + operations = [] diff --git a/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py index 24bea6716..f160deeb2 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py @@ -17,21 +17,11 @@ from fecfiler.transactions.utils_aggregation_queries import ( filter_queryset_for_previous_transactions_in_aggregation, ) -from django_migration_linter import IgnoreMigration logger = structlog.get_logger(__name__) -# Inlined from 0004_report_transactions_link_table -def add_link_table(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - - for t in transaction.objects.all(): - t.reports.add(t.report) - t.save() - - # Inlined from 0005_schedulec_report_coverage_from_date_and_more def set_coverage_date(apps, schema_editor): transaction_model = apps.get_model("transactions", "Transaction") @@ -996,7 +986,6 @@ class Migration(migrations.Migration): ] operations = [ - IgnoreMigration(), migrations.AlterField( model_name="transaction", name="parent_transaction", @@ -1015,14 +1004,6 @@ class Migration(migrations.Migration): through="reports.ReportTransaction", to="reports.report" ), ), - migrations.RunPython( - code=add_link_table, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RemoveField( - model_name="transaction", - name="report", - ), migrations.AddField( model_name="schedulec", name="report_coverage_through_date", diff --git a/django-backend/fecfiler/user/migrations/0001_initial.py b/django-backend/fecfiler/user/migrations/0001_initial.py index b8db93a85..9abe4469a 100644 --- a/django-backend/fecfiler/user/migrations/0001_initial.py +++ b/django-backend/fecfiler/user/migrations/0001_initial.py @@ -44,7 +44,6 @@ class Migration(migrations.Migration): verbose_name="superuser status", ), ), - ("cmtee_id", models.CharField(max_length=9)), ( "username", models.CharField( diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py index 23d044d9d..5d533c444 100644 --- a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py +++ b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py @@ -4,7 +4,6 @@ import fecfiler.user.managers from django.db import migrations, models from django.db.models import Q -from django_migration_linter import IgnoreMigration def remove_old_login_accounts(apps, schema_editor): @@ -38,11 +37,6 @@ class Migration(migrations.Migration): ] operations = [ - IgnoreMigration(), - migrations.RemoveField( - model_name="user", - name="cmtee_id", - ), migrations.AlterField( model_name="user", name="first_name", @@ -55,7 +49,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name="user", - name="security_consent_date", + name="security_consent_exp_date", field=models.DateField(blank=True, null=True), ), migrations.AlterModelManagers( @@ -64,11 +58,6 @@ class Migration(migrations.Migration): ("objects", fecfiler.user.managers.UserManager()), ], ), - migrations.RenameField( - model_name="user", - old_name="security_consent_date", - new_name="security_consent_exp_date", - ), migrations.RunPython( code=remove_old_login_accounts, reverse_code=django.db.migrations.operations.special.RunPython.noop, diff --git a/django-backend/fecfiler/web_services/migrations/0001_initial.py b/django-backend/fecfiler/web_services/migrations/0001_initial.py index b1b05f1d0..491cb3063 100644 --- a/django-backend/fecfiler/web_services/migrations/0001_initial.py +++ b/django-backend/fecfiler/web_services/migrations/0001_initial.py @@ -64,6 +64,8 @@ class Migration(migrations.Migration): ("fec_image_url", models.CharField(max_length=255, null=True)), ("fec_batch_id", models.CharField(max_length=255, null=True)), ("fec_email", models.CharField(max_length=255, null=True)), + ("task_completed", models.DateTimeField(null=True)), + ("fecfile_polling_attempts", models.IntegerField(default=0)), ( "dot_fec", models.ForeignKey( @@ -98,6 +100,8 @@ class Migration(migrations.Migration): ("created", models.DateTimeField(auto_now_add=True)), ("updated", models.DateTimeField(auto_now=True)), ("fec_report_id", models.CharField(max_length=255, null=True)), + ("task_completed", models.DateTimeField(null=True)), + ("fecfile_polling_attempts", models.IntegerField(default=0)), ( "dot_fec", models.ForeignKey( diff --git a/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py b/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py index e2ad18be6..6c5d52a71 100644 --- a/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py +++ b/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py @@ -1,9 +1,8 @@ # Generated by Django 5.2.11 on 2026-03-07 03:21 import django.db.migrations.operations.special -from django.db import migrations, models +from django.db import migrations from django.db.models import F -from django_migration_linter import IgnoreMigration def set_default_task_completed_times(apps, schema_editor): @@ -26,29 +25,8 @@ class Migration(migrations.Migration): ] operations = [ - IgnoreMigration(), - migrations.AddField( - model_name="uploadsubmission", - name="task_completed", - field=models.DateTimeField(null=True), - ), - migrations.AddField( - model_name="webprintsubmission", - name="task_completed", - field=models.DateTimeField(null=True), - ), migrations.RunPython( code=set_default_task_completed_times, reverse_code=django.db.migrations.operations.special.RunPython.noop, ), - migrations.AddField( - model_name="uploadsubmission", - name="fecfile_polling_attempts", - field=models.IntegerField(default=0), - ), - migrations.AddField( - model_name="webprintsubmission", - name="fecfile_polling_attempts", - field=models.IntegerField(default=0), - ), ] From b04fd60a22578505b4ba51205caeceb9bbed8b0a Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 10 Mar 2026 18:58:56 -0400 Subject: [PATCH 17/52] FECFILE-2734: Squashed no-op migrations. --- ..._remove_form1m_iii_candidate_district_and_more.py | 12 ------------ ...remove_form1m_squashed_0006_reporttransaction.py} | 9 +++++++-- ..._report_deleted_squashed_00019_form24_name_fix.py | 2 +- ...002_remove_schedulea_contributor_city_and_more.py | 12 ------------ ...ulea_squashed_0026_alter_transaction_itemized.py} | 5 +++-- 5 files changed, 11 insertions(+), 29 deletions(-) delete mode 100644 django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py rename django-backend/fecfiler/reports/migrations/{0006_reporttransaction.py => 0005_remove_form1m_squashed_0006_reporttransaction.py} (88%) delete mode 100644 django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py rename django-backend/fecfiler/transactions/migrations/{0003_alter_parent_squashed_0026_alter_transaction_itemized.py => 0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py} (99%) diff --git a/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py deleted file mode 100644 index 8c35add58..000000000 --- a/django-backend/fecfiler/reports/migrations/0005_remove_form1m_iii_candidate_district_and_more.py +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-24 09:51 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('reports', '0004_form1m_date_committee_met_requirements_and_more'), - ] - - operations = [] diff --git a/django-backend/fecfiler/reports/migrations/0006_reporttransaction.py b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_squashed_0006_reporttransaction.py similarity index 88% rename from django-backend/fecfiler/reports/migrations/0006_reporttransaction.py rename to django-backend/fecfiler/reports/migrations/0005_remove_form1m_squashed_0006_reporttransaction.py index b632cad43..00a442e3e 100644 --- a/django-backend/fecfiler/reports/migrations/0006_reporttransaction.py +++ b/django-backend/fecfiler/reports/migrations/0005_remove_form1m_squashed_0006_reporttransaction.py @@ -7,13 +7,18 @@ class Migration(migrations.Migration): + replaces = [ + ('reports', '0005_remove_form1m_iii_candidate_district_and_more'), + ('reports', '0006_reporttransaction'), + ] + dependencies = [ ( "committee_accounts", "0001_squashed_0007_alter_committeeaccount_members", ), - ('transactions', '0002_remove_schedulea_contributor_city_and_more'), - ('reports', '0005_remove_form1m_iii_candidate_district_and_more'), + ('transactions', '0001_initial'), + ('reports', '0004_form1m_date_committee_met_requirements_and_more'), ] operations = [ diff --git a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py index 332daee34..b2513a527 100644 --- a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -193,7 +193,7 @@ class Migration(migrations.Migration): ] dependencies = [ - ("reports", "0006_reporttransaction"), + ("reports", "0005_remove_form1m_squashed_0006_reporttransaction"), ] operations = [ diff --git a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py deleted file mode 100644 index f17af0076..000000000 --- a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_contributor_city_and_more.py +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-24 09:51 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('transactions', '0001_initial'), - ] - - operations = [] diff --git a/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py similarity index 99% rename from django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py rename to django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py index f160deeb2..e1118f06f 100644 --- a/django-backend/fecfiler/transactions/migrations/0003_alter_parent_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py @@ -952,6 +952,7 @@ def run_aggregations_for_all_debts(apps, schema_editor): class Migration(migrations.Migration): replaces = [ + ("transactions", "0002_remove_schedulea_contributor_city_and_more"), ("transactions", "0003_alter_transaction_parent_transaction"), ("transactions", "0004_report_transactions_link_table"), ("transactions", "0005_schedulec_report_coverage_from_date_and_more"), @@ -980,9 +981,9 @@ class Migration(migrations.Migration): dependencies = [ ("contacts", "0001_initial"), - ("reports", "0006_reporttransaction"), + ("reports", "0005_remove_form1m_squashed_0006_reporttransaction"), ("reports", "0007_remove_report_deleted_squashed_00019_form24_name_fix"), - ("transactions", "0002_remove_schedulea_contributor_city_and_more"), + ("transactions", "0001_initial"), ] operations = [ From 2375bad06b634fe639cc62481a3ce039d01ade8a Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 11 Mar 2026 11:01:43 -0400 Subject: [PATCH 18/52] FECFILE-2734: Rearranging lines to shrink diff. --- .../0002_initial_squashed_0003_memotext_text_prefix.py | 4 ++-- ...07_remove_report_deleted_squashed_00019_form24_name_fix.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py b/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py index 69bcd9ebc..188e4a59f 100644 --- a/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py +++ b/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py @@ -1,7 +1,7 @@ # Generated by Django 5.2.11 on 2026-03-07 03:21 -import django.db.models.deletion from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -9,8 +9,8 @@ class Migration(migrations.Migration): replaces = [("memo_text", "0002_initial"), ("memo_text", "0003_memotext_text_prefix")] dependencies = [ - ("memo_text", "0001_initial"), ("reports", "0001_initial"), + ("memo_text", "0001_initial"), ] operations = [ diff --git a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py index b2513a527..fea40255d 100644 --- a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -2,8 +2,8 @@ import django.db.models.deletion import uuid -import structlog from django.db import connection, migrations, models +import structlog logger = structlog.get_logger(__name__) From 3a148eead036f3eb0dde5e883e7c989292b3bc89 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 11 Mar 2026 11:05:08 -0400 Subject: [PATCH 19/52] FECFILE-2734: Rearranging lines to shrink diff (part two). --- ...quashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py | 2 +- ...0007_remove_report_deleted_squashed_00019_form24_name_fix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py b/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py index e53712657..d1b9f1996 100644 --- a/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py +++ b/django-backend/fecfiler/cash_on_hand/migrations/0001_squashed_0002_alter_cashonhandyearly_cash_on_hand_and_more.py @@ -1,8 +1,8 @@ # Generated by Django 5.2.11 on 2026-03-06 21:37 +from django.db import migrations, models import django.db.models.deletion import uuid -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py index fea40255d..30b1fbc20 100644 --- a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -2,7 +2,7 @@ import django.db.models.deletion import uuid -from django.db import connection, migrations, models +from django.db import migrations, models, connection import structlog logger = structlog.get_logger(__name__) From 0e629431b5ff8bd5f13c157ca7c0c3c6d0c1ef5a Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 11 Mar 2026 18:05:11 -0400 Subject: [PATCH 20/52] FECFILE-2734: Refactored transaction squashed migration to remove migrations related to obviated triggers. --- ...quashed_0026_alter_transaction_itemized.py | 3148 +---------------- 1 file changed, 21 insertions(+), 3127 deletions(-) diff --git a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py index e1118f06f..857cdcfc3 100644 --- a/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py +++ b/django-backend/fecfiler/transactions/migrations/0002_remove_schedulea_squashed_0026_alter_transaction_itemized.py @@ -1,102 +1,19 @@ # Generated by Django 5.2.11 on 2026-03-07 03:25 import django.contrib.postgres.fields -import django.db.migrations.operations.special import django.db.models.deletion -import structlog import uuid from django.db import connection, migrations, models -from django.db.models import F, Q -from fecfiler.transactions.aggregation import process_aggregation_for_debts from fecfiler.transactions.schedule_a.managers import ( over_two_hundred_types as schedule_a_over_two_hundred_types, ) from fecfiler.transactions.schedule_b.managers import ( over_two_hundred_types as schedule_b_over_two_hundred_types, ) -from fecfiler.transactions.utils_aggregation_queries import ( - filter_queryset_for_previous_transactions_in_aggregation, -) - - -logger = structlog.get_logger(__name__) - - -# Inlined from 0005_schedulec_report_coverage_from_date_and_more -def set_coverage_date(apps, schema_editor): - transaction_model = apps.get_model("transactions", "Transaction") - - for transaction in transaction_model.objects.filter( - Q(schedule_c__isnull=False) | Q(schedule_d__isnull=False) - ): - for report in transaction.reports.all(): - if report.coverage_from_date and transaction.schedule_d: - transaction.schedule_d.report_coverage_from_date = ( - report.coverage_from_date - ) - transaction.schedule_d.save() - if report.coverage_through_date and transaction.schedule_c: - transaction.schedule_c.report_coverage_through_date = ( - report.coverage_through_date - ) - transaction.schedule_c.save() - transaction.save() - - -# Inlined from 0006_independent_expenditure_memos_no_aggregation_group -def set_aggregation_group_to_none_for_ie_memos(apps, schema_editor): - transaction_model = apps.get_model("transactions", "Transaction") - - for transaction in transaction_model.objects.filter( - Q( - transaction_type_identifier__in=[ - "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", - "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", - "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO", - ] - ) - ): - transaction.aggregation_group = None - transaction.save() - - -def reverse_removing_aggregation_group_for_ie_memos(apps, schema_editor): - transaction_model = apps.get_model("transactions", "Transaction") - - for transaction in transaction_model.objects.filter( - Q( - transaction_type_identifier__in=[ - "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", - "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", - "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO", - ] - ) - ): - transaction.aggregation_group = "INDEPENDENT_EXPENDITURE" - transaction.save() - - -# Inlined from 0008_transaction__calendar_ytd_per_election_office_and_more -def populate_existing_rows(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - for row in transaction.objects.all(): - row.aggregate = 0.0 - row.save() - - -# Inlined from 0009_update_calculate_loan_payment_to_date -def update_existing_rows(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - types = [ - "LOAN_RECEIVED_FROM_INDIVIDUAL", - "LOAN_RECEIVED_FROM BANK", - "LOAN_BY_COMMITTEE", - ] - for row in transaction.objects.filter(transaction_type_identifier__in=types): - row.save() # Inlined from 0011_transaction_can_delete +# Keep the final live trigger that maintains Transaction.blocking_reports. def create_trigger_function(apps, schema_editor): with connection.cursor() as cursor: cursor.execute( @@ -115,7 +32,7 @@ def create_trigger_function(apps, schema_editor): FROM reports_reporttransaction WHERE report_id = NEW.id ) - -- all transactions that are reattributed in the submtited report + -- all transactions that are reattributed in the submitted report OR id IN ( SELECT reatt_redes_id FROM reports_reporttransaction @@ -184,769 +101,25 @@ def drop_trigger(apps, schema_editor): cursor.execute("DROP TRIGGER IF EXISTS report_status_update ON reports_report;") -# Inlined from 0012_alter_transactions_blocking_reports -def update_blocking_reports_default(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - transaction._meta.get_field("blocking_reports").default = list - - # Inlined from 0013_transaction_itemized_and_associated_triggers +# This data is used by itemization logic and should exist on fresh installs. def populate_over_two_hundred_types(apps, schema_editor): OverTwoHundredTypesScheduleA = apps.get_model( # noqa: N806 "transactions", "OverTwoHundredTypesScheduleA" ) OverTwoHundredTypesScheduleB = apps.get_model( # noqa: N806 - "transactions", "OverTwoHundredTypesScheduleB" - ) - scha_types_to_create = [ - OverTwoHundredTypesScheduleA(type=type_to_create) - for type_to_create in schedule_a_over_two_hundred_types - ] - OverTwoHundredTypesScheduleA.objects.bulk_create(scha_types_to_create) - schb_types_to_create = [ - OverTwoHundredTypesScheduleB(type=type_to_create) - for type_to_create in schedule_b_over_two_hundred_types - ] - OverTwoHundredTypesScheduleB.objects.bulk_create(schb_types_to_create) - - -def drop_over_two_hundred_types(apps, schema_editor): - print("this reverses migration automatically.") - - -def create_itemized_triggers_and_functions(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION before_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - needs_itemized_set boolean; - itemization boolean; - BEGIN - needs_itemized_set := needs_itemized_set(OLD, NEW); - IF needs_itemized_set THEN - NEW.itemized := calculate_itemization(NEW); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION calculate_itemization( - txn RECORD - ) - RETURNS BOOLEAN AS $$ - DECLARE - itemized boolean; - BEGIN - itemized := TRUE; - IF txn.force_itemized IS NOT NULL THEN - itemized := txn.force_itemized; - ELSIF txn.aggregate < 0 THEN - itemized := TRUE; - ELSIF EXISTS ( - SELECT type from ( - SELECT type - FROM over_two_hundred_types_schedulea - UNION - SELECT type - FROM over_two_hundred_types_scheduleb - ) as scha_schb_types - WHERE type = txn.transaction_type_identifier - ) THEN - IF txn.aggregate > 200 THEN - itemized := TRUE; - ELSE - itemized := FALSE; - END IF; - END IF; - return itemized; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - parent_and_grandparent_ids uuid[]; - children_and_grandchildren_ids uuid[]; - BEGIN - IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN - IF NEW.itemized is TRUE THEN - parent_and_grandparent_ids := - get_parent_grandparent_transaction_ids(NEW); - PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); - ELSE - children_and_grandchildren_ids := - get_children_and_grandchildren_transaction_ids(NEW); - PERFORM set_itemization_for_ids( - FALSE,children_and_grandchildren_ids - ); - END IF; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION needs_itemized_set( - OLD RECORD, - NEW RECORD - ) - RETURNS BOOLEAN AS $$ - BEGIN - return OLD IS NULL OR ( - OLD.force_itemized IS DISTINCT FROM NEW.force_itemized - OR OLD.aggregate IS DISTINCT FROM NEW.aggregate - ); - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION set_itemization_for_ids( - itemization boolean, - ids uuid[] - ) - RETURNS VOID AS $$ - BEGIN - IF cardinality(ids) > 0 THEN - UPDATE transactions_transaction - SET - itemized = itemization - WHERE id = ANY (ids); - END IF; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION get_parent_grandparent_transaction_ids( - txn RECORD - ) - RETURNS uuid[] AS $$ - DECLARE - ids uuid[]; - BEGIN - SELECT array( - SELECT id - FROM transactions_transaction - WHERE id IN ( - txn.parent_transaction_id, - ( - SELECT parent_transaction_id - FROM transactions_transaction - WHERE id = txn.parent_transaction_id - ) - ) - ) into ids; - RETURN ids; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION get_children_and_grandchildren_transaction_ids( - txn RECORD - ) - RETURNS uuid[] AS $$ - DECLARE - ids uuid[]; - BEGIN - SELECT array( - SELECT id - FROM transactions_transaction - WHERE parent_transaction_id = ANY ( - array_prepend(txn.id, - array( - SELECT id - FROM transactions_transaction - WHERE parent_transaction_id = txn.id - ) - ) - ) - ) into ids; - RETURN ids; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER before_transactions_transaction_insert_or_update_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction_insert_or_update(); - - CREATE TRIGGER zafter_transactions_transaction_insert_or_update_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_insert_or_update(); - """ - ) - - -def drop_itemized_triggers_and_functions(apps, schema_editor): - schema_editor.execute( - """ - DROP TRIGGER - IF EXISTS zafter_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS before_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP FUNCTION IF EXISTS before_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS calculate_itemization; - DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS needs_itemized_set; - DROP FUNCTION IF EXISTS set_itemization_for_ids; - DROP FUNCTION IF EXISTS get_parent_grandparent_transaction_ids; - DROP FUNCTION IF EXISTS get_children_and_grandchildren_transaction_ids; - """ - ) - - -# Inlined from 0015_merge_transaction_triggers -def create_triggers(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE TRIGGER before_transactions_transaction_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction(); - - CREATE TRIGGER after_transactions_transaction_infinite_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_infinite(); - - CREATE TRIGGER after_transactions_transaction_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) - EXECUTE FUNCTION after_transactions_transaction(); - """ - ) - - -def reverse_create_triggers(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP TRIGGER - IF EXISTS before_transactions_transaction_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS after_transactions_transaction_infinite_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS after_transactions_transaction_trigger - ON transactions_transaction; - """ - ) - - -def before_transactions_transaction(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION before_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - NEW := process_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def after_transactions_transaction(apps, schema_editor): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION after_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - IF TG_OP = 'UPDATE' - THEN - NEW := calculate_aggregates(OLD, NEW, TG_OP); - NEW := update_can_unamend(NEW); - ELSE - NEW := calculate_aggregates(OLD, NEW, TG_OP); - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite() - RETURNS TRIGGER AS $$ - BEGIN - NEW := handle_parent_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - """ - ) - - -def reverse_after_transactions_transaction(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS after_transactions_transaction - DROP FUNCTION IF EXISTS after_transactions_transaction_infinite - """ - ) - - -def drop_old_triggers(apps, schema_editor): - schema_editor.execute( - """ - DROP TRIGGER - IF EXISTS zafter_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS before_transactions_transaction_insert_or_update_trigger - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS transaction_updated - ON transactions_transaction; - - DROP TRIGGER - IF EXISTS calculate_aggregates_trigger - ON transactions_transaction; - """ - ) - - -def reverse_drop_old_triggers(apps, schema_editor): - schema_editor.execute( - """ - CREATE TRIGGER zafter_transactions_transaction_insert_or_update_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_insert_or_update(); - - CREATE TRIGGER before_transactions_transaction_insert_or_update_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction_insert_or_update(); - - CREATE TRIGGER transaction_updated - AFTER UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION update_can_unamend(); - - CREATE TRIGGER calculate_aggregates_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION calculate_aggregates(); - """ - ) - - -def drop_old_functions(apps, schema_editor): - schema_editor.execute( - """ - DROP FUNCTION IF EXISTS before_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS after_transactions_transaction_insert_or_update; - DROP FUNCTION IF EXISTS calculate_aggregates; - DROP FUNCTION IF EXISTS update_can_unamend; - """ - ) - - -def reverse_drop_old_functions(apps, schema_editor): - schema_editor.execute( - """ - CREATE OR REPLACE FUNCTION before_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - needs_itemized_set boolean; - itemization boolean; - BEGIN - needs_itemized_set := needs_itemized_set(OLD, NEW); - IF needs_itemized_set THEN - NEW.itemized := calculate_itemization(NEW); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - - CREATE OR REPLACE FUNCTION after_transactions_transaction_insert_or_update() - RETURNS TRIGGER AS $$ - DECLARE - parent_and_grandparent_ids uuid[]; - children_and_grandchildren_ids uuid[]; - BEGIN - IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN - IF NEW.itemized is TRUE THEN - parent_and_grandparent_ids := - get_parent_grandparent_transaction_ids(NEW); - PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); - ELSE - children_and_grandchildren_ids := - get_children_and_grandchildren_transaction_ids(NEW); - PERFORM set_itemization_for_ids( - FALSE,children_and_grandchildren_ids - ); - END IF; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - - CREATE OR REPLACE FUNCTION update_can_unamend() - RETURNS TRIGGER AS $$ - BEGIN - UPDATE reports_report - SET can_unamend = FALSE - WHERE id IN ( - SELECT report_id - FROM reports_reporttransaction - WHERE transaction_id = NEW.id - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def process_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION process_itemization( - OLD RECORD, - NEW RECORD - ) - RETURNS RECORD AS $$ - DECLARE - needs_itemized_set boolean; - itemization boolean; - BEGIN - needs_itemized_set := needs_itemized_set(OLD, NEW); - IF needs_itemized_set THEN - NEW.itemized := calculate_itemization(NEW); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_process_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS process_itemization - """ - ) - - -def handle_parent_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION handle_parent_itemization( - OLD RECORD, - NEW RECORD - ) - RETURNS RECORD AS $$ - DECLARE - parent_and_grandparent_ids uuid[]; - children_and_grandchildren_ids uuid[]; - BEGIN - IF OLD IS NULL OR OLD.itemized <> NEW.itemized THEN - IF NEW.itemized is TRUE THEN - parent_and_grandparent_ids := - get_parent_grandparent_transaction_ids(NEW); - PERFORM set_itemization_for_ids(TRUE, parent_and_grandparent_ids); - ELSE - children_and_grandchildren_ids := - get_children_and_grandchildren_transaction_ids(NEW); - PERFORM set_itemization_for_ids( - FALSE,children_and_grandchildren_ids - ); - END IF; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_handle_parent_itemization(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS handle_parent_itemization - """ - ) - - -def calculate_aggregates(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates( - OLD RECORD, - NEW RECORD, - TG_OP TEXT - ) - RETURNS RECORD AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_calculate_aggregates(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS calculate_aggregates - """ - ) - - -def update_can_unamend(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - CREATE OR REPLACE FUNCTION update_can_unamend( - NEW RECORD - ) - RETURNS RECORD AS $$ - BEGIN - UPDATE reports_report - SET can_unamend = FALSE - WHERE id IN ( - SELECT report_id - FROM reports_reporttransaction - WHERE transaction_id = NEW.id - ); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ) - - -def reverse_update_can_unamend(apps, schema): - with connection.cursor() as cursor: - cursor.execute( - """ - DROP FUNCTION IF EXISTS update_can_unamend - """ - ) - - -# Inlined from 0020_trigger_save_on_transactions -def trigger_save_on_transactions(apps, schema_editor): - transactions = apps.get_model("transactions", "transaction") - committees = apps.get_model("committee_accounts", "committeeaccount") - contacts = apps.get_model("contacts", "contact") - - # Update transactions for each committee - for committee in committees.objects.all(): - logger.info(f"Committee:{committee.committee_id}") - - # For each contact, update the first schedule A transaction - for contact in contacts.objects.filter(committee_account=committee): - logger.info(f"Contact: {contact.id}") - first_schedule_a = ( - transactions.objects.filter( - schedule_a__isnull=False, - contact_1=contact, - committee_account=committee, - ) - .order_by("schedule_a__contribution_date", "created") - .first() - ) - if first_schedule_a: - logger.info(f"Saving first Schedule A: {first_schedule_a.id}") - first_schedule_a.save() - - # Election Aggregates - elections = transactions.objects.filter( - schedule_e__isnull=False, - committee_account=committee, - ).values( - "contact_2__candidate_office", - "contact_2__candidate_state", - "contact_2__candidate_district", - "schedule_e__election_code", - ) - for election in elections: - logger.info("Finding first schedule E for election") - first_schedule_e = ( - transactions.objects.filter( - schedule_e__isnull=False, - contact_2__candidate_office=election["contact_2__candidate_office"], - contact_2__candidate_state=election["contact_2__candidate_state"], - contact_2__candidate_district=election[ - "contact_2__candidate_district" - ], - schedule_e__election_code=election["schedule_e__election_code"], - committee_account=committee, - ) - .order_by( - "schedule_e__disbursement_date", - "created", - ) - .first() - ) - if first_schedule_e: - logger.info(f"Saving first Schedule E: {first_schedule_e.id}") - first_schedule_e.save() - - -# Inlined from 0022_schedule_f_aggregation -def calculate_schedule_f_aggregates(apps, schema_editor): - CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") # noqa - Transaction = apps.get_model("transactions", "Transaction") # noqa - - for committee in CommitteeAccount.objects.all(): - schedule_f_transactions = ( - Transaction.objects.all() - .filter( - committee_account=committee, - schedule_f__isnull=False, - ) - .annotate( - date=F("schedule_f__expenditure_date"), - amount=F("schedule_f__expenditure_amount"), - ) - .order_by("date") - ) - - for trans in schedule_f_transactions: - previous_transactions = ( - filter_queryset_for_previous_transactions_in_aggregation( # noqa: E501 - schedule_f_transactions, - trans.date, - trans.aggregation_group, - trans.id, - None, - trans.contact_2.id, - None, - trans.schedule_f.general_election_year, - ) - ) - - previous_transaction = previous_transactions.first() - previous_aggregate = 0 - if previous_transaction: - previous_aggregate = ( - previous_transaction.schedule_f.aggregate_general_elec_expended - ) - - trans.schedule_f.aggregate_general_elec_expended = ( - trans.amount + previous_aggregate - ) - trans.schedule_f.save() - - -# Inlined from 0024_scheduled_balance_at_close_and_more -def run_aggregations_for_all_debts(apps, schema_editor): - transaction = apps.get_model("transactions", "Transaction") - all_root_debts = transaction.objects.filter( - schedule_d__isnull=False, debt__isnull=True + "transactions", "OverTwoHundredTypesScheduleB" ) - for debt in all_root_debts: - process_aggregation_for_debts(debt) + scha_types_to_create = [ + OverTwoHundredTypesScheduleA(type=type_to_create) + for type_to_create in schedule_a_over_two_hundred_types + ] + OverTwoHundredTypesScheduleA.objects.bulk_create(scha_types_to_create) + schb_types_to_create = [ + OverTwoHundredTypesScheduleB(type=type_to_create) + for type_to_create in schedule_b_over_two_hundred_types + ] + OverTwoHundredTypesScheduleB.objects.bulk_create(schb_types_to_create) class Migration(migrations.Migration): @@ -987,22 +160,14 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name="transaction", - name="parent_transaction", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="children", - to="transactions.transaction", - ), - ), migrations.AddField( model_name="transaction", name="reports", field=models.ManyToManyField( - through="reports.ReportTransaction", to="reports.report" + related_name="transactions", + through="reports.ReportTransaction", + through_fields=["transaction", "report"], + to="reports.report", ), ), migrations.AddField( @@ -1015,33 +180,6 @@ class Migration(migrations.Migration): name="report_coverage_from_date", field=models.DateField(blank=True, null=True), ), - migrations.AlterField( - model_name="transaction", - name="parent_transaction", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="transactions.transaction", - ), - ), - migrations.AlterField( - model_name="transaction", - name="reports", - field=models.ManyToManyField( - related_name="transactions", - through="reports.ReportTransaction", - to="reports.report", - ), - ), - migrations.RunPython( - code=set_coverage_date, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RunPython( - code=set_aggregation_group_to_none_for_ie_memos, - reverse_code=reverse_removing_aggregation_group_for_ie_memos, - ), migrations.AddField( model_name="schedulee", name="so_candidate_state", @@ -1068,669 +206,11 @@ class Migration(migrations.Migration): blank=True, decimal_places=2, max_digits=11, null=True ), ), - migrations.RunSQL( - """ - CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - v_election_code TEXT; - v_candidate_office TEXT; - v_candidate_state TEXT; - v_candidate_district TEXT; - BEGIN - SELECT COALESCE(disbursement_date, dissemination_date) - INTO schedule_date FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT election_code - INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT candidate_office, candidate_state, candidate_district - INTO v_candidate_office, v_candidate_state, v_candidate_district - FROM contacts WHERE id = txn.contact_2_id; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - t.id, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - BEGIN - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT transaction_id FROM transactions_transaction - WHERE id = $1 - ) || ''%%''; -- Match the loan_key with a transaction_id prefix - - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id - AND tt.loan_key LIKE ''%%LOAN''; - ' - USING txn.id; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - temp_table_name TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - temp_table_name := 'temp_' || - REPLACE(uuid_generate_v4()::TEXT, '-', '_'); - RAISE NOTICE 'TESTING TRIGGER'; - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates( - NEW, sql_committee_id, temp_table_name || 'NEW'); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates( - OLD, sql_committee_id, temp_table_name || 'OLD'); - END IF; - - ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL - THEN - PERFORM calculate_loan_payment_to_date( - NEW, sql_committee_id, temp_table_name || 'NEW'); - - ELSIF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id, temp_table_name || 'NEW'); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id, temp_table_name || 'OLD'); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER calculate_aggregates_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop - EXECUTE FUNCTION calculate_aggregates(); - """ - ), - migrations.RunPython( - code=populate_existing_rows, - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION get_temp_tablename() - RETURNS TEXT AS $$ - BEGIN - RETURN 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - v_election_code TEXT; - v_candidate_office TEXT; - v_candidate_state TEXT; - v_candidate_district TEXT; - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - SELECT COALESCE(disbursement_date, dissemination_date) - INTO schedule_date FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT election_code - INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - SELECT candidate_office, candidate_state, candidate_district - INTO v_candidate_office, v_candidate_state, v_candidate_district - FROM contacts WHERE id = txn.contact_2_id; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - t.id, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, - sql_committee_id TEXT - ) - RETURNS VOID AS $$ - DECLARE - temp_table_name TEXT; - BEGIN - temp_table_name := get_temp_tablename(); - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT - CASE - WHEN loan_id IS NULL THEN transaction_id - ELSE ( - SELECT transaction_id - FROM transactions_transaction - WHERE id = t.loan_id - ) - END - FROM transactions_transaction t - WHERE id = $1 - ) || ''%%''; -- Match the loan_key with a transaction_id prefix - - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id - AND tt.loan_key LIKE ''%%LOAN''; - ' - USING txn.id; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_aggregates() - RETURNS TRIGGER AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - -- Drop prior versions of these functions - DROP FUNCTION calculate_entity_aggregates(RECORD, TEXT, TEXT); - DROP FUNCTION calculate_calendar_ytd_per_election_office( - RECORD, TEXT, TEXT - ); - DROP FUNCTION calculate_loan_payment_to_date(RECORD, TEXT, TEXT); - """ - ), - migrations.RunPython( - code=update_existing_rows, - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT - contribution_date INTO schedule_date - FROM - transactions_schedulea - WHERE - id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT - expenditure_date INTO schedule_date - FROM - transactions_scheduleb - WHERE - id = txn.schedule_b_id; - END IF; - - EXECUTE ' - UPDATE transactions_transaction AS t - SET aggregate = tc.new_sum - FROM ( - SELECT - id, - aggregate, - date, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $4 - OR ( - tc.date = $4 - AND t.created >= $5 - ) - ); - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) - INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - t.date, - SUM(t.effective_amount) OVER - (ORDER BY t.date, t.created) AS new_sum - FROM transactions_schedulee e - JOIN transaction_view__' || sql_committee_id || ' t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM t.date) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - id, - loan_key, - SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE loan_key LIKE ( - SELECT - CASE - WHEN loan_id IS NULL THEN transaction_id - ELSE ( - SELECT transaction_id - FROM transactions_transaction - WHERE id = t.loan_id - ) - END - FROM transactions_transaction t - WHERE id = $1 - ) || ''%%'' -- Match the loan_key with a transaction_id prefix - ) AS tc - WHERE t.id = tc.id - AND tc.loan_key LIKE ''%%LOAN'' - ; - ' - USING txn.id; - END; - $$ - LANGUAGE plpgsql; - """ - ), migrations.AddField( model_name="transaction", name="blocking_reports", field=django.contrib.postgres.fields.ArrayField( - base_field=models.UUIDField(), default=[], size=None + base_field=models.UUIDField(), default=list, size=None ), ), migrations.RunPython( @@ -1741,20 +221,10 @@ class Migration(migrations.Migration): code=create_trigger, reverse_code=drop_trigger, ), - migrations.AlterField( - model_name="transaction", - name="blocking_reports", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.UUIDField(), default=list, size=None - ), - ), - migrations.RunPython( - code=update_blocking_reports_default, - ), migrations.AddField( model_name="transaction", name="itemized", - field=models.BooleanField(db_default=True), + field=models.BooleanField(db_default=False), ), migrations.CreateModel( name="OverTwoHundredTypesScheduleA", @@ -1802,661 +272,7 @@ class Migration(migrations.Migration): ), migrations.RunPython( code=populate_over_two_hundred_types, - reverse_code=drop_over_two_hundred_types, - ), - migrations.RunPython( - code=create_itemized_triggers_and_functions, - reverse_code=drop_itemized_triggers_and_functions, - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT - contribution_date INTO schedule_date - FROM - transactions_schedulea - WHERE - id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT - expenditure_date INTO schedule_date - FROM - transactions_scheduleb - WHERE - id = txn.schedule_b_id; - END IF; - - EXECUTE ' - UPDATE transactions_transaction AS t - SET aggregate = tc.new_sum - FROM ( - SELECT - t.id, - COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - ) AS date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id) - ) OVER ( - ORDER BY - COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - ), - t.created - ) AS new_sum - FROM transactions_transaction t - LEFT JOIN transactions_schedulea AS sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb AS sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec AS sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 AS sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee AS se - ON t.schedule_e_id = se.id - LEFT JOIN transactions_scheduled AS sd - ON t.schedule_d_id = sd.id - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM COALESCE( - sa.contribution_date, - sb.expenditure_date, - sc.loan_incurred_date, - se.disbursement_date, - se.dissemination_date - )) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE - AND deleted IS NULL - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $4 - OR ( - tc.date = $4 - AND t.created >= $5 - ) - ); - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) - INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_effective_amount( - transaction_type_identifier TEXT, - amount NUMERIC, - schedule_c_id UUID - ) - RETURNS NUMERIC - AS $$ - DECLARE - effective_amount NUMERIC; - BEGIN - -- Case 1: transaction type is a refund - IF transaction_type_identifier IN ( - 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', - 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', - 'REFUND_PARTY_CONTRIBUTION', - 'REFUND_PARTY_CONTRIBUTION_VOID', - 'REFUND_PAC_CONTRIBUTION', - 'REFUND_PAC_CONTRIBUTION_VOID', - 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'REFUND_UNREGISTERED_CONTRIBUTION', - 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', - 'REFUND_INDIVIDUAL_CONTRIBUTION', - 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' - ) THEN - effective_amount := amount * -1; - - -- Case 2: schedule_c exists (return NULL) - ELSIF schedule_c_id IS NOT NULL THEN - effective_amount := NULL; - - -- Default case: return the original amount - ELSE - effective_amount := amount; - END IF; - - RETURN effective_amount; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_is_loan( - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID - ) - RETURNS TEXT - AS $$ - DECLARE - loan_key TEXT; - BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - loan_key := 'F'; - - ELSIF schedule_c_id IS NOT NULL THEN - loan_key := 'T'; - - ELSE - loan_key := 'F'; - END IF; - - RETURN loan_key; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_original_loan_id( - transaction_id text, - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID, - schedule_b_id UUID - ) - RETURNS TEXT - AS $$ - DECLARE - loan_transaction_id text; - BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - SELECT t.transaction_id - INTO loan_transaction_id - FROM transactions_transaction t - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - WHERE t.id = loan; - - ELSIF schedule_c_id IS NOT NULL THEN - loan_transaction_id := transaction_id; - - ELSE - loan_transaction_id := NULL; - END IF; - - RETURN loan_transaction_id; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_date( - trans_id TEXT, - loan UUID, - transaction_type_identifier TEXT, - schedule_c_id UUID, - schedule_b_id UUID - ) - RETURNS DATE - AS $$ - DECLARE - date DATE; - BEGIN - IF loan IS NOT NULL AND transaction_type_identifier - IN ('LOAN_REPAYMENT_RECEIVED', 'LOAN_REPAYMENT_MADE') - THEN - SELECT sb.expenditure_date - INTO date - FROM transactions_transaction t - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - WHERE t.transaction_id = trans_id; - - ELSIF schedule_c_id IS NOT NULL THEN - SELECT s.report_coverage_through_date INTO date - FROM transactions_schedulec s - WHERE s.id = schedule_c_id; - - ELSE - date := NULL; - END IF; - - RETURN date; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_amount( - schedule_a_contribution_amount NUMERIC, - schedule_b_expenditure_amount NUMERIC, - schedule_c_loan_amount NUMERIC, - schedule_c2_guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt UUID, -- Reference to another transaction - schedule_d_id UUID - ) - RETURNS NUMERIC - AS $$ - DECLARE - debt_incurred_amount NUMERIC; - BEGIN - IF debt IS NOT NULL THEN - SELECT sd.incurred_amount - INTO debt_incurred_amount - FROM transactions_transaction t - LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id - WHERE t.id = debt; - ELSE - debt_incurred_amount := NULL; - END IF; - - RETURN COALESCE( - schedule_a_contribution_amount, - schedule_b_expenditure_amount, - schedule_c_loan_amount, - schedule_c2_guaranteed_amount, - schedule_e_expenditure_amount, - debt_incurred_amount, - ( - SELECT incurred_amount - FROM transactions_scheduled - WHERE id = schedule_d_id - ) - ); - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunSQL( - """ - CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( - txn RECORD, sql_committee_id TEXT - ) - RETURNS VOID - AS $$ - DECLARE - pulled_forward_loans RECORD; - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se - ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - ( - SELECT loan_id - FROM transactions_transaction - WHERE id = $1 - ), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - ( - SELECT loan_id - FROM transactions_transaction - WHERE id = $1 - ), - $1 - ) - ) - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING txn.id; - - -- Handle pulled-forward loans - FOR pulled_forward_loans IN - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = txn.loan_id - LOOP - -- Recalculate loan_payment_to_date for each pulled-forward loan - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se - ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE data.original_loan_id = $1 - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING pulled_forward_loans.transaction_id; - END LOOP; - END; - $$ - LANGUAGE plpgsql; - """ - ), - migrations.RunPython( - code=drop_old_triggers, - reverse_code=reverse_drop_old_triggers, - ), - migrations.RunPython( - code=drop_old_functions, - reverse_code=reverse_drop_old_functions, - ), - migrations.RunPython( - code=process_itemization, - reverse_code=reverse_process_itemization, - ), - migrations.RunPython( - code=handle_parent_itemization, - reverse_code=reverse_handle_parent_itemization, - ), - migrations.RunPython( - code=calculate_aggregates, - reverse_code=reverse_calculate_aggregates, - ), - migrations.RunPython( - code=update_can_unamend, - reverse_code=reverse_update_can_unamend, - ), - migrations.RunPython( - code=before_transactions_transaction, - reverse_code=before_transactions_transaction, - ), - migrations.RunPython( - code=after_transactions_transaction, - reverse_code=reverse_after_transactions_transaction, - ), - migrations.RunPython( - code=create_triggers, - reverse_code=reverse_create_triggers, + reverse_code=django.db.migrations.operations.special.RunPython.noop, ), migrations.CreateModel( name="ScheduleF", @@ -2524,367 +340,6 @@ class Migration(migrations.Migration): to="contacts.contact", ), ), - migrations.RunSQL( - sql=""" - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) - INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - AND t.committee_account_id = $9 - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created, - txn.committee_account_id; - END; - $$ - LANGUAGE plpgsql; - """, - reverse_sql=""" - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID - AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) - INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created; - END; - $$ - LANGUAGE plpgsql; - """, - ), - migrations.RunPython( - code=trigger_save_on_transactions, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.AlterField( - model_name="transaction", - name="reports", - field=models.ManyToManyField( - related_name="transactions", - through="reports.ReportTransaction", - through_fields=["transaction", "report"], - to="reports.report", - ), - ), - migrations.RunPython( - code=calculate_schedule_f_aggregates, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RunSQL( - sql=""" - CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( - txn record, sql_committee_id text - ) - RETURNS VOID AS $$ - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se - ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE (data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - ( - SELECT loan_id - FROM transactions_transaction - WHERE id = $1 - ), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - ( - SELECT loan_id - FROM transactions_transaction - WHERE id = $1 - ), - $1 - ) - )) - OR data.original_loan_id IN ( - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = $2 - ) - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING txn.id, txn.loan_id; - END; - $$ - LANGUAGE plpgsql; - """, - ), migrations.AddField( model_name="scheduled", name="balance_at_close", @@ -2920,565 +375,4 @@ class Migration(migrations.Migration): blank=True, decimal_places=2, max_digits=11, null=True ), ), - migrations.RunPython( - code=run_aggregations_for_all_debts, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RunSQL( - sql=""" - -- Drop all aggregate-related triggers from - -- transactions_transaction table - DROP TRIGGER IF EXISTS calculate_aggregates_trigger - ON transactions_transaction; - DROP TRIGGER IF EXISTS after_transactions_transaction_trigger - ON transactions_transaction; - DROP TRIGGER IF EXISTS - after_transactions_transaction_infinite_trigger - ON transactions_transaction; - DROP TRIGGER IF EXISTS before_transactions_transaction_trigger - ON transactions_transaction; - """, - reverse_sql=""" - -- Recreate triggers for aggregate calculation - CREATE TRIGGER before_transactions_transaction_trigger - BEFORE INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION before_transactions_transaction(); - - CREATE TRIGGER after_transactions_transaction_infinite_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - EXECUTE FUNCTION after_transactions_transaction_infinite(); - - CREATE TRIGGER after_transactions_transaction_trigger - AFTER INSERT OR UPDATE ON transactions_transaction - FOR EACH ROW - WHEN (pg_trigger_depth() = 0) - EXECUTE FUNCTION after_transactions_transaction(); - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the calculate_entity_aggregates function - DROP FUNCTION IF EXISTS calculate_entity_aggregates( - txn RECORD, sql_committee_id TEXT, temp_table_name TEXT - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_entity_aggregates function - CREATE OR REPLACE FUNCTION calculate_entity_aggregates( - txn RECORD, - sql_committee_id TEXT, - temp_table_name TEXT - ) - RETURNS VOID AS $$ - DECLARE - schedule_date DATE; - BEGIN - IF txn.schedule_a_id IS NOT NULL THEN - SELECT contribution_date - INTO schedule_date - FROM transactions_schedulea - WHERE id = txn.schedule_a_id; - ELSIF txn.schedule_b_id IS NOT NULL THEN - SELECT expenditure_date - INTO schedule_date - FROM transactions_scheduleb - WHERE id = txn.schedule_b_id; - END IF; - - EXECUTE ' - CREATE TEMPORARY TABLE ' || temp_table_name || ' - ON COMMIT DROP AS - SELECT - id, - SUM(effective_amount) OVER (ORDER BY date, created) - AS new_sum - FROM transaction_view__' || sql_committee_id || ' - WHERE - contact_1_id = $1 - AND EXTRACT(YEAR FROM date) = $2 - AND aggregation_group = $3 - AND force_unaggregated IS NOT TRUE; - - UPDATE transactions_transaction AS t - SET aggregate = tt.new_sum - FROM ' || temp_table_name || ' AS tt - WHERE t.id = tt.id; - ' - USING - txn.contact_1_id, - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the calculate_calendar_ytd_per_election_office function - DROP FUNCTION IF EXISTS calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id TEXT, temp_table_name TEXT - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_calendar_ytd_per_election_office function - CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( - txn RECORD, sql_committee_id text - ) - RETURNS VOID AS $$ - DECLARE - schedule_date date; - v_election_code text; - v_candidate_office text; - v_candidate_state text; - v_candidate_district text; - BEGIN - SELECT - COALESCE(disbursement_date, dissemination_date) - INTO schedule_date - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT election_code INTO v_election_code - FROM transactions_schedulee - WHERE id = txn.schedule_e_id; - - SELECT - candidate_office, - candidate_state, - candidate_district INTO v_candidate_office, - v_candidate_state, - v_candidate_district - FROM contacts - WHERE id = txn.contact_2_id; - EXECUTE ' - UPDATE transactions_transaction AS t - SET _calendar_ytd_per_election_office = tc.new_sum - FROM ( - SELECT - t.id, - Coalesce( - e.disbursement_date, - e.dissemination_date - ) as date, - SUM( - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - NULL, - NULL, - NULL, - NULL, - e.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) - ) OVER (ORDER BY Coalesce( - e.disbursement_date, - e.dissemination_date - ), t.created) AS new_sum - FROM transactions_schedulee e - JOIN transactions_transaction t - ON e.id = t.schedule_e_id - JOIN contacts c - ON t.contact_2_id = c.id - WHERE - e.election_code = $1 - AND c.candidate_office = $2 - AND ( - c.candidate_state = $3 - OR ( - c.candidate_state IS NULL - AND $3 = '''' - ) - ) - AND ( - c.candidate_district = $4 - OR ( - c.candidate_district IS NULL - AND $4 = '''' - ) - ) - AND EXTRACT(YEAR FROM Coalesce( - e.disbursement_date, - e.dissemination_date - )) = $5 - AND aggregation_group = $6 - AND force_unaggregated IS NOT TRUE - AND t.committee_account_id = $9 - ) AS tc - WHERE t.id = tc.id - AND ( - tc.date > $7 - OR ( - tc.date = $7 - AND t.created >= $8 - ) - ); - ' - USING - v_election_code, - v_candidate_office, - COALESCE(v_candidate_state, ''), - COALESCE(v_candidate_district, ''), - EXTRACT(YEAR FROM schedule_date), - txn.aggregation_group, - schedule_date, - txn.created, - txn.committee_account_id; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the calculate_effective_amount function - DROP FUNCTION IF EXISTS calculate_effective_amount( - transaction_type_identifier TEXT, amount NUMERIC, - schedule_c_id UUID - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_effective_amount function - CREATE OR REPLACE FUNCTION calculate_effective_amount( - transaction_type_identifier TEXT, - amount NUMERIC, - schedule_c_id UUID - ) - RETURNS NUMERIC AS $$ - DECLARE - effective_amount NUMERIC; - BEGIN - -- Case 1: transaction type is a refund - IF transaction_type_identifier IN ( - 'TRIBAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'TRIBAL_REFUND_NP_CONVENTION_ACCOUNT', - 'TRIBAL_REFUND_NP_RECOUNT_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_CONVENTION_ACCOUNT', - 'INDIVIDUAL_REFUND_NP_RECOUNT_ACCOUNT', - 'REFUND_PARTY_CONTRIBUTION', - 'REFUND_PARTY_CONTRIBUTION_VOID', - 'REFUND_PAC_CONTRIBUTION', - 'REFUND_PAC_CONTRIBUTION_VOID', - 'INDIVIDUAL_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'BUSINESS_LABOR_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_NON_CONTRIBUTION_ACCOUNT', - 'REFUND_UNREGISTERED_CONTRIBUTION', - 'REFUND_UNREGISTERED_CONTRIBUTION_VOID', - 'REFUND_INDIVIDUAL_CONTRIBUTION', - 'REFUND_INDIVIDUAL_CONTRIBUTION_VOID', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_HEADQUARTERS_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_CONVENTION_ACCOUNT', - 'OTHER_COMMITTEE_REFUND_REFUND_NP_RECOUNT_ACCOUNT' - ) THEN - effective_amount := amount * -1; - - -- Case 2: schedule_c exists (return NULL) - ELSIF schedule_c_id IS NOT NULL THEN - effective_amount := NULL; - - -- Default case: return the original amount - ELSE - effective_amount := amount; - END IF; - - RETURN effective_amount; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the calculate_amount function if it exists - DROP FUNCTION IF EXISTS calculate_amount( - contribution_amount NUMERIC, - expenditure_amount NUMERIC, - loan_amount NUMERIC, - guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt_id UUID, - schedule_d_id UUID - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_amount function - CREATE OR REPLACE FUNCTION calculate_amount( - schedule_a_contribution_amount NUMERIC, - schedule_b_expenditure_amount NUMERIC, - schedule_c_loan_amount NUMERIC, - schedule_c2_guaranteed_amount NUMERIC, - schedule_e_expenditure_amount NUMERIC, - debt UUID, - schedule_d_id UUID - ) - RETURNS NUMERIC AS $$ - DECLARE - debt_incurred_amount NUMERIC; - BEGIN - IF debt IS NOT NULL THEN - SELECT sd.incurred_amount - INTO debt_incurred_amount - FROM transactions_transaction t - LEFT JOIN transactions_scheduled sd ON t.schedule_d_id = sd.id - WHERE t.id = debt; - ELSE - debt_incurred_amount := NULL; - END IF; - - RETURN COALESCE( - schedule_a_contribution_amount, - schedule_b_expenditure_amount, - schedule_c_loan_amount, - schedule_c2_guaranteed_amount, - schedule_e_expenditure_amount, - debt_incurred_amount, - (SELECT incurred_amount - FROM transactions_scheduled - WHERE id = schedule_d_id) - ); - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the calculate_loan_payment_to_date function if it exists - DROP FUNCTION IF EXISTS calculate_loan_payment_to_date( - txn RECORD, sql_committee_id TEXT, temp_table_name TEXT - ) CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_loan_payment_to_date function - CREATE OR REPLACE FUNCTION public.calculate_loan_payment_to_date( - txn record, sql_committee_id text - ) - RETURNS VOID AS $$ - BEGIN - EXECUTE ' - UPDATE transactions_transaction AS t - SET loan_payment_to_date = tc.new_sum - FROM ( - SELECT - data.id, - data.original_loan_id, - data.is_loan, - SUM(data.effective_amount) OVER ( - PARTITION BY data.original_loan_id - ORDER BY data.date - ) AS new_sum - FROM ( - SELECT - t.id, - calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS date, - calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) AS original_loan_id, - calculate_is_loan( - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id - ) AS is_loan, - calculate_effective_amount( - t.transaction_type_identifier, - calculate_amount( - sa.contribution_amount, - sb.expenditure_amount, - sc.loan_amount, - sc2.guaranteed_amount, - se.expenditure_amount, - t.debt_id, - t.schedule_d_id - ), - t.schedule_c_id - ) AS effective_amount - FROM transactions_transaction t - LEFT JOIN transactions_schedulea sa - ON t.schedule_a_id = sa.id - LEFT JOIN transactions_scheduleb sb - ON t.schedule_b_id = sb.id - LEFT JOIN transactions_schedulec sc - ON t.schedule_c_id = sc.id - LEFT JOIN transactions_schedulec2 sc2 - ON t.schedule_c2_id = sc2.id - LEFT JOIN transactions_schedulee se - ON t.schedule_e_id = se.id - WHERE t.deleted IS NULL - ) AS data - WHERE (data.original_loan_id = ( - SELECT calculate_original_loan_id( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id - FROM transactions_transaction - WHERE id = $1), - $1 - ) - ) - AND data.date <= ( - SELECT calculate_loan_date( - t.transaction_id, - t.loan_id, - t.transaction_type_identifier, - t.schedule_c_id, - t.schedule_b_id - ) - FROM transactions_transaction t - WHERE t.id = COALESCE( - (SELECT loan_id - FROM transactions_transaction - WHERE id = $1), - $1 - ) - )) - OR data.original_loan_id IN ( - SELECT t.transaction_id - FROM transactions_transaction t - WHERE t.schedule_c_id IS NOT NULL - AND t.loan_id = $2 - ) - ) AS tc - WHERE t.id = tc.id - AND tc.is_loan = ''T''; - ' - USING txn.id, txn.loan_id; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the trigger handler functions - DROP FUNCTION IF EXISTS after_transactions_transaction() - CASCADE; - DROP FUNCTION IF EXISTS - after_transactions_transaction_infinite() CASCADE; - DROP FUNCTION IF EXISTS before_transactions_transaction() - CASCADE; - DROP FUNCTION IF EXISTS - before_transactions_transaction_insert_or_update() - CASCADE; - DROP FUNCTION IF EXISTS - after_transactions_transaction_insert_or_update() - CASCADE; - """, - reverse_sql=""" - -- Recreate trigger handler functions - CREATE OR REPLACE FUNCTION before_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - NEW := process_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction() - RETURNS TRIGGER AS $$ - BEGIN - IF TG_OP = 'UPDATE' - THEN - NEW := calculate_aggregates(OLD, NEW, TG_OP); - NEW := update_can_unamend(NEW); - ELSE - NEW := calculate_aggregates(OLD, NEW, TG_OP); - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION after_transactions_transaction_infinite() - RETURNS TRIGGER AS $$ - BEGIN - NEW := handle_parent_itemization(OLD, NEW); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.RunSQL( - sql=""" - -- Drop the main calculate_aggregates function that was - -- called by triggers - DROP FUNCTION IF EXISTS calculate_aggregates( - old RECORD, new RECORD, tg_op TEXT - ) CASCADE; - DROP FUNCTION IF EXISTS calculate_aggregates() CASCADE; - """, - reverse_sql=""" - -- Recreate calculate_aggregates function - CREATE OR REPLACE FUNCTION calculate_aggregates( - OLD RECORD, - NEW RECORD, - TG_OP TEXT - ) - RETURNS RECORD AS $$ - DECLARE - sql_committee_id TEXT; - BEGIN - sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); - - -- If schedule_c2_id or schedule_d_id is not null, stop processing - IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL - THEN - RETURN NEW; - END IF; - - IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL - THEN - PERFORM calculate_entity_aggregates(NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - AND NEW.contact_1_id <> OLD.contact_1_id - THEN - PERFORM calculate_entity_aggregates(OLD, sql_committee_id); - END IF; - END IF; - - IF NEW.schedule_c_id IS NOT NULL - OR NEW.schedule_c1_id IS NOT NULL - OR NEW.transaction_type_identifier = 'LOAN_REPAYMENT_MADE' - THEN - PERFORM calculate_loan_payment_to_date(NEW, sql_committee_id); - END IF; - - IF NEW.schedule_e_id IS NOT NULL - THEN - PERFORM calculate_calendar_ytd_per_election_office( - NEW, sql_committee_id); - IF TG_OP = 'UPDATE' - THEN - PERFORM calculate_calendar_ytd_per_election_office( - OLD, sql_committee_id); - END IF; - END IF; - - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """, - ), - migrations.AlterField( - model_name="transaction", - name="itemized", - field=models.BooleanField(db_default=False), - ), ] From 088ca60a37d4fb51233531864f4a4b5f8d71e092 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 12 Mar 2026 16:32:46 -0400 Subject: [PATCH 21/52] FECFILE-2734: Further manually pruned and squashed no-ops and reversed ops. --- ...hed_0007_alter_committeeaccount_members.py | 76 ------------------- ...ial_squashed_0003_memotext_text_prefix.py} | 19 ++++- ...tial_squashed_0003_memotext_text_prefix.py | 32 -------- .../reports/migrations/0002_initial.py | 2 +- .../0003_update_f24_amendment_date.py | 28 ------- ...ate_committee_met_requirements_and_more.py | 2 +- ..._deleted_squashed_00019_form24_name_fix.py | 35 --------- .../transactions/migrations/0001_initial.py | 2 +- ...shed_0007_user_security_consent_version.py | 17 ----- ...initial_squashed_0003_polling_attempts.py} | 8 +- ...ompleted_squashed_0003_polling_attempts.py | 32 -------- 11 files changed, 28 insertions(+), 225 deletions(-) rename django-backend/fecfiler/memo_text/migrations/{0001_initial.py => 0001_initial_squashed_0003_memotext_text_prefix.py} (71%) delete mode 100644 django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py delete mode 100644 django-backend/fecfiler/reports/migrations/0003_update_f24_amendment_date.py rename django-backend/fecfiler/web_services/migrations/{0001_initial.py => 0001_initial_squashed_0003_polling_attempts.py} (93%) delete mode 100644 django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py diff --git a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py index c25d66720..ae61695d5 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0001_squashed_0007_alter_committeeaccount_members.py @@ -1,76 +1,12 @@ # Generated by Django 5.2.11 on 2026-03-06 22:00 import django.core.validators -import django.db.migrations.operations.special import django.db.models.deletion import uuid from django.conf import settings from django.db import migrations, models -def create_memberships(apps, schema_editor): - CommitteeAccount = apps.get_model("committee_accounts", "CommitteeAccount") - User = apps.get_model("user", "User") - Membership = apps.get_model("committee_accounts", "Membership") - db_alias = schema_editor.connection.alias - users = User.objects.using(db_alias).all() - for user in users: - committee = ( - CommitteeAccount.objects.using(db_alias) - .filter(committee_id=user.cmtee_id) - .first() - ) - if committee: - Membership.objects.create( - user=user, - committee_account=committee, - role="COMMITTEE_ADMINISTRATOR", - ) - - -def delete_pending_memberships(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") - Membership.objects.filter(user=None).delete() - - -def generate_new_uuid(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") - for membership in Membership.objects.all(): - membership.uuid = uuid.uuid4() - membership.save() - - -def delete_memberships_with_overlapping_emails(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") - Committee = apps.get_model("committee_accounts", "CommitteeAccount") - - for committee in Committee.objects.all(): - committee_memberships = Membership.objects.filter(committee_account=committee) - unique_pending_emails = set() - emails_to_prune = set() - - for membership in committee_memberships: - pending_email = str(membership.pending_email).lower() - if pending_email not in unique_pending_emails: - unique_pending_emails.add(pending_email) - else: - emails_to_prune.add(pending_email) - - for email in list(emails_to_prune): - overlapping_memberships = list( - committee_memberships.filter(pending_email__iexact=email).order_by("user") - ) - for membership_to_delete in overlapping_memberships[1:]: - membership_to_delete.delete() - - -def remove_pending_emails(apps, schema_editor): - Membership = apps.get_model("committee_accounts", "Membership") - Membership.objects.filter(pending_email__isnull=False, user_id__isnull=False).update( - pending_email=None - ) - - class Migration(migrations.Migration): replaces = [ @@ -183,16 +119,4 @@ class Migration(migrations.Migration): to=settings.AUTH_USER_MODEL, ), ), - migrations.RunPython( - code=create_memberships, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RunPython( - code=delete_memberships_with_overlapping_emails, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.RunPython( - code=remove_pending_emails, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), ] diff --git a/django-backend/fecfiler/memo_text/migrations/0001_initial.py b/django-backend/fecfiler/memo_text/migrations/0001_initial_squashed_0003_memotext_text_prefix.py similarity index 71% rename from django-backend/fecfiler/memo_text/migrations/0001_initial.py rename to django-backend/fecfiler/memo_text/migrations/0001_initial_squashed_0003_memotext_text_prefix.py index 0a493cc37..4f8bd422f 100644 --- a/django-backend/fecfiler/memo_text/migrations/0001_initial.py +++ b/django-backend/fecfiler/memo_text/migrations/0001_initial_squashed_0003_memotext_text_prefix.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-16 20:15 +# Manually squashed by Dan-go 48.0.12 on 2026-03-12 16:15 from django.db import migrations, models import django.db.models.deletion @@ -8,11 +8,18 @@ class Migration(migrations.Migration): initial = True + replaces = [ + ("memo_text", "0001_initial"), + ("memo_text", "0002_initial"), + ("memo_text", "0003_memotext_text_prefix") + ] + dependencies = [ ( "committee_accounts", "0001_squashed_0007_alter_committeeaccount_members", ), + ("reports", "0001_initial"), ] operations = [ @@ -35,6 +42,7 @@ class Migration(migrations.Migration): ("transaction_id_number", models.TextField(blank=True, null=True)), ("transaction_uuid", models.TextField(blank=True, null=True)), ("text4000", models.TextField(blank=True, null=True)), + ("text_prefix", models.TextField(blank=True, null=True)), ( "committee_account", models.ForeignKey( @@ -43,6 +51,15 @@ class Migration(migrations.Migration): to="committee_accounts.committeeaccount", ), ), + ( + "report", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="reports.report", + ), + ), ], options={ "db_table": "memo_text", diff --git a/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py b/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py deleted file mode 100644 index 188e4a59f..000000000 --- a/django-backend/fecfiler/memo_text/migrations/0002_initial_squashed_0003_memotext_text_prefix.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 5.2.11 on 2026-03-07 03:21 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - replaces = [("memo_text", "0002_initial"), ("memo_text", "0003_memotext_text_prefix")] - - dependencies = [ - ("reports", "0001_initial"), - ("memo_text", "0001_initial"), - ] - - operations = [ - migrations.AddField( - model_name="memotext", - name="report", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="reports.report", - ), - ), - migrations.AddField( - model_name="memotext", - name="text_prefix", - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0002_initial.py b/django-backend/fecfiler/reports/migrations/0002_initial.py index 0a6885c0e..1139e029b 100644 --- a/django-backend/fecfiler/reports/migrations/0002_initial.py +++ b/django-backend/fecfiler/reports/migrations/0002_initial.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ ("contacts", "0001_initial"), ("reports", "0001_initial"), - ("web_services", "0001_initial"), + ("web_services", "0001_initial_squashed_0003_polling_attempts"), ] operations = [ diff --git a/django-backend/fecfiler/reports/migrations/0003_update_f24_amendment_date.py b/django-backend/fecfiler/reports/migrations/0003_update_f24_amendment_date.py deleted file mode 100644 index 581769426..000000000 --- a/django-backend/fecfiler/reports/migrations/0003_update_f24_amendment_date.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.db import migrations - - -def update_f24_original_amendment_date(apps, schema_editor): - Report = apps.get_model("reports", "Report") # noqa - reports_to_update = Report.objects.filter( - form_type="F24A", - form_24__isnull=False, - upload_submission__isnull=False - ) - for report in reports_to_update: - report.form_24.original_amendment_date = report.upload_submission.created - report.form_24.save() - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("reports", "0002_initial"), - ] - - operations = [ - migrations.RunPython( - update_f24_original_amendment_date, - migrations.RunPython.noop - ), - ] diff --git a/django-backend/fecfiler/reports/migrations/0004_form1m_date_committee_met_requirements_and_more.py b/django-backend/fecfiler/reports/migrations/0004_form1m_date_committee_met_requirements_and_more.py index b5bcb8d2a..0e67708d9 100644 --- a/django-backend/fecfiler/reports/migrations/0004_form1m_date_committee_met_requirements_and_more.py +++ b/django-backend/fecfiler/reports/migrations/0004_form1m_date_committee_met_requirements_and_more.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('reports', '0003_update_f24_amendment_date'), + ("reports", "0002_initial"), ] operations = [ diff --git a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py index 30b1fbc20..a76f2dae9 100644 --- a/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py +++ b/django-backend/fecfiler/reports/migrations/0007_remove_report_deleted_squashed_00019_form24_name_fix.py @@ -3,9 +3,6 @@ import django.db.models.deletion import uuid from django.db import migrations, models, connection -import structlog - -logger = structlog.get_logger(__name__) def _create_can_delete_trigger(apps, schema_editor): @@ -121,13 +118,6 @@ def _reverse_can_delete_trigger(apps, schema_editor): ) -def _populate_can_delete(apps, schema_editor): - report_model = apps.get_model("reports", "Report") - for row in report_model.objects.all(): - row.can_delete = True - row.save() - - def _create_can_unamend_trigger(apps, schema_editor): schema_editor.execute( """ @@ -240,9 +230,6 @@ class Migration(migrations.Migration): code=_create_can_delete_trigger, reverse_code=_reverse_can_delete_trigger, ), - migrations.RunPython( - code=_populate_can_delete, - ), migrations.RunPython( code=_create_can_unamend_trigger, ), @@ -719,28 +706,6 @@ class Migration(migrations.Migration): name="report_type_category", field=models.TextField(blank=True, null=True), ), - migrations.RunSQL( - sql=""" - UPDATE reports_form3x f - SET filing_frequency = CASE - WHEN r.report_code IN ( - 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', - 'M8', 'M9', 'M10', 'M11', 'M12' - ) THEN 'M' - ELSE 'Q' - END, - report_type_category = CASE - WHEN r.report_code IN ( - 'M11', 'M12', 'MY' - ) THEN 'Non-Election Year' - ELSE 'Election Year' - END - FROM reports_report as r - WHERE r.form_3x_id = f.id - AND filing_frequency IS NULL AND report_type_category IS NULL; - """, - reverse_sql="", - ), migrations.AddField( model_name="form99", name="pdf_attachment", diff --git a/django-backend/fecfiler/transactions/migrations/0001_initial.py b/django-backend/fecfiler/transactions/migrations/0001_initial.py index 84f130f89..bbdbcbc55 100644 --- a/django-backend/fecfiler/transactions/migrations/0001_initial.py +++ b/django-backend/fecfiler/transactions/migrations/0001_initial.py @@ -16,7 +16,7 @@ class Migration(migrations.Migration): ), ("contacts", "0001_initial"), ("reports", "0001_initial"), - ("memo_text", "0001_initial"), + ("memo_text", "0001_initial_squashed_0003_memotext_text_prefix"), ] operations = [ diff --git a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py index 5d533c444..3a6007397 100644 --- a/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py +++ b/django-backend/fecfiler/user/migrations/0002_remove_user_cmtee_id_squashed_0007_user_security_consent_version.py @@ -1,20 +1,7 @@ # Generated by Django 5.2.11 on 2026-03-06 22:06 -import django.db.migrations.operations.special import fecfiler.user.managers from django.db import migrations, models -from django.db.models import Q - - -def remove_old_login_accounts(apps, schema_editor): - User = apps.get_model("user", "User") - - users_to_delete = User.objects.filter( - Q(username__contains="@") | Q(username="adminnxg") | Q(username="tt") - ) - for user in users_to_delete: - user.membership_set.all().delete() - users_to_delete.delete() class Migration(migrations.Migration): @@ -58,10 +45,6 @@ class Migration(migrations.Migration): ("objects", fecfiler.user.managers.UserManager()), ], ), - migrations.RunPython( - code=remove_old_login_accounts, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), migrations.AddField( model_name="user", name="security_consent_version", diff --git a/django-backend/fecfiler/web_services/migrations/0001_initial.py b/django-backend/fecfiler/web_services/migrations/0001_initial_squashed_0003_polling_attempts.py similarity index 93% rename from django-backend/fecfiler/web_services/migrations/0001_initial.py rename to django-backend/fecfiler/web_services/migrations/0001_initial_squashed_0003_polling_attempts.py index 491cb3063..2678ce9ef 100644 --- a/django-backend/fecfiler/web_services/migrations/0001_initial.py +++ b/django-backend/fecfiler/web_services/migrations/0001_initial_squashed_0003_polling_attempts.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-16 20:15 +# Manually squashed by Dan-go 48.0.12 on 2026-03-12 16:15 from django.db import migrations, models import django.db.models.deletion @@ -8,6 +8,12 @@ class Migration(migrations.Migration): initial = True + replaces = [ + ("web_services", "0001_initial"), + ("web_services", "0002_uploadsubmission_task_completed_and_more"), + ("web_services", "0003_uploadsubmission_fecfile_polling_attempts_and_more"), + ] + dependencies = [ ("reports", "0001_initial"), ] diff --git a/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py b/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py deleted file mode 100644 index 6c5d52a71..000000000 --- a/django-backend/fecfiler/web_services/migrations/0002_uploadsubmission_task_completed_squashed_0003_polling_attempts.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 5.2.11 on 2026-03-07 03:21 - -import django.db.migrations.operations.special -from django.db import migrations -from django.db.models import F - - -def set_default_task_completed_times(apps, schema_editor): - uploads = apps.get_model("web_services", "UploadSubmission") - web_prints = apps.get_model("web_services", "WebPrintSubmission") - - uploads.objects.all().update(task_completed=F("updated")) - web_prints.objects.all().update(task_completed=F("updated")) - - -class Migration(migrations.Migration): - - replaces = [ - ("web_services", "0002_uploadsubmission_task_completed_and_more"), - ("web_services", "0003_uploadsubmission_fecfile_polling_attempts_and_more"), - ] - - dependencies = [ - ("web_services", "0001_initial"), - ] - - operations = [ - migrations.RunPython( - code=set_default_task_completed_times, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - ] From 90db06f33225b653d0bdefa319c3ea5b5aacd9ca Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Mon, 16 Mar 2026 11:04:24 -0400 Subject: [PATCH 22/52] FECFILE-2913: Revised comments to add clarity. --- performance-testing/locust_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index 3c5093c39..43fe5448b 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -239,7 +239,7 @@ def create_schedule_a_transaction(self): if response.status_code != 200: raise Exception("Failed to POST new Schedule A transaction") - # if LONG_CHAINS is true then we only want to save the first transaction, + # if LONG_CHAINS is true then we only want to save a "first" transaction, # otherwise we always save the current one so we have the last transaction if not self.saved_schedule_a or not LONG_CHAINS: self.saved_schedule_a = response.json() @@ -423,6 +423,7 @@ def delete_schedule_a_transaction(self): name="delete_schedule_a_transaction", ) if response.status_code == 204: + # if we happened to delete our saved pointer, clear it so it gets reset if transaction == self.saved_schedule_a: self.saved_schedule_a = None return From f87e306d4a63984112fb33d7d360d24643de6600 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Mon, 16 Mar 2026 11:19:00 -0400 Subject: [PATCH 23/52] FECFILE-2913: Linted revised comments. --- performance-testing/locust_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance-testing/locust_run.py b/performance-testing/locust_run.py index 43fe5448b..e59679c8c 100644 --- a/performance-testing/locust_run.py +++ b/performance-testing/locust_run.py @@ -423,7 +423,7 @@ def delete_schedule_a_transaction(self): name="delete_schedule_a_transaction", ) if response.status_code == 204: - # if we happened to delete our saved pointer, clear it so it gets reset + # if we happened to delete our saved pointer, clear it so it'll reset if transaction == self.saved_schedule_a: self.saved_schedule_a = None return From 23be10c40d12f0def86db0f1d37e40cc06684061 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Tue, 17 Mar 2026 15:08:19 -0400 Subject: [PATCH 24/52] Create disable and enable committee account commands. Update activate endpoint to not allow activating a disabled committee. --- .../committee_accounts/utils/accounts.py | 36 ++++++++++++ .../fecfiler/committee_accounts/views.py | 2 +- .../commands/disable_committee_account.py | 30 ++++++++++ .../commands/enable_committee_account.py | 19 +++++++ .../tests/test_disable_committee_account.py | 57 +++++++++++++++++++ .../tests/test_enable_committee_account.py | 31 ++++++++++ django-backend/fecfiler/soft_delete/models.py | 4 ++ django-backend/manage.py | 2 + 8 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/devops/management/commands/disable_committee_account.py create mode 100644 django-backend/fecfiler/devops/management/commands/enable_committee_account.py create mode 100644 django-backend/fecfiler/devops/tests/test_disable_committee_account.py create mode 100644 django-backend/fecfiler/devops/tests/test_enable_committee_account.py diff --git a/django-backend/fecfiler/committee_accounts/utils/accounts.py b/django-backend/fecfiler/committee_accounts/utils/accounts.py index 0931dcaa0..16e67870f 100644 --- a/django-backend/fecfiler/committee_accounts/utils/accounts.py +++ b/django-backend/fecfiler/committee_accounts/utils/accounts.py @@ -6,6 +6,8 @@ import json import structlog from fecfiler.shared.utilities import query_fec_api_single +from django.contrib.sessions.models import Session +from django.utils import timezone logger = structlog.getLogger(__name__) @@ -93,6 +95,40 @@ def delete_committee_account(committee_id): logger.error(f"An error occurred while deleting the committee account: {e}") +def disable_committee_account(committee_id): + committee_account = CommitteeAccount.objects.get(committee_id=committee_id) + committee_account.delete() + logger.info(f"Committee account with ID {committee_id} has been disabled.") + + +def logout_committee_sessions(committee_id): + active_sessions = Session.objects.filter(expire_date__gte=timezone.now()) + sessions_to_delete = [] + + for session in active_sessions: + data = session.get_decoded() + if data.get("committee_id") == committee_id: + sessions_to_delete.append(session.pk) + + Session.objects.filter(pk__in=sessions_to_delete).delete() + logger.info( + f""" + Successfully logged out {len(sessions_to_delete)} users from {committee_id} + """ + ) + + +def enable_committee_account(committee_id): + try: + committee_account = CommitteeAccount.all_objects.get(committee_id=committee_id) + committee_account.undelete() + logger.info(f"Committee account with ID {committee_id} has been enabled.") + except CommitteeAccount.DoesNotExist: + logger.error(f"Committee account with ID {committee_id} does not exist.") + except Exception as e: + logger.error(f"An error occurred while enabling the committee account: {e}") + + def get_committee_emails(committee_id): match settings.FLAG__COMMITTEE_DATA_SOURCE: case "PRODUCTION": diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 89d933224..67dbc140f 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -54,7 +54,7 @@ def get_queryset(self): @action(detail=True, methods=["post"]) def activate(self, request, pk): committee = self.get_object() - if not committee: + if not committee or committee.deleted is not None: return Response("Committee could not be activated", status=403) request.session["committee_id"] = str(committee.committee_id) request.session["committee_uuid"] = str(committee.id) diff --git a/django-backend/fecfiler/devops/management/commands/disable_committee_account.py b/django-backend/fecfiler/devops/management/commands/disable_committee_account.py new file mode 100644 index 000000000..fd09c35ce --- /dev/null +++ b/django-backend/fecfiler/devops/management/commands/disable_committee_account.py @@ -0,0 +1,30 @@ +from .fecfile_base import FECCommand +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.utils.accounts import ( + disable_committee_account, + logout_committee_sessions, +) +import structlog + +logger = structlog.get_logger(__name__) + + +class Command(FECCommand): + help = "Disables a committee account" + command_name = "disable_committee_account" + + def add_arguments(self, parser): + parser.add_argument("committee_id", type=str) + + def command(self, *args, **options): + try: + committee_id = options["committee_id"] + logger.info(f"Disabling committee {committee_id}") + disable_committee_account(committee_id) + logout_committee_sessions(committee_id) + + except CommitteeAccount.DoesNotExist: + logger.error(f"Committee account with ID {committee_id} does not exist.") + except Exception as e: + logger.error(f"Error occurred while disabling committee {committee_id}: {e}") + raise diff --git a/django-backend/fecfiler/devops/management/commands/enable_committee_account.py b/django-backend/fecfiler/devops/management/commands/enable_committee_account.py new file mode 100644 index 000000000..b8920bfee --- /dev/null +++ b/django-backend/fecfiler/devops/management/commands/enable_committee_account.py @@ -0,0 +1,19 @@ +from fecfiler.devops.management.commands.fecfile_base import FECCommand +from fecfiler.committee_accounts.utils.accounts import enable_committee_account +import structlog + +logger = structlog.get_logger(__name__) + + +class Command(FECCommand): + help = "Enables a committee account" + command_name = "enable_committee_account" + + def add_arguments(self, parser): + parser.add_argument( + "committee_id", type=str, help="The ID of the committee account to enable" + ) + + def command(self, *args, **options): + committee_id = options["committee_id"] + enable_committee_account(committee_id) diff --git a/django-backend/fecfiler/devops/tests/test_disable_committee_account.py b/django-backend/fecfiler/devops/tests/test_disable_committee_account.py new file mode 100644 index 000000000..f4c0ed264 --- /dev/null +++ b/django-backend/fecfiler/devops/tests/test_disable_committee_account.py @@ -0,0 +1,57 @@ +from django.test import TestCase +from django.core.management import call_command +from django.contrib.sessions.models import Session +from django.utils import timezone +from fecfiler.committee_accounts.models import CommitteeAccount + +COMMITTEE_ID_TO_DISABLE = "C12345678" +OTHER_COMMITTEE_ID = "C87654321" + + +class DisableCommitteeAccountCommandTest(TestCase): + def setUp(self): + self.committee = CommitteeAccount.objects.create( + committee_id=COMMITTEE_ID_TO_DISABLE + ) + self.other_committee = CommitteeAccount.objects.create( + committee_id=OTHER_COMMITTEE_ID + ) + + def test_disable_committee_command(self): + s1 = Session.objects.create( + session_key="target_session", + expire_date=timezone.now() + timezone.timedelta(days=1), + ) + s1.session_data = Session.objects.encode( + {"committee_id": COMMITTEE_ID_TO_DISABLE} + ) + s1.save() + + s2 = Session.objects.create( + session_key="safe_session", + expire_date=timezone.now() + timezone.timedelta(days=1), + ) + s2.session_data = Session.objects.encode({"committee_id": OTHER_COMMITTEE_ID}) + s2.save() + + self.assertTrue( + CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID_TO_DISABLE).exists() + ) + self.assertEqual(Session.objects.count(), 2) + + call_command("disable_committee_account", COMMITTEE_ID_TO_DISABLE) + + self.assertFalse( + CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID_TO_DISABLE).exists() + ) + self.assertFalse(Session.objects.filter(session_key="target_session").exists()) + self.assertTrue( + CommitteeAccount.objects.filter(committee_id=OTHER_COMMITTEE_ID).exists() + ) + self.assertTrue(Session.objects.filter(session_key="safe_session").exists()) + + def test_disable_non_existent_committee(self): + try: + call_command("disable_committee_account", "C00000000") + except Exception as e: + self.fail(f"Command crashed with error: {e}") diff --git a/django-backend/fecfiler/devops/tests/test_enable_committee_account.py b/django-backend/fecfiler/devops/tests/test_enable_committee_account.py new file mode 100644 index 000000000..3a197e2fd --- /dev/null +++ b/django-backend/fecfiler/devops/tests/test_enable_committee_account.py @@ -0,0 +1,31 @@ +from django.test import TestCase +from django.core.management import call_command +from fecfiler.committee_accounts.models import CommitteeAccount + +COMMITTEE_ID = "C12345678" + + +class EnableCommitteeAccountCommandTest(TestCase): + def setUp(self): + self.committee = CommitteeAccount.objects.create(committee_id=COMMITTEE_ID) + self.committee.delete() + + self.assertFalse( + CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID).exists() + ) + + def test_enable_committee_command(self): + call_command("enable_committee_account", COMMITTEE_ID) + + self.assertTrue( + CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID).exists() + ) + + re_enabled_committee = CommitteeAccount.objects.get(committee_id=COMMITTEE_ID) + self.assertIsNone(re_enabled_committee.deleted) + + def test_enable_non_existent_committee(self): + try: + call_command("enable_committee_account", "C00000000") + except Exception as e: + self.fail(f"Command crashed on non-existent ID: {e}") diff --git a/django-backend/fecfiler/soft_delete/models.py b/django-backend/fecfiler/soft_delete/models.py index f11d1a67f..201f3d7a9 100644 --- a/django-backend/fecfiler/soft_delete/models.py +++ b/django-backend/fecfiler/soft_delete/models.py @@ -23,5 +23,9 @@ def delete(self): self.deleted = datetime.now(timezone.utc) self.save() + def undelete(self): + self.deleted = None + self.save() + def hard_delete(self): super(SoftDeleteModel, self).delete() diff --git a/django-backend/manage.py b/django-backend/manage.py index f2b41e946..c78f75a5e 100755 --- a/django-backend/manage.py +++ b/django-backend/manage.py @@ -26,6 +26,8 @@ "fail_open_submissions", "dump_committee_data", "get_overview", + "disable_committee_account", + "enable_committee_account", ] restricted_commands = [ "loaddata", From 2d5ffd8293b90e597c3f5a47b730a4f23ef2bf60 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 19 Mar 2026 09:46:21 -0400 Subject: [PATCH 25/52] FECFILE-2734: Updated web_services migration 0004 dependency post-squash. --- .../web_services/migrations/0004_uploadsubmission_date_filed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py b/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py index 47959e42c..e057f5db8 100644 --- a/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py +++ b/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('web_services', '0003_uploadsubmission_fecfile_polling_attempts_and_more'), + ('web_services', '0001_initial_squashed_0003_polling_attempts'), ] operations = [ From 489d75f0b7580353e491e2897da14d99e201fde2 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Thu, 19 Mar 2026 13:50:39 -0400 Subject: [PATCH 26/52] Update PyJWT --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6857ba96..e1109f5d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ gunicorn==23.0.0 invoke==2.2.0 jwcrypto==1.5.6 psycopg[pool]==3.3.0 -PyJWT==2.10.1 +PyJWT==2.12.0 redis==4.5.5 requests==2.32.4 structlog==25.3.0 From 10d0bc1a2b776efc6038593bbb1a51b0a6a89046 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Fri, 20 Mar 2026 09:21:02 -0400 Subject: [PATCH 27/52] Update Django to 5.2.12 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6857ba96..261d78f20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ django-cors-headers==4.7.0 django-deprecate-fields==0.2.1 django-migration-linter==5.2.0 django-structlog==9.1.1 -Django==5.2.11 +Django==5.2.12 djangorestframework==3.16.0 drf-spectacular==0.28.0 Faker==37.6.0 From 5b6477851affaee0f8325dabc0b2aa2f509592e5 Mon Sep 17 00:00:00 2001 From: Max Zaremba Date: Fri, 20 Mar 2026 12:36:27 -0400 Subject: [PATCH 28/52] Updated python to 3.13 --- Dockerfile-e2e | 2 +- Scheduler_Dockerfile | 2 +- Worker_Dockerfile | 2 +- Worker_Dockerfile-e2e | 2 +- runtime.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index 2b2666c8e..00b11422d 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -1,4 +1,4 @@ -FROM python:3.12 +FROM python:3.13 ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec_e2e diff --git a/Scheduler_Dockerfile b/Scheduler_Dockerfile index 5a281158d..b60194fde 100644 --- a/Scheduler_Dockerfile +++ b/Scheduler_Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12 +FROM python:3.13 ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec diff --git a/Worker_Dockerfile b/Worker_Dockerfile index 3cb8dd4a1..a5aecbf99 100644 --- a/Worker_Dockerfile +++ b/Worker_Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12 +FROM python:3.13 ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec diff --git a/Worker_Dockerfile-e2e b/Worker_Dockerfile-e2e index 513a74a58..f9aa08beb 100644 --- a/Worker_Dockerfile-e2e +++ b/Worker_Dockerfile-e2e @@ -1,4 +1,4 @@ -FROM python:3.12 +FROM python:3.13 ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec_e2e diff --git a/runtime.txt b/runtime.txt index 64f28603a..c94f67640 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.12.x +python-3.13.x From f24c50271a0932f4e50483433fa7559434eb03d2 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Fri, 20 Mar 2026 14:35:50 -0400 Subject: [PATCH 29/52] Update fecfile-validate hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6010f6744..4aa7e6e6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ Django==5.2.12 djangorestframework==3.16.0 drf-spectacular==0.28.0 Faker==37.6.0 -git+https://github.com/fecgov/fecfile-validate@0cf017439a59a08a7bed02e2069045708887209c#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@0322bb973333f175d45d2b4b06de470a0d5c48d7#egg=fecfile_validate&subdirectory=fecfile_validate_python github3.py==4.0.1 GitPython==3.1.43 gunicorn==23.0.0 From d2b37e10025ed07d65e8c05f24a209c88c876d1b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 23 Mar 2026 02:37:55 -0400 Subject: [PATCH 30/52] Removes the can_unamend triggers and their associated db functions and replaces them with native django code --- django-backend/fecfiler/reports/managers.py | 9 ++ .../0008_remove_can_unamend_trigger.py | 84 +++++++++++++++ django-backend/fecfiler/reports/models.py | 4 +- .../fecfiler/transactions/models.py | 5 + .../transactions/tests/test_models.py | 101 ++++++++++++++++++ django-backend/fecfiler/transactions/views.py | 1 + 6 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/reports/migrations/0008_remove_can_unamend_trigger.py diff --git a/django-backend/fecfiler/reports/managers.py b/django-backend/fecfiler/reports/managers.py index ac3601bf2..a1e5f9b63 100644 --- a/django-backend/fecfiler/reports/managers.py +++ b/django-backend/fecfiler/reports/managers.py @@ -66,6 +66,15 @@ def get_queryset(self): return queryset +class ReportTransactionManager(Manager): + def create(self, **kwargs): + created = super(Manager, self).create(**kwargs) + created.report.can_unamend = False + created.report.save() + + return created + + class ReportType(Enum): F3 = Value("F3") F3X = Value("F3X") diff --git a/django-backend/fecfiler/reports/migrations/0008_remove_can_unamend_trigger.py b/django-backend/fecfiler/reports/migrations/0008_remove_can_unamend_trigger.py new file mode 100644 index 000000000..3c441f49a --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0008_remove_can_unamend_trigger.py @@ -0,0 +1,84 @@ +from django.db import connection, migrations + + +def drop_trigger_functions(_apps, _schema_editor): + with connection.cursor() as cursor: + cursor.execute( + "DROP TRIGGER IF EXISTS transaction_updated ON transactions_transaction;" + ) + cursor.execute( + "DROP TRIGGER IF EXISTS transaction_created ON reports_reporttransaction;" + ) + + +def drop_db_functions(_apps, _schema_editor): + with connection.cursor() as cursor: + cursor.execute("DROP FUNCTION IF EXISTS update_can_unamend();") + cursor.execute("DROP FUNCTION IF EXISTS update_can_unamend_new_transaction();") + + +def _reverse_create_can_unamend_trigger(apps, schema_editor): + schema_editor.execute( + """ + CREATE OR REPLACE FUNCTION update_can_unamend() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE reports_report + SET can_unamend = FALSE + WHERE id IN ( + SELECT report_id + FROM reports_reporttransaction + WHERE transaction_id = NEW.id + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER transaction_updated + AFTER UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION update_can_unamend(); + + CREATE OR REPLACE FUNCTION update_can_unamend_new_transaction() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE reports_report + SET can_unamend = FALSE + WHERE id IN ( + SELECT report_id + FROM reports_reporttransaction + WHERE id = NEW.id + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER transaction_created + AFTER INSERT ON reports_reporttransaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION update_can_unamend_new_transaction(); + """ + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("reports", "0007_remove_report_deleted_squashed_00019_form24_name_fix"), + ( + "transactions", + "0002_remove_schedulea_squashed_0026_alter_transaction_itemized" + ), + ] + + operations = [ + migrations.RunPython( + code=drop_trigger_functions, + reverse_code=_reverse_create_can_unamend_trigger, + ), + migrations.RunPython( + code=drop_db_functions, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 1c993704a..3c3a32d3d 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -4,7 +4,7 @@ from django.db import models, transaction as db_transaction from django.db.models import Q from fecfiler.committee_accounts.models import CommitteeOwnedModel -from .managers import ReportManager +from .managers import ReportManager, ReportTransactionManager from .form_3.models import Form3 from .form_3x.models import Form3X from .form_24.models import Form24 @@ -235,3 +235,5 @@ class ReportTransaction(models.Model): report = models.ForeignKey(Report, on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + + objects = ReportTransactionManager() diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 71cb68509..319d0e251 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -500,6 +500,9 @@ def save(self, *args, **kwargs): error=str(e), exc_info=True, ) + + # Handle report unamending + self.reports.update(can_unamend=False) elif skip_aggregation: # Clear the flag for next save self._skip_aggregation = False @@ -534,6 +537,8 @@ def delete(self): current_date = self.get_date() if current_date: process_aggregation_for_election(self, current_date) + + self.reports.update(can_unamend=False) except Exception as e: logger.error( "Failed to update aggregates via service on delete", diff --git a/django-backend/fecfiler/transactions/tests/test_models.py b/django-backend/fecfiler/transactions/tests/test_models.py index 82709a41d..88f3cb20e 100644 --- a/django-backend/fecfiler/transactions/tests/test_models.py +++ b/django-backend/fecfiler/transactions/tests/test_models.py @@ -1465,6 +1465,107 @@ def test_reaggregation_after_report_deletion(self): # Transaction 4 should now have aggregate of $20 (10 + 10) self.assertEqual(transaction_4.aggregate, Decimal("20.00")) + def test_creating_a_transaction_resets_can_unamend(self): + # Create a report that can be unamended + report_1 = create_form3x(self.committee, "2024-01-01", "2024-01-31", {}) + report_1.can_unamend = True + report_1.save() + + # Create a new transaction + transaction_1 = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_3, + "2024-01-05", + "100.00", + report=report_1, + ) + transaction_1.refresh_from_db() + + # The report should no longer be able to be unamended + report_1.refresh_from_db() + self.assertFalse(report_1.can_unamend) + + def test_updating_a_transaction_resets_can_unamend(self): + # Create a report that can be unamended + report_1 = create_form3x(self.committee, "2024-01-01", "2024-01-31", {}) + + # Create a new transaction + transaction_1 = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_3, + "2024-01-05", + "100.00", + report=report_1, + ) + transaction_1.refresh_from_db() + + # Set can_unamend to True + report_1.can_unamend = True + report_1.save() + + # Saving the transaction should reset can_unamend + transaction_1.save() + report_1.refresh_from_db() + self.assertFalse(report_1.can_unamend) + + def test_deleting_a_transaction_resets_can_unamend(self): + # Create a report that can be unamended + report_1 = create_form3x(self.committee, "2024-01-01", "2024-01-31", {}) + + # Create a new transaction + transaction_1 = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_3, + "2024-01-05", + "100.00", + report=report_1, + ) + transaction_1.refresh_from_db() + + # Set can_unamend to True + report_1.can_unamend = True + report_1.save() + + # Deleting the transaction should reset can_unamend + transaction_1.delete() + report_1.refresh_from_db() + self.assertFalse(report_1.can_unamend) + + def test_updating_a_transaction_resets_can_unamend_in_all_affiliated_reports(self): + # Create a report that can be unamended + report_1 = create_form3x(self.committee, "2024-01-01", "2024-01-31", {}) + + # Create a new transaction + transaction_1 = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_3, + "2024-01-05", + "100.00", + report=report_1, + ) + transaction_1.refresh_from_db() + + # Set can_unamend to True + report_1.can_unamend = True + report_1.save() + + # Updating a transaction should reset can_unamend in all affiliated reports + report_2 = create_form3x(self.committee, "2024-02-01", "2024-02-28", {}) + report_2.can_unamend = True + report_2.save() + + transaction_1.reports.add(report_2) + + transaction_1.save() + report_1.refresh_from_db() + report_2.refresh_from_db() + self.assertFalse(report_1.can_unamend) + self.assertFalse(report_2.can_unamend) + def undelete(transaction): transaction.deleted = None diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index ed6346308..1aabaf225 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -155,6 +155,7 @@ def create(self, request, *args, **kwargs): saved_transaction = self.save_transaction(request.data, request) logger.info(f"Created new transaction: {saved_transaction.id}") update_dependent_parent(saved_transaction) + saved_transaction.reports.update(can_unamend=False) return Response(saved_transaction.id) def update(self, request, *args, **kwargs): From a0c6e7a5437f35f2b8970a1ef61603d7c44d1bf0 Mon Sep 17 00:00:00 2001 From: Max Zaremba Date: Mon, 23 Mar 2026 14:31:10 -0400 Subject: [PATCH 31/52] Updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6e58397e9..96b0a1f6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12 +FROM python:3.13 ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec From cdc80b587f97790325e132406d09623271e0aa65 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Mon, 23 Mar 2026 16:45:45 -0400 Subject: [PATCH 32/52] Make disabled a property for committee_accounts --- .../0008_committeeaccount_disabled.py | 18 ++++++++++++++++++ .../fecfiler/committee_accounts/models.py | 10 ++++++++++ .../committee_accounts/utils/accounts.py | 6 +++--- .../fecfiler/committee_accounts/views.py | 4 ++-- .../tests/test_disable_committee_account.py | 14 ++++---------- .../tests/test_enable_committee_account.py | 12 +++--------- django-backend/fecfiler/soft_delete/models.py | 4 ---- 7 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 django-backend/fecfiler/committee_accounts/migrations/0008_committeeaccount_disabled.py diff --git a/django-backend/fecfiler/committee_accounts/migrations/0008_committeeaccount_disabled.py b/django-backend/fecfiler/committee_accounts/migrations/0008_committeeaccount_disabled.py new file mode 100644 index 000000000..9898c3e49 --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/migrations/0008_committeeaccount_disabled.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.12 on 2026-03-23 20:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('committee_accounts', '0001_squashed_0007_alter_committeeaccount_members'), + ] + + operations = [ + migrations.AddField( + model_name='committeeaccount', + name='disabled', + field=models.DateTimeField(blank=True, default=None, null=True), + ), + ] diff --git a/django-backend/fecfiler/committee_accounts/models.py b/django-backend/fecfiler/committee_accounts/models.py index 7bbf50b52..e3765812d 100644 --- a/django-backend/fecfiler/committee_accounts/models.py +++ b/django-backend/fecfiler/committee_accounts/models.py @@ -1,3 +1,4 @@ +from django.utils import timezone import uuid from django.db import models from django.core.validators import RegexValidator @@ -27,6 +28,15 @@ class CommitteeAccount(SoftDeleteModel): ) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + disabled = models.DateTimeField(blank=True, null=True, default=None) + + def disable(self): + self.disabled = timezone.now() + self.save() + + def enable(self): + self.disabled = None + self.save() class Meta: db_table = "committee_accounts" diff --git a/django-backend/fecfiler/committee_accounts/utils/accounts.py b/django-backend/fecfiler/committee_accounts/utils/accounts.py index 16e67870f..efff89388 100644 --- a/django-backend/fecfiler/committee_accounts/utils/accounts.py +++ b/django-backend/fecfiler/committee_accounts/utils/accounts.py @@ -97,7 +97,7 @@ def delete_committee_account(committee_id): def disable_committee_account(committee_id): committee_account = CommitteeAccount.objects.get(committee_id=committee_id) - committee_account.delete() + committee_account.disable() logger.info(f"Committee account with ID {committee_id} has been disabled.") @@ -120,8 +120,8 @@ def logout_committee_sessions(committee_id): def enable_committee_account(committee_id): try: - committee_account = CommitteeAccount.all_objects.get(committee_id=committee_id) - committee_account.undelete() + committee_account = CommitteeAccount.objects.get(committee_id=committee_id) + committee_account.enable() logger.info(f"Committee account with ID {committee_id} has been enabled.") except CommitteeAccount.DoesNotExist: logger.error(f"Committee account with ID {committee_id} does not exist.") diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 67dbc140f..69a2150ff 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -53,8 +53,8 @@ def get_queryset(self): ) @action(detail=True, methods=["post"]) def activate(self, request, pk): - committee = self.get_object() - if not committee or committee.deleted is not None: + committee: CommitteeAccount = self.get_object() + if not committee or committee.disabled is not None: return Response("Committee could not be activated", status=403) request.session["committee_id"] = str(committee.committee_id) request.session["committee_uuid"] = str(committee.id) diff --git a/django-backend/fecfiler/devops/tests/test_disable_committee_account.py b/django-backend/fecfiler/devops/tests/test_disable_committee_account.py index f4c0ed264..e93ff752c 100644 --- a/django-backend/fecfiler/devops/tests/test_disable_committee_account.py +++ b/django-backend/fecfiler/devops/tests/test_disable_committee_account.py @@ -34,20 +34,14 @@ def test_disable_committee_command(self): s2.session_data = Session.objects.encode({"committee_id": OTHER_COMMITTEE_ID}) s2.save() - self.assertTrue( - CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID_TO_DISABLE).exists() - ) self.assertEqual(Session.objects.count(), 2) call_command("disable_committee_account", COMMITTEE_ID_TO_DISABLE) - - self.assertFalse( - CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID_TO_DISABLE).exists() - ) + self.committee.refresh_from_db() + self.other_committee.refresh_from_db() + self.assertIsNotNone(self.committee.disabled) self.assertFalse(Session.objects.filter(session_key="target_session").exists()) - self.assertTrue( - CommitteeAccount.objects.filter(committee_id=OTHER_COMMITTEE_ID).exists() - ) + self.assertIsNone(self.other_committee.disabled) self.assertTrue(Session.objects.filter(session_key="safe_session").exists()) def test_disable_non_existent_committee(self): diff --git a/django-backend/fecfiler/devops/tests/test_enable_committee_account.py b/django-backend/fecfiler/devops/tests/test_enable_committee_account.py index 3a197e2fd..370eca78a 100644 --- a/django-backend/fecfiler/devops/tests/test_enable_committee_account.py +++ b/django-backend/fecfiler/devops/tests/test_enable_committee_account.py @@ -8,21 +8,15 @@ class EnableCommitteeAccountCommandTest(TestCase): def setUp(self): self.committee = CommitteeAccount.objects.create(committee_id=COMMITTEE_ID) - self.committee.delete() - + self.committee.disable() self.assertFalse( - CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID).exists() + CommitteeAccount.objects.get(committee_id=COMMITTEE_ID).disabled is None ) def test_enable_committee_command(self): call_command("enable_committee_account", COMMITTEE_ID) - - self.assertTrue( - CommitteeAccount.objects.filter(committee_id=COMMITTEE_ID).exists() - ) - re_enabled_committee = CommitteeAccount.objects.get(committee_id=COMMITTEE_ID) - self.assertIsNone(re_enabled_committee.deleted) + self.assertIsNone(re_enabled_committee.disabled) def test_enable_non_existent_committee(self): try: diff --git a/django-backend/fecfiler/soft_delete/models.py b/django-backend/fecfiler/soft_delete/models.py index 201f3d7a9..f11d1a67f 100644 --- a/django-backend/fecfiler/soft_delete/models.py +++ b/django-backend/fecfiler/soft_delete/models.py @@ -23,9 +23,5 @@ def delete(self): self.deleted = datetime.now(timezone.utc) self.save() - def undelete(self): - self.deleted = None - self.save() - def hard_delete(self): super(SoftDeleteModel, self).delete() From 7de5531c477aeae3de33eff8e3c46e51b71f83a5 Mon Sep 17 00:00:00 2001 From: sVmsepi0l Date: Tue, 24 Mar 2026 11:13:40 -0400 Subject: [PATCH 33/52] optimize API test duration --- .circleci/config.yml | 167 +++++++++++++++++++++++++++---------- django-backend/.coveragerc | 3 + requirements-test.txt | 1 + 3 files changed, 129 insertions(+), 42 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f78e53562..8374fef10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,9 +5,8 @@ version: 2.1 # See: https://circleci.com/docs/2.0/orb-intro/ orbs: # See the orb documentation here: https://circleci.com/developer/orbs/orb/circleci/python - python: circleci/python@3.2.0 - browser-tools: circleci/browser-tools@2.2.1 - node: circleci/node@7.1.1 + python: circleci/python@4.0.0 + node: circleci/node@7.2.1 # See: https://circleci.com/docs/2.0/configuration-reference/#jobs jobs: @@ -15,9 +14,10 @@ jobs: # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/ # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python docker: - - image: cimg/python:3.13-node + - image: &python313_image cimg/python:3.13-node - image: cimg/postgres:14.13 - image: cimg/redis:6.2.6 + resource_class: large steps: - run: @@ -48,43 +48,14 @@ jobs: pip-dependency-file: requirements-all.txt - run: - name: Load test database fixure - command: | - psql ${DATABASE_URL} < e2e-test-db.sql - working_directory: ~/project/db - - - run: - name: Check for missing migrations - command: | - python manage.py makemigrations --check - working_directory: ~/project/django-backend/ - - - run: - name: Check for breaking change migrations - # After December 2024 - command: | - python manage.py lintmigrations --git-commit-id d73068e23fc5b035af2b224b16d4726b7b20d67c --project-root-path '.' - working_directory: ~/project/django-backend/ - - - run: - name: Run migrations - command: | - python manage.py migrate --no-input --traceback --verbosity 3 + name: Run tests + # Use built-in Django test module + command: coverage run --rcfile=.coveragerc manage.py test --parallel 4 --timing working_directory: ~/project/django-backend/ - run: - name: Run lint - command: | - flake8 --config django-backend/.flake8 - - - run: - name: Run deptry - command: deptry ~/project/ - - - run: - name: Run tests - # Use built-in Django test module - command: coverage run --source='.' --rcfile=.coveragerc manage.py test + name: Combine coverage data + command: coverage combine --rcfile=.coveragerc working_directory: ~/project/django-backend/ - run: @@ -133,14 +104,119 @@ jobs: SONARQUBE_SCANNER_PARAMS: '{"sonar.host.url":"https://sonarcloud.io"}' - save_cache: key: v1-sonarcloud-scanner-7.1.0.4889 - paths: /tmp/cache/scanner + paths: + - /tmp/cache/scanner - store_artifacts: path: /tmp/sonar_report destination: sonar_report + schema-checks: + docker: + - image: *python313_image + - image: cimg/postgres:14.13 + - image: cimg/redis:6.2.6 + + steps: + - run: + name: Check for necessary environment variables + command: | + while read var; do + [ -z "${!var}" ] && { echo "Environment variable $var is needed for a successful test run, there is no default."; exit 1; } + done \<< EOF + FECFILE_FEC_WEBSITE_API_KEY + EOF + exit 0 + + - checkout: + method: full + + - run: + name: Create unified requirements so CircleCI can cache them + command: | + cd ~/project/ + ls -l + cat requirements.txt > requirements-all.txt + echo >> requirements-all.txt # blank in case new newline at end of requirements.txt + cat requirements-test.txt >> requirements-all.txt + + - python/install-packages: + pkg-manager: pip + app-dir: ~/project/ + pip-dependency-file: requirements-all.txt + + - run: + name: Check for missing migrations + command: | + python manage.py makemigrations --check + working_directory: ~/project/django-backend/ + + - run: + name: Check for breaking change migrations + # After December 2024 + command: | + python manage.py lintmigrations --git-commit-id d73068e23fc5b035af2b224b16d4726b7b20d67c --project-root-path '.' + working_directory: ~/project/django-backend/ + + - run: + name: Run migrations + command: | + python manage.py migrate --no-input --traceback --verbosity 3 + working_directory: ~/project/django-backend/ + + lint: + docker: + - image: *python313_image + + steps: + - checkout + + - run: + name: Create unified requirements so CircleCI can cache them + command: | + cd ~/project/ + ls -l + cat requirements.txt > requirements-all.txt + echo >> requirements-all.txt # blank in case new newline at end of requirements.txt + cat requirements-test.txt >> requirements-all.txt + + - python/install-packages: + pkg-manager: pip + app-dir: ~/project/ + pip-dependency-file: requirements-all.txt + + - run: + name: Run lint + command: | + flake8 --config django-backend/.flake8 + + dependency-check: + docker: + - image: *python313_image + + steps: + - checkout + + - run: + name: Create unified requirements so CircleCI can cache them + command: | + cd ~/project/ + ls -l + cat requirements.txt > requirements-all.txt + echo >> requirements-all.txt # blank in case new newline at end of requirements.txt + cat requirements-test.txt >> requirements-all.txt + + - python/install-packages: + pkg-manager: pip + app-dir: ~/project/ + pip-dependency-file: requirements-all.txt + + - run: + name: Run deptry + command: deptry ~/project/ + deploy-job: docker: - - image: cimg/python:3.13 + - image: *python313_image steps: - checkout @@ -164,7 +240,7 @@ jobs: docs-build: docker: - - image: cimg/python:3.13 + - image: *python313_image steps: - checkout @@ -190,7 +266,8 @@ jobs: - persist_to_workspace: root: ~/project/docs/_build - paths: html + paths: + - html docs-deploy: docker: @@ -244,6 +321,9 @@ jobs: workflows: primary: # This is the name of the workflow, feel free to change it to better match your workflow. jobs: + - lint + - dependency-check + - schema-checks - test # This job is triggered whenever a commit is made to the dev/stage/test/prod branches. # It kicks off the e2e-test pipeline in the fecfile-web-app project. @@ -253,6 +333,9 @@ workflows: only: /develop|release\/sprint-[\.\d]+|release\/test|main/ - deploy-job: requires: + - lint + - dependency-check + - schema-checks - test filters: branches: diff --git a/django-backend/.coveragerc b/django-backend/.coveragerc index 32d237e78..4762eff38 100644 --- a/django-backend/.coveragerc +++ b/django-backend/.coveragerc @@ -1,6 +1,9 @@ # .coveragerc to control coverage.py # https://coverage.readthedocs.io/en/coverage-4.3.3/source.html#execution [run] +source = . +parallel = True +concurrency = multiprocessing omit = # Excluding dev scripts from code coverage ./scripts/json_schema_to_django_model.py \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index 9db851fad..212d63187 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -7,3 +7,4 @@ sphinx==7.2.6 django-silk==5.4.3 static3==0.7.0 dj-static==0.0.6 +tblib==3.1.0 From b8a50b2f7c04f924df2a9303f584afec824aa42a Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Mar 2026 12:33:29 -0400 Subject: [PATCH 34/52] Adds a signal receiver to handle the m2m_changed signal for report_transaction_changed --- django-backend/fecfiler/transactions/models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 319d0e251..3a11ca4cc 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -1,5 +1,7 @@ from django.db import models from django.db.models import Q +from django.db.models.signals import m2m_changed +from django.dispatch import receiver from django.contrib.postgres.fields import ArrayField from fecfiler.soft_delete.models import SoftDeleteModel from fecfiler.committee_accounts.models import CommitteeOwnedModel @@ -738,3 +740,16 @@ class OverTwoHundredTypesScheduleB(models.Model): class Meta: db_table = "over_two_hundred_types_scheduleb" indexes = [models.Index(fields=["type"])] + + +@receiver(m2m_changed, sender=Transaction.reports.through) +def report_transaction_changed(sender, **kwargs): + instance = kwargs.pop('instance', None) + pk_set = kwargs.pop('pk_set', None) + action = kwargs.pop('action', None) + if action == "post_add": + instance.reports.update(can_unamend=False) + if action == "post_remove" or action == "post_clear": + from fecfiler.reports.models import Report + removed_reports = Report.objects.filter(id__in=pk_set) + removed_reports.update(can_unamend=False) From ecc200ba27b39cc2d95a7ddd870bd40c604e33d2 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Mar 2026 12:33:58 -0400 Subject: [PATCH 35/52] Removes the new ReportTransactionManager which is now redundant --- django-backend/fecfiler/reports/managers.py | 9 --------- django-backend/fecfiler/reports/models.py | 6 ++---- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/django-backend/fecfiler/reports/managers.py b/django-backend/fecfiler/reports/managers.py index a1e5f9b63..ac3601bf2 100644 --- a/django-backend/fecfiler/reports/managers.py +++ b/django-backend/fecfiler/reports/managers.py @@ -66,15 +66,6 @@ def get_queryset(self): return queryset -class ReportTransactionManager(Manager): - def create(self, **kwargs): - created = super(Manager, self).create(**kwargs) - created.report.can_unamend = False - created.report.save() - - return created - - class ReportType(Enum): F3 = Value("F3") F3X = Value("F3X") diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 3c3a32d3d..4fede5cc6 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -4,7 +4,7 @@ from django.db import models, transaction as db_transaction from django.db.models import Q from fecfiler.committee_accounts.models import CommitteeOwnedModel -from .managers import ReportManager, ReportTransactionManager +from .managers import ReportManager from .form_3.models import Form3 from .form_3x.models import Form3X from .form_24.models import Form24 @@ -234,6 +234,4 @@ class ReportTransaction(models.Model): transaction = models.ForeignKey("transactions.Transaction", on_delete=models.CASCADE) report = models.ForeignKey(Report, on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) - updated = models.DateTimeField(auto_now=True) - - objects = ReportTransactionManager() + updated = models.DateTimeField(auto_now=True) \ No newline at end of file From 636e9c5bf1441a1d234cbfb825b0fa77f7e979bc Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Mar 2026 12:35:43 -0400 Subject: [PATCH 36/52] Removes the manual setting of can_unamend in transaction creation since that's now redundant --- django-backend/fecfiler/transactions/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 1aabaf225..ed6346308 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -155,7 +155,6 @@ def create(self, request, *args, **kwargs): saved_transaction = self.save_transaction(request.data, request) logger.info(f"Created new transaction: {saved_transaction.id}") update_dependent_parent(saved_transaction) - saved_transaction.reports.update(can_unamend=False) return Response(saved_transaction.id) def update(self, request, *args, **kwargs): From b46f845871d65866f158b70e213999af682a8b6b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Mar 2026 12:36:06 -0400 Subject: [PATCH 37/52] Updates unit tests to be more consistent with behavior in the api --- django-backend/fecfiler/transactions/tests/test_models.py | 2 +- django-backend/fecfiler/transactions/tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/tests/test_models.py b/django-backend/fecfiler/transactions/tests/test_models.py index 88f3cb20e..ef9576d49 100644 --- a/django-backend/fecfiler/transactions/tests/test_models.py +++ b/django-backend/fecfiler/transactions/tests/test_models.py @@ -1558,7 +1558,7 @@ def test_updating_a_transaction_resets_can_unamend_in_all_affiliated_reports(sel report_2.can_unamend = True report_2.save() - transaction_1.reports.add(report_2) + transaction_1.reports.set([report_1, report_2]) transaction_1.save() report_1.refresh_from_db() diff --git a/django-backend/fecfiler/transactions/tests/utils.py b/django-backend/fecfiler/transactions/tests/utils.py index 06ba64db8..f39f96d88 100644 --- a/django-backend/fecfiler/transactions/tests/utils.py +++ b/django-backend/fecfiler/transactions/tests/utils.py @@ -298,7 +298,7 @@ def create_test_transaction( **(transaction_data or {}) ) if report: - create_report_transaction(report, transaction) + transaction.reports.set([report]) return transaction From 5f3a205c9ea673bfe4b01f213060eb562d28bc3e Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Mar 2026 12:42:45 -0400 Subject: [PATCH 38/52] Restores the report models.py file to develop --- django-backend/fecfiler/reports/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 4fede5cc6..1c993704a 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -234,4 +234,4 @@ class ReportTransaction(models.Model): transaction = models.ForeignKey("transactions.Transaction", on_delete=models.CASCADE) report = models.ForeignKey(Report, on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) - updated = models.DateTimeField(auto_now=True) \ No newline at end of file + updated = models.DateTimeField(auto_now=True) From 2ab5a88e7f5f271269f948f0feed281ff43a53c7 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Mar 2026 12:44:34 -0400 Subject: [PATCH 39/52] Reseting can_unamend in delete() is also now redundant --- django-backend/fecfiler/transactions/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 3a11ca4cc..55648aebf 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -540,7 +540,6 @@ def delete(self): if current_date: process_aggregation_for_election(self, current_date) - self.reports.update(can_unamend=False) except Exception as e: logger.error( "Failed to update aggregates via service on delete", From 100b19eb0197944d6a02801ef9c64f0ca811d7c3 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Tue, 24 Mar 2026 12:38:45 -0400 Subject: [PATCH 40/52] Drop unused functions --- .../0003_remove_unused_functions.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 django-backend/fecfiler/transactions/migrations/0003_remove_unused_functions.py diff --git a/django-backend/fecfiler/transactions/migrations/0003_remove_unused_functions.py b/django-backend/fecfiler/transactions/migrations/0003_remove_unused_functions.py new file mode 100644 index 000000000..9f2c0c2e0 --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0003_remove_unused_functions.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2.12 on 2026-03-24 16:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "transactions", + "0002_remove_schedulea_squashed_0026_alter_transaction_itemized", + ), + ] + + operations = [ + migrations.RunSQL( + sql=""" + DROP FUNCTION IF EXISTS + public.calculate_calendar_ytd_per_election_office CASCADE; + DROP FUNCTION IF EXISTS public.calculate_entity_aggregates CASCADE; + DROP FUNCTION IF EXISTS public.calculate_is_loan CASCADE; + DROP FUNCTION IF EXISTS public.calculate_itemization CASCADE; + DROP FUNCTION IF EXISTS public.calculate_loan_date CASCADE; + DROP FUNCTION IF EXISTS public.calculate_loan_payment_to_date CASCADE; + DROP FUNCTION IF EXISTS public.calculate_original_loan_id CASCADE; + DROP FUNCTION IF EXISTS + public.get_children_and_grandchildren_transaction_ids CASCADE; + DROP FUNCTION IF EXISTS + public.get_parent_grandparent_transaction_ids CASCADE; + DROP FUNCTION IF EXISTS public.get_temp_tablename CASCADE; + DROP FUNCTION IF EXISTS public.handle_parent_itemization CASCADE; + DROP FUNCTION IF EXISTS public.needs_itemized_set CASCADE; + DROP FUNCTION IF EXISTS public.process_itemization CASCADE; + DROP FUNCTION IF EXISTS public.set_itemization_for_ids CASCADE; + """, + ), + ] From bef8a9ec96715bf780cf73cbb1fbcf3d3d0ba9a2 Mon Sep 17 00:00:00 2001 From: sVmsepi0l Date: Tue, 24 Mar 2026 18:53:47 -0400 Subject: [PATCH 41/52] reusable command for caching unified requirements --- .circleci/config.yml | 65 ++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 48 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8374fef10..d63f382c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,18 @@ orbs: python: circleci/python@4.0.0 node: circleci/node@7.2.1 +commands: + unified-requirements-cache: + steps: + - run: + name: Create unified requirements so CircleCI can cache them + command: | + cd ~/project/ + ls -l + cat requirements.txt > requirements-all.txt + echo >> requirements-all.txt # blank in case new newline at end of requirements.txt + cat requirements-test.txt >> requirements-all.txt + # See: https://circleci.com/docs/2.0/configuration-reference/#jobs jobs: test: @@ -33,15 +45,7 @@ jobs: - checkout: method: full - - run: - name: Create unified requirements so CircleCI can cache them - command: | - cd ~/project/ - ls -l - cat requirements.txt > requirements-all.txt - echo >> requirements-all.txt # blank in case new newline at end of requirements.txt - cat requirements-test.txt >> requirements-all.txt - + - unified-requirements-cache - python/install-packages: pkg-manager: pip app-dir: ~/project/ @@ -130,15 +134,7 @@ jobs: - checkout: method: full - - run: - name: Create unified requirements so CircleCI can cache them - command: | - cd ~/project/ - ls -l - cat requirements.txt > requirements-all.txt - echo >> requirements-all.txt # blank in case new newline at end of requirements.txt - cat requirements-test.txt >> requirements-all.txt - + - unified-requirements-cache - python/install-packages: pkg-manager: pip app-dir: ~/project/ @@ -169,16 +165,7 @@ jobs: steps: - checkout - - - run: - name: Create unified requirements so CircleCI can cache them - command: | - cd ~/project/ - ls -l - cat requirements.txt > requirements-all.txt - echo >> requirements-all.txt # blank in case new newline at end of requirements.txt - cat requirements-test.txt >> requirements-all.txt - + - unified-requirements-cache - python/install-packages: pkg-manager: pip app-dir: ~/project/ @@ -195,16 +182,7 @@ jobs: steps: - checkout - - - run: - name: Create unified requirements so CircleCI can cache them - command: | - cd ~/project/ - ls -l - cat requirements.txt > requirements-all.txt - echo >> requirements-all.txt # blank in case new newline at end of requirements.txt - cat requirements-test.txt >> requirements-all.txt - + - unified-requirements-cache - python/install-packages: pkg-manager: pip app-dir: ~/project/ @@ -244,16 +222,7 @@ jobs: steps: - checkout - - - run: - name: Create unified requirements so CircleCI can cache them - command: | - cd ~/project/ - ls -l - cat requirements.txt > requirements-all.txt - echo >> requirements-all.txt # blank in case new newline at end of requirements.txt - cat requirements-test.txt >> requirements-all.txt - + - unified-requirements-cache - python/install-packages: pkg-manager: pip app-dir: ~/project/ From 8549fd90df57116fba6c728fa7f9ce159d063dbf Mon Sep 17 00:00:00 2001 From: sVmsepi0l Date: Tue, 24 Mar 2026 18:58:43 -0400 Subject: [PATCH 42/52] reusable commands for unified-requirements-cache & check-env-vars --- .circleci/config.yml | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d63f382c9..34679fb95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,6 +20,17 @@ commands: echo >> requirements-all.txt # blank in case new newline at end of requirements.txt cat requirements-test.txt >> requirements-all.txt + check-env-vars: + steps: + - run: + name: Check for necessary environment variables + command: | + while read var; do + [ -z "${!var}" ] && { echo "Environment variable $var is needed for a successful test run, there is no default."; exit 1; } + done \<< EOF + FECFILE_FEC_WEBSITE_API_KEY + EOF + exit 0 # See: https://circleci.com/docs/2.0/configuration-reference/#jobs jobs: test: @@ -32,16 +43,7 @@ jobs: resource_class: large steps: - - run: - name: Check for necessary environment variables - command: | - while read var; do - [ -z "${!var}" ] && { echo "Environment variable $var is needed for a successful test run, there is no default."; exit 1; } - done \<< EOF - FECFILE_FEC_WEBSITE_API_KEY - EOF - exit 0 - + - check-env-vars - checkout: method: full @@ -121,16 +123,7 @@ jobs: - image: cimg/redis:6.2.6 steps: - - run: - name: Check for necessary environment variables - command: | - while read var; do - [ -z "${!var}" ] && { echo "Environment variable $var is needed for a successful test run, there is no default."; exit 1; } - done \<< EOF - FECFILE_FEC_WEBSITE_API_KEY - EOF - exit 0 - + - check-env-vars - checkout: method: full From 730d36eaf9abc5d92679dd23f0d57e6878376cca Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 11:12:18 -0400 Subject: [PATCH 43/52] Moves ReportTransaction creation override to the model --- django-backend/fecfiler/reports/managers.py | 9 --------- django-backend/fecfiler/reports/models.py | 9 +++++++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/django-backend/fecfiler/reports/managers.py b/django-backend/fecfiler/reports/managers.py index a1e5f9b63..ac3601bf2 100644 --- a/django-backend/fecfiler/reports/managers.py +++ b/django-backend/fecfiler/reports/managers.py @@ -66,15 +66,6 @@ def get_queryset(self): return queryset -class ReportTransactionManager(Manager): - def create(self, **kwargs): - created = super(Manager, self).create(**kwargs) - created.report.can_unamend = False - created.report.save() - - return created - - class ReportType(Enum): F3 = Value("F3") F3X = Value("F3X") diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 3c3a32d3d..e0f1585fc 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -4,7 +4,7 @@ from django.db import models, transaction as db_transaction from django.db.models import Q from fecfiler.committee_accounts.models import CommitteeOwnedModel -from .managers import ReportManager, ReportTransactionManager +from .managers import ReportManager from .form_3.models import Form3 from .form_3x.models import Form3X from .form_24.models import Form24 @@ -236,4 +236,9 @@ class ReportTransaction(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) - objects = ReportTransactionManager() + def save(self, *args, **kwargs): + with db_transaction.atomic(): + super(ReportTransaction, self).save(*args, **kwargs) + if self.report.can_unamend: + self.report.can_unamend = False + self.report.save() From c52cef185bf963939a43bc5e55ca565632be2e82 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 11:17:31 -0400 Subject: [PATCH 44/52] Resets can_unamend on ReportTransaction deletion --- django-backend/fecfiler/reports/models.py | 7 +++++++ django-backend/fecfiler/transactions/views.py | 9 ++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index e0f1585fc..a2547137b 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -242,3 +242,10 @@ def save(self, *args, **kwargs): if self.report.can_unamend: self.report.can_unamend = False self.report.save() + + def delete(self, *args, **kwargs): + with db_transaction.atomic(): + super(ReportTransaction, self).delete(*args, **kwargs) + if self.report.can_unamend: + self.report.can_unamend = False + self.report.save() diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 1aabaf225..95463adbf 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -31,7 +31,7 @@ process_aggregation_for_entity_contact, process_aggregation_for_election, ) -from fecfiler.reports.models import Report +from fecfiler.reports.models import Report, ReportTransaction from fecfiler.contacts.models import Contact from fecfiler.contacts.serializers import create_or_update_contact from fecfiler.transactions.schedule_c.views import save_hook as schedule_c_save_hook @@ -155,7 +155,6 @@ def create(self, request, *args, **kwargs): saved_transaction = self.save_transaction(request.data, request) logger.info(f"Created new transaction: {saved_transaction.id}") update_dependent_parent(saved_transaction) - saved_transaction.reports.update(can_unamend=False) return Response(saved_transaction.id) def update(self, request, *args, **kwargs): @@ -515,7 +514,11 @@ def save_transaction(self, transaction_data, request): transaction_instance = transaction_serializer.save(**save_kwargs) # Link the transaction to all the reports it references in report_ids - transaction_instance.reports.set(report_ids) + for report_id in report_ids: + ReportTransaction.objects.create( + transaction=transaction_instance, + report_id=report_id + ) if transaction_instance.schedule_c or transaction_instance.schedule_d: reports = Report.objects.filter(id__in=report_ids) coverage_through_date = None From 41cb3b002e8e2df7b019c1a9c816a0e4301e7dad Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 11:17:48 -0400 Subject: [PATCH 45/52] Updates unit tests --- django-backend/fecfiler/transactions/tests/test_models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/tests/test_models.py b/django-backend/fecfiler/transactions/tests/test_models.py index 88f3cb20e..57bf27e9d 100644 --- a/django-backend/fecfiler/transactions/tests/test_models.py +++ b/django-backend/fecfiler/transactions/tests/test_models.py @@ -1,6 +1,7 @@ from decimal import Decimal from django.test import TestCase from fecfiler.reports.tests.utils import create_form3x +from fecfiler.reports.models import ReportTransaction from fecfiler.committee_accounts.models import CommitteeAccount from fecfiler.transactions.models import Transaction from fecfiler.memo_text.models import MemoText @@ -1558,7 +1559,10 @@ def test_updating_a_transaction_resets_can_unamend_in_all_affiliated_reports(sel report_2.can_unamend = True report_2.save() - transaction_1.reports.add(report_2) + ReportTransaction.objects.create( + transaction=transaction_1, + report=report_2 + ) transaction_1.save() report_1.refresh_from_db() From 59864ebe76b56d115c33f9cbfdbf030652ae8608 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 11:29:28 -0400 Subject: [PATCH 46/52] Removes redundant can_unamend reset in the transaction model for the delete case --- django-backend/fecfiler/transactions/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 319d0e251..1080d0cc9 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -537,8 +537,6 @@ def delete(self): current_date = self.get_date() if current_date: process_aggregation_for_election(self, current_date) - - self.reports.update(can_unamend=False) except Exception as e: logger.error( "Failed to update aggregates via service on delete", From 83948521fe0b95c86eb68dfb8623e520837877f4 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 12:16:42 -0400 Subject: [PATCH 47/52] Updates transaction saving (in the viewset) to ensure that ReportTransactions are created and deleted appropriately --- django-backend/fecfiler/transactions/views.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 95463adbf..d39cdbecc 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -514,11 +514,31 @@ def save_transaction(self, transaction_data, request): transaction_instance = transaction_serializer.save(**save_kwargs) # Link the transaction to all the reports it references in report_ids - for report_id in report_ids: + current_report_ids = set() + current_report_id_dicts = list(transaction_instance.reports.values("id")) + for report_id_dict in current_report_id_dicts: + current_report_ids.add(str(report_id_dict["id"])) + + updated_report_ids = set(report_ids) + + new_report_ids = updated_report_ids - current_report_ids + removed_report_ids = current_report_ids - updated_report_ids + + for report_id in new_report_ids: ReportTransaction.objects.create( transaction=transaction_instance, report_id=report_id ) + + for report_id in removed_report_ids: + report_transaction = ReportTransaction.objects.filter( + transaction=transaction_instance, + report_id=report_id + ).first() + if report_transaction is not None: + report_transaction.delete() + + # handle loans and debts if transaction_instance.schedule_c or transaction_instance.schedule_d: reports = Report.objects.filter(id__in=report_ids) coverage_through_date = None From bfee9416d3ede690e3851f9fc8c9da97b378e9cf Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 13:18:13 -0400 Subject: [PATCH 48/52] All adding and removing of reports from a transaction is handled in the transaction model --- .../fecfiler/reports/tests/test_models.py | 3 +- .../fecfiler/transactions/models.py | 56 ++++++++++++++----- .../schedule_c/tests/test_views.py | 2 +- .../schedule_c2/tests/test_views.py | 4 +- .../schedule_d/tests/test_views.py | 2 +- .../transactions/tests/test_manager.py | 2 +- .../transactions/tests/test_models.py | 2 +- .../fecfiler/transactions/tests/utils.py | 2 +- django-backend/fecfiler/transactions/views.py | 31 ++-------- .../dot_fec/tests/test_dot_fec_composer.py | 4 +- .../dot_fec/tests/test_dot_fec_serializer.py | 2 +- .../web_services/summary/test_tasks.py | 6 +- .../web_services/summary/tests/utils.py | 10 ++-- 13 files changed, 65 insertions(+), 61 deletions(-) diff --git a/django-backend/fecfiler/reports/tests/test_models.py b/django-backend/fecfiler/reports/tests/test_models.py index 6eaf17ff9..9cf08a298 100644 --- a/django-backend/fecfiler/reports/tests/test_models.py +++ b/django-backend/fecfiler/reports/tests/test_models.py @@ -83,8 +83,7 @@ def test_delete(self): "H2024", candidate_a, ) - ie.reports.set([f24_report_id, f3x_report_id]) - ie.save() + ie.set_reports([f24_report_id, f3x_report_id]) ie_id = ie.id f24_report.delete() diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 8a1fdecbc..55fd1bbf8 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -1,7 +1,6 @@ from django.db import models from django.db.models import Q -from django.db.models.signals import m2m_changed -from django.dispatch import receiver +from django.apps import apps from django.contrib.postgres.fields import ArrayField from fecfiler.soft_delete.models import SoftDeleteModel from fecfiler.committee_accounts.models import CommitteeOwnedModel @@ -660,6 +659,46 @@ def delete_coupled_transactions(self): parent.refresh_from_db(fields=["deleted"]) parent.delete() + def add_to_report(self, report_id): + ReportTransaction = apps.get_model("reports.ReportTransaction") + report_transaction = ReportTransaction.objects.filter( + transaction=self, + report_id=report_id + ).first() + + if report_transaction is None: + ReportTransaction.objects.create( + transaction=self, + report_id=report_id + ) + + def remove_from_report(self, report_id): + ReportTransaction = apps.get_model("reports.ReportTransaction") + report_transaction = ReportTransaction.objects.filter( + transaction=self, + report_id=report_id + ).first() + + if report_transaction is not None: + report_transaction.delete() + + def set_reports(self, report_ids): + current_report_ids = set() + current_report_id_dicts = list(self.reports.values("id")) + for report_id_dict in current_report_id_dicts: + current_report_ids.add(str(report_id_dict["id"])) + + updated_report_ids = set(report_ids) + + new_report_ids = updated_report_ids - current_report_ids + removed_report_ids = current_report_ids - updated_report_ids + + for report_id in new_report_ids: + self.add_to_report(report_id) + + for report_id in removed_report_ids: + self.remove_from_report(report_id) + class Meta: indexes = [models.Index(fields=["_form_type"])] @@ -738,16 +777,3 @@ class OverTwoHundredTypesScheduleB(models.Model): class Meta: db_table = "over_two_hundred_types_scheduleb" indexes = [models.Index(fields=["type"])] - - -@receiver(m2m_changed, sender=Transaction.reports.through) -def report_transaction_changed(sender, **kwargs): - instance = kwargs.pop('instance', None) - pk_set = kwargs.pop('pk_set', None) - action = kwargs.pop('action', None) - if action == "post_add": - instance.reports.update(can_unamend=False) - if action == "post_remove" or action == "post_clear": - from fecfiler.reports.models import Report - removed_reports = Report.objects.filter(id__in=pk_set) - removed_reports.update(can_unamend=False) diff --git a/django-backend/fecfiler/transactions/schedule_c/tests/test_views.py b/django-backend/fecfiler/transactions/schedule_c/tests/test_views.py index 1d50d20d6..f7ad747f0 100644 --- a/django-backend/fecfiler/transactions/schedule_c/tests/test_views.py +++ b/django-backend/fecfiler/transactions/schedule_c/tests/test_views.py @@ -58,7 +58,7 @@ def setUp(self): self.loan.save() self.loan.memo_text.transaction_uuid = self.loan.id self.loan.memo_text.save() - self.loan.reports.add(self.report_1) + self.loan.add_to_report(self.report_1.id) def test_create_loan_in_future_report(self): save_hook(self.loan, False) diff --git a/django-backend/fecfiler/transactions/schedule_c2/tests/test_views.py b/django-backend/fecfiler/transactions/schedule_c2/tests/test_views.py index bd0cc65ee..c95c55951 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/tests/test_views.py +++ b/django-backend/fecfiler/transactions/schedule_c2/tests/test_views.py @@ -40,7 +40,7 @@ def setUp(self): self.schedule_c.save() self.loan.schedule_c = self.schedule_c self.loan.save() - self.loan.reports.add(self.report_1) + self.loan.add_to_report(self.report_1.id) self.report_2 = Report( form_type="F3XN", @@ -63,7 +63,7 @@ def setUp(self): self.schedule_c2.save() self.guarantor.schedule_c2 = self.schedule_c2 self.guarantor.save() - self.guarantor.reports.add(self.report_1) + self.guarantor.add_to_report(self.report_1.id) def test_create_guarantor_in_future_report(self): c2_hook(self.guarantor, False) diff --git a/django-backend/fecfiler/transactions/schedule_d/tests/test_views.py b/django-backend/fecfiler/transactions/schedule_d/tests/test_views.py index 41cfb476a..2d4be47c8 100644 --- a/django-backend/fecfiler/transactions/schedule_d/tests/test_views.py +++ b/django-backend/fecfiler/transactions/schedule_d/tests/test_views.py @@ -42,7 +42,7 @@ def setUp(self): self.schedule_d.save() self.debt.schedule_d = self.schedule_d self.debt.save() - self.debt.reports.add(self.report_1) + self.debt.add_to_report(self.report_1.id) def test_create_debt_in_future_report(self): save_hook(self.debt, False) diff --git a/django-backend/fecfiler/transactions/tests/test_manager.py b/django-backend/fecfiler/transactions/tests/test_manager.py index a9442419d..d30eab8ea 100644 --- a/django-backend/fecfiler/transactions/tests/test_manager.py +++ b/django-backend/fecfiler/transactions/tests/test_manager.py @@ -289,7 +289,7 @@ def test_debts(self): q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) original_debt = create_debt(self.committee, self.contact_1, Decimal("123.00")) original_debt.save() - original_debt.reports.add(q1_report) + original_debt.add_to_report(q1_report.id) first_repayment = create_schedule_b( "OPERATING_EXPENDITURE", self.committee, diff --git a/django-backend/fecfiler/transactions/tests/test_models.py b/django-backend/fecfiler/transactions/tests/test_models.py index 57bf27e9d..2cbd92dc8 100644 --- a/django-backend/fecfiler/transactions/tests/test_models.py +++ b/django-backend/fecfiler/transactions/tests/test_models.py @@ -240,7 +240,7 @@ def test_delete_debt_transactions(self): carried forward copies of it (along with repayments to them)""" original_debt = create_debt(self.committee, self.contact_1, Decimal("123.00")) original_debt.save() - original_debt.reports.add(self.q1_report) + original_debt.add_to_report(self.q1_report.id) carried_forward_debt = carry_forward_debt(original_debt, self.m1_report) first_repayment = create_schedule_b( "OPERATING_EXPENDITURE", diff --git a/django-backend/fecfiler/transactions/tests/utils.py b/django-backend/fecfiler/transactions/tests/utils.py index f39f96d88..386209153 100644 --- a/django-backend/fecfiler/transactions/tests/utils.py +++ b/django-backend/fecfiler/transactions/tests/utils.py @@ -298,7 +298,7 @@ def create_test_transaction( **(transaction_data or {}) ) if report: - transaction.reports.set([report]) + transaction.set_reports([report.id]) return transaction diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index d39cdbecc..8a0ac895a 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -31,7 +31,7 @@ process_aggregation_for_entity_contact, process_aggregation_for_election, ) -from fecfiler.reports.models import Report, ReportTransaction +from fecfiler.reports.models import Report from fecfiler.contacts.models import Contact from fecfiler.contacts.serializers import create_or_update_contact from fecfiler.transactions.schedule_c.views import save_hook as schedule_c_save_hook @@ -236,7 +236,7 @@ def add_transaction_to_report(self, request): transaction = Transaction.objects.get(id=request.data.get("transaction_id")) transactions = transaction.get_transaction_family() for t in transactions: - t.reports.add(report) + t.add_to_report(report.id) except Transaction.DoesNotExist: return Response("No transaction matching id provided", status=404) @@ -254,7 +254,8 @@ def remove_transaction_from_report(self, request): except Transaction.DoesNotExist: return Response("No transaction matching id provided", status=404) - transaction.reports.remove(report) + transaction.remove_from_report(report.id) + return Response("Transaction removed from report") @action(detail=False, methods=["get"], url_path=r"previous/entity") @@ -514,29 +515,7 @@ def save_transaction(self, transaction_data, request): transaction_instance = transaction_serializer.save(**save_kwargs) # Link the transaction to all the reports it references in report_ids - current_report_ids = set() - current_report_id_dicts = list(transaction_instance.reports.values("id")) - for report_id_dict in current_report_id_dicts: - current_report_ids.add(str(report_id_dict["id"])) - - updated_report_ids = set(report_ids) - - new_report_ids = updated_report_ids - current_report_ids - removed_report_ids = current_report_ids - updated_report_ids - - for report_id in new_report_ids: - ReportTransaction.objects.create( - transaction=transaction_instance, - report_id=report_id - ) - - for report_id in removed_report_ids: - report_transaction = ReportTransaction.objects.filter( - transaction=transaction_instance, - report_id=report_id - ).first() - if report_transaction is not None: - report_transaction.delete() + transaction_instance.set_reports(report_ids) # handle loans and debts if transaction_instance.schedule_c or transaction_instance.schedule_d: diff --git a/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_composer.py index decb5e3b0..a76badfb6 100644 --- a/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_composer.py @@ -84,7 +84,7 @@ def setUp(self): "GENERAL", "SA11AI", ) - self.transaction.reports.add(self.f3x) + self.transaction.add_to_report(self.f3x.id) self.transaction.save() self.report_level_memo = create_report_memo( self.committee, @@ -137,7 +137,7 @@ def test_row_contains_aggregate(self): "SA11AI", ) for transaction in [earlier_transaction, later_transaction]: - transaction.reports.add(self.f3x) + transaction.add_to_report(self.f3x.id) transaction.save() later_transaction.refresh_from_db() diff --git a/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py index 04cae5ce0..2bf7ed3ad 100644 --- a/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py @@ -51,7 +51,7 @@ def setUp(self): "GENERAL", "SA11AI", ) - self.transaction.reports.add(self.f3x) + self.transaction.add_to_report(self.f3x.id) self.transaction.save() self.schc_transaction1 = create_loan( diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 4167f4ff5..ee0d6ccd7 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -193,7 +193,7 @@ def test_report_group_recalculation_year_to_year(self): data["memo"], data["itemized"], ) - scha.reports.add(data["report"]) + scha.add_to_report(data["report"].id) scha.save() calculate_summary(next_year_report.id) @@ -325,7 +325,7 @@ def test_cash_on_hand_override_between_reports(self): data["memo"], data["itemized"], ) - scha.reports.add(data["report"]) + scha.add_to_report(data["report"].id) scha.save() years_later_report = create_form3x( @@ -369,7 +369,7 @@ def test_cash_on_hand_override_for_last_year(self): False, True, ) - schedule_a.reports.add(first_report) + schedule_a.add_to_report(first_report.id) schedule_a.save() next_year_report = create_form3x( diff --git a/django-backend/fecfiler/web_services/summary/tests/utils.py b/django-backend/fecfiler/web_services/summary/tests/utils.py index 96fc1e89b..407e7d662 100644 --- a/django-backend/fecfiler/web_services/summary/tests/utils.py +++ b/django-backend/fecfiler/web_services/summary/tests/utils.py @@ -860,7 +860,7 @@ def gen_schedule_a(transaction_data, f3x, committee, contact): data["memo"], data["itemized"], ) - scha.reports.add(f3x) + scha.add_to_report(f3x.id) scha.save() if data["form_type"] == "SA11AII": debt = scha @@ -879,7 +879,7 @@ def gen_schedule_b(transaction_data, f3x, committee, contact): data["form_type"], ) - schb.reports.add(f3x) + schb.add_to_report(f3x.id) schb.save() @@ -895,7 +895,7 @@ def gen_schedule_c(transaction_data, f3x, committee, contact): "LOAN_RECEIVED_FROM_INDIVIDUAL", data["form_type"], ) - schc.reports.add(f3x) + schc.add_to_report(f3x.id) def gen_schedule_d(transaction_data, f3x, committee, contact): @@ -924,7 +924,7 @@ def gen_schedule_e(transaction_data, f3x, committee, contact, candidate): candidate, data["memo_code"], ) - sche.reports.add(f3x) + sche.add_to_report(f3x.id) def gen_schedule_f( @@ -958,5 +958,5 @@ def gen_schedule_f( }, ) - schf.reports.add(f3x) + schf.add_to_report(f3x.id) schf.save() From 2d156ebdeca038dcd5797f7ef399234c088a11f8 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 15:39:56 -0400 Subject: [PATCH 49/52] set_reports uses reports.set() for efficiency and manually updates can_unamend on the side --- django-backend/fecfiler/transactions/models.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 55fd1bbf8..a18fc46f4 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -689,15 +689,13 @@ def set_reports(self, report_ids): current_report_ids.add(str(report_id_dict["id"])) updated_report_ids = set(report_ids) + report_ids_to_reset_can_unamend = current_report_ids ^ updated_report_ids - new_report_ids = updated_report_ids - current_report_ids - removed_report_ids = current_report_ids - updated_report_ids + Transaction.objects.filter( + id__in=report_ids_to_reset_can_unamend + ).update(can_unamend = False) - for report_id in new_report_ids: - self.add_to_report(report_id) - - for report_id in removed_report_ids: - self.remove_from_report(report_id) + self.reports.set(report_ids) class Meta: indexes = [models.Index(fields=["_form_type"])] From f69b8aaae36271f458c794cccf629eb858b37a99 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 Mar 2026 15:44:52 -0400 Subject: [PATCH 50/52] fixes a typo --- django-backend/fecfiler/transactions/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index a18fc46f4..3a6829cae 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -691,7 +691,8 @@ def set_reports(self, report_ids): updated_report_ids = set(report_ids) report_ids_to_reset_can_unamend = current_report_ids ^ updated_report_ids - Transaction.objects.filter( + Report = apps.get_model("reports.Report") + Report.objects.filter( id__in=report_ids_to_reset_can_unamend ).update(can_unamend = False) From 9da708d30e14bf805d2ef556a83eb106e038a881 Mon Sep 17 00:00:00 2001 From: Elaine Krauss <104506225+Elaine-Krauss-TCG@users.noreply.github.com> Date: Wed, 25 Mar 2026 16:10:59 -0400 Subject: [PATCH 51/52] Update models.py for linting --- django-backend/fecfiler/transactions/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 3a6829cae..04a0be0e2 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -694,7 +694,7 @@ def set_reports(self, report_ids): Report = apps.get_model("reports.Report") Report.objects.filter( id__in=report_ids_to_reset_can_unamend - ).update(can_unamend = False) + ).update(can_unamend=False) self.reports.set(report_ids) From b74923ef8584616380e55295690630b18fbe90b1 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Fri, 27 Mar 2026 13:29:09 -0400 Subject: [PATCH 52/52] Update software name in .fec to FECfile+ --- .../fecfiler/web_services/dot_fec/dot_fec_composer.py | 2 +- .../web_services/dot_fec/tests/test_dot_fec_serializer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py index 8010bb157..cd5f1a799 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py @@ -120,7 +120,7 @@ def compose_header(report_id): "HDR", "FEC", FEC_FORMAT_VERSION, - "FECFile Online", + "FECfile+", "0.0.1", report.report_id, report.report_version, diff --git a/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py index 2bf7ed3ad..f164840b1 100644 --- a/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/tests/test_dot_fec_serializer.py @@ -75,7 +75,7 @@ def setUp(self): self.committee, self.f3x, "dahtest2" ) - self.header = Header("HDR", "FEC", "8.5", "FECFile Online", "0.0.1") + self.header = Header("HDR", "FEC", "8.5", "FECfile+", "0.0.1") def test_serialize_field(self): f3x_field_mappings = get_field_mappings("F3X") @@ -186,7 +186,7 @@ def test_serialize_header_instance(self): self.assertEqual(split_row[0], "HDR") self.assertEqual(split_row[1], "FEC") self.assertEqual(split_row[2], "8.5") - self.assertEqual(split_row[3], "FECFile Online") + self.assertEqual(split_row[3], "FECfile+") self.assertEqual(split_row[4], "0.0.1") def test_get_value_from_path(self):