diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 7086a5834..ff966121a 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -198,7 +198,7 @@ jobs: shell: bash {0} run: | cd lcfs/charts/lcfs-postgres-pr - helm -n ${{ env.DEV_NAMESPACE }} -f ./values-dev-pr.yaml upgrade --install lcfs-postgres-dev-${{ env.PR_NUMBER }} oci://registry-1.docker.io/bitnamicharts/postgresql --version 15.5.17 + helm -n ${{ env.DEV_NAMESPACE }} -f ./values-dev-pr.yaml upgrade --install lcfs-postgres-dev-${{ env.PR_NUMBER }} oci://registry-1.docker.io/bitnamicharts/postgresql --version 16.7.16 --set image.tag=17.0.0 cd ../lcfs-redis-pr helm -n ${{ env.DEV_NAMESPACE }} -f ./values-dev-pr.yaml upgrade --install lcfs-redis-dev-${{ env.PR_NUMBER }} oci://registry-1.docker.io/bitnamicharts/redis --version 19.6.1 cd ../lcfs-minio-pr diff --git a/backend/lcfs/db/migrations/versions/2025-04-17-10-06_7a1f5f52793c.py b/backend/lcfs/db/migrations/versions/2025-04-17-10-06_7a1f5f52793c.py index ff5e31a88..1c62913fa 100644 --- a/backend/lcfs/db/migrations/versions/2025-04-17-10-06_7a1f5f52793c.py +++ b/backend/lcfs/db/migrations/versions/2025-04-17-10-06_7a1f5f52793c.py @@ -17,7 +17,7 @@ MV_NAME = "mv_credit_ledger" CREATE_VIEW_SQL = f""" -CREATE MATERIALIZED VIEW {MV_NAME} AS +CREATE MATERIALIZED VIEW IF NOT EXISTS {MV_NAME} AS WITH base AS ( SELECT t.transaction_id, @@ -110,11 +110,12 @@ IDX_ORG_DATE = f"{MV_NAME}_org_date_idx" CREATE_IDX_ORG_YEAR = ( - f"CREATE INDEX {IDX_ORG_YEAR} " + f"CREATE INDEX IF NOT EXISTS {IDX_ORG_YEAR} " f"ON {MV_NAME} (organization_id, compliance_period);" ) CREATE_IDX_ORG_DATE = ( - f"CREATE INDEX {IDX_ORG_DATE} " f"ON {MV_NAME} (organization_id, update_date DESC);" + f"CREATE INDEX IF NOT EXISTS {IDX_ORG_DATE} " + f"ON {MV_NAME} (organization_id, update_date DESC);" ) diff --git a/backend/lcfs/db/migrations/versions/2025-06-02-09-36_67c82d9397dd.py b/backend/lcfs/db/migrations/versions/2025-06-02-09-36_67c82d9397dd.py index b16e459c9..9ed7d37dd 100644 --- a/backend/lcfs/db/migrations/versions/2025-06-02-09-36_67c82d9397dd.py +++ b/backend/lcfs/db/migrations/versions/2025-06-02-09-36_67c82d9397dd.py @@ -22,6 +22,8 @@ depends_on = None # Specify which sections to execute from the SQL file +# Note: Excluded views that reference organization_early_issuance_by_year table +# as that table doesn't exist yet - will be created in migration a1b2c3d4e5f7 SECTIONS_TO_EXECUTE = [ "Compliance Reports Analytics View", "Compliance Reports Waiting review", @@ -36,11 +38,12 @@ "Fuel Supply Fuel Code Base View", "Fuel Supply Base View", "Compliance Report Fuel Supply Base View", - "Compliance Report Chained View", - "Compliance Report Base View", - "Allocation Agreement Chained View", - "Allocation Agreement Base View", - "Fuel Code Base View" + "Compliance Report Chained View", # Safe - doesn't reference organization_early_issuance_by_year + "Allocation Agreement Chained View", # Safe - doesn't reference organization_early_issuance_by_year + "Fuel Code Base View", + # Excluded until organization_early_issuance_by_year table is created: + # "Compliance Report Base View", # References organization_early_issuance_by_year + # "Allocation Agreement Base View", # Depends on Compliance Report Base View ] diff --git a/backend/lcfs/db/migrations/versions/2025-06-24-12-02_fcba2790c890.py b/backend/lcfs/db/migrations/versions/2025-06-24-12-02_fcba2790c890.py index b2254900c..0efbf9bf2 100644 --- a/backend/lcfs/db/migrations/versions/2025-06-24-12-02_fcba2790c890.py +++ b/backend/lcfs/db/migrations/versions/2025-06-24-12-02_fcba2790c890.py @@ -7,6 +7,7 @@ """ from lcfs.db.dependencies import ( + create_role_if_not_exists, execute_sql_sections, find_and_read_sql_file, parse_sql_sections, @@ -23,6 +24,9 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### """Recreate the v_compliance_report view from metabase.sql""" try: + # Create the role if it doesn't exist + create_role_if_not_exists() + # Read the metabase.sql file content = find_and_read_sql_file(sqlFile="metabase.sql") sections = parse_sql_sections(content) diff --git a/backend/lcfs/db/migrations/versions/2025-06-28-16-41_413eef467edd.py b/backend/lcfs/db/migrations/versions/2025-06-28-16-41_413eef467edd.py index 18d3eebfe..a6f43969e 100644 --- a/backend/lcfs/db/migrations/versions/2025-06-28-16-41_413eef467edd.py +++ b/backend/lcfs/db/migrations/versions/2025-06-28-16-41_413eef467edd.py @@ -9,6 +9,7 @@ import sqlalchemy as sa from alembic import op from lcfs.db.dependencies import ( + create_role_if_not_exists, execute_sql_sections, find_and_read_sql_file, parse_sql_sections, @@ -24,6 +25,9 @@ def recreate_compliance_reports_view(): """Recreate the v_compliance_report view from metabase.sql""" try: + # Ensure role exists before creating views + create_role_if_not_exists() + # Read the metabase.sql file content = find_and_read_sql_file(sqlFile="metabase.sql") sections = parse_sql_sections(content) diff --git a/backend/lcfs/db/migrations/versions/2025-07-10-10-10_a1b2c3d4e5f7.py b/backend/lcfs/db/migrations/versions/2025-07-10-10-10_a1b2c3d4e5f7.py index 6489eacca..3a860ba0e 100644 --- a/backend/lcfs/db/migrations/versions/2025-07-10-10-10_a1b2c3d4e5f7.py +++ b/backend/lcfs/db/migrations/versions/2025-07-10-10-10_a1b2c3d4e5f7.py @@ -34,82 +34,87 @@ def upgrade() -> None: """ ) - # Create organization_early_issuance_by_year table - op.create_table( - "organization_early_issuance_by_year", - sa.Column( - "early_issuance_by_year_id", - sa.Integer(), - autoincrement=True, - nullable=False, - comment="Unique identifier for the early issuance by year record", - ), - sa.Column( - "organization_id", - sa.Integer(), - nullable=False, - comment="Foreign key to the organization", - ), - sa.Column( - "compliance_period_id", - sa.Integer(), - nullable=False, - comment="Foreign key to the compliance period", - ), - sa.Column( - "has_early_issuance", - sa.Boolean(), - nullable=False, - server_default=sa.text("FALSE"), - comment="True if the organization can create early issuance reports for this compliance year", - ), - sa.Column( - "create_date", - sa.TIMESTAMP(timezone=True), - server_default=sa.text("now()"), - nullable=True, - comment="Date and time (UTC) when the physical record was created in the database.", - ), - sa.Column( - "update_date", - sa.TIMESTAMP(timezone=True), - server_default=sa.text("now()"), - nullable=True, - comment="Date and time (UTC) when the physical record was updated in the database.", - ), - sa.Column( - "create_user", - sa.String(), - nullable=True, - comment="The user who created this record in the database.", - ), - sa.Column( - "update_user", - sa.String(), - nullable=True, - comment="The user who last updated this record in the database.", - ), - sa.PrimaryKeyConstraint("early_issuance_by_year_id"), - sa.ForeignKeyConstraint( - ["organization_id"], - ["organization.organization_id"], - name="fk_organization_early_issuance_by_year_organization_id", - ), - sa.ForeignKeyConstraint( - ["compliance_period_id"], - ["compliance_period.compliance_period_id"], - name="fk_organization_early_issuance_by_year_compliance_period_id", - ), - sa.UniqueConstraint( - "organization_id", - "compliance_period_id", - name="uq_organization_early_issuance_by_year", - ), - comment="Tracks early issuance reporting eligibility by organization and compliance year", - ) + # Create organization_early_issuance_by_year table (check if exists first) + connection = op.get_bind() + inspector = sa.inspect(connection) + + if not inspector.has_table("organization_early_issuance_by_year"): + op.create_table( + "organization_early_issuance_by_year", + sa.Column( + "early_issuance_by_year_id", + sa.Integer(), + autoincrement=True, + nullable=False, + comment="Unique identifier for the early issuance by year record", + ), + sa.Column( + "organization_id", + sa.Integer(), + nullable=False, + comment="Foreign key to the organization", + ), + sa.Column( + "compliance_period_id", + sa.Integer(), + nullable=False, + comment="Foreign key to the compliance period", + ), + sa.Column( + "has_early_issuance", + sa.Boolean(), + nullable=False, + server_default=sa.text("FALSE"), + comment="True if the organization can create early issuance reports for this compliance year", + ), + sa.Column( + "create_date", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=True, + comment="Date and time (UTC) when the physical record was created in the database.", + ), + sa.Column( + "update_date", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=True, + comment="Date and time (UTC) when the physical record was updated in the database.", + ), + sa.Column( + "create_user", + sa.String(), + nullable=True, + comment="The user who created this record in the database.", + ), + sa.Column( + "update_user", + sa.String(), + nullable=True, + comment="The user who last updated this record in the database.", + ), + sa.PrimaryKeyConstraint("early_issuance_by_year_id"), + sa.ForeignKeyConstraint( + ["organization_id"], + ["organization.organization_id"], + name="fk_organization_early_issuance_by_year_organization_id", + ), + sa.ForeignKeyConstraint( + ["compliance_period_id"], + ["compliance_period.compliance_period_id"], + name="fk_organization_early_issuance_by_year_compliance_period_id", + ), + sa.UniqueConstraint( + "organization_id", + "compliance_period_id", + name="uq_organization_early_issuance_by_year", + ), + comment="Tracks early issuance reporting eligibility by organization and compliance year", + ) # Copy all has_early_issuance values from organization table # to organization_early_issuance_by_year table for the current year (2025) + # Use ON CONFLICT to handle case where data already exists op.execute( """ INSERT INTO organization_early_issuance_by_year ( @@ -148,4 +153,9 @@ def downgrade() -> None: """ ) - op.drop_table("organization_early_issuance_by_year") + # Only drop table if it exists + connection = op.get_bind() + inspector = sa.inspect(connection) + + if inspector.has_table("organization_early_issuance_by_year"): + op.drop_table("organization_early_issuance_by_year") diff --git a/backend/lcfs/db/migrations/versions/2025-07-24-10-12_c3d4e5f6g7h8.py b/backend/lcfs/db/migrations/versions/2025-07-24-10-12_c3d4e5f6g7h8.py index 6235fc0f9..d224d1ac2 100644 --- a/backend/lcfs/db/migrations/versions/2025-07-24-10-12_c3d4e5f6g7h8.py +++ b/backend/lcfs/db/migrations/versions/2025-07-24-10-12_c3d4e5f6g7h8.py @@ -21,6 +21,8 @@ depends_on = None # Specify which sections to execute from the SQL file +# These sections create the views that were excluded from the earlier migration +# because they reference organization_early_issuance_by_year table SECTIONS_TO_EXECUTE = [ "Compliance Report Base View With Early Issuance By Year", "Allocation Agreement Base View With Early Issuance By Year", diff --git a/backend/lcfs/db/migrations/versions/2025-07-29-23-23_3eb97134895a.py b/backend/lcfs/db/migrations/versions/2025-07-29-23-23_3eb97134895a.py index f4a68979b..453b26579 100644 --- a/backend/lcfs/db/migrations/versions/2025-07-29-23-23_3eb97134895a.py +++ b/backend/lcfs/db/migrations/versions/2025-07-29-23-23_3eb97134895a.py @@ -8,6 +8,7 @@ import sqlalchemy as sa from alembic import op +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision = "3eb97134895a" @@ -17,17 +18,33 @@ def upgrade() -> None: - op.add_column( - "compliance_report", - sa.Column( - "is_non_assessment", - sa.Boolean(), - nullable=False, - server_default="false", - comment="Flag indicating if report is not subject to assessment under the Low Carbon Fuels Act", - ), - ) + # Check if the column already exists before adding it + conn = op.get_bind() + inspector = inspect(conn) + columns = [col['name'] for col in inspector.get_columns('compliance_report')] + + if 'is_non_assessment' not in columns: + op.add_column( + "compliance_report", + sa.Column( + "is_non_assessment", + sa.Boolean(), + nullable=False, + server_default="false", + comment="Flag indicating if report is not subject to assessment under the Low Carbon Fuels Act", + ), + ) + else: + print("Column 'is_non_assessment' already exists in compliance_report table, skipping...") def downgrade() -> None: - op.drop_column("compliance_report", "is_non_assessment") + # Check if the column exists before dropping it + conn = op.get_bind() + inspector = inspect(conn) + columns = [col['name'] for col in inspector.get_columns('compliance_report')] + + if 'is_non_assessment' in columns: + op.drop_column("compliance_report", "is_non_assessment") + else: + print("Column 'is_non_assessment' does not exist in compliance_report table, skipping...") diff --git a/backend/lcfs/db/migrations/versions/2025-07-30-14-31_1c0b3bed4671.py b/backend/lcfs/db/migrations/versions/2025-07-30-14-31_1c0b3bed4671.py index 5a19db3ad..8b5ffc6ba 100644 --- a/backend/lcfs/db/migrations/versions/2025-07-30-14-31_1c0b3bed4671.py +++ b/backend/lcfs/db/migrations/versions/2025-07-30-14-31_1c0b3bed4671.py @@ -23,7 +23,9 @@ # Sections to recreate after altering columns SECTIONS_TO_EXECUTE = [ - "Allocation Agreement Base View", + "Compliance Reports Analytics View", # Must come first - dependency for Fuel Export Analytics Base View + "Allocation Agreement Chained View", + "Allocation Agreement Base View With Early Issuance By Year", "Fuel Export Analytics Base View", "Fuel Supply Analytics Base View", "Fuel Supply Base View", @@ -362,6 +364,7 @@ def upgrade() -> None: # Using CASCADE to drop dependent views automatically op.execute("DROP MATERIALIZED VIEW IF EXISTS mv_credit_ledger CASCADE;") op.execute("DROP MATERIALIZED VIEW IF EXISTS mv_transaction_aggregate CASCADE;") + op.execute("DROP VIEW IF EXISTS vw_allocation_agreement_chained CASCADE;") op.execute("DROP VIEW IF EXISTS vw_allocation_agreement_base CASCADE;") op.execute("DROP VIEW IF EXISTS vw_fuel_export_analytics_base CASCADE;") op.execute("DROP VIEW IF EXISTS vw_fuel_supply_analytics_base CASCADE;") @@ -532,6 +535,7 @@ def downgrade() -> None: # Drop views and materialized views before altering columns back op.execute("DROP MATERIALIZED VIEW IF EXISTS mv_credit_ledger CASCADE;") op.execute("DROP MATERIALIZED VIEW IF EXISTS mv_transaction_aggregate CASCADE;") + op.execute("DROP VIEW IF EXISTS vw_allocation_agreement_chained CASCADE;") op.execute("DROP VIEW IF EXISTS vw_allocation_agreement_base CASCADE;") op.execute("DROP VIEW IF EXISTS vw_fuel_export_analytics_base CASCADE;") op.execute("DROP VIEW IF EXISTS vw_fuel_supply_analytics_base CASCADE;") diff --git a/backend/lcfs/db/migrations/versions/2025-10-29-13-12_a3290902296b.py b/backend/lcfs/db/migrations/versions/2025-10-29-13-12_a3290902296b.py new file mode 100644 index 000000000..d02456d42 --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2025-10-29-13-12_a3290902296b.py @@ -0,0 +1,56 @@ +"""add renewable diesel fuel type + +Revision ID: a3290902296b +Revises: 1909a3e5fafd +Create Date: 2025-04-11 13:29:03.149771 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a3290902296b" +down_revision = "1909a3e5fafd" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + INSERT INTO fuel_type ( + fuel_type_id, + fuel_type, + fossil_derived, + provision_1_id, + provision_2_id, + default_carbon_intensity, + units, + unrecognized, + is_legacy, + renewable + ) + VALUES ( + 24, + 'Renewable diesel', + FALSE, + NULL, + NULL, + 100.21, + 'Litres', + FALSE, + TRUE, + TRUE + ) + ON CONFLICT (fuel_type_id) DO NOTHING; + """ + ) + + +def downgrade() -> None: + op.execute( + """ + DELETE FROM fuel_type + WHERE fuel_type_id = 24; + """ + ) diff --git a/backend/lcfs/db/migrations/versions/2025-10-29-13-13_54d55e878dad.py b/backend/lcfs/db/migrations/versions/2025-10-29-13-13_54d55e878dad.py new file mode 100644 index 000000000..159bac8d7 --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2025-10-29-13-13_54d55e878dad.py @@ -0,0 +1,100 @@ +"""compliance summary historical update + +Revision ID: 54d55e878dad +Revises: a3290902296b +Create Date: 2025-04-04 13:52:08.981318 + +""" + +import sqlalchemy as sa +from alembic import op +from alembic_postgresql_enum import TableReference +from sqlalchemy.dialects import postgresql +from lcfs.db.dependencies import ( + create_role_if_not_exists, + execute_sql_sections, + find_and_read_sql_file, + parse_sql_sections, +) + +# revision identifiers, used by Alembic. +revision = "54d55e878dad" +down_revision = "a3290902296b" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Get all dependent views that need to be dropped + dependent_views = [ + "vw_allocation_agreement_chained", + "vw_compliance_report_chained", + "vw_compliance_report_fuel_supply_base", + "vw_allocation_agreement_base", + "vw_compliance_report_base", + ] + + # Drop all dependent views in reverse dependency order + for view in dependent_views: + try: + op.execute(f"DROP VIEW IF EXISTS {view} CASCADE") + except Exception as e: + print(f"Note: Could not drop view {view} (may not exist): {e}") + + # Add new column + op.add_column( + "compliance_report_summary", + sa.Column( + "historical_snapshot", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Contains historical data from pre-2024 TFRS system for data retention and analysis purposes.", + ), + ) + + # Drop the columns + op.drop_column("compliance_report_summary", "credits_offset_b") + op.drop_column("compliance_report_summary", "credits_offset_c") + op.drop_column("compliance_report_summary", "credits_offset_a") + + # Ensure role exists before recreating views + create_role_if_not_exists() + + # Recreate all the views + try: + content = find_and_read_sql_file(sqlFile="metabase.sql") + sections = parse_sql_sections(content) + + # Recreate views in dependency order + views_to_recreate = [ + "Compliance Report Base View", + "Allocation Agreement Base View", + "Compliance Report Fuel Supply Base View", + "Compliance Report Chained View", + "Allocation Agreement Chained View", + ] + + for view in views_to_recreate: + try: + execute_sql_sections(sections, [view]) + except Exception as e: + print(f"Warning: Could not recreate view '{view}': {e}") + + except Exception as e: + print(f"Error recreating views: {e}") + + +def downgrade() -> None: + op.add_column( + "compliance_report_summary", + sa.Column("credits_offset_a", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.add_column( + "compliance_report_summary", + sa.Column("credits_offset_c", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.add_column( + "compliance_report_summary", + sa.Column("credits_offset_b", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.drop_column("compliance_report_summary", "historical_snapshot") diff --git a/backend/lcfs/db/migrations/versions/2025-10-29-13-14_5c3de27f158b.py b/backend/lcfs/db/migrations/versions/2025-10-29-13-14_5c3de27f158b.py new file mode 100644 index 000000000..514e08f47 --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2025-10-29-13-14_5c3de27f158b.py @@ -0,0 +1,62 @@ +"""Add Beta Tester role + +Revision ID: 5c3de27f158b +Revises: 54d55e878dad +Create Date: 2025-04-17 23:46:58.106919 + +""" + +from alembic import op +from alembic_postgresql_enum import TableReference + +# revision identifiers, used by Alembic. +revision = "5c3de27f158b" +down_revision = "54d55e878dad" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.sync_enum_values( + "public", + "role_enum", + [ + "GOVERNMENT", + "ADMINISTRATOR", + "ANALYST", + "COMPLIANCE_MANAGER", + "DIRECTOR", + "SUPPLIER", + "MANAGE_USERS", + "TRANSFER", + "COMPLIANCE_REPORTING", + "SIGNING_AUTHORITY", + "READ_ONLY", + "BETA_TESTER", + ], + [TableReference(table_schema="public", table_name="role", column_name="name")], + enum_values_to_rename=[], + ) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.sync_enum_values( + "public", + "role_enum", + [ + "GOVERNMENT", + "ADMINISTRATOR", + "ANALYST", + "COMPLIANCE_MANAGER", + "DIRECTOR", + "SUPPLIER", + "MANAGE_USERS", + "TRANSFER", + "COMPLIANCE_REPORTING", + "SIGNING_AUTHORITY", + "READ_ONLY", + ], + [TableReference(table_schema="public", table_name="role", column_name="name")], + enum_values_to_rename=[], + ) diff --git a/backend/lcfs/db/migrations/versions/2025-10-29-13-15_87592f5136b3.py b/backend/lcfs/db/migrations/versions/2025-10-29-13-15_87592f5136b3.py new file mode 100644 index 000000000..54445fbc2 --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2025-10-29-13-15_87592f5136b3.py @@ -0,0 +1,218 @@ +"""historical target carbon intensities + +Revision ID: 87592f5136b3 +Revises: 5c3de27f158b +Create Date: 2025-04-07 15:25:24.384935 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "87592f5136b3" +down_revision = "5c3de27f158b" +branch_labels = None +depends_on = None + + +def upgrade(): + # Insert historical gasoline values + op.execute( + sa.text( + """ + INSERT INTO target_carbon_intensity ( + compliance_period_id, + fuel_category_id, + target_carbon_intensity, + reduction_target_percentage, + effective_status, + create_date, + update_date + ) VALUES + -- 2013 Gasoline + (4, 2, 92.38, 0, true, NOW(), NOW()), + -- 2014 Gasoline + (5, 2, 92.38, 0, true, NOW(), NOW()), + -- 2015 Gasoline + (6, 2, 91.21, 0, true, NOW(), NOW()), + -- 2016 Gasoline + (7, 2, 90.28, 0, true, NOW(), NOW()), + -- 2017 Gasoline + (8, 2, 90.02, 0, true, NOW(), NOW()), + -- 2018 Gasoline + (9, 2, 88.60, 0, true, NOW(), NOW()), + -- 2019 Gasoline + (10, 2, 87.18, 0, true, NOW(), NOW()), + -- 2020 Gasoline + (11, 2, 85.28, 0, true, NOW(), NOW()), + -- 2021 Gasoline + (12, 2, 85.11, 0, true, NOW(), NOW()), + -- 2022 Gasoline + (13, 2, 84.00, 0, true, NOW(), NOW()), + -- 2023 Gasoline + (14, 2, 81.86, 0, true, NOW(), NOW()); + """ + ) + ) + + # Insert historical diesel values + op.execute( + sa.text( + """ + INSERT INTO target_carbon_intensity ( + compliance_period_id, + fuel_category_id, + target_carbon_intensity, + reduction_target_percentage, + effective_status, + create_date, + update_date + ) VALUES + -- 2013 Diesel + (4, 1, 86.20, 0, true, NOW(), NOW()), + -- 2014 Diesel + (5, 1, 86.20, 0, true, NOW(), NOW()), + -- 2015 Diesel + (6, 1, 85.11, 0, true, NOW(), NOW()), + -- 2016 Diesel + (7, 1, 84.23, 0, true, NOW(), NOW()), + -- 2017 Diesel + (8, 1, 83.74, 0, true, NOW(), NOW()), + -- 2018 Diesel + (9, 1, 82.41, 0, true, NOW(), NOW()), + -- 2019 Diesel + (10, 1, 81.09, 0, true, NOW(), NOW()), + -- 2020 Diesel + (11, 1, 79.33, 0, true, NOW(), NOW()), + -- 2021 Diesel + (12, 1, 79.17, 0, true, NOW(), NOW()), + -- 2022 Diesel + (13, 1, 78.00, 0, true, NOW(), NOW()), + -- 2023 Diesel + (14, 1, 76.14, 0, true, NOW(), NOW()); + """ + ) + ) + + op.execute( + sa.text( + """ + INSERT INTO fuel_instance ( + fuel_type_id, + fuel_category_id, + create_date, + update_date + ) VALUES + -- Natural gas-based gasoline (21) -> Gasoline (2) + (21, 1, NOW(), NOW()), + -- Petroleum-based diesel (22) -> Diesel (1) + (22, 2, NOW(), NOW()), + -- Petroleum-based gasoline (23) -> Gasoline (2) + (23, 1, NOW(), NOW()); + """ + ) + ) + + # Add EER records for legacy fuel types (2013-2023) + op.execute( + sa.text( + """ + INSERT INTO energy_effectiveness_ratio ( + fuel_category_id, + fuel_type_id, + end_use_type_id, + ratio, + create_date, + update_date, + effective_status, + compliance_period_id + ) VALUES + -- Natural gas-based gasoline (21) records + (2, 21, 24, 1.0, NOW(), NOW(), true, 4), + (2, 21, 24, 1.0, NOW(), NOW(), true, 5), + (2, 21, 24, 1.0, NOW(), NOW(), true, 6), + (2, 21, 24, 1.0, NOW(), NOW(), true, 7), + (2, 21, 24, 1.0, NOW(), NOW(), true, 8), + (2, 21, 24, 1.0, NOW(), NOW(), true, 9), + (2, 21, 24, 1.0, NOW(), NOW(), true, 10), + (2, 21, 24, 1.0, NOW(), NOW(), true, 11), + (2, 21, 24, 1.0, NOW(), NOW(), true, 12), + (2, 21, 24, 1.0, NOW(), NOW(), true, 13), + (2, 21, 24, 1.0, NOW(), NOW(), true, 14), + -- Petroleum-based diesel (22) records + (1, 22, 24, 1.0, NOW(), NOW(), true, 4), + (1, 22, 24, 1.0, NOW(), NOW(), true, 5), + (1, 22, 24, 1.0, NOW(), NOW(), true, 6), + (1, 22, 24, 1.0, NOW(), NOW(), true, 7), + (1, 22, 24, 1.0, NOW(), NOW(), true, 8), + (1, 22, 24, 1.0, NOW(), NOW(), true, 9), + (1, 22, 24, 1.0, NOW(), NOW(), true, 10), + (1, 22, 24, 1.0, NOW(), NOW(), true, 11), + (1, 22, 24, 1.0, NOW(), NOW(), true, 12), + (1, 22, 24, 1.0, NOW(), NOW(), true, 13), + (1, 22, 24, 1.0, NOW(), NOW(), true, 14), + -- Petroleum-based gasoline (23) records + (2, 23, 24, 1.0, NOW(), NOW(), true, 4), + (2, 23, 24, 1.0, NOW(), NOW(), true, 5), + (2, 23, 24, 1.0, NOW(), NOW(), true, 6), + (2, 23, 24, 1.0, NOW(), NOW(), true, 7), + (2, 23, 24, 1.0, NOW(), NOW(), true, 8), + (2, 23, 24, 1.0, NOW(), NOW(), true, 9), + (2, 23, 24, 1.0, NOW(), NOW(), true, 10), + (2, 23, 24, 1.0, NOW(), NOW(), true, 11), + (2, 23, 24, 1.0, NOW(), NOW(), true, 12), + (2, 23, 24, 1.0, NOW(), NOW(), true, 13), + (2, 23, 24, 1.0, NOW(), NOW(), true, 14); + """ + ) + ) + + # Set fossil_derived to true for legacy fuel types + op.execute( + """ + UPDATE fuel_type + SET fossil_derived = true + WHERE fuel_type_id IN (21, 22, 23) + """ + ) + + +def downgrade(): + # Remove the fossil_derived flag + op.execute( + """ + UPDATE fuel_type + SET fossil_derived = false + WHERE fuel_type_id IN (21, 22, 23) + """ + ) + # Remove the added EER records + op.execute( + sa.text( + """ + DELETE FROM energy_effectiveness_ratio + WHERE fuel_type_id IN (21, 22, 23); + """ + ) + ) + + # Remove the added fuel_instance records + op.execute( + sa.text( + """ + DELETE FROM fuel_instance + WHERE fuel_type_id IN (21, 22, 23); + """ + ) + ) + + # Remove the imported historical values + op.execute( + sa.text( + """ + DELETE FROM target_carbon_intensity + WHERE compliance_period_id IN (4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14); + """ + ) + ) diff --git a/backend/lcfs/db/migrations/versions/2025-10-29-13-16_8e530edb155f.py b/backend/lcfs/db/migrations/versions/2025-10-29-13-16_8e530edb155f.py new file mode 100644 index 000000000..f5400151b --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2025-10-29-13-16_8e530edb155f.py @@ -0,0 +1,199 @@ +"""Populate missing organization snapshots for compliance reports + +Revision ID: 8e530edb155f +Revises: 87592f5136b3 +Create Date: 2025-04-19 08:50:58.195991 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.sql import text + +# revision identifiers, used by Alembic. +revision = "8e530edb155f" +down_revision = "87592f5136b3" +branch_labels = None +depends_on = None + +# Define a unique user identifier for this migration +MIGRATION_USER = "ALEMBIC_8e530edb155f" + + +def upgrade() -> None: + # Get connection bind for execution + conn = op.get_bind() + + # Find compliance reports missing a snapshot + missing_reports_query = text( + """ + SELECT cr.compliance_report_id, cr.organization_id + FROM compliance_report cr + LEFT JOIN compliance_report_organization_snapshot cros + ON cr.compliance_report_id = cros.compliance_report_id + WHERE cros.organization_snapshot_id IS NULL + """ + ) + result = conn.execute(missing_reports_query) + missing_reports = result.fetchall() + + print( + f"Found {len(missing_reports)} compliance reports missing organization snapshots." + ) + + # SQL to fetch organization details + get_org_query = text( + """ + SELECT name, operating_name, email, phone, records_address, + organization_address_id, organization_attorney_address_id + FROM organization + WHERE organization_id = :org_id + """ + ) + + # SQL to fetch address details + get_addr_query = text( + """ + SELECT street_address, address_other, city, province_state, country, \"postalCode_zipCode\" + FROM {table_name} + WHERE {id_column} = :addr_id + """ + ) + + # SQL to insert snapshot + insert_snapshot_query = text( + """ + INSERT INTO compliance_report_organization_snapshot ( + compliance_report_id, name, operating_name, email, phone, + head_office_address, records_address, service_address, + is_edited, create_date, update_date, create_user, update_user + ) + VALUES ( + :cr_id, :name, :op_name, :email, :phone, + :head_addr, :rec_addr, :svc_addr, + false, NOW(), NOW(), :user, :user + ) + """ + ) + + snapshots_created = 0 + for report in missing_reports: + cr_id = report.compliance_report_id + org_id = report.organization_id + + print(f"Processing report ID: {cr_id}, Org ID: {org_id}") + + # Fetch organization details using conn.execute + org_result = conn.execute(get_org_query, {"org_id": org_id}) + org_row = org_result.fetchone() + + if not org_row: + print( + f"WARN: Organization data not found for org_id: {org_id}. Skipping report {cr_id}." + ) + continue + + org_data = dict(org_row._mapping) # Convert RowProxy to dict + + # Fetch addresses using conn.execute + service_address_data = None + if org_data.get("organization_address_id"): + # Manually format the query string + addr_sql_str = str(get_addr_query).format( + table_name="organization_address", id_column="organization_address_id" + ) + service_address_result = conn.execute( + text(addr_sql_str), {"addr_id": org_data["organization_address_id"]} + ) + service_address_row = service_address_result.fetchone() + if service_address_row: + service_address_data = dict(service_address_row._mapping) + else: + print( + f"WARN: Service address data not found for organization_address_id: {org_data['organization_address_id']}" + ) + else: + print(f"WARN: No organization_address_id found for org_id: {org_id}") + + attorney_address_data = None + if org_data.get("organization_attorney_address_id"): + addr_sql_str = str(get_addr_query).format( + table_name="organization_attorney_address", + id_column="organization_attorney_address_id", + ) + attorney_address_result = conn.execute( + text(addr_sql_str), + {"addr_id": org_data["organization_attorney_address_id"]}, + ) + attorney_address_row = attorney_address_result.fetchone() + if attorney_address_row: + attorney_address_data = dict(attorney_address_row._mapping) + else: + print( + f"WARN: Attorney address data not found for organization_attorney_address_id: {org_data['organization_attorney_address_id']}" + ) + else: + print( + f"WARN: No organization_attorney_address_id found for org_id: {org_id}" + ) + + # Build address strings + def build_address_string(addr_dict): + if not addr_dict: + return None + parts = [ + addr_dict.get("street_address"), + addr_dict.get("address_other"), + addr_dict.get("city"), + addr_dict.get("province_state"), + addr_dict.get("country"), + addr_dict.get("postalCode_zipCode"), + ] + return ", ".join(filter(None, [p.strip() if p else None for p in parts])) + + service_address = build_address_string(service_address_data) + head_office_address = build_address_string(attorney_address_data) + + # Insert the snapshot using conn.execute + result = conn.execute( + insert_snapshot_query, + { + "cr_id": cr_id, + "name": org_data.get("name"), + "op_name": org_data.get("operating_name") or org_data.get("name"), + "email": org_data.get("email"), + "phone": org_data.get("phone"), + "head_addr": head_office_address, + "rec_addr": org_data.get("records_address"), + "svc_addr": service_address, + "user": MIGRATION_USER, + }, + ) + # Check rowcount for confirmation, although ON CONFLICT might affect it + if result.rowcount > 0: + snapshots_created += 1 + print(f"OK: Created snapshot for report ID: {cr_id}") + else: + print( + f"WARN: Snapshot might already exist or insert failed (0 rows affected) for report ID: {cr_id}" + ) + + print(f"Finished. Successfully created {snapshots_created} organization snapshots.") + + +def downgrade() -> None: + # Delete only the snapshots created by this specific migration + conn = op.get_bind() + delete_query = text( + """ + DELETE FROM compliance_report_organization_snapshot + WHERE create_user = :user + """ + ) + try: + result = conn.execute(delete_query, {"user": MIGRATION_USER}) + print( + f"Downgrade: Deleted {result.rowcount} organization snapshots created by migration {revision}." + ) + except Exception as e: + print(f"ERROR during downgrade delete for migration {revision}: {e}") diff --git a/backend/lcfs/db/models/compliance/ComplianceReportSummary.py b/backend/lcfs/db/models/compliance/ComplianceReportSummary.py index 08645711c..97b16d47d 100644 --- a/backend/lcfs/db/models/compliance/ComplianceReportSummary.py +++ b/backend/lcfs/db/models/compliance/ComplianceReportSummary.py @@ -1,5 +1,6 @@ from sqlalchemy import Column, Integer, Float, ForeignKey, Boolean, DateTime from sqlalchemy.orm import relationship +from sqlalchemy.dialects.postgresql import JSONB from lcfs.db.base import BaseModel, Auditable @@ -109,23 +110,41 @@ class ComplianceReportSummary(BaseModel, Auditable): line_21_non_compliance_penalty_payable = Column(Float, nullable=False, default=0) total_non_compliance_penalty_payable = Column(Float, nullable=False, default=0) + historical_snapshot = Column( + JSONB, + nullable=True, + comment="Contains historical data from pre-2024 TFRS system for data retention and analysis purposes.", + ) + # Penalty override fields for IDIR Director - penalty_override_enabled = Column(Boolean, nullable=False, default=False, comment="Whether penalty override is enabled (checkbox state)") - renewable_penalty_override = Column(Float, nullable=True, comment="Director override value for Line 11 renewable fuel penalty") - low_carbon_penalty_override = Column(Float, nullable=True, comment="Director override value for Line 21 low carbon fuel penalty") - penalty_override_date = Column(DateTime(timezone=True), nullable=True, comment="Timestamp when penalty was last overridden") + penalty_override_enabled = Column( + Boolean, + nullable=False, + default=False, + comment="Whether penalty override is enabled (checkbox state)", + ) + renewable_penalty_override = Column( + Float, + nullable=True, + comment="Director override value for Line 11 renewable fuel penalty", + ) + low_carbon_penalty_override = Column( + Float, + nullable=True, + comment="Director override value for Line 21 low carbon fuel penalty", + ) + penalty_override_date = Column( + DateTime(timezone=True), + nullable=True, + comment="Timestamp when penalty was last overridden", + ) penalty_override_user = Column( Integer, ForeignKey("user_profile.user_profile_id"), nullable=True, - comment="User who made the penalty override" + comment="User who made the penalty override", ) - # Legacy TFRS Columns - credits_offset_a = Column(Integer) - credits_offset_b = Column(Integer) - credits_offset_c = Column(Integer) - # Early Issuance Columns early_issuance_credits_q1 = Column(Integer) early_issuance_credits_q2 = Column(Integer) @@ -133,7 +152,9 @@ class ComplianceReportSummary(BaseModel, Auditable): early_issuance_credits_q4 = Column(Integer) compliance_report = relationship("ComplianceReport", back_populates="summary") - penalty_override_user_profile = relationship("UserProfile", foreign_keys=[penalty_override_user]) + penalty_override_user_profile = relationship( + "UserProfile", foreign_keys=[penalty_override_user] + ) @property def total_renewable_fuel_supplied(self): diff --git a/backend/lcfs/db/models/user/Role.py b/backend/lcfs/db/models/user/Role.py index e6d216b99..768fe2967 100644 --- a/backend/lcfs/db/models/user/Role.py +++ b/backend/lcfs/db/models/user/Role.py @@ -17,6 +17,7 @@ class RoleEnum(enum.Enum): COMPLIANCE_REPORTING = "Compliance Reporting" SIGNING_AUTHORITY = "Signing Authority" READ_ONLY = "Read Only" + BETA_TESTER = "Beta Tester" class Role(BaseModel, Auditable): diff --git a/backend/lcfs/db/sql/views/metabase.sql b/backend/lcfs/db/sql/views/metabase.sql index e1fd16096..4f85c928a 100644 --- a/backend/lcfs/db/sql/views/metabase.sql +++ b/backend/lcfs/db/sql/views/metabase.sql @@ -588,6 +588,1168 @@ GROUP BY user_type; GRANT SELECT ON vw_login_failures_analysis TO basic_lcfs_reporting_role; + +-- ========================================== +-- Fuel Supply Analytics Base View +-- ========================================== +drop view if exists vw_fuel_supply_analytics_base; +CREATE OR REPLACE VIEW vw_fuel_supply_analytics_base AS +WITH + latest_fs AS ( + SELECT + fs.group_uuid, + MAX(fs.version) AS max_version + FROM + fuel_supply fs + WHERE + action_type <> 'DELETE' + GROUP BY + fs.group_uuid + ), + selected_fs AS ( + SELECT + fs.* + FROM + fuel_supply fs + JOIN latest_fs lfs ON fs.group_uuid = lfs.group_uuid + AND fs.version = lfs.max_version + WHERE + fs.action_type != 'DELETE' + ), + finished_fuel_transport_modes_agg AS ( + SELECT + fc.fuel_code_id, + ARRAY_AGG(tm.transport_mode ORDER BY tm.transport_mode) AS transport_modes + FROM + fuel_code fc + JOIN finished_fuel_transport_mode fftm ON fc.fuel_code_id = fftm.fuel_code_id + JOIN transport_mode tm ON fftm.transport_mode_id = tm.transport_mode_id + GROUP BY + fc.fuel_code_id + ), + feedstock_fuel_transport_modes_agg AS ( + SELECT + fc.fuel_code_id, + ARRAY_AGG(tm.transport_mode ORDER BY tm.transport_mode) AS transport_modes + FROM + fuel_code fc + JOIN feedstock_fuel_transport_mode fftm ON fc.fuel_code_id = fftm.fuel_code_id + JOIN transport_mode tm ON fftm.transport_mode_id = tm.transport_mode_id + GROUP BY + fc.fuel_code_id + ), + grouped_reports AS ( + SELECT + compliance_report_id, + compliance_report_group_uuid, + VERSION, + compliance_period_id, + current_status_id, + organization_id + FROM + compliance_report + WHERE + compliance_report_group_uuid IN ( + SELECT + vcrb.compliance_report_group_uuid + FROM + vw_compliance_report_analytics_base vcrb + ) + ) +SELECT DISTINCT + gr.compliance_report_group_uuid, + vcrb.report_status, + vcrb.compliance_report_id, + org.organization_id, + org.name AS supplier_name, + cp.description AS compliance_year, + ft.fuel_type, + fcat.category as fuel_category, + eut.type AS end_use_type, + pa.description as provision_description, + fs.compliance_units, + fs.quantity, + ft.units as fuel_units, + fs.target_ci, + fs.ci_of_fuel as rci, + fs.uci, + fs.energy_density, + fs.eer, + fs.energy as energy_content, + concat(fcp.prefix, fc.fuel_suffix) AS fuel_code, + fc.company as fuel_code_company, + fc.feedstock, + fc.feedstock_location, + fc.feedstock_misc, + fc.effective_date, + fc.application_date, + fc.approval_date, + fc.expiration_date, + ft.renewable, + ft.fossil_derived, + fc.carbon_intensity, + fc.fuel_production_facility_city, + fc.fuel_production_facility_province_state, + fc.fuel_production_facility_country, + fc.facility_nameplate_capacity, + fc.facility_nameplate_capacity_unit, + finishedftma.transport_modes as Finished_fuel_transport_modes, + feedstockftma.transport_modes as Feedstock_fuel_transport_modes, + fcs.status as fuel_code_status, + fs.fuel_supply_id, + fs.fuel_type_id, + fs.fuel_code_id, + fs.provision_of_the_act_id, + fs.fuel_category_id, + fs.end_use_id +FROM + selected_fs fs + JOIN grouped_reports gr ON fs.compliance_report_id = gr.compliance_report_id + JOIN vw_compliance_report_analytics_base vcrb ON vcrb.compliance_report_group_uuid = gr.compliance_report_group_uuid + JOIN compliance_period cp ON gr.compliance_period_id = cp.compliance_period_id + JOIN organization org ON gr.organization_id = org.organization_id + LEFT JOIN fuel_code fc ON fs.fuel_code_id = fc.fuel_code_id + LEFT JOIN fuel_code_status fcs ON fc.fuel_status_id = fcs.fuel_code_status_id + LEFT JOIN fuel_code_prefix fcp ON fc.prefix_id = fcp.fuel_code_prefix_id + LEFT JOIN fuel_type ft ON fs.fuel_type_id = ft.fuel_type_id + LEFT JOIN fuel_category fcat ON fcat.fuel_category_id = fs.fuel_category_id + LEFT JOIN end_use_type eut ON fs.end_use_id = eut.end_use_type_id + LEFT JOIN provision_of_the_act pa ON fs.provision_of_the_act_id = pa.provision_of_the_act_id + LEFT JOIN finished_fuel_transport_modes_agg finishedftma ON fc.fuel_code_id = finishedftma.fuel_code_id + LEFT JOIN feedstock_fuel_transport_modes_agg feedstockftma ON fc.fuel_code_id = feedstockftma.fuel_code_id; +grant select on vw_fuel_supply_analytics_base to basic_lcfs_reporting_role; +grant select on fuel_category, fuel_type, fuel_code, fuel_code_status, fuel_code_prefix, provision_of_the_act, end_use_type to basic_lcfs_reporting_role; + +-- ========================================== +-- Transaction Base View +-- ========================================== +drop view if exists vw_transaction_base; +CREATE OR REPLACE VIEW vw_transaction_base AS +SELECT * +FROM transaction +WHERE transaction_action != 'Released'; + +GRANT SELECT ON vw_transaction_base TO basic_lcfs_reporting_role; + +-- ========================================== +-- Fuel Supply Fuel Code Base View +-- ========================================== +drop view if exists vw_fuel_supply_fuel_code_base; +CREATE OR REPLACE VIEW vw_fuel_supply_fuel_code_base AS +SELECT + "source"."compliance_report_id" AS "compliance_report_id", + "source"."organization_id" AS "organization_id", + "source"."organization" AS "organization", + "source"."compliance_period" AS "compliance_period", + "source"."report_status" AS "report_status", + "source"."fuel_supply_id" AS "fuel_supply_id", + "source"."fuel_type_id" AS "fuel_type_id", + "source"."fuel_type" AS "fuel_type", + "source"."renewable" AS "renewable", + "source"."fuel_category_id" AS "fuel_category_id", + "source"."fuel_category" AS "fuel_category", + "source"."fuel_code_id" AS "fuel_code_id", + "source"."quantity" AS "quantity", + "source"."units" AS "units", + "source"."compliance_units" AS "compliance_units", + "source"."provision_of_the_act_id" AS "provision_of_the_act_id", + "source"."provision_of_the_act" AS "provision_of_the_act", + "source"."end_use_id" AS "end_use_id", + "source"."end_use_type" AS "end_use_type", + "source"."ci_of_fuel" AS "ci_of_fuel", + "source"."target_ci" AS "target_ci", + "source"."uci" AS "uci", + "source"."energy_density" AS "energy_density", + "source"."eer" AS "eer", + "source"."energy" AS "energy", + "source"."fuel_type_other" AS "fuel_type_other", + "Fuel Code - fuel_code_id"."fuel_code_id" AS "FC - id__fuel_code_id", + "Fuel Code - fuel_code_id"."fuel_status_id" AS "FC - id__fuel_status_id", + "Fuel Code - fuel_code_id"."prefix_id" AS "FC - id__prefix_id", + "Fuel Code - fuel_code_id"."fuel_suffix" AS "FC - id__fuel_suffix", + "Fuel Code - fuel_code_id"."company" AS "FC - id__company", + "Fuel Code - fuel_code_id"."carbon_intensity" AS "FC - id__carbon_intensity", + "Fuel Code - fuel_code_id"."last_updated" AS "FC - id__last_updated", + "Fuel Code - fuel_code_id"."application_date" AS "FC - id__application_date", + "Fuel Code - fuel_code_id"."approval_date" AS "FC - id__approval_date", + "Fuel Code - fuel_code_id"."fuel_type_id" AS "FC - id__fuel_type_id", + "Fuel Code - fuel_code_id"."feedstock" AS "FC - id__feedstock", + "Fuel Code - fuel_code_id"."feedstock_location" AS "FC - id__feedstock_location", + "Fuel Code - fuel_code_id"."feedstock_misc" AS "FC - id__feedstock_misc", + "Fuel Code - fuel_code_id"."fuel_production_facility_city" AS "FC - id__fuel_production_facility_city", + "Fuel Code - fuel_code_id"."fuel_production_facility_province_state" AS "FC - id__fuel_production_facility_province_state", + "Fuel Code - fuel_code_id"."fuel_production_facility_country" AS "FC - id__fuel_production_facility_country", + "Fuel Code - fuel_code_id"."facility_nameplate_capacity" AS "FC - id__facility_nameplate_capacity", + "Fuel Code - fuel_code_id"."effective_date" AS "FC - id__effective_date", + "Fuel Code - fuel_code_id"."effective_status" AS "FC - id__effective_status", + "Fuel Code - fuel_code_id"."expiration_date" AS "FC - id__expiration_date" + FROM + ( + WITH latest_fs AS ( + SELECT + DISTINCT ON (group_uuid) * + FROM + fuel_supply + +ORDER BY + group_uuid, + VERSION DESC + ) + SELECT + ----------------------------------------------------- + -- compliance_report, organization, compliance_period + ----------------------------------------------------- + compliance_report.compliance_report_id, + compliance_report.organization_id, + organization.name AS organization, + compliance_period.description AS compliance_period, + compliance_report_status.status AS report_status, + ------------------------------------------------- + -- fuel_supply + ------------------------------------------------- + fuel_supply.fuel_supply_id, + fuel_supply.fuel_type_id, + fuel_type.fuel_type, + fuel_type.renewable, + fuel_supply.fuel_category_id, + fuel_category.category AS fuel_category, + fuel_supply.fuel_code_id, + fuel_supply.quantity, + fuel_supply.units, + fuel_supply.compliance_units, + fuel_supply.provision_of_the_act_id, + provision_of_the_act.name AS provision_of_the_act, + fuel_supply.end_use_id, + end_use_type.type AS end_use_type, + fuel_supply.ci_of_fuel, + fuel_supply.target_ci, + fuel_supply.uci, + fuel_supply.energy_density, + fuel_supply.eer, + fuel_supply.energy, + fuel_supply.fuel_type_other + FROM + compliance_report + JOIN compliance_report_status ON compliance_report.current_status_id = compliance_report_status.compliance_report_status_id + JOIN organization ON compliance_report.organization_id = organization.organization_id + JOIN compliance_period ON compliance_report.compliance_period_id = compliance_period.compliance_period_id + JOIN latest_fs fuel_supply ON compliance_report.compliance_report_id = fuel_supply.compliance_report_id + JOIN fuel_type ON fuel_supply.fuel_type_id = fuel_type.fuel_type_id + JOIN fuel_category ON fuel_supply.fuel_category_id = fuel_category.fuel_category_id + +LEFT JOIN provision_of_the_act ON fuel_supply.provision_of_the_act_id = provision_of_the_act.provision_of_the_act_id + LEFT JOIN end_use_type ON fuel_supply.end_use_id = end_use_type.end_use_type_id + +WHERE + compliance_report.current_status_id IN (2, 3, 4, 5) + ) AS "source" + LEFT JOIN "fuel_code" AS "Fuel Code - fuel_code_id" ON "source"."fuel_code_id" = "Fuel Code - fuel_code_id"."fuel_code_id"; + + +GRANT SELECT ON vw_fuel_supply_fuel_code_base TO basic_lcfs_reporting_role; + +-- ========================================== +-- Fuel Supply Base View +-- ========================================== +drop view if exists vw_fuel_supply_base; +CREATE OR REPLACE VIEW vw_fuel_supply_base AS +WITH latest_fs AS ( + SELECT + DISTINCT ON (group_uuid) * + FROM + fuel_supply + +ORDER BY + group_uuid, + VERSION DESC + ) + SELECT + compliance_report.compliance_report_id, + compliance_report.organization_id, + organization.name AS organization, + compliance_period.description AS compliance_period, + compliance_report_status.status AS report_status, + fuel_supply.fuel_supply_id, + fuel_supply.fuel_type_id, + fuel_type.fuel_type, + fuel_type.renewable, + fuel_supply.fuel_category_id, + fuel_category.category AS fuel_category, + fuel_supply.fuel_code_id, + fuel_supply.quantity, + fuel_supply.units, + fuel_supply.compliance_units, + fuel_supply.provision_of_the_act_id, + provision_of_the_act.name AS provision_of_the_act, + fuel_supply.end_use_id, + end_use_type.type AS end_use_type, + fuel_supply.ci_of_fuel, + fuel_supply.target_ci, + fuel_supply.uci, + fuel_supply.energy_density, + fuel_supply.eer, + fuel_supply.energy, + fuel_supply.fuel_type_other + FROM + compliance_report + JOIN compliance_report_status ON compliance_report.current_status_id = compliance_report_status.compliance_report_status_id + JOIN organization ON compliance_report.organization_id = organization.organization_id + JOIN compliance_period ON compliance_report.compliance_period_id = compliance_period.compliance_period_id + JOIN latest_fs fuel_supply ON compliance_report.compliance_report_id = fuel_supply.compliance_report_id + JOIN fuel_type ON fuel_supply.fuel_type_id = fuel_type.fuel_type_id + JOIN fuel_category ON fuel_supply.fuel_category_id = fuel_category.fuel_category_id + +LEFT JOIN provision_of_the_act ON fuel_supply.provision_of_the_act_id = provision_of_the_act.provision_of_the_act_id + LEFT JOIN end_use_type ON fuel_supply.end_use_id = end_use_type.end_use_type_id + +WHERE + compliance_report.current_status_id IN (2, 3, 4, 5); + +GRANT SELECT ON vw_fuel_supply_base TO basic_lcfs_reporting_role; + +-- ========================================== +-- Compliance Report Fuel Supply Base View +-- ========================================== +drop view if exists vw_compliance_report_fuel_supply_base; +CREATE OR REPLACE VIEW vw_compliance_report_fuel_supply_base AS +SELECT + "source"."fuel_supply_id" AS "fuel_supply_id", + "source"."compliance_report_id" AS "compliance_report_id", + "source"."quantity" AS "quantity", + "source"."units" AS "units", + "source"."compliance_units" AS "compliance_units", + "source"."target_ci" AS "target_ci", + "source"."ci_of_fuel" AS "ci_of_fuel", + "source"."energy_density" AS "energy_density", + "source"."eer" AS "eer", + "source"."uci" AS "uci", + "source"."energy" AS "energy", + "source"."fuel_type_other" AS "fuel_type_other", + "source"."fuel_category_id" AS "fuel_category_id", + "source"."fuel_code_id" AS "fuel_code_id", + "source"."fuel_type_id" AS "fuel_type_id", + "source"."provision_of_the_act_id" AS "provision_of_the_act_id", + "source"."end_use_id" AS "end_use_id", + "source"."q1_quantity" AS "q1_quantity", + "source"."q2_quantity" AS "q2_quantity", + "source"."q3_quantity" AS "q3_quantity", + "source"."q4_quantity" AS "q4_quantity", + "Compliance Report groups - Compliance Report"."compliance_report_id" AS "CR groups - id", + "Compliance Report groups - Compliance Report"."compliance_report_group_uuid" AS "CR groups - group_uuid", + "Compliance Report groups - Compliance Report"."version" AS "CR groups - version", + "Compliance Report groups - Compliance Report"."compliance_period" AS "CR groups - compliance_period", + "Compliance Report groups - Compliance Report"."organization_name" AS "CR groups - organization_name", + "Compliance Report groups - Compliance Report"."report_status" AS "CR groups - report_status" + FROM + ( + SELECT + "fuel_supply"."fuel_supply_id" AS "fuel_supply_id", + "fuel_supply"."compliance_report_id" AS "compliance_report_id", + "fuel_supply"."quantity" AS "quantity", + "fuel_supply"."units" AS "units", + "fuel_supply"."compliance_units" AS "compliance_units", + "fuel_supply"."target_ci" AS "target_ci", + "fuel_supply"."ci_of_fuel" AS "ci_of_fuel", + "fuel_supply"."energy_density" AS "energy_density", + "fuel_supply"."eer" AS "eer", + "fuel_supply"."uci" AS "uci", + "fuel_supply"."energy" AS "energy", + "fuel_supply"."fuel_type_other" AS "fuel_type_other", + "fuel_supply"."fuel_category_id" AS "fuel_category_id", + "fuel_supply"."fuel_code_id" AS "fuel_code_id", + "fuel_supply"."fuel_type_id" AS "fuel_type_id", + "fuel_supply"."provision_of_the_act_id" AS "provision_of_the_act_id", + "fuel_supply"."end_use_id" AS "end_use_id", + "fuel_supply"."q1_quantity" AS "q1_quantity", + "fuel_supply"."q2_quantity" AS "q2_quantity", + "fuel_supply"."q3_quantity" AS "q3_quantity", + "fuel_supply"."q4_quantity" AS "q4_quantity" + FROM + "fuel_supply" + INNER JOIN ( + SELECT + "fuel_supply"."group_uuid" AS "group_uuid", + MAX("fuel_supply"."version") AS "max" + FROM + "fuel_supply" + +GROUP BY + "fuel_supply"."group_uuid" + +ORDER BY + "fuel_supply"."group_uuid" ASC + ) AS "Latest Fuel Supply per Group - Version" ON ( + "fuel_supply"."group_uuid" = "Latest Fuel Supply per Group - Version"."group_uuid" + ) + + AND ( + "fuel_supply"."version" = "Latest Fuel Supply per Group - Version"."max" + ) + +WHERE + "fuel_supply"."action_type" = CAST('CREATE' AS "actiontypeenum") + ) AS "source" + +LEFT JOIN ( + SELECT + "source"."compliance_report_id" AS "compliance_report_id", + "source"."compliance_report_group_uuid" AS "compliance_report_group_uuid", + "source"."version" AS "version", + "source"."compliance_period" AS "compliance_period", + "source"."organization_name" AS "organization_name", + "source"."report_status" AS "report_status", + "Compliance Report - Compliance Report Group UUID"."compliance_report_id" AS "CR - Group UUID__compliance_report_id", + "Compliance Report - Compliance Report Group UUID"."transaction_id" AS "CR - Group UUID__transaction_id", + "Compliance Report - Compliance Report Group UUID"."legacy_id" AS "CR - Group UUID__legacy_id", + "Compliance Report - Compliance Report Group UUID"."version" AS "CR - Group UUID__version", + "Compliance Report - Compliance Report Group UUID"."supplemental_initiator" AS "CR - Group UUID__supplemental_initiator", + "Compliance Report - Compliance Report Group UUID"."current_status_id" AS "CR - Group UUID__current_status_id" + FROM + ( + SELECT + "v_compliance_report"."compliance_report_id" AS "compliance_report_id", + "v_compliance_report"."compliance_report_group_uuid" AS "compliance_report_group_uuid", + "v_compliance_report"."version" AS "version", + "v_compliance_report"."compliance_period_id" AS "compliance_period_id", + "v_compliance_report"."compliance_period" AS "compliance_period", + "v_compliance_report"."organization_id" AS "organization_id", + "v_compliance_report"."organization_name" AS "organization_name", + "v_compliance_report"."report_type" AS "report_type", + "v_compliance_report"."report_status_id" AS "report_status_id", + "v_compliance_report"."report_status" AS "report_status", + "v_compliance_report"."update_date" AS "update_date", + "v_compliance_report"."supplemental_initiator" AS "supplemental_initiator" + FROM + "v_compliance_report" + WHERE + ( + "v_compliance_report"."report_status" <> CAST('Draft' AS "compliancereportstatusenum") + ) + + OR ( + "v_compliance_report"."report_status" IS NULL + ) + ) AS "source" + INNER JOIN "compliance_report" AS "Compliance Report - Compliance Report Group UUID" ON "source"."compliance_report_group_uuid" = "Compliance Report - Compliance Report Group UUID"."compliance_report_group_uuid" + ) AS "Compliance Report groups - Compliance Report" ON "source"."compliance_report_id" = "Compliance Report groups - Compliance Report"."compliance_report_id"; + +GRANT SELECT ON vw_compliance_report_fuel_supply_base TO basic_lcfs_reporting_role; + +-- ========================================== +-- Compliance Report Chained View +-- ========================================== +drop view if exists vw_compliance_report_chained; +CREATE OR REPLACE VIEW vw_compliance_report_chained AS +SELECT + compliance_report_group_uuid AS group_uuid, + max(VERSION) AS max_version + FROM + COMPLIANCE_REPORT + +GROUP BY + COMPLIANCE_REPORT.compliance_report_group_uuid; + +-- Grant SELECT privileges to the reporting role +GRANT SELECT ON vw_compliance_report_chained TO basic_lcfs_reporting_role; + +-- ========================================== +-- Compliance Report Base View +-- ========================================== +drop view if exists vw_compliance_report_base cascade; +CREATE OR REPLACE VIEW vw_compliance_report_base AS +SELECT + "compliance_report"."compliance_report_id" AS "compliance_report_id", + "compliance_report"."compliance_period_id" AS "compliance_period_id", + "compliance_report"."organization_id" AS "organization_id", + "compliance_report"."current_status_id" AS "current_status_id", + "compliance_report"."transaction_id" AS "transaction_id", + "compliance_report"."compliance_report_group_uuid" AS "compliance_report_group_uuid", + "compliance_report"."legacy_id" AS "legacy_id", + "compliance_report"."version" AS "version", + "compliance_report"."supplemental_initiator" AS "supplemental_initiator", + "compliance_report"."reporting_frequency" AS "reporting_frequency", + "compliance_report"."nickname" AS "nickname", + "compliance_report"."supplemental_note" AS "supplemental_note", + "compliance_report"."assessment_statement" AS "assessment_statement", + CASE + WHEN "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" > 0 THEN 'Not Met' + ELSE 'Met' + END AS "Renewable Requirements", + CASE + WHEN "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" > 0 THEN 'Not Met' + ELSE 'Met' + END AS "Low Carbon Requirements", + "Compliance Period"."compliance_period_id" AS "Compliance Period__compliance_period_id", + "Compliance Period"."description" AS "Compliance Period__description", + "Compliance Period"."display_order" AS "Compliance Period__display_order", + "Compliance Period"."create_date" AS "Compliance Period__create_date", + "Compliance Period"."update_date" AS "Compliance Period__update_date", + "Compliance Period"."effective_date" AS "Compliance Period__effective_date", + "Compliance Period"."effective_status" AS "Compliance Period__effective_status", + "Compliance Period"."expiration_date" AS "Compliance Period__expiration_date", + "Compliance Report Status - Current Status"."compliance_report_status_id" AS "Compliance Report Status - Current Status__complian_8aca39b7", + "Compliance Report Status - Current Status"."display_order" AS "Compliance Report Status - Current Status__display_order", + "Compliance Report Status - Current Status"."status" AS "Compliance Report Status - Current Status__status", + "Compliance Report Status - Current Status"."create_date" AS "Compliance Report Status - Current Status__create_date", + "Compliance Report Status - Current Status"."update_date" AS "Compliance Report Status - Current Status__update_date", + "Compliance Report Status - Current Status"."effective_date" AS "Compliance Report Status - Current Status__effective_date", + "Compliance Report Status - Current Status"."effective_status" AS "Compliance Report Status - Current Status__effective_status", + "Compliance Report Status - Current Status"."expiration_date" AS "Compliance Report Status - Current Status__expiration_date", + "Compliance Report Summary - Compliance Report"."summary_id" AS "Compliance Report Summary - Compliance Report__summary_id", + "Compliance Report Summary - Compliance Report"."compliance_report_id" AS "Compliance Report Summary - Compliance Report__comp_1db2e1e9", + "Compliance Report Summary - Compliance Report"."quarter" AS "Compliance Report Summary - Compliance Report__quarter", + "Compliance Report Summary - Compliance Report"."is_locked" AS "Compliance Report Summary - Compliance Report__is_locked", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_gasoline" AS "Compliance Report Summary - Compliance Report__line_2c0818fb", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_diesel" AS "Compliance Report Summary - Compliance Report__line_2ff66c5b", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_1fcf7a18", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_d70f8aef", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_2773c83c", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_e4c8e80c", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_9cec896d", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_489bea32", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_af2beb8e", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_gasoline" AS "Compliance Report Summary - Compliance Report__line_a26a000d", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_diesel" AS "Compliance Report Summary - Compliance Report__line_0ef43e75", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_91ad62ee", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_gasoline" AS "Compliance Report Summary - Compliance Report__line_b1027537", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_diesel" AS "Compliance Report Summary - Compliance Report__line_38be33f3", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_82c517d4", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_gasoline" AS "Compliance Report Summary - Compliance Report__line_6927f733", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_diesel" AS "Compliance Report Summary - Compliance Report__line_93d805cb", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_5ae095d0", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_gasoline" AS "Compliance Report Summary - Compliance Report__line_157d6973", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_diesel" AS "Compliance Report Summary - Compliance Report__line_31fd1f1b", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_26ba0b90", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_gasoline" AS "Compliance Report Summary - Compliance Report__line_27419684", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_diesel" AS "Compliance Report Summary - Compliance Report__line_ac263897", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_1486f467", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_gasoline" AS "Compliance Report Summary - Compliance Report__line_c12cb8c0", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_diesel" AS "Compliance Report Summary - Compliance Report__line_05eb459f", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_f2ebda23", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_1be763e7", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_b72177b0", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_28200104", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_gasoline" AS "Compliance Report Summary - Compliance Report__line_53735f1d", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_diesel" AS "Compliance Report Summary - Compliance Report__line_64a07c80", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_60b43dfe", + "Compliance Report Summary - Compliance Report"."line_12_low_carbon_fuel_required" AS "Compliance Report Summary - Compliance Report__line_da2710ad", + "Compliance Report Summary - Compliance Report"."line_13_low_carbon_fuel_supplied" AS "Compliance Report Summary - Compliance Report__line_b25fca1c", + "Compliance Report Summary - Compliance Report"."line_14_low_carbon_fuel_surplus" AS "Compliance Report Summary - Compliance Report__line_4b98033f", + "Compliance Report Summary - Compliance Report"."line_15_banked_units_used" AS "Compliance Report Summary - Compliance Report__line_1d7d6a31", + "Compliance Report Summary - Compliance Report"."line_16_banked_units_remaining" AS "Compliance Report Summary - Compliance Report__line_684112bb", + "Compliance Report Summary - Compliance Report"."line_17_non_banked_units_used" AS "Compliance Report Summary - Compliance Report__line_b1d3ad5e", + "Compliance Report Summary - Compliance Report"."line_18_units_to_be_banked" AS "Compliance Report Summary - Compliance Report__line_e173956e", + "Compliance Report Summary - Compliance Report"."line_19_units_to_be_exported" AS "Compliance Report Summary - Compliance Report__line_9a885574", + "Compliance Report Summary - Compliance Report"."line_20_surplus_deficit_units" AS "Compliance Report Summary - Compliance Report__line_8e71546f", + "Compliance Report Summary - Compliance Report"."line_21_surplus_deficit_ratio" AS "Compliance Report Summary - Compliance Report__line_00d2728d", + "Compliance Report Summary - Compliance Report"."line_22_compliance_units_issued" AS "Compliance Report Summary - Compliance Report__line_29d9cb9c", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_gasoline" AS "Compliance Report Summary - Compliance Report__line_d8942234", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_diesel" AS "Compliance Report Summary - Compliance Report__line_125e31fc", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_eb5340d7", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" AS "Compliance Report Summary - Compliance Report__line_bff5157d", + "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__line_7c9c21b1", + "Compliance Report Summary - Compliance Report"."total_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__tota_0e1e6fb3", + "Compliance Report Summary - Compliance Report"."create_date" AS "Compliance Report Summary - Compliance Report__create_date", + "Compliance Report Summary - Compliance Report"."update_date" AS "Compliance Report Summary - Compliance Report__update_date", + "Compliance Report Summary - Compliance Report"."create_user" AS "Compliance Report Summary - Compliance Report__create_user", + "Compliance Report Summary - Compliance Report"."update_user" AS "Compliance Report Summary - Compliance Report__update_user", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q1" AS "Compliance Report Summary - Compliance Report__earl_6d4994a4", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q2" AS "Compliance Report Summary - Compliance Report__earl_f440c51e", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q3" AS "Compliance Report Summary - Compliance Report__earl_8347f588", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q4" AS "Compliance Report Summary - Compliance Report__earl_1d23602b", + "Transaction"."transaction_id" AS "Transaction__transaction_id", + "Transaction"."compliance_units" AS "Transaction__compliance_units", + "Transaction"."organization_id" AS "Transaction__organization_id", + "Transaction"."transaction_action" AS "Transaction__transaction_action", + "Transaction"."create_date" AS "Transaction__create_date", + "Transaction"."update_date" AS "Transaction__update_date", + "Transaction"."create_user" AS "Transaction__create_user", + "Transaction"."update_user" AS "Transaction__update_user", + "Transaction"."effective_date" AS "Transaction__effective_date", + "Transaction"."effective_status" AS "Transaction__effective_status", + "Transaction"."expiration_date" AS "Transaction__expiration_date", + "Organization"."organization_id" AS "Organization__organization_id", + "Organization"."organization_code" AS "Organization__organization_code", + "Organization"."name" AS "Organization__name", + "Organization"."operating_name" AS "Organization__operating_name", + "Organization"."email" AS "Organization__email", + "Organization"."phone" AS "Organization__phone", + "Organization"."edrms_record" AS "Organization__edrms_record", + "Organization"."total_balance" AS "Organization__total_balance", + "Organization"."reserved_balance" AS "Organization__reserved_balance", + "Organization"."count_transfers_in_progress" AS "Organization__count_transfers_in_progress", + "Organization"."organization_status_id" AS "Organization__organization_status_id", + "Organization"."organization_type_id" AS "Organization__organization_type_id", + "Organization"."organization_address_id" AS "Organization__organization_address_id", + "Organization"."organization_attorney_address_id" AS "Organization__organization_attorney_address_id", + "Organization"."create_date" AS "Organization__create_date", + "Organization"."update_date" AS "Organization__update_date", + "Organization"."create_user" AS "Organization__create_user", + "Organization"."update_user" AS "Organization__update_user", + "Organization"."effective_date" AS "Organization__effective_date", + "Organization"."effective_status" AS "Organization__effective_status", + "Organization"."expiration_date" AS "Organization__expiration_date", + COALESCE("Organization Early Issuance"."has_early_issuance", false) AS "Organization__has_early_issuance", + "Organization"."records_address" AS "Organization__records_address", + "Compliance Reports Chained - Compliance Report Group UUID"."group_uuid" AS "CR Chained - CR Group UUID__group_uuid", + "Compliance Reports Chained - Compliance Report Group UUID"."max_version" AS "CR Chained - CR Group UUID__max_version" + FROM + "compliance_report" + INNER JOIN "compliance_period" AS "Compliance Period" ON "compliance_report"."compliance_period_id" = "Compliance Period"."compliance_period_id" + INNER JOIN "compliance_report_status" AS "Compliance Report Status - Current Status" ON "compliance_report"."current_status_id" = "Compliance Report Status - Current Status"."compliance_report_status_id" + INNER JOIN "compliance_report_summary" AS "Compliance Report Summary - Compliance Report" ON "compliance_report"."compliance_report_id" = "Compliance Report Summary - Compliance Report"."compliance_report_id" + +LEFT JOIN "transaction" AS "Transaction" ON "compliance_report"."transaction_id" = "Transaction"."transaction_id" + LEFT JOIN "organization" AS "Organization" ON "compliance_report"."organization_id" = "Organization"."organization_id" + LEFT JOIN "organization_early_issuance_by_year" AS "Organization Early Issuance" ON "compliance_report"."organization_id" = "Organization Early Issuance"."organization_id" AND "compliance_report"."compliance_period_id" = "Organization Early Issuance"."compliance_period_id" + INNER JOIN ( + SELECT + compliance_report_group_uuid AS group_uuid, + max(VERSION) AS max_version + FROM + COMPLIANCE_REPORT + +GROUP BY + COMPLIANCE_REPORT.compliance_report_group_uuid + ) AS "Compliance Reports Chained - Compliance Report Group UUID" ON ( + "compliance_report"."compliance_report_group_uuid" = "Compliance Reports Chained - Compliance Report Group UUID"."group_uuid" + ) + + AND ( + "compliance_report"."version" = "Compliance Reports Chained - Compliance Report Group UUID"."max_version" + ); + +GRANT SELECT ON vw_compliance_report_base TO basic_lcfs_reporting_role; + +-- ========================================== +-- Allocation Agreement Chained View +-- ========================================== +drop view if exists vw_allocation_agreement_chained; +CREATE OR REPLACE VIEW vw_allocation_agreement_chained AS +SELECT + "allocation_agreement"."group_uuid" AS "group_uuid", + MAX("allocation_agreement"."version") AS "max" + FROM + "allocation_agreement" + +GROUP BY + "allocation_agreement"."group_uuid" + +ORDER BY + "allocation_agreement"."group_uuid" ASC; + +GRANT SELECT ON vw_allocation_agreement_chained TO basic_lcfs_reporting_role; + +-- ========================================== +-- Allocation Agreement Base View +-- ========================================== +drop view if exists vw_allocation_agreement_base; +CREATE OR REPLACE VIEW vw_allocation_agreement_base AS +WITH latest_aa AS ( + SELECT DISTINCT ON (group_uuid) * + FROM allocation_agreement + ORDER BY group_uuid, version DESC +) +SELECT + "source"."group_uuid" AS "group_uuid", + "source"."max" AS "max", + CASE + WHEN "Allocation Agreement - Group UUID"."allocation_transaction_type_id" = 1 THEN 'Allocated From' + WHEN "Allocation Agreement - Group UUID"."allocation_transaction_type_id" = 2 THEN 'Allocation To' + END AS "Allocation transaction type", + "Allocation Agreement - Group UUID"."allocation_agreement_id" AS "Allocation Agreement - Group UUID__allocation_agreement_id", + "Allocation Agreement - Group UUID"."transaction_partner" AS "Allocation Agreement - Group UUID__transaction_partner", + "Allocation Agreement - Group UUID"."postal_address" AS "Allocation Agreement - Group UUID__postal_address", + "Allocation Agreement - Group UUID"."transaction_partner_email" AS "Allocation Agreement - Group UUID__transaction_partner_email", + "Allocation Agreement - Group UUID"."transaction_partner_phone" AS "Allocation Agreement - Group UUID__transaction_partner_phone", + "Allocation Agreement - Group UUID"."ci_of_fuel" AS "Allocation Agreement - Group UUID__ci_of_fuel", + "Allocation Agreement - Group UUID"."quantity" AS "Allocation Agreement - Group UUID__quantity", + "Allocation Agreement - Group UUID"."units" AS "Allocation Agreement - Group UUID__units", + "Allocation Agreement - Group UUID"."fuel_type_other" AS "Allocation Agreement - Group UUID__fuel_type_other", + "Allocation Agreement - Group UUID"."allocation_transaction_type_id" AS "Allocation Agreement - Group UUID__allocation_trans_ad55ef48", + "Allocation Agreement - Group UUID"."fuel_type_id" AS "Allocation Agreement - Group UUID__fuel_type_id", + "Allocation Agreement - Group UUID"."fuel_category_id" AS "Allocation Agreement - Group UUID__fuel_category_id", + "Allocation Agreement - Group UUID"."provision_of_the_act_id" AS "Allocation Agreement - Group UUID__provision_of_the_act_id", + "Allocation Agreement - Group UUID"."fuel_code_id" AS "Allocation Agreement - Group UUID__fuel_code_id", + "Allocation Agreement - Group UUID"."compliance_report_id" AS "Allocation Agreement - Group UUID__compliance_report_id", + "Allocation Agreement - Group UUID"."create_date" AS "Allocation Agreement - Group UUID__create_date", + "Allocation Agreement - Group UUID"."update_date" AS "Allocation Agreement - Group UUID__update_date", + "Allocation Agreement - Group UUID"."create_user" AS "Allocation Agreement - Group UUID__create_user", + "Allocation Agreement - Group UUID"."update_user" AS "Allocation Agreement - Group UUID__update_user", + "Allocation Agreement - Group UUID"."display_order" AS "Allocation Agreement - Group UUID__display_order", + "Allocation Agreement - Group UUID"."group_uuid" AS "Allocation Agreement - Group UUID__group_uuid", + "Allocation Agreement - Group UUID"."version" AS "Allocation Agreement - Group UUID__version", + "Allocation Agreement - Group UUID"."action_type" AS "Allocation Agreement - Group UUID__action_type", + "Allocation Agreement - Group UUID"."quantity_not_sold" AS "Allocation Agreement - Group UUID__quantity_not_sold", + "Compliance Report Base - Compliance Report"."compliance_report_id" AS "Compliance Report Base - Compliance Report__complia_6afb9aaa", + "Compliance Report Base - Compliance Report"."compliance_period_id" AS "Compliance Report Base - Compliance Report__complia_cda244b4", + "Compliance Report Base - Compliance Report"."organization_id" AS "Compliance Report Base - Compliance Report__organization_id", + "Compliance Report Base - Compliance Report"."current_status_id" AS "Compliance Report Base - Compliance Report__current_4315b3a2", + "Compliance Report Base - Compliance Report"."transaction_id" AS "Compliance Report Base - Compliance Report__transaction_id", + "Compliance Report Base - Compliance Report"."compliance_report_group_uuid" AS "Compliance Report Base - Compliance Report__complia_8e1217db", + "Compliance Report Base - Compliance Report"."legacy_id" AS "Compliance Report Base - Compliance Report__legacy_id", + "Compliance Report Base - Compliance Report"."version" AS "Compliance Report Base - Compliance Report__version", + "Compliance Report Base - Compliance Report"."supplemental_initiator" AS "Compliance Report Base - Compliance Report__supplem_3e383c17", + "Compliance Report Base - Compliance Report"."reporting_frequency" AS "Compliance Report Base - Compliance Report__reporti_c3204642", + "Compliance Report Base - Compliance Report"."nickname" AS "Compliance Report Base - Compliance Report__nickname", + "Compliance Report Base - Compliance Report"."supplemental_note" AS "Compliance Report Base - Compliance Report__supplem_76c93d97", + "Compliance Report Base - Compliance Report"."create_date" AS "Compliance Report Base - Compliance Report__create_date", + "Compliance Report Base - Compliance Report"."update_date" AS "Compliance Report Base - Compliance Report__update_date", + "Compliance Report Base - Compliance Report"."create_user" AS "Compliance Report Base - Compliance Report__create_user", + "Compliance Report Base - Compliance Report"."update_user" AS "Compliance Report Base - Compliance Report__update_user", + "Compliance Report Base - Compliance Report"."assessment_statement" AS "Compliance Report Base - Compliance Report__assessm_7b8d860b", + "Compliance Report Base - Compliance Report"."Renewable Requirements" AS "Compliance Report Base - Compliance Report__Renewab_f11c34b5", + "Compliance Report Base - Compliance Report"."Low Carbon Requirements" AS "Compliance Report Base - Compliance Report__Low Car_035b150f", + "Compliance Report Base - Compliance Report"."Compliance Period__compliance_period_id" AS "Compliance Report Base - Compliance Report__Complia_dd118a33", + "Compliance Report Base - Compliance Report"."Compliance Period__description" AS "Compliance Report Base - Compliance Report__Complia_cb30ad19", + "Compliance Report Base - Compliance Report"."Compliance Period__display_order" AS "Compliance Report Base - Compliance Report__Complia_721c9a5e", + "Compliance Report Base - Compliance Report"."Compliance Period__create_date" AS "Compliance Report Base - Compliance Report__Complia_8deb472c", + "Compliance Report Base - Compliance Report"."Compliance Period__update_date" AS "Compliance Report Base - Compliance Report__Complia_d8268dda", + "Compliance Report Base - Compliance Report"."Compliance Period__effective_date" AS "Compliance Report Base - Compliance Report__Complia_6a450a4b", + "Compliance Report Base - Compliance Report"."Compliance Period__effective_status" AS "Compliance Report Base - Compliance Report__Complia_e535ee64", + "Compliance Report Base - Compliance Report"."Compliance Period__expiration_date" AS "Compliance Report Base - Compliance Report__Complia_27d99d4c", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__complian_8aca39b7" AS "Compliance Report Base - Compliance Report__Complia_35a08ff4", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__display_order" AS "Compliance Report Base - Compliance Report__Complia_617d3c08", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__status" AS "Compliance Report Base - Compliance Report__Complia_f6d97a34", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__create_date" AS "Compliance Report Base - Compliance Report__Complia_de8161a6", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__update_date" AS "Compliance Report Base - Compliance Report__Complia_8b4cab50", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__effective_date" AS "Compliance Report Base - Compliance Report__Complia_e85e9f2c", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__effective_status" AS "Compliance Report Base - Compliance Report__Complia_4fecf6d4", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__expiration_date" AS "Compliance Report Base - Compliance Report__Complia_f48d7222", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__summary_id" AS "Compliance Report Base - Compliance Report__Complia_fe5bffa9", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__comp_1db2e1e9" AS "Compliance Report Base - Compliance Report__Complia_ac5855d3", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__quarter" AS "Compliance Report Base - Compliance Report__Complia_716d2691", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__is_locked" AS "Compliance Report Base - Compliance Report__Complia_9880522d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_2c0818fb" AS "Compliance Report Base - Compliance Report__Complia_e084f95e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_2ff66c5b" AS "Compliance Report Base - Compliance Report__Complia_bbf3740d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1fcf7a18" AS "Compliance Report Base - Compliance Report__Complia_0948e373", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_d70f8aef" AS "Compliance Report Base - Compliance Report__Complia_1151727e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_2773c83c" AS "Compliance Report Base - Compliance Report__Complia_8c046bfd", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_e4c8e80c" AS "Compliance Report Base - Compliance Report__Complia_0c86fc50", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_9cec896d" AS "Compliance Report Base - Compliance Report__Complia_817c5f4a", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_489bea32" AS "Compliance Report Base - Compliance Report__Complia_d930be19", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_af2beb8e" AS "Compliance Report Base - Compliance Report__Complia_c9668257", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_a26a000d" AS "Compliance Report Base - Compliance Report__Complia_eef8c8e5", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_0ef43e75" AS "Compliance Report Base - Compliance Report__Complia_469410d8", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_91ad62ee" AS "Compliance Report Base - Compliance Report__Complia_169c9f01", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b1027537" AS "Compliance Report Base - Compliance Report__Complia_ba262a42", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_38be33f3" AS "Compliance Report Base - Compliance Report__Complia_3609a003", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_82c517d4" AS "Compliance Report Base - Compliance Report__Complia_3d0e6803", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_6927f733" AS "Compliance Report Base - Compliance Report__Complia_aad5cfd2", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_93d805cb" AS "Compliance Report Base - Compliance Report__Complia_c437462d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_5ae095d0" AS "Compliance Report Base - Compliance Report__Complia_6bd2da0d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_157d6973" AS "Compliance Report Base - Compliance Report__Complia_ed316fbb", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_31fd1f1b" AS "Compliance Report Base - Compliance Report__Complia_2c880057", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_26ba0b90" AS "Compliance Report Base - Compliance Report__Complia_84fd8648", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_27419684" AS "Compliance Report Base - Compliance Report__Complia_c8859563", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_ac263897" AS "Compliance Report Base - Compliance Report__Complia_45fe3e7a", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1486f467" AS "Compliance Report Base - Compliance Report__Complia_256023d8", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_c12cb8c0" AS "Compliance Report Base - Compliance Report__Complia_eee3c998", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_05eb459f" AS "Compliance Report Base - Compliance Report__Complia_c90c8a63", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_f2ebda23" AS "Compliance Report Base - Compliance Report__Complia_c7afbbd2", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1be763e7" AS "Compliance Report Base - Compliance Report__Complia_b7c1da7c", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b72177b0" AS "Compliance Report Base - Compliance Report__Complia_eddee97b", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_28200104" AS "Compliance Report Base - Compliance Report__Complia_656afc20", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_53735f1d" AS "Compliance Report Base - Compliance Report__Complia_5698e212", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_64a07c80" AS "Compliance Report Base - Compliance Report__Complia_91c16e81", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_60b43dfe" AS "Compliance Report Base - Compliance Report__Complia_a38eca8f", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_da2710ad" AS "Compliance Report Base - Compliance Report__Complia_83fb46a6", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b25fca1c" AS "Compliance Report Base - Compliance Report__Complia_677a2191", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_4b98033f" AS "Compliance Report Base - Compliance Report__Complia_d3db4f29", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1d7d6a31" AS "Compliance Report Base - Compliance Report__Complia_805e5698", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_684112bb" AS "Compliance Report Base - Compliance Report__Complia_ba662a5c", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b1d3ad5e" AS "Compliance Report Base - Compliance Report__Complia_407ecab4", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_e173956e" AS "Compliance Report Base - Compliance Report__Complia_884b89fd", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_9a885574" AS "Compliance Report Base - Compliance Report__Complia_6030ea9c", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_8e71546f" AS "Compliance Report Base - Compliance Report__Complia_392f751b", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_00d2728d" AS "Compliance Report Base - Compliance Report__Complia_889450ac", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_29d9cb9c" AS "Compliance Report Base - Compliance Report__Complia_875531b4", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_d8942234" AS "Compliance Report Base - Compliance Report__Complia_ea067573", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_125e31fc" AS "Compliance Report Base - Compliance Report__Complia_a93845e5", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_eb5340d7" AS "Compliance Report Base - Compliance Report__Complia_2ef7cee4", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_bff5157d" AS "Compliance Report Base - Compliance Report__Complia_87c65017", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_7c9c21b1" AS "Compliance Report Base - Compliance Report__Complia_4bc20903", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__tota_0e1e6fb3" AS "Compliance Report Base - Compliance Report__Complia_0dd060c1", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__create_date" AS "Compliance Report Base - Compliance Report__Complia_e63c35ab", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__update_date" AS "Compliance Report Base - Compliance Report__Complia_b3f1ff5d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__create_user" AS "Compliance Report Base - Compliance Report__Complia_c131d498", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__update_user" AS "Compliance Report Base - Compliance Report__Complia_94fc1e6e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_6d4994a4" AS "Compliance Report Base - Compliance Report__Complia_df1f10d1", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_f440c51e" AS "Compliance Report Base - Compliance Report__Complia_a759d116", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_8347f588" AS "Compliance Report Base - Compliance Report__Complia_63b4b44e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_1d23602b" AS "Compliance Report Base - Compliance Report__Complia_05c7a0a8", + "Compliance Report Base - Compliance Report"."Transaction__transaction_id" AS "Compliance Report Base - Compliance Report__Transac_cf3e7bc2", + "Compliance Report Base - Compliance Report"."Transaction__compliance_units" AS "Compliance Report Base - Compliance Report__Transac_2787ccf9", + "Compliance Report Base - Compliance Report"."Transaction__organization_id" AS "Compliance Report Base - Compliance Report__Transac_d7fde363", + "Compliance Report Base - Compliance Report"."Transaction__transaction_action" AS "Compliance Report Base - Compliance Report__Transac_5afd1852", + "Compliance Report Base - Compliance Report"."Transaction__create_date" AS "Compliance Report Base - Compliance Report__Transac_1a243093", + "Compliance Report Base - Compliance Report"."Transaction__update_date" AS "Compliance Report Base - Compliance Report__Transac_4fe9fa65", + "Compliance Report Base - Compliance Report"."Transaction__create_user" AS "Compliance Report Base - Compliance Report__Transac_3d29d1a0", + "Compliance Report Base - Compliance Report"."Transaction__update_user" AS "Compliance Report Base - Compliance Report__Transac_68e41b56", + "Compliance Report Base - Compliance Report"."Transaction__effective_date" AS "Compliance Report Base - Compliance Report__Transac_cfe283e1", + "Compliance Report Base - Compliance Report"."Transaction__effective_status" AS "Compliance Report Base - Compliance Report__Transac_25b92424", + "Compliance Report Base - Compliance Report"."Transaction__expiration_date" AS "Compliance Report Base - Compliance Report__Transac_117f7033", + "Compliance Report Base - Compliance Report"."Organization__organization_id" AS "Compliance Report Base - Compliance Report__Organiz_ea2b8583", + "Compliance Report Base - Compliance Report"."Organization__organization_code" AS "Compliance Report Base - Compliance Report__Organiz_93047002", + "Compliance Report Base - Compliance Report"."Organization__name" AS "Compliance Report Base - Compliance Report__Organiz_825c76de", + "Compliance Report Base - Compliance Report"."Organization__operating_name" AS "Compliance Report Base - Compliance Report__Organiz_1e26fe43", + "Compliance Report Base - Compliance Report"."Organization__email" AS "Compliance Report Base - Compliance Report__Organiz_6f46599a", + "Compliance Report Base - Compliance Report"."Organization__phone" AS "Compliance Report Base - Compliance Report__Organiz_cc9bb233", + "Compliance Report Base - Compliance Report"."Organization__edrms_record" AS "Compliance Report Base - Compliance Report__Organiz_f96b3406", + "Compliance Report Base - Compliance Report"."Organization__total_balance" AS "Compliance Report Base - Compliance Report__Organiz_5b0e3a8e", + "Compliance Report Base - Compliance Report"."Organization__reserved_balance" AS "Compliance Report Base - Compliance Report__Organiz_002dae56", + "Compliance Report Base - Compliance Report"."Organization__count_transfers_in_progress" AS "Compliance Report Base - Compliance Report__Organiz_ddd4cd6e", + "Compliance Report Base - Compliance Report"."Organization__organization_status_id" AS "Compliance Report Base - Compliance Report__Organiz_4ca3989e", + "Compliance Report Base - Compliance Report"."Organization__organization_type_id" AS "Compliance Report Base - Compliance Report__Organiz_af01dea6", + "Compliance Report Base - Compliance Report"."Organization__organization_address_id" AS "Compliance Report Base - Compliance Report__Organiz_57f78a9f", + "Compliance Report Base - Compliance Report"."Organization__organization_attorney_address_id" AS "Compliance Report Base - Compliance Report__Organiz_b438bcc2", + "Compliance Report Base - Compliance Report"."Organization__create_date" AS "Compliance Report Base - Compliance Report__Organiz_48401463", + "Compliance Report Base - Compliance Report"."Organization__update_date" AS "Compliance Report Base - Compliance Report__Organiz_1d8dde95", + "Compliance Report Base - Compliance Report"."Organization__create_user" AS "Compliance Report Base - Compliance Report__Organiz_6f4df550", + "Compliance Report Base - Compliance Report"."Organization__update_user" AS "Compliance Report Base - Compliance Report__Organiz_3a803fa6", + "Compliance Report Base - Compliance Report"."Organization__effective_date" AS "Compliance Report Base - Compliance Report__Organiz_c111b484", + "Compliance Report Base - Compliance Report"."Organization__effective_status" AS "Compliance Report Base - Compliance Report__Organiz_858e103a", + "Compliance Report Base - Compliance Report"."Organization__expiration_date" AS "Compliance Report Base - Compliance Report__Organiz_2ca916d3", + "Compliance Report Base - Compliance Report"."Organization__has_early_issuance" AS "Compliance Report Base - Compliance Report__Organiz_23acfb32", + "Compliance Report Base - Compliance Report"."Organization__records_address" AS "Compliance Report Base - Compliance Report__Organiz_eb230b97", + "Compliance Report Base - Compliance Report"."Compliance Reports Chained - Compliance Report Grou_1a77e4cb" AS "Compliance Report Base - Compliance Report__Complia_a81d65f5", + "Compliance Report Base - Compliance Report"."Compliance Reports Chained - Compliance Report Grou_480bb7b1" AS "Compliance Report Base - Compliance Report__Complia_cfae0019" + FROM + ( + SELECT + "allocation_agreement"."group_uuid" AS "group_uuid", + MAX("allocation_agreement"."version") AS "max" + FROM + "allocation_agreement" + +GROUP BY + "allocation_agreement"."group_uuid" + +ORDER BY + "allocation_agreement"."group_uuid" ASC + ) AS "source" + +LEFT JOIN "allocation_agreement" AS "Allocation Agreement - Group UUID" ON ( + "source"."group_uuid" = "Allocation Agreement - Group UUID"."group_uuid" + ) + + AND ( + "source"."max" = "Allocation Agreement - Group UUID"."version" + ) + LEFT JOIN ( + SELECT + "compliance_report"."compliance_report_id" AS "compliance_report_id", + "compliance_report"."compliance_period_id" AS "compliance_period_id", + "compliance_report"."organization_id" AS "organization_id", + "compliance_report"."current_status_id" AS "current_status_id", + "compliance_report"."transaction_id" AS "transaction_id", + "compliance_report"."compliance_report_group_uuid" AS "compliance_report_group_uuid", + "compliance_report"."legacy_id" AS "legacy_id", + "compliance_report"."version" AS "version", + "compliance_report"."supplemental_initiator" AS "supplemental_initiator", + "compliance_report"."reporting_frequency" AS "reporting_frequency", + "compliance_report"."nickname" AS "nickname", + "compliance_report"."supplemental_note" AS "supplemental_note", + "compliance_report"."create_date" AS "create_date", + "compliance_report"."update_date" AS "update_date", + "compliance_report"."create_user" AS "create_user", + "compliance_report"."update_user" AS "update_user", + "compliance_report"."assessment_statement" AS "assessment_statement", + CASE + WHEN "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" > 0 THEN 'Not Met' + ELSE 'Met' + END AS "Renewable Requirements", + CASE + WHEN "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" > 0 THEN 'Not Met' + ELSE 'Met' + END AS "Low Carbon Requirements", + "Compliance Period"."compliance_period_id" AS "Compliance Period__compliance_period_id", + "Compliance Period"."description" AS "Compliance Period__description", + "Compliance Period"."display_order" AS "Compliance Period__display_order", + "Compliance Period"."create_date" AS "Compliance Period__create_date", + "Compliance Period"."update_date" AS "Compliance Period__update_date", + "Compliance Period"."effective_date" AS "Compliance Period__effective_date", + "Compliance Period"."effective_status" AS "Compliance Period__effective_status", + "Compliance Period"."expiration_date" AS "Compliance Period__expiration_date", + "Compliance Report Status - Current Status"."compliance_report_status_id" AS "Compliance Report Status - Current Status__complian_8aca39b7", + "Compliance Report Status - Current Status"."display_order" AS "Compliance Report Status - Current Status__display_order", + "Compliance Report Status - Current Status"."status" AS "Compliance Report Status - Current Status__status", + "Compliance Report Status - Current Status"."create_date" AS "Compliance Report Status - Current Status__create_date", + "Compliance Report Status - Current Status"."update_date" AS "Compliance Report Status - Current Status__update_date", + "Compliance Report Status - Current Status"."effective_date" AS "Compliance Report Status - Current Status__effective_date", + "Compliance Report Status - Current Status"."effective_status" AS "Compliance Report Status - Current Status__effective_status", + "Compliance Report Status - Current Status"."expiration_date" AS "Compliance Report Status - Current Status__expiration_date", + "Compliance Report Summary - Compliance Report"."summary_id" AS "Compliance Report Summary - Compliance Report__summary_id", + "Compliance Report Summary - Compliance Report"."compliance_report_id" AS "Compliance Report Summary - Compliance Report__comp_1db2e1e9", + "Compliance Report Summary - Compliance Report"."quarter" AS "Compliance Report Summary - Compliance Report__quarter", + "Compliance Report Summary - Compliance Report"."is_locked" AS "Compliance Report Summary - Compliance Report__is_locked", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_gasoline" AS "Compliance Report Summary - Compliance Report__line_2c0818fb", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_diesel" AS "Compliance Report Summary - Compliance Report__line_2ff66c5b", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_1fcf7a18", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_d70f8aef", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_2773c83c", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_e4c8e80c", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_9cec896d", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_489bea32", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_af2beb8e", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_gasoline" AS "Compliance Report Summary - Compliance Report__line_a26a000d", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_diesel" AS "Compliance Report Summary - Compliance Report__line_0ef43e75", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_91ad62ee", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_gasoline" AS "Compliance Report Summary - Compliance Report__line_b1027537", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_diesel" AS "Compliance Report Summary - Compliance Report__line_38be33f3", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_82c517d4", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_gasoline" AS "Compliance Report Summary - Compliance Report__line_6927f733", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_diesel" AS "Compliance Report Summary - Compliance Report__line_93d805cb", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_5ae095d0", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_gasoline" AS "Compliance Report Summary - Compliance Report__line_157d6973", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_diesel" AS "Compliance Report Summary - Compliance Report__line_31fd1f1b", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_26ba0b90", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_gasoline" AS "Compliance Report Summary - Compliance Report__line_27419684", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_diesel" AS "Compliance Report Summary - Compliance Report__line_ac263897", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_1486f467", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_gasoline" AS "Compliance Report Summary - Compliance Report__line_c12cb8c0", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_diesel" AS "Compliance Report Summary - Compliance Report__line_05eb459f", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_f2ebda23", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_1be763e7", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_b72177b0", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_28200104", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_gasoline" AS "Compliance Report Summary - Compliance Report__line_53735f1d", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_diesel" AS "Compliance Report Summary - Compliance Report__line_64a07c80", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_60b43dfe", + "Compliance Report Summary - Compliance Report"."line_12_low_carbon_fuel_required" AS "Compliance Report Summary - Compliance Report__line_da2710ad", + "Compliance Report Summary - Compliance Report"."line_13_low_carbon_fuel_supplied" AS "Compliance Report Summary - Compliance Report__line_b25fca1c", + "Compliance Report Summary - Compliance Report"."line_14_low_carbon_fuel_surplus" AS "Compliance Report Summary - Compliance Report__line_4b98033f", + "Compliance Report Summary - Compliance Report"."line_15_banked_units_used" AS "Compliance Report Summary - Compliance Report__line_1d7d6a31", + "Compliance Report Summary - Compliance Report"."line_16_banked_units_remaining" AS "Compliance Report Summary - Compliance Report__line_684112bb", + "Compliance Report Summary - Compliance Report"."line_17_non_banked_units_used" AS "Compliance Report Summary - Compliance Report__line_b1d3ad5e", + "Compliance Report Summary - Compliance Report"."line_18_units_to_be_banked" AS "Compliance Report Summary - Compliance Report__line_e173956e", + "Compliance Report Summary - Compliance Report"."line_19_units_to_be_exported" AS "Compliance Report Summary - Compliance Report__line_9a885574", + "Compliance Report Summary - Compliance Report"."line_20_surplus_deficit_units" AS "Compliance Report Summary - Compliance Report__line_8e71546f", + "Compliance Report Summary - Compliance Report"."line_21_surplus_deficit_ratio" AS "Compliance Report Summary - Compliance Report__line_00d2728d", + "Compliance Report Summary - Compliance Report"."line_22_compliance_units_issued" AS "Compliance Report Summary - Compliance Report__line_29d9cb9c", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_gasoline" AS "Compliance Report Summary - Compliance Report__line_d8942234", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_diesel" AS "Compliance Report Summary - Compliance Report__line_125e31fc", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_eb5340d7", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" AS "Compliance Report Summary - Compliance Report__line_bff5157d", + "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__line_7c9c21b1", + "Compliance Report Summary - Compliance Report"."total_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__tota_0e1e6fb3", + "Compliance Report Summary - Compliance Report"."create_date" AS "Compliance Report Summary - Compliance Report__create_date", + "Compliance Report Summary - Compliance Report"."update_date" AS "Compliance Report Summary - Compliance Report__update_date", + "Compliance Report Summary - Compliance Report"."create_user" AS "Compliance Report Summary - Compliance Report__create_user", + "Compliance Report Summary - Compliance Report"."update_user" AS "Compliance Report Summary - Compliance Report__update_user", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q1" AS "Compliance Report Summary - Compliance Report__earl_6d4994a4", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q2" AS "Compliance Report Summary - Compliance Report__earl_f440c51e", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q3" AS "Compliance Report Summary - Compliance Report__earl_8347f588", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q4" AS "Compliance Report Summary - Compliance Report__earl_1d23602b", + "Transaction"."transaction_id" AS "Transaction__transaction_id", + "Transaction"."compliance_units" AS "Transaction__compliance_units", + "Transaction"."organization_id" AS "Transaction__organization_id", + "Transaction"."transaction_action" AS "Transaction__transaction_action", + "Transaction"."create_date" AS "Transaction__create_date", + "Transaction"."update_date" AS "Transaction__update_date", + "Transaction"."create_user" AS "Transaction__create_user", + "Transaction"."update_user" AS "Transaction__update_user", + "Transaction"."effective_date" AS "Transaction__effective_date", + "Transaction"."effective_status" AS "Transaction__effective_status", + "Transaction"."expiration_date" AS "Transaction__expiration_date", + "Organization"."organization_id" AS "Organization__organization_id", + "Organization"."organization_code" AS "Organization__organization_code", + "Organization"."name" AS "Organization__name", + "Organization"."operating_name" AS "Organization__operating_name", + "Organization"."email" AS "Organization__email", + "Organization"."phone" AS "Organization__phone", + "Organization"."edrms_record" AS "Organization__edrms_record", + "Organization"."total_balance" AS "Organization__total_balance", + "Organization"."reserved_balance" AS "Organization__reserved_balance", + "Organization"."count_transfers_in_progress" AS "Organization__count_transfers_in_progress", + "Organization"."organization_status_id" AS "Organization__organization_status_id", + "Organization"."organization_type_id" AS "Organization__organization_type_id", + "Organization"."organization_address_id" AS "Organization__organization_address_id", + "Organization"."organization_attorney_address_id" AS "Organization__organization_attorney_address_id", + "Organization"."create_date" AS "Organization__create_date", + "Organization"."update_date" AS "Organization__update_date", + "Organization"."create_user" AS "Organization__create_user", + "Organization"."update_user" AS "Organization__update_user", + "Organization"."effective_date" AS "Organization__effective_date", + "Organization"."effective_status" AS "Organization__effective_status", + "Organization"."expiration_date" AS "Organization__expiration_date", + COALESCE("Organization Early Issuance"."has_early_issuance", false) AS "Organization__has_early_issuance", + "Organization"."records_address" AS "Organization__records_address", + "Compliance Reports Chained - Compliance Report Group UUID"."group_uuid" AS "Compliance Reports Chained - Compliance Report Grou_1a77e4cb", + "Compliance Reports Chained - Compliance Report Group UUID"."max_version" AS "Compliance Reports Chained - Compliance Report Grou_480bb7b1" + FROM + "compliance_report" + INNER JOIN "compliance_period" AS "Compliance Period" ON "compliance_report"."compliance_period_id" = "Compliance Period"."compliance_period_id" + INNER JOIN "compliance_report_status" AS "Compliance Report Status - Current Status" ON "compliance_report"."current_status_id" = "Compliance Report Status - Current Status"."compliance_report_status_id" + INNER JOIN "compliance_report_summary" AS "Compliance Report Summary - Compliance Report" ON "compliance_report"."compliance_report_id" = "Compliance Report Summary - Compliance Report"."compliance_report_id" + LEFT JOIN "transaction" AS "Transaction" ON "compliance_report"."transaction_id" = "Transaction"."transaction_id" + LEFT JOIN "organization" AS "Organization" ON "compliance_report"."organization_id" = "Organization"."organization_id" + LEFT JOIN "organization_early_issuance_by_year" AS "Organization Early Issuance" ON "compliance_report"."organization_id" = "Organization Early Issuance"."organization_id" AND "compliance_report"."compliance_period_id" = "Organization Early Issuance"."compliance_period_id" + INNER JOIN ( + SELECT + compliance_report_group_uuid AS group_uuid, + max(VERSION) AS max_version + FROM + COMPLIANCE_REPORT + GROUP BY + COMPLIANCE_REPORT.compliance_report_group_uuid + ) AS "Compliance Reports Chained - Compliance Report Group UUID" ON ( + "compliance_report"."compliance_report_group_uuid" = "Compliance Reports Chained - Compliance Report Group UUID"."group_uuid" + ) + AND ( + "compliance_report"."version" = "Compliance Reports Chained - Compliance Report Group UUID"."max_version" + ) + ) AS "Compliance Report Base - Compliance Report" ON "Allocation Agreement - Group UUID"."compliance_report_id" = "Compliance Report Base - Compliance Report"."compliance_report_id"; + +GRANT SELECT ON vw_allocation_agreement_base TO basic_lcfs_reporting_role; + +-- ========================================== +-- Fuel Code Base View +-- ========================================== +drop view if exists vw_fuel_code_base cascade; +CREATE OR REPLACE VIEW vw_fuel_code_base AS +SELECT + fc.fuel_code_id, + fcp.fuel_code_prefix_id, + fcp.prefix, + fc.fuel_suffix, + fcs.fuel_code_status_id, + fcs.status, + ft.fuel_type_id, + ft.fuel_type, + fc.company, + fc.contact_name, + fc.contact_email, + fc.carbon_intensity, + fc.edrms, + fc.last_updated, + fc.application_date, + fc.approval_date, + fc.create_date, + fc.effective_date, + fc.expiration_date, + fc.effective_status, + fc.feedstock, + fc.feedstock_location, + fc.feedstock_misc, + fc.fuel_production_facility_city, + fc.fuel_production_facility_province_state, + fc.fuel_production_facility_country, + fc.facility_nameplate_capacity, + fc.facility_nameplate_capacity_unit, + fc.former_company, + finished_modes.transport_modes AS finished_fuel_transport_modes, + feedstock_modes.transport_modes AS feedstock_fuel_transport_modes, + fc.notes +FROM fuel_code fc +JOIN fuel_code_prefix fcp ON fc.prefix_id = fcp.fuel_code_prefix_id +JOIN fuel_code_status fcs ON fc.fuel_status_id = fcs.fuel_code_status_id +JOIN fuel_type ft ON fc.fuel_type_id = ft.fuel_type_id +LEFT JOIN LATERAL ( + SELECT ARRAY_AGG(tm.transport_mode ORDER BY tm.transport_mode) AS transport_modes + FROM finished_fuel_transport_mode fftm + JOIN transport_mode tm ON fftm.transport_mode_id = tm.transport_mode_id + WHERE fftm.fuel_code_id = fc.fuel_code_id +) finished_modes ON TRUE +LEFT JOIN LATERAL ( + SELECT ARRAY_AGG(tm.transport_mode ORDER BY tm.transport_mode) AS transport_modes + FROM feedstock_fuel_transport_mode fftm + JOIN transport_mode tm ON fftm.transport_mode_id = tm.transport_mode_id + WHERE fftm.fuel_code_id = fc.fuel_code_id +) feedstock_modes ON TRUE +WHERE fcs.status != 'Deleted'; +GRANT SELECT ON vw_fuel_code_base TO basic_lcfs_reporting_role; +GRANT SELECT ON fuel_code, fuel_code_prefix, fuel_code_status, fuel_type, fuel_category TO basic_lcfs_reporting_role; +-- ========================================== +-- Compliance Reports List View +-- ========================================== +create or replace view v_compliance_report as +WITH latest_versions AS ( + -- Use window function instead of GROUP BY for better performance + SELECT DISTINCT + compliance_report_group_uuid, + FIRST_VALUE(version) OVER ( + PARTITION BY compliance_report_group_uuid + ORDER BY version DESC + ) as max_version, + FIRST_VALUE(current_status_id) OVER ( + PARTITION BY compliance_report_group_uuid + ORDER BY version DESC + ) as latest_status_id, + FIRST_VALUE(crs.status) OVER ( + PARTITION BY compliance_report_group_uuid + ORDER BY version DESC + ) as latest_status, + FIRST_VALUE(supplemental_initiator) OVER ( + PARTITION BY compliance_report_group_uuid + ORDER BY version DESC + ) as latest_supplemental_initiator, + FIRST_VALUE(cr.create_date) OVER ( + PARTITION BY compliance_report_group_uuid + ORDER BY version DESC + ) as latest_supplemental_create_date + FROM compliance_report cr + JOIN compliance_report_status crs ON cr.current_status_id = crs.compliance_report_status_id +), +versioned_reports AS ( + -- Single scan to get both latest and second-latest with their statuses + SELECT + cr.*, + crs.status, + ROW_NUMBER() OVER ( + PARTITION BY cr.compliance_report_group_uuid + ORDER BY cr.version DESC + ) as version_rank, + lws.latest_supplemental_initiator, + lws.latest_supplemental_create_date, + lws.latest_status + FROM compliance_report cr + JOIN compliance_report_status crs ON cr.current_status_id = crs.compliance_report_status_id + JOIN latest_versions lws ON cr.compliance_report_group_uuid = lws.compliance_report_group_uuid +), +selected_reports AS ( + SELECT * + FROM versioned_reports vr + WHERE version_rank = 1 -- Always include latest + OR (version_rank = 2 -- Include second-latest when LATEST has these conditions + AND (vr.latest_status IN ('Draft','Analyst_adjustment') + OR vr.latest_supplemental_initiator = 'GOVERNMENT_REASSESSMENT')) +) +SELECT DISTINCT + sr.compliance_report_id, + sr.compliance_report_group_uuid, + sr.version, + cp.compliance_period_id, + cp.description AS compliance_period, + o.organization_id, + o.name AS organization_name, + sr.nickname AS report_type, + sr.current_status_id AS report_status_id, + case when sr.latest_status = 'Draft' + and sr.version_rank = 2 -- not the latest report + and sr.latest_supplemental_initiator = 'GOVERNMENT_INITIATED'::supplementalinitiatortype + then 'Supplemental_requested'::compliancereportstatusenum + else sr.status + end as report_status, + sr.update_date, + sr.supplemental_initiator, + sr.reporting_frequency, + sr.legacy_id, + sr.transaction_id, + sr.assessment_statement, + (sr.version_rank = 1) as is_latest, + sr.latest_supplemental_initiator as latest_report_supplemental_initiator, + sr.latest_supplemental_create_date, + sr.latest_status, + sr.assigned_analyst_id, + up.first_name AS assigned_analyst_first_name, + up.last_name AS assigned_analyst_last_name +FROM selected_reports sr +JOIN compliance_period cp ON sr.compliance_period_id = cp.compliance_period_id +JOIN organization o ON sr.organization_id = o.organization_id +LEFT JOIN user_profile up ON sr.assigned_analyst_id = up.user_profile_id +ORDER BY sr.compliance_report_group_uuid, sr.version DESC; + -- ========================================== -- Compliance Report Base View With Early Issuance By Year -- ========================================== @@ -689,9 +1851,6 @@ SELECT "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" AS "Compliance Report Summary - Compliance Report__line_bff5157d", "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__line_7c9c21b1", "Compliance Report Summary - Compliance Report"."total_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__tota_0e1e6fb3", - "Compliance Report Summary - Compliance Report"."credits_offset_a" AS "Compliance Report Summary - Compliance Report__cred_0f455db6", - "Compliance Report Summary - Compliance Report"."credits_offset_b" AS "Compliance Report Summary - Compliance Report__cred_964c0c0c", - "Compliance Report Summary - Compliance Report"."credits_offset_c" AS "Compliance Report Summary - Compliance Report__cred_e14b3c9a", "Compliance Report Summary - Compliance Report"."create_date" AS "Compliance Report Summary - Compliance Report__create_date", "Compliance Report Summary - Compliance Report"."update_date" AS "Compliance Report Summary - Compliance Report__update_date", "Compliance Report Summary - Compliance Report"."create_user" AS "Compliance Report Summary - Compliance Report__create_user", @@ -1067,6 +2226,358 @@ CREATE INDEX IF NOT EXISTS idx_fse_organization_name_norm -- ========================================== -- LEVEL 3: Views depending on Level 2 views -- ========================================== +-- Allocation Agreement Extended Base View (from HEAD) +-- ========================================== +drop view if exists vw_allocation_agreement_extended_base; +CREATE OR REPLACE VIEW vw_allocation_agreement_extended_base AS +SELECT + "source"."group_uuid" AS "group_uuid", + "source"."max" AS "max", + CASE + WHEN "Allocation Agreement - Group UUID"."allocation_transaction_type_id" = 1 THEN 'Allocated From' + WHEN "Allocation Agreement - Group UUID"."allocation_transaction_type_id" = 2 THEN 'Allocation To' + END AS "Allocation transaction type", + "Allocation Agreement - Group UUID"."allocation_agreement_id" AS "Allocation Agreement - Group UUID__allocation_agreement_id", + "Allocation Agreement - Group UUID"."transaction_partner" AS "Allocation Agreement - Group UUID__transaction_partner", + "Allocation Agreement - Group UUID"."postal_address" AS "Allocation Agreement - Group UUID__postal_address", + "Allocation Agreement - Group UUID"."transaction_partner_email" AS "Allocation Agreement - Group UUID__transaction_partner_email", + "Allocation Agreement - Group UUID"."transaction_partner_phone" AS "Allocation Agreement - Group UUID__transaction_partner_phone", + "Allocation Agreement - Group UUID"."ci_of_fuel" AS "Allocation Agreement - Group UUID__ci_of_fuel", + "Allocation Agreement - Group UUID"."quantity" AS "Allocation Agreement - Group UUID__quantity", + "Allocation Agreement - Group UUID"."units" AS "Allocation Agreement - Group UUID__units", + "Allocation Agreement - Group UUID"."fuel_type_other" AS "Allocation Agreement - Group UUID__fuel_type_other", + "Allocation Agreement - Group UUID"."allocation_transaction_type_id" AS "Allocation Agreement - Group UUID__allocation_trans_ad55ef48", + "Allocation Agreement - Group UUID"."fuel_type_id" AS "Allocation Agreement - Group UUID__fuel_type_id", + "Allocation Agreement - Group UUID"."fuel_category_id" AS "Allocation Agreement - Group UUID__fuel_category_id", + "Allocation Agreement - Group UUID"."provision_of_the_act_id" AS "Allocation Agreement - Group UUID__provision_of_the_act_id", + "Allocation Agreement - Group UUID"."fuel_code_id" AS "Allocation Agreement - Group UUID__fuel_code_id", + "Allocation Agreement - Group UUID"."compliance_report_id" AS "Allocation Agreement - Group UUID__compliance_report_id", + "Allocation Agreement - Group UUID"."create_date" AS "Allocation Agreement - Group UUID__create_date", + "Allocation Agreement - Group UUID"."update_date" AS "Allocation Agreement - Group UUID__update_date", + "Allocation Agreement - Group UUID"."create_user" AS "Allocation Agreement - Group UUID__create_user", + "Allocation Agreement - Group UUID"."update_user" AS "Allocation Agreement - Group UUID__update_user", + "Allocation Agreement - Group UUID"."display_order" AS "Allocation Agreement - Group UUID__display_order", + "Allocation Agreement - Group UUID"."group_uuid" AS "Allocation Agreement - Group UUID__group_uuid", + "Allocation Agreement - Group UUID"."version" AS "Allocation Agreement - Group UUID__version", + "Allocation Agreement - Group UUID"."action_type" AS "Allocation Agreement - Group UUID__action_type", + "Allocation Agreement - Group UUID"."quantity_not_sold" AS "Allocation Agreement - Group UUID__quantity_not_sold", + "Compliance Report Base - Compliance Report"."compliance_report_id" AS "Compliance Report Base - Compliance Report__complia_6afb9aaa", + "Compliance Report Base - Compliance Report"."compliance_period_id" AS "Compliance Report Base - Compliance Report__complia_cda244b4", + "Compliance Report Base - Compliance Report"."organization_id" AS "Compliance Report Base - Compliance Report__organization_id", + "Compliance Report Base - Compliance Report"."current_status_id" AS "Compliance Report Base - Compliance Report__current_4315b3a2", + "Compliance Report Base - Compliance Report"."transaction_id" AS "Compliance Report Base - Compliance Report__transaction_id", + "Compliance Report Base - Compliance Report"."compliance_report_group_uuid" AS "Compliance Report Base - Compliance Report__complia_8e1217db", + "Compliance Report Base - Compliance Report"."legacy_id" AS "Compliance Report Base - Compliance Report__legacy_id", + "Compliance Report Base - Compliance Report"."version" AS "Compliance Report Base - Compliance Report__version", + "Compliance Report Base - Compliance Report"."supplemental_initiator" AS "Compliance Report Base - Compliance Report__supplem_3e383c17", + "Compliance Report Base - Compliance Report"."reporting_frequency" AS "Compliance Report Base - Compliance Report__reporti_c3204642", + "Compliance Report Base - Compliance Report"."nickname" AS "Compliance Report Base - Compliance Report__nickname", + "Compliance Report Base - Compliance Report"."supplemental_note" AS "Compliance Report Base - Compliance Report__supplem_76c93d97", + "Compliance Report Base - Compliance Report"."create_date" AS "Compliance Report Base - Compliance Report__create_date", + "Compliance Report Base - Compliance Report"."update_date" AS "Compliance Report Base - Compliance Report__update_date", + "Compliance Report Base - Compliance Report"."create_user" AS "Compliance Report Base - Compliance Report__create_user", + "Compliance Report Base - Compliance Report"."update_user" AS "Compliance Report Base - Compliance Report__update_user", + "Compliance Report Base - Compliance Report"."assessment_statement" AS "Compliance Report Base - Compliance Report__assessm_7b8d860b", + "Compliance Report Base - Compliance Report"."Renewable Requirements" AS "Compliance Report Base - Compliance Report__Renewab_f11c34b5", + "Compliance Report Base - Compliance Report"."Low Carbon Requirements" AS "Compliance Report Base - Compliance Report__Low Car_035b150f", + "Compliance Report Base - Compliance Report"."Compliance Period__compliance_period_id" AS "Compliance Report Base - Compliance Report__Complia_dd118a33", + "Compliance Report Base - Compliance Report"."Compliance Period__description" AS "Compliance Report Base - Compliance Report__Complia_cb30ad19", + "Compliance Report Base - Compliance Report"."Compliance Period__display_order" AS "Compliance Report Base - Compliance Report__Complia_721c9a5e", + "Compliance Report Base - Compliance Report"."Compliance Period__create_date" AS "Compliance Report Base - Compliance Report__Complia_8deb472c", + "Compliance Report Base - Compliance Report"."Compliance Period__update_date" AS "Compliance Report Base - Compliance Report__Complia_d8268dda", + "Compliance Report Base - Compliance Report"."Compliance Period__effective_date" AS "Compliance Report Base - Compliance Report__Complia_6a450a4b", + "Compliance Report Base - Compliance Report"."Compliance Period__effective_status" AS "Compliance Report Base - Compliance Report__Complia_e535ee64", + "Compliance Report Base - Compliance Report"."Compliance Period__expiration_date" AS "Compliance Report Base - Compliance Report__Complia_27d99d4c", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__complian_8aca39b7" AS "Compliance Report Base - Compliance Report__Complia_35a08ff4", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__display_order" AS "Compliance Report Base - Compliance Report__Complia_617d3c08", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__status" AS "Compliance Report Base - Compliance Report__Complia_f6d97a34", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__create_date" AS "Compliance Report Base - Compliance Report__Complia_de8161a6", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__update_date" AS "Compliance Report Base - Compliance Report__Complia_8b4cab50", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__effective_date" AS "Compliance Report Base - Compliance Report__Complia_e85e9f2c", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__effective_status" AS "Compliance Report Base - Compliance Report__Complia_4fecf6d4", + "Compliance Report Base - Compliance Report"."Compliance Report Status - Current Status__expiration_date" AS "Compliance Report Base - Compliance Report__Complia_f48d7222", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__summary_id" AS "Compliance Report Base - Compliance Report__Complia_fe5bffa9", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__comp_1db2e1e9" AS "Compliance Report Base - Compliance Report__Complia_ac5855d3", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__quarter" AS "Compliance Report Base - Compliance Report__Complia_716d2691", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__is_locked" AS "Compliance Report Base - Compliance Report__Complia_9880522d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_2c0818fb" AS "Compliance Report Base - Compliance Report__Complia_e084f95e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_2ff66c5b" AS "Compliance Report Base - Compliance Report__Complia_bbf3740d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1fcf7a18" AS "Compliance Report Base - Compliance Report__Complia_0948e373", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_d70f8aef" AS "Compliance Report Base - Compliance Report__Complia_1151727e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_2773c83c" AS "Compliance Report Base - Compliance Report__Complia_8c046bfd", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_e4c8e80c" AS "Compliance Report Base - Compliance Report__Complia_0c86fc50", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_9cec896d" AS "Compliance Report Base - Compliance Report__Complia_817c5f4a", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_489bea32" AS "Compliance Report Base - Compliance Report__Complia_d930be19", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_af2beb8e" AS "Compliance Report Base - Compliance Report__Complia_c9668257", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_a26a000d" AS "Compliance Report Base - Compliance Report__Complia_eef8c8e5", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_0ef43e75" AS "Compliance Report Base - Compliance Report__Complia_469410d8", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_91ad62ee" AS "Compliance Report Base - Compliance Report__Complia_169c9f01", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b1027537" AS "Compliance Report Base - Compliance Report__Complia_ba262a42", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_38be33f3" AS "Compliance Report Base - Compliance Report__Complia_3609a003", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_82c517d4" AS "Compliance Report Base - Compliance Report__Complia_3d0e6803", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_6927f733" AS "Compliance Report Base - Compliance Report__Complia_aad5cfd2", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_93d805cb" AS "Compliance Report Base - Compliance Report__Complia_c437462d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_5ae095d0" AS "Compliance Report Base - Compliance Report__Complia_6bd2da0d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_157d6973" AS "Compliance Report Base - Compliance Report__Complia_ed316fbb", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_31fd1f1b" AS "Compliance Report Base - Compliance Report__Complia_2c880057", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_26ba0b90" AS "Compliance Report Base - Compliance Report__Complia_84fd8648", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_27419684" AS "Compliance Report Base - Compliance Report__Complia_c8859563", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_ac263897" AS "Compliance Report Base - Compliance Report__Complia_45fe3e7a", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1486f467" AS "Compliance Report Base - Compliance Report__Complia_256023d8", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_c12cb8c0" AS "Compliance Report Base - Compliance Report__Complia_eee3c998", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_05eb459f" AS "Compliance Report Base - Compliance Report__Complia_c90c8a63", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_f2ebda23" AS "Compliance Report Base - Compliance Report__Complia_c7afbbd2", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1be763e7" AS "Compliance Report Base - Compliance Report__Complia_b7c1da7c", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b72177b0" AS "Compliance Report Base - Compliance Report__Complia_eddee97b", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_28200104" AS "Compliance Report Base - Compliance Report__Complia_656afc20", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_53735f1d" AS "Compliance Report Base - Compliance Report__Complia_5698e212", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_64a07c80" AS "Compliance Report Base - Compliance Report__Complia_91c16e81", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_60b43dfe" AS "Compliance Report Base - Compliance Report__Complia_a38eca8f", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_da2710ad" AS "Compliance Report Base - Compliance Report__Complia_83fb46a6", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b25fca1c" AS "Compliance Report Base - Compliance Report__Complia_677a2191", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_4b98033f" AS "Compliance Report Base - Compliance Report__Complia_d3db4f29", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_1d7d6a31" AS "Compliance Report Base - Compliance Report__Complia_805e5698", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_684112bb" AS "Compliance Report Base - Compliance Report__Complia_ba662a5c", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_b1d3ad5e" AS "Compliance Report Base - Compliance Report__Complia_407ecab4", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_e173956e" AS "Compliance Report Base - Compliance Report__Complia_884b89fd", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_9a885574" AS "Compliance Report Base - Compliance Report__Complia_6030ea9c", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_8e71546f" AS "Compliance Report Base - Compliance Report__Complia_392f751b", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_00d2728d" AS "Compliance Report Base - Compliance Report__Complia_889450ac", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_29d9cb9c" AS "Compliance Report Base - Compliance Report__Complia_875531b4", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_d8942234" AS "Compliance Report Base - Compliance Report__Complia_ea067573", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_125e31fc" AS "Compliance Report Base - Compliance Report__Complia_a93845e5", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_eb5340d7" AS "Compliance Report Base - Compliance Report__Complia_2ef7cee4", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_bff5157d" AS "Compliance Report Base - Compliance Report__Complia_87c65017", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__line_7c9c21b1" AS "Compliance Report Base - Compliance Report__Complia_4bc20903", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__tota_0e1e6fb3" AS "Compliance Report Base - Compliance Report__Complia_0dd060c1", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__create_date" AS "Compliance Report Base - Compliance Report__Complia_e63c35ab", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__update_date" AS "Compliance Report Base - Compliance Report__Complia_b3f1ff5d", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__create_user" AS "Compliance Report Base - Compliance Report__Complia_c131d498", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__update_user" AS "Compliance Report Base - Compliance Report__Complia_94fc1e6e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_6d4994a4" AS "Compliance Report Base - Compliance Report__Complia_df1f10d1", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_f440c51e" AS "Compliance Report Base - Compliance Report__Complia_a759d116", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_8347f588" AS "Compliance Report Base - Compliance Report__Complia_63b4b44e", + "Compliance Report Base - Compliance Report"."Compliance Report Summary - Compliance Report__earl_1d23602b" AS "Compliance Report Base - Compliance Report__Complia_05c7a0a8", + "Compliance Report Base - Compliance Report"."Transaction__transaction_id" AS "Compliance Report Base - Compliance Report__Transac_cf3e7bc2", + "Compliance Report Base - Compliance Report"."Transaction__compliance_units" AS "Compliance Report Base - Compliance Report__Transac_2787ccf9", + "Compliance Report Base - Compliance Report"."Transaction__organization_id" AS "Compliance Report Base - Compliance Report__Transac_d7fde363", + "Compliance Report Base - Compliance Report"."Transaction__transaction_action" AS "Compliance Report Base - Compliance Report__Transac_5afd1852", + "Compliance Report Base - Compliance Report"."Transaction__create_date" AS "Compliance Report Base - Compliance Report__Transac_1a243093", + "Compliance Report Base - Compliance Report"."Transaction__update_date" AS "Compliance Report Base - Compliance Report__Transac_4fe9fa65", + "Compliance Report Base - Compliance Report"."Transaction__create_user" AS "Compliance Report Base - Compliance Report__Transac_3d29d1a0", + "Compliance Report Base - Compliance Report"."Transaction__update_user" AS "Compliance Report Base - Compliance Report__Transac_68e41b56", + "Compliance Report Base - Compliance Report"."Transaction__effective_date" AS "Compliance Report Base - Compliance Report__Transac_cfe283e1", + "Compliance Report Base - Compliance Report"."Transaction__effective_status" AS "Compliance Report Base - Compliance Report__Transac_25b92424", + "Compliance Report Base - Compliance Report"."Transaction__expiration_date" AS "Compliance Report Base - Compliance Report__Transac_117f7033", + "Compliance Report Base - Compliance Report"."Organization__organization_id" AS "Compliance Report Base - Compliance Report__Organiz_ea2b8583", + "Compliance Report Base - Compliance Report"."Organization__organization_code" AS "Compliance Report Base - Compliance Report__Organiz_93047002", + "Compliance Report Base - Compliance Report"."Organization__name" AS "Compliance Report Base - Compliance Report__Organiz_825c76de", + "Compliance Report Base - Compliance Report"."Organization__operating_name" AS "Compliance Report Base - Compliance Report__Organiz_1e26fe43", + "Compliance Report Base - Compliance Report"."Organization__email" AS "Compliance Report Base - Compliance Report__Organiz_6f46599a", + "Compliance Report Base - Compliance Report"."Organization__phone" AS "Compliance Report Base - Compliance Report__Organiz_cc9bb233", + "Compliance Report Base - Compliance Report"."Organization__edrms_record" AS "Compliance Report Base - Compliance Report__Organiz_f96b3406", + "Compliance Report Base - Compliance Report"."Organization__total_balance" AS "Compliance Report Base - Compliance Report__Organiz_5b0e3a8e", + "Compliance Report Base - Compliance Report"."Organization__reserved_balance" AS "Compliance Report Base - Compliance Report__Organiz_002dae56", + "Compliance Report Base - Compliance Report"."Organization__count_transfers_in_progress" AS "Compliance Report Base - Compliance Report__Organiz_ddd4cd6e", + "Compliance Report Base - Compliance Report"."Organization__organization_status_id" AS "Compliance Report Base - Compliance Report__Organiz_4ca3989e", + "Compliance Report Base - Compliance Report"."Organization__organization_type_id" AS "Compliance Report Base - Compliance Report__Organiz_af01dea6", + "Compliance Report Base - Compliance Report"."Organization__organization_address_id" AS "Compliance Report Base - Compliance Report__Organiz_57f78a9f", + "Compliance Report Base - Compliance Report"."Organization__organization_attorney_address_id" AS "Compliance Report Base - Compliance Report__Organiz_b438bcc2", + "Compliance Report Base - Compliance Report"."Organization__create_date" AS "Compliance Report Base - Compliance Report__Organiz_48401463", + "Compliance Report Base - Compliance Report"."Organization__update_date" AS "Compliance Report Base - Compliance Report__Organiz_1d8dde95", + "Compliance Report Base - Compliance Report"."Organization__create_user" AS "Compliance Report Base - Compliance Report__Organiz_6f4df550", + "Compliance Report Base - Compliance Report"."Organization__update_user" AS "Compliance Report Base - Compliance Report__Organiz_3a803fa6", + "Compliance Report Base - Compliance Report"."Organization__effective_date" AS "Compliance Report Base - Compliance Report__Organiz_c111b484", + "Compliance Report Base - Compliance Report"."Organization__effective_status" AS "Compliance Report Base - Compliance Report__Organiz_858e103a", + "Compliance Report Base - Compliance Report"."Organization__expiration_date" AS "Compliance Report Base - Compliance Report__Organiz_2ca916d3", + "Compliance Report Base - Compliance Report"."Organization__has_early_issuance" AS "Compliance Report Base - Compliance Report__Organiz_23acfb32", + "Compliance Report Base - Compliance Report"."Organization__records_address" AS "Compliance Report Base - Compliance Report__Organiz_eb230b97", + "Compliance Report Base - Compliance Report"."Compliance Reports Chained - Compliance Report Grou_1a77e4cb" AS "Compliance Report Base - Compliance Report__Complia_a81d65f5", + "Compliance Report Base - Compliance Report"."Compliance Reports Chained - Compliance Report Grou_480bb7b1" AS "Compliance Report Base - Compliance Report__Complia_cfae0019" + FROM + ( + SELECT + "allocation_agreement"."group_uuid" AS "group_uuid", + MAX("allocation_agreement"."version") AS "max" + FROM + "allocation_agreement" + GROUP BY + "allocation_agreement"."group_uuid" + ORDER BY + "allocation_agreement"."group_uuid" ASC + ) AS "source" + LEFT JOIN "allocation_agreement" AS "Allocation Agreement - Group UUID" ON ( + "source"."group_uuid" = "Allocation Agreement - Group UUID"."group_uuid" + ) + AND ( + "source"."max" = "Allocation Agreement - Group UUID"."version" + ) + LEFT JOIN ( + SELECT + "compliance_report"."compliance_report_id" AS "compliance_report_id", + "compliance_report"."compliance_period_id" AS "compliance_period_id", + "compliance_report"."organization_id" AS "organization_id", + "compliance_report"."current_status_id" AS "current_status_id", + "compliance_report"."transaction_id" AS "transaction_id", + "compliance_report"."compliance_report_group_uuid" AS "compliance_report_group_uuid", + "compliance_report"."legacy_id" AS "legacy_id", + "compliance_report"."version" AS "version", + "compliance_report"."supplemental_initiator" AS "supplemental_initiator", + "compliance_report"."reporting_frequency" AS "reporting_frequency", + "compliance_report"."nickname" AS "nickname", + "compliance_report"."supplemental_note" AS "supplemental_note", + "compliance_report"."create_date" AS "create_date", + "compliance_report"."update_date" AS "update_date", + "compliance_report"."create_user" AS "create_user", + "compliance_report"."update_user" AS "update_user", + "compliance_report"."assessment_statement" AS "assessment_statement", + CASE + WHEN "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" > 0 THEN 'Not Met' + ELSE 'Met' + END AS "Renewable Requirements", + CASE + WHEN "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" > 0 THEN 'Not Met' + ELSE 'Met' + END AS "Low Carbon Requirements", + "Compliance Period"."compliance_period_id" AS "Compliance Period__compliance_period_id", + "Compliance Period"."description" AS "Compliance Period__description", + "Compliance Period"."display_order" AS "Compliance Period__display_order", + "Compliance Period"."create_date" AS "Compliance Period__create_date", + "Compliance Period"."update_date" AS "Compliance Period__update_date", + "Compliance Period"."effective_date" AS "Compliance Period__effective_date", + "Compliance Period"."effective_status" AS "Compliance Period__effective_status", + "Compliance Period"."expiration_date" AS "Compliance Period__expiration_date", + "Compliance Report Status - Current Status"."compliance_report_status_id" AS "Compliance Report Status - Current Status__complian_8aca39b7", + "Compliance Report Status - Current Status"."display_order" AS "Compliance Report Status - Current Status__display_order", + "Compliance Report Status - Current Status"."status" AS "Compliance Report Status - Current Status__status", + "Compliance Report Status - Current Status"."create_date" AS "Compliance Report Status - Current Status__create_date", + "Compliance Report Status - Current Status"."update_date" AS "Compliance Report Status - Current Status__update_date", + "Compliance Report Status - Current Status"."effective_date" AS "Compliance Report Status - Current Status__effective_date", + "Compliance Report Status - Current Status"."effective_status" AS "Compliance Report Status - Current Status__effective_status", + "Compliance Report Status - Current Status"."expiration_date" AS "Compliance Report Status - Current Status__expiration_date", + "Compliance Report Summary - Compliance Report"."summary_id" AS "Compliance Report Summary - Compliance Report__summary_id", + "Compliance Report Summary - Compliance Report"."compliance_report_id" AS "Compliance Report Summary - Compliance Report__comp_1db2e1e9", + "Compliance Report Summary - Compliance Report"."quarter" AS "Compliance Report Summary - Compliance Report__quarter", + "Compliance Report Summary - Compliance Report"."is_locked" AS "Compliance Report Summary - Compliance Report__is_locked", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_gasoline" AS "Compliance Report Summary - Compliance Report__line_2c0818fb", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_diesel" AS "Compliance Report Summary - Compliance Report__line_2ff66c5b", + "Compliance Report Summary - Compliance Report"."line_1_fossil_derived_base_fuel_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_1fcf7a18", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_d70f8aef", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_2773c83c", + "Compliance Report Summary - Compliance Report"."line_2_eligible_renewable_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_e4c8e80c", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_9cec896d", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_489bea32", + "Compliance Report Summary - Compliance Report"."line_3_total_tracked_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_af2beb8e", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_gasoline" AS "Compliance Report Summary - Compliance Report__line_a26a000d", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_diesel" AS "Compliance Report Summary - Compliance Report__line_0ef43e75", + "Compliance Report Summary - Compliance Report"."line_4_eligible_renewable_fuel_required_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_91ad62ee", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_gasoline" AS "Compliance Report Summary - Compliance Report__line_b1027537", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_diesel" AS "Compliance Report Summary - Compliance Report__line_38be33f3", + "Compliance Report Summary - Compliance Report"."line_5_net_notionally_transferred_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_82c517d4", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_gasoline" AS "Compliance Report Summary - Compliance Report__line_6927f733", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_diesel" AS "Compliance Report Summary - Compliance Report__line_93d805cb", + "Compliance Report Summary - Compliance Report"."line_6_renewable_fuel_retained_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_5ae095d0", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_gasoline" AS "Compliance Report Summary - Compliance Report__line_157d6973", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_diesel" AS "Compliance Report Summary - Compliance Report__line_31fd1f1b", + "Compliance Report Summary - Compliance Report"."line_7_previously_retained_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_26ba0b90", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_gasoline" AS "Compliance Report Summary - Compliance Report__line_27419684", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_diesel" AS "Compliance Report Summary - Compliance Report__line_ac263897", + "Compliance Report Summary - Compliance Report"."line_8_obligation_deferred_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_1486f467", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_gasoline" AS "Compliance Report Summary - Compliance Report__line_c12cb8c0", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_diesel" AS "Compliance Report Summary - Compliance Report__line_05eb459f", + "Compliance Report Summary - Compliance Report"."line_9_obligation_added_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_f2ebda23", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_gasoline" AS "Compliance Report Summary - Compliance Report__line_1be763e7", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_diesel" AS "Compliance Report Summary - Compliance Report__line_b72177b0", + "Compliance Report Summary - Compliance Report"."line_10_net_renewable_fuel_supplied_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_28200104", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_gasoline" AS "Compliance Report Summary - Compliance Report__line_53735f1d", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_diesel" AS "Compliance Report Summary - Compliance Report__line_64a07c80", + "Compliance Report Summary - Compliance Report"."line_11_non_compliance_penalty_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_60b43dfe", + "Compliance Report Summary - Compliance Report"."line_12_low_carbon_fuel_required" AS "Compliance Report Summary - Compliance Report__line_da2710ad", + "Compliance Report Summary - Compliance Report"."line_13_low_carbon_fuel_supplied" AS "Compliance Report Summary - Compliance Report__line_b25fca1c", + "Compliance Report Summary - Compliance Report"."line_14_low_carbon_fuel_surplus" AS "Compliance Report Summary - Compliance Report__line_4b98033f", + "Compliance Report Summary - Compliance Report"."line_15_banked_units_used" AS "Compliance Report Summary - Compliance Report__line_1d7d6a31", + "Compliance Report Summary - Compliance Report"."line_16_banked_units_remaining" AS "Compliance Report Summary - Compliance Report__line_684112bb", + "Compliance Report Summary - Compliance Report"."line_17_non_banked_units_used" AS "Compliance Report Summary - Compliance Report__line_b1d3ad5e", + "Compliance Report Summary - Compliance Report"."line_18_units_to_be_banked" AS "Compliance Report Summary - Compliance Report__line_e173956e", + "Compliance Report Summary - Compliance Report"."line_19_units_to_be_exported" AS "Compliance Report Summary - Compliance Report__line_9a885574", + "Compliance Report Summary - Compliance Report"."line_20_surplus_deficit_units" AS "Compliance Report Summary - Compliance Report__line_8e71546f", + "Compliance Report Summary - Compliance Report"."line_21_surplus_deficit_ratio" AS "Compliance Report Summary - Compliance Report__line_00d2728d", + "Compliance Report Summary - Compliance Report"."line_22_compliance_units_issued" AS "Compliance Report Summary - Compliance Report__line_29d9cb9c", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_gasoline" AS "Compliance Report Summary - Compliance Report__line_d8942234", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_diesel" AS "Compliance Report Summary - Compliance Report__line_125e31fc", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_jet_fuel" AS "Compliance Report Summary - Compliance Report__line_eb5340d7", + "Compliance Report Summary - Compliance Report"."line_11_fossil_derived_base_fuel_total" AS "Compliance Report Summary - Compliance Report__line_bff5157d", + "Compliance Report Summary - Compliance Report"."line_21_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__line_7c9c21b1", + "Compliance Report Summary - Compliance Report"."total_non_compliance_penalty_payable" AS "Compliance Report Summary - Compliance Report__tota_0e1e6fb3", + "Compliance Report Summary - Compliance Report"."create_date" AS "Compliance Report Summary - Compliance Report__create_date", + "Compliance Report Summary - Compliance Report"."update_date" AS "Compliance Report Summary - Compliance Report__update_date", + "Compliance Report Summary - Compliance Report"."create_user" AS "Compliance Report Summary - Compliance Report__create_user", + "Compliance Report Summary - Compliance Report"."update_user" AS "Compliance Report Summary - Compliance Report__update_user", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q1" AS "Compliance Report Summary - Compliance Report__earl_6d4994a4", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q2" AS "Compliance Report Summary - Compliance Report__earl_f440c51e", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q3" AS "Compliance Report Summary - Compliance Report__earl_8347f588", + "Compliance Report Summary - Compliance Report"."early_issuance_credits_q4" AS "Compliance Report Summary - Compliance Report__earl_1d23602b", + "Transaction"."transaction_id" AS "Transaction__transaction_id", + "Transaction"."compliance_units" AS "Transaction__compliance_units", + "Transaction"."organization_id" AS "Transaction__organization_id", + "Transaction"."transaction_action" AS "Transaction__transaction_action", + "Transaction"."create_date" AS "Transaction__create_date", + "Transaction"."update_date" AS "Transaction__update_date", + "Transaction"."create_user" AS "Transaction__create_user", + "Transaction"."update_user" AS "Transaction__update_user", + "Transaction"."effective_date" AS "Transaction__effective_date", + "Transaction"."effective_status" AS "Transaction__effective_status", + "Transaction"."expiration_date" AS "Transaction__expiration_date", + "Organization"."organization_id" AS "Organization__organization_id", + "Organization"."organization_code" AS "Organization__organization_code", + "Organization"."name" AS "Organization__name", + "Organization"."operating_name" AS "Organization__operating_name", + "Organization"."email" AS "Organization__email", + "Organization"."phone" AS "Organization__phone", + "Organization"."edrms_record" AS "Organization__edrms_record", + "Organization"."total_balance" AS "Organization__total_balance", + "Organization"."reserved_balance" AS "Organization__reserved_balance", + "Organization"."count_transfers_in_progress" AS "Organization__count_transfers_in_progress", + "Organization"."organization_status_id" AS "Organization__organization_status_id", + "Organization"."organization_type_id" AS "Organization__organization_type_id", + "Organization"."organization_address_id" AS "Organization__organization_address_id", + "Organization"."organization_attorney_address_id" AS "Organization__organization_attorney_address_id", + "Organization"."create_date" AS "Organization__create_date", + "Organization"."update_date" AS "Organization__update_date", + "Organization"."create_user" AS "Organization__create_user", + "Organization"."update_user" AS "Organization__update_user", + "Organization"."effective_date" AS "Organization__effective_date", + "Organization"."effective_status" AS "Organization__effective_status", + "Organization"."expiration_date" AS "Organization__expiration_date", + COALESCE("Organization Early Issuance"."has_early_issuance", false) AS "Organization__has_early_issuance", + "Organization"."records_address" AS "Organization__records_address", + "Compliance Reports Chained - Compliance Report Group UUID"."group_uuid" AS "Compliance Reports Chained - Compliance Report Grou_1a77e4cb", + "Compliance Reports Chained - Compliance Report Group UUID"."max_version" AS "Compliance Reports Chained - Compliance Report Grou_480bb7b1" + FROM + "compliance_report" + INNER JOIN "compliance_period" AS "Compliance Period" ON "compliance_report"."compliance_period_id" = "Compliance Period"."compliance_period_id" + INNER JOIN "compliance_report_status" AS "Compliance Report Status - Current Status" ON "compliance_report"."current_status_id" = "Compliance Report Status - Current Status"."compliance_report_status_id" + INNER JOIN "compliance_report_summary" AS "Compliance Report Summary - Compliance Report" ON "compliance_report"."compliance_report_id" = "Compliance Report Summary - Compliance Report"."compliance_report_id" + LEFT JOIN "transaction" AS "Transaction" ON "compliance_report"."transaction_id" = "Transaction"."transaction_id" + LEFT JOIN "organization" AS "Organization" ON "compliance_report"."organization_id" = "Organization"."organization_id" + LEFT JOIN "organization_early_issuance_by_year" AS "Organization Early Issuance" ON "compliance_report"."organization_id" = "Organization Early Issuance"."organization_id" AND "compliance_report"."compliance_period_id" = "Organization Early Issuance"."compliance_period_id" + INNER JOIN ( + SELECT + compliance_report_group_uuid AS group_uuid, + max(VERSION) AS max_version + FROM + COMPLIANCE_REPORT + GROUP BY + COMPLIANCE_REPORT.compliance_report_group_uuid + ) AS "Compliance Reports Chained - Compliance Report Group UUID" ON ( + "compliance_report"."compliance_report_group_uuid" = "Compliance Reports Chained - Compliance Report Group UUID"."group_uuid" + ) + AND ( + "compliance_report"."version" = "Compliance Reports Chained - Compliance Report Group UUID"."max_version" + ) + ) AS "Compliance Report Base - Compliance Report" ON "Allocation Agreement - Group UUID"."compliance_report_id" = "Compliance Report Base - Compliance Report"."compliance_report_id"; + +GRANT SELECT ON vw_allocation_agreement_extended_base TO basic_lcfs_reporting_role; -- ========================================== -- Compliance Reports Waiting review @@ -1542,4 +3053,4 @@ GRANT SELECT ON vw_allocation_agreement_base TO basic_lcfs_reporting_role; -- Additional permissions for base tables -- ========================================== GRANT SELECT ON organization, transfer_status, transfer_category, compliance_period, compliance_report_status TO basic_lcfs_reporting_role; -GRANT SELECT ON fuel_category, fuel_type, fuel_code, fuel_code_status, fuel_code_prefix, provision_of_the_act, end_use_type TO basic_lcfs_reporting_role; \ No newline at end of file +GRANT SELECT ON fuel_category, fuel_type, fuel_code, fuel_code_status, fuel_code_prefix, provision_of_the_act, end_use_type TO basic_lcfs_reporting_role; diff --git a/backend/lcfs/tests/allocation_agreement/conftest.py b/backend/lcfs/tests/allocation_agreement/conftest.py index c6beb4018..34be0629e 100644 --- a/backend/lcfs/tests/allocation_agreement/conftest.py +++ b/backend/lcfs/tests/allocation_agreement/conftest.py @@ -120,7 +120,7 @@ def create_mock_response_schema(overrides: dict): allocation_agreement_id=1, allocation_transaction_type=AllocationTransactionTypeSchema( allocation_transaction_type_id=1, type="Allocated from" - ), + ).type, transaction_partner="LCFS Org 2", postal_address="789 Stellar Lane Floor 10", transaction_partner_email="tfrs@gov.bc.ca", @@ -131,8 +131,10 @@ def create_mock_response_schema(overrides: dict): default_carbon_intensity=10.0, units="gCO2e/MJ", unrecognized=False, - ), - fuel_category=FuelCategorySchema(fuel_category_id=1, category="Diesel"), + ).fuel_type, + fuel_category=FuelCategorySchema( + fuel_category_id=1, category="Diesel" + ).category, fuel_type_other=None, ci_of_fuel=100.21, provision_of_the_act=ProvisionOfTheActSchema( @@ -163,7 +165,7 @@ def create_mock_update_response_schema(overrides: dict): allocation_agreement_id=1, allocation_transaction_type=AllocationTransactionTypeSchema( allocation_transaction_type_id=1, type="Allocated from" - ), + ).type, transaction_partner="LCFS Org 2", postal_address="789 Stellar Lane Floor 10", transaction_partner_email="tfrs@gov.bc.ca", @@ -174,8 +176,10 @@ def create_mock_update_response_schema(overrides: dict): default_carbon_intensity=10.0, units="gCO2e/MJ", unrecognized=False, - ), - fuel_category=FuelCategorySchema(fuel_category_id=1, category="Diesel"), + ).fuel_type, + fuel_category=FuelCategorySchema( + fuel_category_id=1, category="Diesel" + ).category, fuel_type_other=None, ci_of_fuel=100.21, provision_of_the_act=ProvisionOfTheActSchema( diff --git a/backend/lcfs/tests/allocation_agreement/test_alllocation_agreement_repo.py b/backend/lcfs/tests/allocation_agreement/test_alllocation_agreement_repo.py index 9400c29f2..fa7f5eeca 100644 --- a/backend/lcfs/tests/allocation_agreement/test_alllocation_agreement_repo.py +++ b/backend/lcfs/tests/allocation_agreement/test_alllocation_agreement_repo.py @@ -144,9 +144,10 @@ async def test_get_effective_allocation_agreements( # Verify the result assert len(result) == 1 assert result[0].allocation_agreement_id == 1 - # Compare against the enum's value instead of the enum itself - assert result[0].action_type == ActionTypeEnum.CREATE.value - assert result[0].fuel_type == "Biodiesel" + # Compare against the enum itself + assert result[0].action_type == ActionTypeEnum.CREATE + # fuel_type is a related object with a fuel_type property + assert result[0].fuel_type.fuel_type == "Biodiesel" assert result[0].transaction_partner == "LCFS Org 2" assert result[0].postal_address == "789 Stellar Lane Floor 10" diff --git a/backend/lcfs/tests/allocation_agreement/test_allocation_agreement_views.py b/backend/lcfs/tests/allocation_agreement/test_allocation_agreement_views.py index 3012b010b..104e1979b 100644 --- a/backend/lcfs/tests/allocation_agreement/test_allocation_agreement_views.py +++ b/backend/lcfs/tests/allocation_agreement/test_allocation_agreement_views.py @@ -74,7 +74,7 @@ async def test_save_allocation_agreement_create( data = response.json() assert "allocationAgreementId" in data assert data["quantity"] == 100 - assert data["fuelType"]["fuelType"] == "Biodiesel" + assert data["fuelType"] == "Biodiesel" @pytest.mark.anyio diff --git a/backend/lcfs/tests/compliance_report/test_summary_service.py b/backend/lcfs/tests/compliance_report/test_summary_service.py index 2fbec9a86..7aa1c4486 100644 --- a/backend/lcfs/tests/compliance_report/test_summary_service.py +++ b/backend/lcfs/tests/compliance_report/test_summary_service.py @@ -286,7 +286,6 @@ async def test_supplemental_low_carbon_fuel_target_summary( mock_assessed_report ) - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = ( 1000 # Expected to be called ) @@ -313,9 +312,7 @@ async def test_supplemental_low_carbon_fuel_target_summary( assert line_values[12] == 500 assert line_values[13] == 300 assert line_values[14] == 200 - assert ( - line_values[15] == 15 - ) # From assessed_report_mock.line_18_units_to_be_banked + assert line_values[15] == 15 # From assessed_report_mock.line_18_units_to_be_banked assert ( line_values[16] == 15 ) # From assessed_report_mock.line_19_units_to_be_exported @@ -890,7 +887,9 @@ async def test_calculate_renewable_fuel_target_summary_no_renewables( # Penalty should be applied due to no renewables, checking for decimal values assert result[10].gasoline == 15.08 # 50.25 L shortfall * $0.30/L = 15.075 rounded - assert result[10].diesel == 72.18 # 160.4 L shortfall * $0.45/L = 72.18 (8% of 2005 = 160.4) + assert ( + result[10].diesel == 72.18 + ) # 160.4 L shortfall * $0.45/L = 72.18 (8% of 2005 = 160.4) assert result[10].jet_fuel == 45.08 # 90.15 L shortfall * $0.50/L = 45.075 rounded assert result[10].total_value == (15.08 + 72.18 + 45.08) # 132.34 @@ -1595,8 +1594,13 @@ async def test_calculate_fuel_supply_compliance_units_parametrized( compliance_report_summary_service.fuel_supply_repo.get_effective_fuel_supplies = ( AsyncMock(return_value=[mock_fuel_supply]) ) + # Mock the compliance report and its period description dummy_report = MagicMock() dummy_report.compliance_report_group_uuid = "dummy-group" + # Ensure compliance_period and description are mocked for non-historical check + dummy_report.compliance_period = MagicMock() + dummy_report.compliance_period.description = "2024" # Use a non-historical year + result = ( await compliance_report_summary_service.calculate_fuel_supply_compliance_units( dummy_report @@ -1640,8 +1644,13 @@ async def test_calculate_fuel_export_compliance_units_parametrized( compliance_report_summary_service.fuel_export_repo.get_effective_fuel_exports = ( AsyncMock(return_value=[mock_fuel_export]) ) + # Mock the compliance report and its period description dummy_report = MagicMock() dummy_report.compliance_report_group_uuid = "dummy-group" + # Ensure compliance_period and description are mocked for non-historical check + dummy_report.compliance_period = MagicMock() + dummy_report.compliance_period.description = "2024" # Use a non-historical year + result = ( await compliance_report_summary_service.calculate_fuel_export_compliance_units( dummy_report @@ -1651,869 +1660,131 @@ async def test_calculate_fuel_export_compliance_units_parametrized( @pytest.mark.anyio -async def test_line_17_method_called_during_summary_calculation( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test that the Line 17 TFRS method is called during low carbon fuel target summary calculation""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 123 - - # Mock compliance report (non-supplemental) - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 0 - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None - compliance_report.summary.is_locked = False # Add is_locked attribute - - # Setup repository mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 100 - mock_summary_repo.get_received_compliance_units.return_value = 200 - mock_summary_repo.get_issued_compliance_units.return_value = 300 - - # Mock the TFRS Line 17 calculation - expected_line_17_balance = 1500 - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = ( - expected_line_17_balance - ) - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=400) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=50) - ) - - # Call the low carbon fuel target summary calculation - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - ) - - # Verify the Line 17 method was called with correct parameters - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_called_once_with( - organization_id, compliance_period_start.year - ) - - # Verify Line 17 value appears in the summary - line_values = _get_line_values(summary) - assert line_values[17] == expected_line_17_balance - - -@pytest.mark.anyio -async def test_line_17_different_compliance_periods( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test Line 17 calculation with different compliance periods""" - organization_id = 456 - - # Test 2023 compliance period - compliance_period_start_2023 = datetime(2023, 1, 1) - compliance_period_end_2023 = datetime(2023, 12, 31) - - compliance_report_2023 = MagicMock(spec=ComplianceReport) - compliance_report_2023.version = 0 - compliance_report_2023.summary = MagicMock() - compliance_report_2023.summary.line_17_non_banked_units_used = None - compliance_report_2023.summary.is_locked = False # Add is_locked attribute - - # Setup mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 50 - mock_summary_repo.get_received_compliance_units.return_value = 100 - mock_summary_repo.get_issued_compliance_units.return_value = 150 - - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = 800 - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=200) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=25) - ) - - # Call for 2023 - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start_2023, - compliance_period_end_2023, - organization_id, - compliance_report_2023, - ) - - # Verify 2023 was used - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_called_with( - organization_id, 2023 - ) - - # Reset mock - mock_trxn_repo.reset_mock() - - # Test 2025 compliance period - compliance_period_start_2025 = datetime(2025, 1, 1) - compliance_period_end_2025 = datetime(2025, 12, 31) - - compliance_report_2025 = MagicMock(spec=ComplianceReport) - compliance_report_2025.version = 0 - compliance_report_2025.summary = MagicMock() - compliance_report_2025.summary.line_17_non_banked_units_used = None - compliance_report_2025.summary.is_locked = False # Add is_locked attribute - - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = 1200 - - # Call for 2025 - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start_2025, - compliance_period_end_2025, - organization_id, - compliance_report_2025, - ) - - # Verify 2025 was used - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_called_with( - organization_id, 2025 - ) - - -@pytest.mark.anyio -async def test_supplemental_report_preserves_existing_line_17_value( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test that supplemental reports preserve existing Line 17 values""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 789 - - # Mock supplemental report with existing Line 17 value - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 1 # Supplemental - compliance_report.summary = MagicMock() - existing_line_17_value = 2500 - compliance_report.summary.line_17_non_banked_units_used = existing_line_17_value - compliance_report.summary.is_locked = True # Add is_locked attribute - - # Setup repository mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 150 - mock_summary_repo.get_received_compliance_units.return_value = 250 - mock_summary_repo.get_issued_compliance_units.return_value = 350 - - # Mock previous summary for supplemental reports - previous_summary_mock = MagicMock() - previous_summary_mock.line_18_units_to_be_banked = 75 - previous_summary_mock.line_19_units_to_be_exported = 125 - mock_summary_repo.get_previous_summary = AsyncMock( - return_value=previous_summary_mock - ) - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=450) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=65) - ) - - # Call the method - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - ) - - # Verify the Line 17 method was NOT called for supplemental with existing value - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_not_called() - - # Verify existing Line 17 value is preserved - line_values = _get_line_values(summary) - assert line_values[17] == existing_line_17_value - - -@pytest.mark.anyio -async def test_supplemental_report_calculates_line_17_when_missing( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test that supplemental reports calculate Line 17 when no existing value is present""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 321 - - # Mock supplemental report WITHOUT existing Line 17 value - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 2 # Supplemental - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None # No existing value - compliance_report.summary.is_locked = False # Add is_locked attribute - - # Setup repository mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 80 - mock_summary_repo.get_received_compliance_units.return_value = 160 - mock_summary_repo.get_issued_compliance_units.return_value = 240 - - # Mock previous summary for supplemental reports - previous_summary_mock = MagicMock() - previous_summary_mock.line_18_units_to_be_banked = 40 - previous_summary_mock.line_19_units_to_be_exported = 60 - mock_summary_repo.get_previous_summary = AsyncMock( - return_value=previous_summary_mock - ) - - # Mock the TFRS Line 17 calculation - expected_line_17_balance = 1800 - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = ( - expected_line_17_balance - ) - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=320) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=45) - ) - - # Call the method - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - ) - - # Verify the Line 17 method was called for supplemental without existing value - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_called_once_with( - organization_id, compliance_period_start.year - ) - - # Verify Line 17 value in summary - line_values = _get_line_values(summary) - assert line_values[17] == expected_line_17_balance - - -@pytest.mark.anyio -async def test_line_17_error_handling( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test error handling when Line 17 calculation fails""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 999 - - # Mock compliance report - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 0 - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None - compliance_report.summary.is_locked = False # Add is_locked attribute - - # Setup repository mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 100 - mock_summary_repo.get_received_compliance_units.return_value = 200 - mock_summary_repo.get_issued_compliance_units.return_value = 300 - - # Mock the TFRS Line 17 calculation to raise an exception - mock_trxn_repo.calculate_line_17_available_balance_for_period.side_effect = ( - Exception("Database connection failed") - ) - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=400) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=50) - ) - - # Test that the exception is properly propagated - with pytest.raises(Exception, match="Database connection failed"): - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - - -@pytest.mark.anyio -async def test_line_17_zero_balance_handling( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test handling of zero balance from Line 17 calculation""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 111 - - # Mock compliance report - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 0 - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None - compliance_report.summary.is_locked = False # Add is_locked attribute - - # Setup repository mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 75 - mock_summary_repo.get_received_compliance_units.return_value = 125 - mock_summary_repo.get_issued_compliance_units.return_value = 175 - - # Mock the TFRS Line 17 calculation to return 0 - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = 0 - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=250) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=35) - ) - - # Call the method - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - ) - - # Verify Line 17 is 0 - line_values = _get_line_values(summary) - assert line_values[17] == 0 - - # Verify other calculations proceed normally - assert line_values[12] == 75 # Transferred out - assert line_values[13] == 125 # Received - assert line_values[14] == 175 # Issued - - -@pytest.mark.anyio -async def test_supplemental_report_unlocked_recalculates_line_17( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test that unlocked supplemental reports recalculate Line 17 even with existing value""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 654 - - # Mock supplemental report with existing Line 17 value but NOT locked - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 1 # Supplemental - compliance_report.summary = MagicMock() - existing_line_17_value = 1500 # This should be ignored since not locked - compliance_report.summary.line_17_non_banked_units_used = existing_line_17_value - compliance_report.summary.is_locked = False # NOT locked - should recalculate - - # Setup repository mocks - mock_summary_repo.get_transferred_out_compliance_units.return_value = 200 - mock_summary_repo.get_received_compliance_units.return_value = 400 - mock_summary_repo.get_issued_compliance_units.return_value = 600 - - # Mock previous summary for supplemental reports - previous_summary_mock = MagicMock() - previous_summary_mock.line_18_units_to_be_banked = 100 - previous_summary_mock.line_19_units_to_be_exported = 150 - mock_summary_repo.get_previous_summary = AsyncMock( - return_value=previous_summary_mock - ) - - # Mock the TFRS Line 17 calculation - this should be called and used - expected_line_17_balance = 2200 # Different from existing value - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = ( - expected_line_17_balance - ) - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=500) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=75) - ) - - # Call the method - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - ) - - # Verify the Line 17 method WAS called for unlocked supplemental - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_called_once_with( - organization_id, compliance_period_start.year - ) - - # Verify NEW calculated Line 17 value is used, not the existing one - line_values = _get_line_values(summary) - assert line_values[17] == expected_line_17_balance - assert line_values[17] != existing_line_17_value # Should be different - - -@pytest.mark.anyio -async def test_line_17_integration_with_compliance_report_creation( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test that Line 17 calculation integrates properly with compliance report summary creation workflow""" - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 555 - - # Mock compliance report - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 0 - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None - compliance_report.summary.is_locked = False # Add is_locked attribute - - # Setup repository mocks with realistic values - mock_summary_repo.get_transferred_out_compliance_units.return_value = 500 - mock_summary_repo.get_received_compliance_units.return_value = 1000 - mock_summary_repo.get_issued_compliance_units.return_value = 1500 - - # Mock TFRS Line 17 calculation with a realistic balance - expected_line_17_balance = 2750 - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = ( - expected_line_17_balance - ) - - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=800) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=150) - ) - - # Call the method - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, - ) - ) - - # Verify Line 17 method was called correctly - mock_trxn_repo.calculate_line_17_available_balance_for_period.assert_called_once_with( - organization_id, 2024 - ) - - # Verify the summary contains the correct Line 17 value - line_values = _get_line_values(summary) - assert line_values[17] == expected_line_17_balance - - # Verify that other summary lines also contain expected values - assert line_values[12] == 500 # Transferred out - assert line_values[13] == 1000 # Received - assert line_values[14] == 1500 # Issued - - # Verify that a summary was returned - assert summary is not None - assert len(summary) > 0 - - # Verify penalty was calculated - assert penalty is not None - - -@pytest.mark.anyio -async def test_calculate_notional_transfers_sum_quarterly_logic( - compliance_report_summary_service, -): - """Test the quarterly notional transfer calculation logic we added""" - - # Test data for notional transfers with quarterly fields - test_notional_transfers = [ - # Regular transfer with quantity field only - NotionalTransferSchema( - notional_transfer_id=1, - compliance_report_id=1, - fuel_category="Gasoline", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=1000, - q1_quantity=None, - q2_quantity=None, - q3_quantity=None, - q4_quantity=None, - legal_name="Test Company 1", - address_for_service="123 Test St", - group_uuid="test-group-1", - version=1, - action_type="create", - ), - # Quarterly transfer with quarterly fields only - NotionalTransferSchema( - notional_transfer_id=2, - compliance_report_id=1, - fuel_category="Diesel", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=None, - q1_quantity=250, - q2_quantity=300, - q3_quantity=200, - q4_quantity=250, - legal_name="Test Company 2", - address_for_service="456 Test Ave", - group_uuid="test-group-2", - version=1, - action_type="create", - ), - # Transferred quarterly transfer - NotionalTransferSchema( - notional_transfer_id=3, - compliance_report_id=1, - fuel_category="Jet fuel", - received_or_transferred=ReceivedOrTransferredEnumSchema.Transferred, - quantity=None, - q1_quantity=100, - q2_quantity=150, - q3_quantity=100, - q4_quantity=150, - legal_name="Test Company 3", - address_for_service="789 Test Blvd", - group_uuid="test-group-3", - version=1, - action_type="create", - ), - # Mixed quarterly transfer (some quarters with values) - NotionalTransferSchema( - notional_transfer_id=4, - compliance_report_id=1, - fuel_category="Gasoline", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=None, - q1_quantity=500, - q2_quantity=None, - q3_quantity=300, - q4_quantity=None, - legal_name="Test Company 4", - address_for_service="321 Test Dr", - group_uuid="test-group-4", - version=1, - action_type="create", - ), - ] - - # Test the logic directly (same as implemented in summary service) - notional_transfers_sums = {"gasoline": 0, "diesel": 0, "jet_fuel": 0} - - for transfer in test_notional_transfers: - # Normalize the fuel category key - normalized_category = transfer.fuel_category.replace(" ", "_").lower() - - # Calculate total quantity - use quarterly fields if main quantity is None - total_quantity = transfer.quantity - if total_quantity is None: - # Sum up quarterly quantities for quarterly notional transfers - quarterly_sum = ( - (transfer.q1_quantity or 0) - + (transfer.q2_quantity or 0) - + (transfer.q3_quantity or 0) - + (transfer.q4_quantity or 0) - ) - total_quantity = quarterly_sum if quarterly_sum > 0 else 0 - - # Update the corresponding category sum - if transfer.received_or_transferred.lower() == "received": - notional_transfers_sums[normalized_category] += total_quantity - elif transfer.received_or_transferred.lower() == "transferred": - notional_transfers_sums[normalized_category] -= total_quantity - - # Verify the calculations - # Expected results: - # Gasoline: 1000 (regular) + 800 (500 + 300 from quarterly) = 1800 - # Diesel: 1000 (250 + 300 + 200 + 250 from quarterly) = 1000 - # Jet fuel: -500 (transferred, so negative: -(100 + 150 + 100 + 150)) = -500 - - assert notional_transfers_sums["gasoline"] == 1800 - assert notional_transfers_sums["diesel"] == 1000 - assert notional_transfers_sums["jet_fuel"] == -500 - - -@pytest.mark.anyio -async def test_quarterly_notional_transfer_edge_cases(): - """Test edge cases for quarterly notional transfer calculations""" - - test_edge_cases = [ - # Transfer with quantity=0 and quarterly fields (quantity takes precedence) - NotionalTransferSchema( - notional_transfer_id=1, - compliance_report_id=1, - fuel_category="Gasoline", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=0, - q1_quantity=100, - q2_quantity=200, - q3_quantity=150, - q4_quantity=50, - legal_name="Test Company 1", - address_for_service="123 Test St", - group_uuid="test-group-1", - version=1, - action_type="create", +@pytest.mark.parametrize( + "fuel_data, expected_legacy_result", + [ + # Test data based on previous failures, using legacy formula expectations + ( + { + "target_ci": 100, + "eer": 1, + "ci_of_fuel": 80, + "uci": 10, # uci ignored in legacy + "quantity": 1_000_000, + "q1_quantity": 0, + "q2_quantity": 0, + "q3_quantity": 0, + "q4_quantity": 0, + "energy_density": 1, + }, + 20, # Was 10 in non-legacy ), - # Transfer with quarterly fields only (at least one non-zero) - NotionalTransferSchema( - notional_transfer_id=2, - compliance_report_id=1, - fuel_category="Diesel", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=None, - q1_quantity=0, - q2_quantity=0, - q3_quantity=100, # At least one non-zero quarterly field - q4_quantity=0, - legal_name="Test Company 2", - address_for_service="456 Test Ave", - group_uuid="test-group-2", - version=1, - action_type="create", + ( + { + "target_ci": 100, + "eer": 1, + "ci_of_fuel": 80, + "uci": 10, # uci ignored in legacy + "quantity": 500_000, + "q1_quantity": 0, + "q2_quantity": 500_000, + "q3_quantity": 0, + "q4_quantity": 0, + "energy_density": 1, + }, + 20, # Was 10 in non-legacy ), - # Transfer with negative quarterly values (unusual but possible) - NotionalTransferSchema( - notional_transfer_id=3, - compliance_report_id=1, - fuel_category="Jet fuel", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=None, - q1_quantity=-50, - q2_quantity=100, - q3_quantity=-25, - q4_quantity=75, - legal_name="Test Company 3", - address_for_service="789 Test Blvd", - group_uuid="test-group-3", - version=1, - action_type="create", + ( + { + "target_ci": 80, + "eer": 1, + "ci_of_fuel": 90, + "uci": 5, # uci ignored in legacy + "quantity": 1_000_000, + "q1_quantity": 0, + "q2_quantity": 0, + "q3_quantity": 0, + "q4_quantity": 0, + "energy_density": 1, + }, + -10, # Was -15 in non-legacy ), - ] - - # Test the quarterly calculation logic - notional_transfers_sums = {"gasoline": 0, "diesel": 0, "jet_fuel": 0} - - for transfer in test_edge_cases: - # Normalize the fuel category key - normalized_category = transfer.fuel_category.replace(" ", "_").lower() - - # Calculate total quantity - use quarterly fields if main quantity is None - total_quantity = transfer.quantity - if total_quantity is None: - # Sum up quarterly quantities for quarterly notional transfers - quarterly_sum = ( - (transfer.q1_quantity or 0) - + (transfer.q2_quantity or 0) - + (transfer.q3_quantity or 0) - + (transfer.q4_quantity or 0) - ) - total_quantity = quarterly_sum if quarterly_sum > 0 else 0 - - # Update the corresponding category sum - if transfer.received_or_transferred.lower() == "received": - notional_transfers_sums[normalized_category] += total_quantity - elif transfer.received_or_transferred.lower() == "transferred": - notional_transfers_sums[normalized_category] -= total_quantity - - # Expected results: - # Gasoline: quantity=0, so use 0 (not quarterly fields) - # Diesel: 0 + 0 + 100 + 0 = 100 - # Jet fuel: -50 + 100 + (-25) + 75 = 100 - - assert notional_transfers_sums["gasoline"] == 0 - assert notional_transfers_sums["diesel"] == 100 - assert notional_transfers_sums["jet_fuel"] == 100 - - -@pytest.mark.anyio -async def test_notional_transfer_summary_integration_with_quarterly( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo + ], +) +async def test_calculate_fuel_supply_compliance_units_parametrized_legacy( + compliance_report_summary_service, mock_summary_repo, fuel_data, expected_legacy_result ): - """Integration test for compliance report summary with quarterly notional transfers""" - - # Setup compliance report - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 1 - - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.version = 0 - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None - compliance_report.summary.is_locked = False - - # Mock quarterly notional transfers - quarterly_notional_transfers = [ - NotionalTransferSchema( - notional_transfer_id=1, - compliance_report_id=1, - fuel_category="Gasoline", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=None, - q1_quantity=500, - q2_quantity=500, - q3_quantity=0, - q4_quantity=0, - legal_name="Test Company", - address_for_service="123 Test St", - group_uuid="test-group-1", - version=1, - action_type="create", - ) - ] - - mock_notional_transfers_response = MagicMock() - mock_notional_transfers_response.notional_transfers = quarterly_notional_transfers - - # Setup repository responses - mock_summary_repo.get_transferred_out_compliance_units.return_value = 100 - mock_summary_repo.get_received_compliance_units.return_value = 200 - mock_summary_repo.get_issued_compliance_units.return_value = 300 - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = 1000 - - # Mock the notional transfer service to return our quarterly data - compliance_report_summary_service.notional_transfer_service.calculate_notional_transfers = AsyncMock( - return_value=mock_notional_transfers_response + """Test calculation for compliance periods before 2024 (legacy formula)""" + mock_fuel_supply = Mock(**fuel_data) + compliance_report_summary_service.fuel_supply_repo.get_effective_fuel_supplies = ( + AsyncMock(return_value=[mock_fuel_supply]) ) + # Mock the compliance report and its period description for LEGACY check + dummy_report = MagicMock() + dummy_report.compliance_report_group_uuid = "dummy-group-legacy" + dummy_report.compliance_period = MagicMock() + dummy_report.compliance_period.description = "2023" # Use a legacy year - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=400) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=50) - ) + # Mock aggregate methods to return empty results + mock_summary_repo.aggregate_quantities.return_value = {} - # Call the low carbon fuel target summary calculation - summary, penalty = ( - await compliance_report_summary_service.calculate_low_carbon_fuel_target_summary( - compliance_period_start, - compliance_period_end, - organization_id, - compliance_report, + # Call the compliance report summary calculation + result = ( + await compliance_report_summary_service.calculate_fuel_supply_compliance_units( + dummy_report ) ) - - # Verify the summary was calculated without errors - assert isinstance(summary, list) - assert len(summary) == 11 - - # Verify that the calculation completed successfully (not asserting the mock call since it's internal) - assert summary is not None + assert result == expected_legacy_result @pytest.mark.anyio -async def test_quarterly_notional_transfer_calculation_logic( - compliance_report_summary_service, mock_trxn_repo, mock_summary_repo -): - """Test the quarterly notional transfer calculation logic in the compliance report summary""" - - # Setup compliance report - compliance_period_start = datetime(2024, 1, 1) - compliance_period_end = datetime(2024, 12, 31) - organization_id = 1 - - compliance_report = MagicMock(spec=ComplianceReport) - compliance_report.compliance_report_id = 1 - compliance_report.version = 0 - compliance_report.summary = MagicMock() - compliance_report.summary.line_17_non_banked_units_used = None - compliance_report.summary.is_locked = False - compliance_report.compliance_report_group_uuid = "test-group-uuid" - compliance_report.compliance_period = MagicMock() - compliance_report.compliance_period.effective_date = compliance_period_start - compliance_report.compliance_period.expiration_date = compliance_period_end - compliance_report.compliance_period.description = "2024" - compliance_report.organization_id = organization_id - - # Create test notional transfers with quarterly data - test_notional_transfers = [ - NotionalTransferSchema( - notional_transfer_id=1, - compliance_report_id=1, - fuel_category="Gasoline", - received_or_transferred=ReceivedOrTransferredEnumSchema.Received, - quantity=None, # Use quarterly fields - q1_quantity=500, - q2_quantity=500, - q3_quantity=0, - q4_quantity=0, - legal_name="Test Company 1", - address_for_service="123 Test St", - group_uuid="test-group-1", - version=1, - action_type="create", +@pytest.mark.parametrize( + "fuel_export_data, expected_legacy_result", + [ + # Test data based on previous failures, using legacy formula expectations + ( + { + "target_ci": 100, + "eer": 1, + "ci_of_fuel": 80, + "uci": 10, # uci ignored in legacy + "quantity": 1_000_000, + "energy_density": 1, + }, + -20, # Was -10 in non-legacy ), - NotionalTransferSchema( - notional_transfer_id=2, - compliance_report_id=1, - fuel_category="Diesel", - received_or_transferred=ReceivedOrTransferredEnumSchema.Transferred, - quantity=None, # Use quarterly fields - q1_quantity=200, - q2_quantity=300, - q3_quantity=100, - q4_quantity=200, - legal_name="Test Company 2", - address_for_service="456 Test Ave", - group_uuid="test-group-2", - version=1, - action_type="create", + ( + { + "target_ci": 80, + "eer": 1, + "ci_of_fuel": 90, + "uci": 5, # uci ignored in legacy + "quantity": 1_000_000, + "energy_density": 1, + }, + 0, # Same as non-legacy because (-10) becomes 0 after export processing ), - ] - - mock_notional_transfers_response = MagicMock() - mock_notional_transfers_response.notional_transfers = test_notional_transfers - - # Setup repository responses - mock_summary_repo.get_transferred_out_compliance_units.return_value = 100 - mock_summary_repo.get_received_compliance_units.return_value = 200 - mock_summary_repo.get_issued_compliance_units.return_value = 300 - mock_trxn_repo.calculate_line_17_available_balance_for_period.return_value = 1000 - - # Mock other required methods - compliance_report_summary_service.notional_transfer_service.calculate_notional_transfers = AsyncMock( - return_value=mock_notional_transfers_response - ) - compliance_report_summary_service.calculate_fuel_supply_compliance_units = ( - AsyncMock(return_value=400) - ) - compliance_report_summary_service.calculate_fuel_export_compliance_units = ( - AsyncMock(return_value=50) - ) - compliance_report_summary_service.fuel_supply_repo.get_effective_fuel_supplies = ( - AsyncMock(return_value=[]) - ) - compliance_report_summary_service.other_uses_repo.get_effective_other_uses = ( - AsyncMock(return_value=[]) - ) + ], +) +async def test_calculate_fuel_export_compliance_units_parametrized_legacy( + compliance_report_summary_service, fuel_export_data, expected_legacy_result +): + """Test calculation for compliance periods before 2024 (legacy formula)""" + mock_fuel_export = MagicMock(**fuel_export_data) compliance_report_summary_service.fuel_export_repo.get_effective_fuel_exports = ( - AsyncMock(return_value=[]) - ) - compliance_report_summary_service.allocation_agreement_repo.get_allocation_agreements = AsyncMock( - return_value=[] - ) - compliance_report_summary_service.repo.get_compliance_report_by_id = AsyncMock( - return_value=compliance_report - ) - compliance_report_summary_service.calculate_quarterly_fuel_supply_compliance_units = AsyncMock( - return_value=[0, 0, 0, 0] + AsyncMock(return_value=[mock_fuel_export]) ) + # Mock the compliance report and its period description for LEGACY check + dummy_report = MagicMock() + dummy_report.compliance_report_group_uuid = "dummy-group-legacy" + dummy_report.compliance_period = MagicMock() + dummy_report.compliance_period.description = "2023" # Use a legacy year - # Mock aggregate methods to return empty results - mock_summary_repo.aggregate_quantities.return_value = {} - - # Call the compliance report summary calculation result = ( - await compliance_report_summary_service.calculate_compliance_report_summary( - compliance_report.compliance_report_id + await compliance_report_summary_service.calculate_fuel_export_compliance_units( + dummy_report ) ) - - # Verify the summary was calculated without errors - assert result is not None - assert hasattr(result, "renewable_fuel_target_summary") - assert hasattr(result, "low_carbon_fuel_target_summary") - # The main test is that the calculation completed successfully with quarterly notional transfers # and the quarterly calculation logic we added didn't cause any errors # This verifies that our quarterly notional transfer schema validation and processing works @@ -2782,6 +2053,7 @@ async def test_penalty_override_with_zero_values(): # Tests for Summary Lines 7 & 9 Auto-population and Locking (Issue #2893) + @pytest.mark.anyio async def test_renewable_fuel_target_summary_contains_lines_7_and_9( compliance_report_summary_service, @@ -2790,13 +2062,24 @@ async def test_renewable_fuel_target_summary_contains_lines_7_and_9( # Mock data fossil_quantities = {"gasoline": 1000, "diesel": 2000, "jet_fuel": 500} renewable_quantities = {"gasoline": 100, "diesel": 200, "jet_fuel": 50} - previous_retained = {"gasoline": 10, "diesel": 20, "jet_fuel": 5} # This should populate Line 7 - previous_obligation = {"gasoline": 5, "diesel": 10, "jet_fuel": 2} # This should populate Line 9 + previous_retained = { + "gasoline": 10, + "diesel": 20, + "jet_fuel": 5, + } # This should populate Line 7 + previous_obligation = { + "gasoline": 5, + "diesel": 10, + "jet_fuel": 2, + } # This should populate Line 9 notional_transfers_sums = {"gasoline": 0, "diesel": 0, "jet_fuel": 0} previous_year_required = {"gasoline": 400, "diesel": 750, "jet_fuel": 0} - + # Create a proper ComplianceReportSummary mock with the actual fields - from lcfs.db.models.compliance.ComplianceReportSummary import ComplianceReportSummary + from lcfs.db.models.compliance.ComplianceReportSummary import ( + ComplianceReportSummary, + ) + mock_prev_summary = ComplianceReportSummary( line_6_renewable_fuel_retained_gasoline=100, line_6_renewable_fuel_retained_diesel=200, @@ -2808,7 +2091,7 @@ async def test_renewable_fuel_target_summary_contains_lines_7_and_9( line_4_eligible_renewable_fuel_required_diesel=80, line_4_eligible_renewable_fuel_required_jet_fuel=0, ) - + # Test that the method includes Lines 7 & 9 in the result result = compliance_report_summary_service.calculate_renewable_fuel_target_summary( fossil_quantities, @@ -2820,23 +2103,27 @@ async def test_renewable_fuel_target_summary_contains_lines_7_and_9( mock_prev_summary, previous_year_required, ) - + # Check that all lines are present in the result - should be 11 lines total assert len(result) == 11, f"Expected 11 lines, got {len(result)}" - + # Find Lines 7 & 9 in the result (handle both legacy and non-legacy formats) line_7_row = next((row for row in result if row.line in [7, "7", "7 | 18"]), None) line_9_row = next((row for row in result if row.line in [9, "9", "9 | 20"]), None) - - assert line_7_row is not None, f"Line 7 should be present in summary. Found lines: {[row.line for row in result]}" - assert line_9_row is not None, f"Line 9 should be present in summary. Found lines: {[row.line for row in result]}" + + assert ( + line_7_row is not None + ), f"Line 7 should be present in summary. Found lines: {[row.line for row in result]}" + assert ( + line_9_row is not None + ), f"Line 9 should be present in summary. Found lines: {[row.line for row in result]}" assert line_7_row.gasoline == previous_retained["gasoline"] # 10 - assert line_7_row.diesel == previous_retained["diesel"] # 20 + assert line_7_row.diesel == previous_retained["diesel"] # 20 assert line_7_row.jet_fuel == previous_retained["jet_fuel"] # 5 assert line_7_row.max_gasoline == 20 assert line_7_row.max_diesel == 38 assert line_7_row.max_jet_fuel == 5 - + assert line_9_row.gasoline == previous_obligation["gasoline"] # 5 - assert line_9_row.diesel == previous_obligation["diesel"] # 10 + assert line_9_row.diesel == previous_obligation["diesel"] # 10 assert line_9_row.jet_fuel == previous_obligation["jet_fuel"] # 2 diff --git a/backend/lcfs/tests/fuel_supply/test_fuel_supplies_services.py b/backend/lcfs/tests/fuel_supply/test_fuel_supplies_services.py index 7ab112174..565e9a115 100644 --- a/backend/lcfs/tests/fuel_supply/test_fuel_supplies_services.py +++ b/backend/lcfs/tests/fuel_supply/test_fuel_supplies_services.py @@ -2,7 +2,7 @@ import pytest from fastapi import HTTPException from types import SimpleNamespace -from unittest.mock import MagicMock, AsyncMock +from unittest.mock import MagicMock, AsyncMock, patch from lcfs.db.base import ActionTypeEnum from lcfs.db.models import ( @@ -23,6 +23,7 @@ FuelCategoryResponseSchema, ) from lcfs.web.api.fuel_supply.services import FuelSupplyServices +from lcfs.web.api.fuel_supply.legacy_repo import LegacyFuelSupplyRepository # Fixture to set up the FuelSupplyServices with mocked dependencies # Mock common fuel type and fuel category for reuse @@ -65,17 +66,44 @@ def fuel_supply_service(): return service, mock_repo, mock_fuel_code_repo -# Asynchronous test for get_fuel_supply_options @pytest.mark.anyio -async def test_get_fuel_supply_options(fuel_supply_service): +async def test_get_fuel_supply_options_non_legacy(fuel_supply_service): + """Test get_fuel_supply_options for a non-legacy year (>= 2024)""" service, mock_repo, mock_fuel_code_repo = fuel_supply_service - mock_repo.get_fuel_supply_table_options = AsyncMock(return_value={"fuel_types": []}) - compliance_period = "2023" + compliance_period = "2024" + # Mocked structure returned by the repo + expected_repo_options = {"fuel_types": [{"db_col": "non_legacy_value"}]} + + # Mock the *non-legacy* repo method call + mock_repo.get_fuel_supply_table_options = AsyncMock( + return_value=expected_repo_options + ) + + # Patch the row mapper method within the service + with patch.object( + service, "fuel_type_row_mapper", return_value=None + ) as mock_mapper: + response = await service.get_fuel_supply_options(compliance_period) + + # Assert the *non-legacy* repo method was called with positional arg + mock_repo.get_fuel_supply_table_options.assert_awaited_once_with( + compliance_period # Check for positional argument + ) + + # Assert the mapper was called with the repo data (now iterates over each row) + mock_mapper.assert_called_once_with( + compliance_period, [], expected_repo_options["fuel_types"][0] + ) + + # Assert the final response structure (mapper is mocked, so fuel_types will be empty) + assert isinstance(response, FuelTypeOptionsResponse) + assert ( + response.fuel_types == [] + ) # Because mapper is mocked to return None and modify list in-place - response = await service.get_fuel_supply_options(compliance_period) - assert isinstance(response, FuelTypeOptionsResponse) - mock_repo.get_fuel_supply_table_options.assert_awaited_once_with(compliance_period) +# Legacy test removed - LegacyFuelSupplyRepository no longer exists +# The service now handles all years uniformly without legacy-specific logic @pytest.mark.anyio diff --git a/backend/lcfs/web/api/allocation_agreement/repo.py b/backend/lcfs/web/api/allocation_agreement/repo.py index c744f4b66..e13df408a 100644 --- a/backend/lcfs/web/api/allocation_agreement/repo.py +++ b/backend/lcfs/web/api/allocation_agreement/repo.py @@ -3,7 +3,7 @@ from sqlalchemy import and_, select, delete, func, or_ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload -from typing import List, Optional +from typing import List, Optional, Tuple from lcfs.db.base import ActionTypeEnum from lcfs.db.dependencies import get_async_db_session @@ -16,9 +16,9 @@ from lcfs.db.models.fuel.FuelType import QuantityUnitsEnum from lcfs.db.models.fuel.ProvisionOfTheAct import ProvisionOfTheAct from lcfs.utils.constants import LCFS_Constants -from lcfs.web.api.allocation_agreement.schema import AllocationAgreementSchema from lcfs.web.api.base import PaginationRequestSchema from lcfs.web.api.fuel_code.repo import FuelCodeRepository +from lcfs.web.api.allocation_agreement.schema import AllocationAgreementSchema from lcfs.web.core.decorators import repo_handler from sqlalchemy import and_, select, delete, func, text @@ -76,9 +76,10 @@ async def get_table_options(self, compliance_period: str) -> dict: @repo_handler async def get_allocation_agreements( self, compliance_report_id: int, changelog: bool = False - ) -> List[AllocationAgreementSchema]: + ) -> List[AllocationAgreement]: """ Queries allocation agreements from the database for a specific compliance report. + Returns raw ORM model objects. """ # Retrieve the compliance report's group UUID report_group_query = await self.db.execute( @@ -101,10 +102,10 @@ async def get_effective_allocation_agreements( compliance_report_group_uuid: str, compliance_report_id: int, changelog: bool = False, - ) -> List[AllocationAgreementSchema]: + ) -> List[AllocationAgreement]: """ - Queries allocation agreements from the database for a specific compliance report. - If changelog=True, includes deleted records to show history. + Queries effective allocation agreements from the database. + Returns raw ORM model objects. """ # Get all compliance report IDs in the group up to the specified report compliance_reports_select = select(ComplianceReport.compliance_report_id).where( @@ -180,43 +181,12 @@ async def get_effective_allocation_agreements( result = await self.db.execute(allocation_agreements_select) allocation_agreements = result.unique().scalars().all() - return [ - AllocationAgreementSchema( - allocation_agreement_id=allocation_agreement.allocation_agreement_id, - transaction_partner=allocation_agreement.transaction_partner, - transaction_partner_email=allocation_agreement.transaction_partner_email, - transaction_partner_phone=allocation_agreement.transaction_partner_phone, - postal_address=allocation_agreement.postal_address, - ci_of_fuel=allocation_agreement.ci_of_fuel, - quantity=allocation_agreement.quantity, - q1_quantity=allocation_agreement.q1_quantity, - q2_quantity=allocation_agreement.q2_quantity, - q3_quantity=allocation_agreement.q3_quantity, - q4_quantity=allocation_agreement.q4_quantity, - units=allocation_agreement.units, - compliance_report_id=allocation_agreement.compliance_report_id, - allocation_transaction_type=allocation_agreement.allocation_transaction_type.type, - fuel_type=allocation_agreement.fuel_type.fuel_type, - fuel_type_other=allocation_agreement.fuel_type_other, - fuel_category=allocation_agreement.fuel_category.category, - provision_of_the_act=allocation_agreement.provision_of_the_act.name, - # Set fuel_code only if it exists - fuel_code=( - allocation_agreement.fuel_code.fuel_code - if allocation_agreement.fuel_code - else None - ), - group_uuid=allocation_agreement.group_uuid, - version=allocation_agreement.version, - action_type=allocation_agreement.action_type, - ) - for allocation_agreement in allocation_agreements - ] + return allocation_agreements @repo_handler async def get_allocation_agreements_paginated( self, pagination: PaginationRequestSchema, compliance_report_id: int - ) -> List[AllocationAgreementSchema]: + ) -> Tuple[List[AllocationAgreement], int]: conditions = [AllocationAgreement.compliance_report_id == compliance_report_id] offset = 0 if pagination.page < 1 else (pagination.page - 1) * pagination.size limit = pagination.size diff --git a/backend/lcfs/web/api/allocation_agreement/schema.py b/backend/lcfs/web/api/allocation_agreement/schema.py index 3e3f4e03b..222afce24 100644 --- a/backend/lcfs/web/api/allocation_agreement/schema.py +++ b/backend/lcfs/web/api/allocation_agreement/schema.py @@ -92,14 +92,14 @@ class AllocationAgreementChangelogFuelTypeSchema(BaseSchema): class AllocationAgreementResponseSchema(BaseSchema): compliance_report_id: int allocation_agreement_id: int - allocation_transaction_type: AllocationTransactionTypeSchema - transaction_partner: str - postal_address: str - transaction_partner_email: str - transaction_partner_phone: str - fuel_type: FuelTypeChangelogSchema + allocation_transaction_type: str + transaction_partner: Optional[str] = None + postal_address: Optional[str] = None + transaction_partner_email: Optional[str] = None + transaction_partner_phone: Optional[str] = None + fuel_type: str fuel_category_id: Optional[int] = None - fuel_category: FuelCategoryResponseSchema + fuel_category: Optional[str] = None fuel_type_other: Optional[str] = None ci_of_fuel: Optional[float] = None provision_of_the_act: Optional[ProvisionOfTheActSchema] = None @@ -109,8 +109,7 @@ class AllocationAgreementResponseSchema(BaseSchema): q3_quantity: Optional[int] = None q4_quantity: Optional[int] = None units: str - fuel_category: FuelCategoryResponseSchema - fuel_code: Optional[FuelCodeResponseSchema] = None + fuel_code: Optional[str] = None group_uuid: str version: int action_type: str @@ -123,19 +122,19 @@ class AllocationAgreementChangelogSchema(BaseSchema): allocation_transaction_type: str transaction_partner: str postal_address: str - transaction_partner_email: str - transaction_partner_phone: str + transaction_partner_email: Optional[str] = None + transaction_partner_phone: Optional[str] = None fuel_type: AllocationAgreementChangelogFuelTypeSchema fuel_type_other: Optional[str] = None ci_of_fuel: float - provision_of_the_act: str + provision_of_the_act: Optional[str] = None quantity: Optional[int] = None q1_quantity: Optional[int] = None q2_quantity: Optional[int] = None q3_quantity: Optional[int] = None q4_quantity: Optional[int] = None units: str - fuel_category: FuelCategorySchema + fuel_category: Optional[FuelCategorySchema] = None fuel_code: Optional[FuelCodeSchema] = None deleted: Optional[bool] = None group_uuid: Optional[str] = None @@ -155,19 +154,19 @@ class AllocationAgreementCreateSchema(BaseSchema): allocation_transaction_type: str transaction_partner: str postal_address: str - transaction_partner_email: str - transaction_partner_phone: str + transaction_partner_email: Optional[str] = None + transaction_partner_phone: Optional[str] = None fuel_type: str fuel_type_other: Optional[str] = None ci_of_fuel: Optional[float] = 0 - provision_of_the_act: str + provision_of_the_act: Optional[str] = None quantity: Optional[int] = None q1_quantity: Optional[int] = None q2_quantity: Optional[int] = None q3_quantity: Optional[int] = None q4_quantity: Optional[int] = None units: Optional[str] = None - fuel_category: str + fuel_category: Optional[str] = None fuel_code: Optional[str] = None deleted: Optional[bool] = None group_uuid: Optional[str] = None @@ -192,12 +191,12 @@ class AllocationAgreementSchema(AllocationAgreementCreateSchema): class AllocationAgreementAllSchema(BaseSchema): - allocation_agreements: List[AllocationAgreementSchema] + allocation_agreements: List[AllocationAgreementResponseSchema] pagination: Optional[PaginationResponseSchema] = {} class AllocationAgreementListSchema(BaseSchema): - allocation_agreements: List[AllocationAgreementSchema] + allocation_agreements: List[AllocationAgreementResponseSchema] pagination: PaginationResponseSchema diff --git a/backend/lcfs/web/api/allocation_agreement/services.py b/backend/lcfs/web/api/allocation_agreement/services.py index 46547fe52..04c3438f9 100644 --- a/backend/lcfs/web/api/allocation_agreement/services.py +++ b/backend/lcfs/web/api/allocation_agreement/services.py @@ -2,7 +2,7 @@ import structlog import uuid from fastapi import Depends, HTTPException, status -from typing import Optional +from typing import Optional, List from lcfs.db.base import ActionTypeEnum from lcfs.db.models.compliance.AllocationAgreement import AllocationAgreement @@ -16,6 +16,8 @@ AllocationAgreementAllSchema, AllocationTransactionTypeSchema, DeleteAllocationAgreementResponseSchema, + AllocationAgreementResponseSchema, + ProvisionOfTheActSchema, ) from lcfs.web.api.base import PaginationRequestSchema, PaginationResponseSchema from lcfs.web.api.compliance_report.repo import ComplianceReportRepository @@ -145,31 +147,93 @@ async def get_table_options( @service_handler async def get_allocation_agreements( - self, compliance_report_id: int, changelog: bool = False - ) -> AllocationAgreementListSchema: - """ - Gets the list of allocation agreements for a specific compliance report. - """ - allocation_agreements = await self.repo.get_allocation_agreements( - compliance_report_id, changelog=changelog + self, + compliance_report_id: int, + changelog: bool = False, + ) -> AllocationAgreementAllSchema: + """Get all Allocation agreements for a compliance report""" + # Expect raw models from repo + allocation_agreements_models: List[AllocationAgreement] = ( + await self.repo.get_allocation_agreements( + compliance_report_id=compliance_report_id, changelog=changelog + ) ) - + allocation_agreements_response = [] + # Map directly from ORM models + for aa in allocation_agreements_models: + aa_response = AllocationAgreementResponseSchema( + allocation_agreement_id=aa.allocation_agreement_id, + compliance_report_id=aa.compliance_report_id, + allocation_transaction_type=aa.allocation_transaction_type.type, + transaction_partner=aa.transaction_partner, + postal_address=aa.postal_address, + transaction_partner_email=aa.transaction_partner_email, + transaction_partner_phone=aa.transaction_partner_phone, + fuel_type=aa.fuel_type.fuel_type, + fuel_category_id=aa.fuel_category_id, + fuel_category=(aa.fuel_category.category if aa.fuel_category else None), + fuel_type_other=aa.fuel_type_other, + ci_of_fuel=aa.ci_of_fuel, + provision_of_the_act=( + ProvisionOfTheActSchema( + provision_of_the_act_id=aa.provision_of_the_act.provision_of_the_act_id, + name=aa.provision_of_the_act.name + ) if aa.provision_of_the_act else None + ), + quantity=aa.quantity, + units=aa.units, + fuel_code=(aa.fuel_code.fuel_code if aa.fuel_code else None), + group_uuid=aa.group_uuid, + version=aa.version, + action_type=aa.action_type, + updated=None, # Assuming this is not mapped + ) + allocation_agreements_response.append(aa_response) return AllocationAgreementAllSchema( - allocation_agreements=[ - AllocationAgreementSchema.model_validate(aa) - for aa in allocation_agreements - ] + allocation_agreements=allocation_agreements_response ) @service_handler async def get_allocation_agreements_paginated( self, pagination: PaginationRequestSchema, compliance_report_id: int ) -> AllocationAgreementListSchema: - allocation_agreements, total_count = ( + # Repo returns raw models and count + allocation_agreements_models, total_count = ( await self.repo.get_allocation_agreements_paginated( pagination, compliance_report_id ) ) + allocation_agreements_response = [] + for aa in allocation_agreements_models: + # Construct the ResponseSchema using strings from ORM model attributes + aa_response = AllocationAgreementResponseSchema( + allocation_agreement_id=aa.allocation_agreement_id, + compliance_report_id=aa.compliance_report_id, + allocation_transaction_type=aa.allocation_transaction_type.type, + transaction_partner=aa.transaction_partner, + postal_address=aa.postal_address, + transaction_partner_email=aa.transaction_partner_email, + transaction_partner_phone=aa.transaction_partner_phone, + fuel_type=aa.fuel_type.fuel_type, + fuel_category_id=aa.fuel_category_id, + fuel_category=(aa.fuel_category.category if aa.fuel_category else None), + fuel_type_other=aa.fuel_type_other, + ci_of_fuel=aa.ci_of_fuel, + provision_of_the_act=( + ProvisionOfTheActSchema( + provision_of_the_act_id=aa.provision_of_the_act.provision_of_the_act_id, + name=aa.provision_of_the_act.name + ) if aa.provision_of_the_act else None + ), + quantity=aa.quantity, + units=aa.units, + fuel_code=(aa.fuel_code.fuel_code if aa.fuel_code else None), + group_uuid=aa.group_uuid, + version=aa.version, + action_type=aa.action_type, + updated=None, + ) + allocation_agreements_response.append(aa_response) return AllocationAgreementListSchema( pagination=PaginationResponseSchema( total=total_count, @@ -191,10 +255,26 @@ async def get_allocation_agreements_paginated( q3_quantity=allocation_agreement.q3_quantity, q4_quantity=allocation_agreement.q4_quantity, units=allocation_agreement.units, - allocation_transaction_type=allocation_agreement.allocation_transaction_type.type, - fuel_type=allocation_agreement.fuel_type.fuel_type, - fuel_category=allocation_agreement.fuel_category.category, - provision_of_the_act=allocation_agreement.provision_of_the_act.name, + allocation_transaction_type=( + allocation_agreement.allocation_transaction_type.type + if allocation_agreement.allocation_transaction_type + else None + ), + fuel_type=( + allocation_agreement.fuel_type.fuel_type + if allocation_agreement.fuel_type + else None + ), + fuel_category=( + allocation_agreement.fuel_category.category + if allocation_agreement.fuel_category + else None + ), + provision_of_the_act=( + allocation_agreement.provision_of_the_act.name + if allocation_agreement.provision_of_the_act + else None + ), # Set fuel_code only if it exists fuel_code=( allocation_agreement.fuel_code.fuel_code @@ -203,7 +283,7 @@ async def get_allocation_agreements_paginated( ), compliance_report_id=allocation_agreement.compliance_report_id, ) - for allocation_agreement in allocation_agreements + for allocation_agreement in allocation_agreements_response ], ) @@ -294,7 +374,11 @@ async def update_allocation_agreement( recalculated_ci = await self.calculate_ci_of_fuel( fuel_type=existing_allocation_agreement.fuel_type, fuel_category=existing_allocation_agreement.fuel_category, - provision_of_the_act=existing_allocation_agreement.provision_of_the_act.name, + provision_of_the_act=( + existing_allocation_agreement.provision_of_the_act.name + if existing_allocation_agreement.provision_of_the_act + else None + ), fuel_code=existing_allocation_agreement.fuel_code, ) existing_allocation_agreement.ci_of_fuel = recalculated_ci @@ -348,8 +432,16 @@ async def update_allocation_agreement( allocation_transaction_type=updated_allocation_agreement.allocation_transaction_type.type, fuel_type=updated_allocation_agreement.fuel_type.fuel_type, fuel_type_other=updated_allocation_agreement.fuel_type_other, - fuel_category=updated_allocation_agreement.fuel_category.category, - provision_of_the_act=updated_allocation_agreement.provision_of_the_act.name, + fuel_category=( + updated_allocation_agreement.fuel_category.category + if updated_allocation_agreement.fuel_category + else None + ), + provision_of_the_act=( + updated_allocation_agreement.provision_of_the_act.name + if updated_allocation_agreement.provision_of_the_act + else None + ), fuel_code=( updated_allocation_agreement.fuel_code.fuel_code if updated_allocation_agreement.fuel_code @@ -427,8 +519,16 @@ async def create_allocation_agreement( allocation_transaction_type=allocation_transaction_type_value, fuel_type=fuel_type_value, fuel_type_other=created_allocation_agreement.fuel_type_other, - fuel_category=fuel_category_value, - provision_of_the_act=provision_of_the_act_value, + fuel_category=( + created_allocation_agreement.fuel_category.category + if created_allocation_agreement.fuel_category + else None + ), + provision_of_the_act=( + created_allocation_agreement.provision_of_the_act.name + if created_allocation_agreement.provision_of_the_act + else None + ), fuel_code=fuel_code_value, group_uuid=created_allocation_agreement.group_uuid, version=created_allocation_agreement.version, diff --git a/backend/lcfs/web/api/compliance_report/constants.py b/backend/lcfs/web/api/compliance_report/constants.py index a62261a0f..e024aef0f 100644 --- a/backend/lcfs/web/api/compliance_report/constants.py +++ b/backend/lcfs/web/api/compliance_report/constants.py @@ -54,6 +54,8 @@ }, } +# DEPRECATED: Legacy PART3 format - keeping for reference but no longer used +# All reports now use LOW_CARBON_FUEL_TARGET_DESCRIPTIONS format PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS = { 12: { "description": "Total credits from fuel supplied (from Schedule B)", @@ -151,7 +153,7 @@ NON_COMPLIANCE_PENALTY_SUMMARY_DESCRIPTIONS = { 11: { - "legacy": "Renewable fuel target non-compliance penalty total (Line 11|22, Gasoline + Diesel)", + "legacy": "Renewable fuel target non-compliance penalty total (Line 11, Gasoline + Diesel)", "description": "Renewable fuel target non-compliance penalty total (Line 11, Gasoline + Diesel + Jet fuel)", "field": "fossil_derived_base_fuel", }, diff --git a/backend/lcfs/web/api/compliance_report/summary_calculations.md b/backend/lcfs/web/api/compliance_report/summary_calculations.md index 23258670c..09fe157d1 100644 --- a/backend/lcfs/web/api/compliance_report/summary_calculations.md +++ b/backend/lcfs/web/api/compliance_report/summary_calculations.md @@ -202,7 +202,8 @@ In the ETL process, the compliance_report_summary table maintains all line value - `diesel_class_previously_retained` → `line_7_previously_retained_diesel` - `gasoline_class_obligation` → `line_9_obligation_added_gasoline` - `gasoline_class_previously_retained` → `line_7_previously_retained_gasoline` -- `credits_offset` → `line_22_compliance_units_issued` + +Note: Line 22 (Available compliance unit balance at period end) is not sourced from `credits_offset` (credits used). It is populated from TFRS snapshots during the summary update step to reflect the end-of-period available balance. These mappings ensure that historical data from the TFRS system is correctly transferred to the new LCFS system while maintaining data integrity and calculation consistency. diff --git a/backend/lcfs/web/api/compliance_report/summary_service.py b/backend/lcfs/web/api/compliance_report/summary_service.py index 1c78102ac..8af537298 100644 --- a/backend/lcfs/web/api/compliance_report/summary_service.py +++ b/backend/lcfs/web/api/compliance_report/summary_service.py @@ -18,7 +18,6 @@ from lcfs.utils.constants import LCFS_Constants from lcfs.web.api.allocation_agreement.repo import AllocationAgreementRepository from lcfs.web.api.compliance_report.constants import ( - PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS, RENEWABLE_FUEL_TARGET_DESCRIPTIONS, LOW_CARBON_FUEL_TARGET_DESCRIPTIONS, NON_COMPLIANCE_PENALTY_SUMMARY_DESCRIPTIONS, @@ -45,6 +44,16 @@ logger = structlog.get_logger(__name__) +# REFACTORING NOTE: Legacy Report Handling +# This service has been refactored to always use modern format for all reports. +# Previously, reports were formatted differently based on compliance year (pre/post 2024). +# Legacy-specific code has been deprecated but kept for reference. +# Key changes: +# - All reports now use LOW_CARBON_FUEL_TARGET_DESCRIPTIONS (modern format) +# - Legacy PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS is deprecated +# - Line formatting always uses modern numbering (no "line | line+11" format) +# - Frontend feature flags control display, not backend logic + ELIGIBLE_GASOLINE_RENEWABLE_TYPES = { "renewable gasoline", "ethanol", @@ -77,7 +86,13 @@ def set_nickname(self, nickname: str): def get_nickname(self) -> Optional[str]: return self.nickname + # DEPRECATED: Legacy year checking - keeping for reference but no longer used + # All reports now use modern format regardless of year def is_legacy_year(self) -> bool: + """ + DEPRECATED: This method is no longer used as all reports now use modern format. + Keeping for reference and potential future use. + """ return ( self.compliance_period < int(LCFS_Constants.LEGISLATION_TRANSITION_YEAR) if self.compliance_period is not None @@ -222,39 +237,19 @@ def convert_summary_to_dict( self._handle_summary_lines(summary, summary_obj, column.key, line) # DB Columns are not in the same order as display, so sort them - summary.low_carbon_fuel_target_summary.sort( - key=lambda row: int( - re.match(r"(\d+)", row.line).group(1) - if compliance_data_service.is_legacy_year() - else row.line - ) - ) + summary.low_carbon_fuel_target_summary.sort(key=lambda row: int(row.line)) return summary def _get_line_value(self, line: int, is_legacy: bool = False) -> Union[str, int]: - """Helper method to format line values based on legacy year status""" - if not is_legacy: - return line - - if line is None: - return line - elif 1 <= line <= 11: - return f"{line} | {line + 11}" - elif 12 <= line <= 22: - mapping = { - 12: "23", - 13: "24", - 14: "25", - 15: "26", - 16: "26a", - 17: "26b", - 18: "26c", - 19: "27", - 20: "28", - } - return mapping.get(line, str(line)) - return str(line) + """ + Helper method to format line values - always use modern format + + DEPRECATED: The is_legacy parameter is no longer used as all reports now use modern format. + Previously this would format legacy lines as "line | line+11" format. + Keeping the parameter for compatibility but it's ignored. + """ + return line def _extract_line_number(self, column_key: str) -> Optional[int]: """Extract the line number (1..N) from a column key like 'line_4_...' using regex.""" @@ -291,16 +286,10 @@ def _handle_low_carbon_line( self, summary, summary_obj, column_key, line: int ) -> None: """Populate the low_carbon_fuel_target_summary section""" - is_legacy = compliance_data_service.is_legacy_year() - if is_legacy and line > 20: - return + # Always use modern format for all reports description = self._format_description( line=line, - descriptions_dict=( - PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS - if is_legacy - else LOW_CARBON_FUEL_TARGET_DESCRIPTIONS - ), + descriptions_dict=LOW_CARBON_FUEL_TARGET_DESCRIPTIONS, ) desc = None if line == 21: @@ -318,29 +307,18 @@ def _handle_low_carbon_line( desc = description.replace( "{{COMPLIANCE_YEAR_PLUS_1}}", str(compliance_year + 1) ) - elif line in [17, 18] and is_legacy: - desc = self._part3_special_description( - line, PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS - ) else: desc = description + summary.low_carbon_fuel_target_summary.append( ComplianceReportSummaryRowSchema( - line=self._get_line_value(line, is_legacy), - format=( - FORMATS.CURRENCY.value - if (line == 21 or (line == 20 and is_legacy)) - else FORMATS.NUMBER.value - ), + line=line, + format=(FORMATS.CURRENCY.value if line == 21 else FORMATS.NUMBER.value), description=desc, field=LOW_CARBON_FUEL_TARGET_DESCRIPTIONS[line]["field"], value=int(getattr(summary_obj, column_key) or 0), - units=( - PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS[line]["units"] - if is_legacy - else "" - ), - bold=True if (is_legacy and line > 18) else False, + units="", + bold=False, ) ) @@ -378,12 +356,7 @@ def _get_or_create_summary_row( The 'line' is stored as a string internally. """ existing_element = next( - ( - el - for el in target_list - if el.line - == self._get_line_value(line, compliance_data_service.is_legacy_year()) - ), + (el for el in target_list if el.line == line), None, ) if existing_element: @@ -399,7 +372,7 @@ def _get_or_create_summary_row( # Create and append the new row new_element = ComplianceReportSummaryRowSchema( - line=self._get_line_value(line, compliance_data_service.is_legacy_year()), + line=line, format=default_format, description=description, field=default_descriptions[line]["field"], @@ -424,12 +397,8 @@ def _assign_fuel_value( def _format_description(self, line, descriptions_dict): """ Builds a description string from the dictionary. - Optionally handle a special line with dynamic formatting. """ - base_desc = descriptions_dict[line].get( - ("legacy" if compliance_data_service.is_legacy_year() else "description"), - descriptions_dict[line].get("description"), - ) + base_desc = descriptions_dict[line].get("description") return base_desc # By default, no fancy placeholders used here. def _line_4_special_description(self, line, summary_obj, descriptions_dict): @@ -454,7 +423,7 @@ def _line_4_special_description(self, line, summary_obj, descriptions_dict): def _renewable_special_description(self, line, summary_obj, descriptions_dict): """ - For lines 6 and 8, your original code does some .format() with three placeholders + For lines 6 and 8, format the description with placeholders (line_4_eligible_renewable_fuel_required_* * 0.05). """ base_desc = descriptions_dict[line].get( @@ -485,25 +454,21 @@ def _renewable_special_description(self, line, summary_obj, descriptions_dict): def _non_compliance_special_description(self, line, summary_obj, descriptions_dict): """ - For line 21, your original code does .format(...) with summary_obj.line_21_non_compliance_penalty_payable / 600 + For line 21, format with summary_obj.line_21_non_compliance_penalty_payable / 600 """ - base_desc = descriptions_dict[line].get( - ("legacy" if compliance_data_service.is_legacy_year() else "description"), - descriptions_dict[line].get("description"), - ) + base_desc = descriptions_dict[line].get("description") return base_desc.format( "{:,}".format(int(summary_obj.line_21_non_compliance_penalty_payable / 600)) ) - def _part3_special_description(self, line, descriptions_dict): - """ - For line 26a and 26b, your original code does .format(...) with compliance report nick name - """ - base_desc = descriptions_dict[line].get( - ("legacy" if compliance_data_service.is_legacy_year() else "description"), - descriptions_dict[line].get("description"), - ) - return base_desc.format("{:}".format(compliance_data_service.get_nickname())) + # DEPRECATED: Legacy PART3 formatting - keeping for reference but no longer used + # def _part3_special_description(self, line, descriptions_dict): + # """ + # DEPRECATED: For line 26a and 26b, your original code does .format(...) with compliance report nick name + # This was used for legacy PART3_LOW_CARBON_FUEL_TARGET_DESCRIPTIONS format + # """ + # base_desc = descriptions_dict[line].get("description") + # return base_desc.format("{:}".format(compliance_data_service.get_nickname())) @service_handler async def update_compliance_report_summary( @@ -1468,9 +1433,7 @@ async def calculate_low_carbon_fuel_target_summary( low_carbon_fuel_target_summary = [ ComplianceReportSummaryRowSchema( - line=self._get_line_value( - line, compliance_data_service.is_legacy_year() - ), + line=self._get_line_value(line), description=( LOW_CARBON_FUEL_TARGET_DESCRIPTIONS[line]["description"].format( "{:,}".format(non_compliance_penalty_payable_units * -1) @@ -1518,9 +1481,7 @@ def calculate_non_compliance_penalty_summary( non_compliance_penalty_summary = [ ComplianceReportSummaryRowSchema( - line=self._get_line_value( - line, compliance_data_service.is_legacy_year() - ), + line=self._get_line_value(line), description=( NON_COMPLIANCE_PENALTY_SUMMARY_DESCRIPTIONS[line][ "description" @@ -1555,9 +1516,11 @@ async def calculate_fuel_supply_compliance_units( # Initialize compliance units sum compliance_units_sum = 0.0 + # Check if this is a historical report (pre-2024) + is_historical = int(report.compliance_period.description) < 2024 + # Calculate compliance units for each fuel supply record for fuel_supply in fuel_supply_records: - TCI = fuel_supply.target_ci or 0 # Target Carbon Intensity EER = fuel_supply.eer or 0 # Energy Effectiveness Ratio RCI = fuel_supply.ci_of_fuel or 0 # Recorded Carbon Intensity @@ -1571,8 +1534,16 @@ async def calculate_fuel_supply_compliance_units( ) ED = fuel_supply.energy_density or 0 # Energy Density - # Apply the compliance units formula - compliance_units = calculate_compliance_units(TCI, EER, RCI, UCI, Q, ED) + # Apply the appropriate compliance units formula + compliance_units = calculate_compliance_units( + TCI=TCI, + EER=EER, + RCI=RCI, + UCI=UCI, + Q=Q, + ED=ED, + is_historical=is_historical, + ) compliance_units_sum += compliance_units return round(compliance_units_sum) @@ -1638,19 +1609,32 @@ async def calculate_fuel_export_compliance_units( report.compliance_report_group_uuid, report.compliance_report_id ) + # Check if this is a historical report (pre-2024) + is_historical = int(report.compliance_period.description) < 2024 + # Initialize compliance units sum compliance_units_sum = 0.0 # Calculate compliance units for each fuel export record for fuel_export in fuel_export_records: - TCI = fuel_export.target_ci or 0 # Target Carbon Intensity + TCI = fuel_export.target_ci or 0 # Target Carbon Intensity / CI class EER = fuel_export.eer or 0 # Energy Effectiveness Ratio - RCI = fuel_export.ci_of_fuel or 0 # Recorded Carbon Intensity - UCI = fuel_export.uci or 0 # Additional Carbon Intensity + RCI = fuel_export.ci_of_fuel or 0 # Recorded Carbon Intensity / CI fuel + UCI = ( + fuel_export.uci or 0 + ) # Additional Carbon Intensity (only used in new calculation) Q = fuel_export.quantity or 0 # Quantity of Fuel Supplied ED = fuel_export.energy_density or 0 # Energy Density - # Apply the compliance units formula - compliance_units = calculate_compliance_units(TCI, EER, RCI, UCI, Q, ED) + # Apply the appropriate compliance units formula + compliance_units = calculate_compliance_units( + TCI=TCI, + EER=EER, + RCI=RCI, + UCI=UCI, + Q=Q, + ED=ED, + is_historical=is_historical, + ) compliance_units = -compliance_units compliance_units = compliance_units if compliance_units < 0 else 0 diff --git a/backend/lcfs/web/api/fuel_supply/legacy_repo.py b/backend/lcfs/web/api/fuel_supply/legacy_repo.py new file mode 100644 index 000000000..0c5f9b0a8 --- /dev/null +++ b/backend/lcfs/web/api/fuel_supply/legacy_repo.py @@ -0,0 +1,300 @@ +import structlog +from datetime import datetime +from fastapi import Depends +from sqlalchemy import and_, or_, select, literal +from sqlalchemy import func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload +from typing import Optional, Sequence, Any +from sqlalchemy.dialects.postgresql import array_agg +from sqlalchemy.sql.expression import distinct +from lcfs.db.dependencies import get_async_db_session +from lcfs.db.models.compliance import CompliancePeriod +from lcfs.db.models.fuel import ( + CategoryCarbonIntensity, + DefaultCarbonIntensity, + EnergyDensity, + EnergyEffectivenessRatio, + FuelCategory, + FuelInstance, + FuelCode, + FuelCodePrefix, + FuelCodeStatus, + FuelType, + ProvisionOfTheAct, + TargetCarbonIntensity, + UnitOfMeasure, + EndUseType, +) +from lcfs.web.core.decorators import repo_handler + +logger = structlog.get_logger(__name__) + + +class LegacyFuelSupplyRepository: + def __init__(self, db: AsyncSession = Depends(get_async_db_session)): + self.db = db + + @repo_handler + async def get_fuel_supply_table_options(self, compliance_period: str): + """ + Retrieve Fuel Type and other static data for LEGACY years (before LCFS transition year) + to use them while populating fuel supply form. + """ + + subquery_compliance_period_id = ( + select(CompliancePeriod.compliance_period_id) + .where(CompliancePeriod.description == compliance_period) + .scalar_subquery() + ) + + subquery_fuel_code_status_id = ( + select(FuelCodeStatus.fuel_code_status_id) + .where(FuelCodeStatus.status == "Approved") + .scalar_subquery() + ) + + try: + current_year = int(compliance_period) + except ValueError as e: + logger.error( + "Invalid compliance_period: not an integer", + compliance_period=compliance_period, + error=str(e), + ) + raise ValueError( + f"""Invalid compliance_period: '{ + compliance_period}' must be an integer.""" + ) from e + + start_of_compliance_year = datetime(current_year, 1, 1) + end_of_compliance_year = datetime(current_year, 12, 31) + + # Define conditions for the query for legacy years: + # - Include fuel types marked as legacy or non‑fossil-derived. + # - Exclude jet fuel from FuelCategory. + # - Provision itself is flagged as legacy. + # - Gasoline (Natural gas-based, Petroleum-based): provision ID 4 + # - Petroleum-based diesel: provision ID 5 + # - Other non-fossil fuels: provision ID not in [4, 5] + fuel_instance_condition = or_( + FuelType.is_legacy == True, + FuelType.fossil_derived == False, + ) + fuel_category_condition = FuelCategory.fuel_category_id != 3 # Exclude Jet Fuel + provision_condition = and_( + ProvisionOfTheAct.is_legacy == True, + or_( + # For gasoline types (Natural gas-based and Petroleum-based) + and_( + FuelType.fuel_type.in_( + ["Natural gas-based gasoline", "Petroleum-based gasoline"] + ), + ProvisionOfTheAct.provision_of_the_act_id == 4, + ), + # For Petroleum-based diesel + and_( + FuelType.fuel_type == "Petroleum-based diesel", + ProvisionOfTheAct.provision_of_the_act_id == 5, + ), + # For all other non-fossil fuels + and_( + ~FuelType.fuel_type.in_( + [ + "Natural gas-based gasoline", + "Petroleum-based gasoline", + "Petroleum-based diesel", + ] + ), + ProvisionOfTheAct.provision_of_the_act_id.notin_([4, 5]), + ), + ), + ) + + # Construct the main query using the above conditions. + query = ( + select( + FuelType.fuel_type_id, + FuelInstance.fuel_instance_id, + EndUseType.end_use_type_id, + FuelType.fuel_type, + FuelType.fossil_derived, + FuelCategory.fuel_category_id, + FuelCategory.category, + DefaultCarbonIntensity.default_carbon_intensity, + CategoryCarbonIntensity.category_carbon_intensity, + EnergyDensity.energy_density_id, + EnergyDensity.density.label("energy_density"), + FuelType.units.label("unit"), + FuelType.unrecognized, + UnitOfMeasure.uom_id, + UnitOfMeasure.name, + func.coalesce( + array_agg( + distinct( + func.jsonb_build_object( + "provision_of_the_act_id", + ProvisionOfTheAct.provision_of_the_act_id, + "name", + ProvisionOfTheAct.name, + ) + ) + ).filter(ProvisionOfTheAct.provision_of_the_act_id.is_not(None)), + [], + ).label("provisions"), + func.coalesce( + array_agg( + distinct( + func.jsonb_build_object( + "fuel_code_id", + FuelCode.fuel_code_id, + "fuel_code", + FuelCodePrefix.prefix + FuelCode.fuel_suffix, + "fuel_code_prefix_id", + FuelCodePrefix.fuel_code_prefix_id, + "fuel_code_carbon_intensity", + FuelCode.carbon_intensity, + ) + ) + ).filter(FuelCode.fuel_code_id.is_not(None)), + [], + ).label("fuel_codes"), + func.coalesce( + array_agg( + distinct( + func.jsonb_build_object( + "eer_id", + EnergyEffectivenessRatio.eer_id, + "energy_effectiveness_ratio", + func.coalesce(EnergyEffectivenessRatio.ratio, 1), + "end_use_type_id", + EndUseType.end_use_type_id, + "end_use_type", + EndUseType.type, + "end_use_sub_type", + EndUseType.sub_type, + ) + ) + ).filter(EnergyEffectivenessRatio.eer_id.is_not(None)), + [], + ).label("eers"), + func.coalesce( + array_agg( + distinct( + func.jsonb_build_object( + "target_carbon_intensity_id", + TargetCarbonIntensity.target_carbon_intensity_id, + "target_carbon_intensity", + TargetCarbonIntensity.target_carbon_intensity, + "reduction_target_percentage", + TargetCarbonIntensity.reduction_target_percentage, + ) + ) + ).filter( + TargetCarbonIntensity.target_carbon_intensity_id.is_not(None) + ), + [], + ).label("target_carbon_intensities"), + ) + .join( + FuelInstance, + and_( + FuelInstance.fuel_type_id == FuelType.fuel_type_id, + fuel_instance_condition, + ), + ) + .join( + FuelCategory, + and_( + FuelCategory.fuel_category_id == FuelInstance.fuel_category_id, + fuel_category_condition, + ), + ) + .outerjoin( + DefaultCarbonIntensity, + and_( + DefaultCarbonIntensity.fuel_type_id == FuelType.fuel_type_id, + DefaultCarbonIntensity.compliance_period_id + == subquery_compliance_period_id, + ), + ) + .outerjoin( + CategoryCarbonIntensity, + and_( + CategoryCarbonIntensity.fuel_category_id + == FuelCategory.fuel_category_id, + CategoryCarbonIntensity.compliance_period_id + == subquery_compliance_period_id, + ), + ) + .outerjoin( + ProvisionOfTheAct, + and_( + ProvisionOfTheAct.name != "Unknown", + provision_condition, + ), + ) + .outerjoin( + EnergyDensity, EnergyDensity.fuel_type_id == FuelType.fuel_type_id + ) + .outerjoin(UnitOfMeasure, UnitOfMeasure.uom_id == EnergyDensity.uom_id) + .outerjoin( + EnergyEffectivenessRatio, + and_( + EnergyEffectivenessRatio.fuel_category_id + == FuelCategory.fuel_category_id, + EnergyEffectivenessRatio.fuel_type_id == FuelInstance.fuel_type_id, + EnergyEffectivenessRatio.compliance_period_id + == subquery_compliance_period_id, + ), + ) + .outerjoin( + EndUseType, + EndUseType.end_use_type_id == EnergyEffectivenessRatio.end_use_type_id, + ) + .outerjoin( + TargetCarbonIntensity, + and_( + TargetCarbonIntensity.fuel_category_id + == FuelCategory.fuel_category_id, + TargetCarbonIntensity.compliance_period_id + == subquery_compliance_period_id, + ), + ) + .outerjoin( + FuelCode, + and_( + FuelCode.fuel_type_id == FuelType.fuel_type_id, + FuelCode.fuel_status_id == subquery_fuel_code_status_id, + FuelCode.expiration_date >= start_of_compliance_year, + FuelCode.effective_date <= end_of_compliance_year, + ), + ) + .outerjoin( + FuelCodePrefix, FuelCodePrefix.fuel_code_prefix_id == FuelCode.prefix_id + ) + .group_by( + FuelType.fuel_type_id, + FuelInstance.fuel_instance_id, + EndUseType.end_use_type_id, + FuelType.fuel_type, + FuelType.fossil_derived, + FuelCategory.fuel_category_id, + FuelCategory.category, + DefaultCarbonIntensity.default_carbon_intensity, + CategoryCarbonIntensity.category_carbon_intensity, + EnergyDensity.energy_density_id, + EnergyDensity.density, + FuelType.units, + FuelType.unrecognized, + UnitOfMeasure.uom_id, + UnitOfMeasure.name, + ) + .order_by(FuelType.fuel_type_id, FuelType.fuel_type) + ) + + fuel_type_results = (await self.db.execute(query)).all() + + return { + "fuel_types": fuel_type_results, + } diff --git a/backend/lcfs/web/api/fuel_supply/schema.py b/backend/lcfs/web/api/fuel_supply/schema.py index c3e2f2e6b..12be21b1a 100644 --- a/backend/lcfs/web/api/fuel_supply/schema.py +++ b/backend/lcfs/web/api/fuel_supply/schema.py @@ -182,9 +182,10 @@ class FuelSupplyResponseSchema(FuelSupplyCreateUpdateSchema): action_type: str fuel_type: str fuel_category: str - end_use_type: str - provision_of_the_act: str = None - compliance_units: int = None + end_use_id: Optional[int] = None + end_use_type: Optional[str] = None + provision_of_the_act: Optional[str] = None + compliance_units: Optional[int] = None fuel_code: Optional[str] uci: Optional[float] = None diff --git a/backend/lcfs/web/api/fuel_supply/services.py b/backend/lcfs/web/api/fuel_supply/services.py index f0d0c8878..ba5d164e8 100644 --- a/backend/lcfs/web/api/fuel_supply/services.py +++ b/backend/lcfs/web/api/fuel_supply/services.py @@ -261,16 +261,32 @@ def map_entity_to_schema(self, fuel_supply: FuelSupply): fuel_type_other=fuel_supply.fuel_type_other, ci_of_fuel=fuel_supply.ci_of_fuel, end_use_id=fuel_supply.end_use_id, - provision_of_the_act=fuel_supply.provision_of_the_act.name, + provision_of_the_act=( + fuel_supply.provision_of_the_act.name + if fuel_supply.provision_of_the_act + else None + ), provision_of_the_act_id=fuel_supply.provision_of_the_act_id, - fuel_type=fuel_supply.fuel_type.fuel_type, - fuel_category=fuel_supply.fuel_category.category, + fuel_type=( + fuel_supply.fuel_type.fuel_type if fuel_supply.fuel_type else None + ), + fuel_category=( + fuel_supply.fuel_category.category + if fuel_supply.fuel_category + else None + ), fuel_code_id=fuel_supply.fuel_code_id, fuel_category_id=fuel_supply.fuel_category_id, fuel_supply_id=fuel_supply.fuel_supply_id, action_type=fuel_supply.action_type, - compliance_units=round(fuel_supply.compliance_units), - end_use_type=fuel_supply.end_use_type.type, + compliance_units=( + round(fuel_supply.compliance_units) + if fuel_supply.compliance_units is not None + else None + ), + end_use_type=( + fuel_supply.end_use_type.type if fuel_supply.end_use_type else None + ), target_ci=fuel_supply.target_ci, version=fuel_supply.version, quantity=fuel_supply.quantity, diff --git a/backend/lcfs/web/api/notional_transfer/services.py b/backend/lcfs/web/api/notional_transfer/services.py index 75cba48f3..5019c7aa3 100644 --- a/backend/lcfs/web/api/notional_transfer/services.py +++ b/backend/lcfs/web/api/notional_transfer/services.py @@ -77,7 +77,9 @@ def model_to_schema(self, model: NotionalTransfer) -> NotionalTransferSchema: q4_quantity=model.q4_quantity, legal_name=model.legal_name, address_for_service=model.address_for_service, - fuel_category=model.fuel_category.category, + fuel_category=( + model.fuel_category.category if model.fuel_category else None + ), is_canada_produced=model.is_canada_produced, is_q1_supplied=model.is_q1_supplied, received_or_transferred=model.received_or_transferred, diff --git a/backend/lcfs/web/api/organizations/repo.py b/backend/lcfs/web/api/organizations/repo.py index 4e49c51e3..008bbae5a 100644 --- a/backend/lcfs/web/api/organizations/repo.py +++ b/backend/lcfs/web/api/organizations/repo.py @@ -175,12 +175,18 @@ async def create_organization(self, org_model: Organization): await self.db.flush() await self.db.refresh(org_model) - # Add year-based early issuance for current year before validation - org_model.has_early_issuance = await self.get_current_year_early_issuance( + # Get early issuance status for current year + has_early_issuance = await self.get_current_year_early_issuance( org_model.organization_id ) - return OrganizationResponseSchema.model_validate(org_model) + # Create response with early issuance data + org_data = { + **{column.name: getattr(org_model, column.name) for column in org_model.__table__.columns}, + "has_early_issuance": has_early_issuance + } + + return OrganizationResponseSchema.model_validate(org_data) @repo_handler async def get_organization(self, organization_id: int) -> Organization: @@ -207,12 +213,18 @@ async def update_organization(self, organization: Organization) -> Organization: await self.db.flush() await self.db.refresh(organization) - # Add year-based early issuance for current year before validation - organization.has_early_issuance = await self.get_current_year_early_issuance( + # Get early issuance status for current year + has_early_issuance = await self.get_current_year_early_issuance( organization.organization_id ) - return OrganizationResponseSchema.model_validate(organization) + # Create response with early issuance data + org_data = { + **{column.name: getattr(organization, column.name) for column in organization.__table__.columns}, + "has_early_issuance": has_early_issuance + } + + return OrganizationResponseSchema.model_validate(org_data) def add(self, entity: BaseModel): self.db.add(entity) @@ -365,11 +377,18 @@ async def get_organizations_paginated(self, offset, limit, conditions, paginatio # Add year-based early issuance for current year to each organization validated_organizations = [] for organization in organizations: - organization.has_early_issuance = ( - await self.get_current_year_early_issuance(organization.organization_id) - ) + has_early_issuance = await self.get_current_year_early_issuance(organization.organization_id) + + # Create organization data with early issuance and relationships + org_data = { + **{column.name: getattr(organization, column.name) for column in organization.__table__.columns}, + "has_early_issuance": has_early_issuance, + "org_type": organization.org_type, + "org_status": organization.org_status + } + validated_organizations.append( - OrganizationSchema.model_validate(organization) + OrganizationSchema.model_validate(org_data) ) return validated_organizations, total_count diff --git a/backend/lcfs/web/api/organizations/schema.py b/backend/lcfs/web/api/organizations/schema.py index cb76e7349..253c1cf25 100644 --- a/backend/lcfs/web/api/organizations/schema.py +++ b/backend/lcfs/web/api/organizations/schema.py @@ -157,8 +157,8 @@ class OrganizationBase(BaseSchema): class OrganizationSchema(OrganizationBase): organization_address_id: Optional[int] = None organization_attorney_address_id: Optional[int] = None - org_type: Optional[OrganizationTypeSchema] = [] - org_status: Optional[OrganizationStatusSchema] = [] + org_type: Optional[OrganizationTypeSchema] = None + org_status: Optional[OrganizationStatusSchema] = None class OrganizationListSchema(BaseSchema): @@ -230,8 +230,8 @@ class OrganizationUpdateSchema(BaseSchema): credit_market_is_buyer: Optional[bool] = False credits_to_sell: Optional[int] = 0 display_in_credit_market: Optional[bool] = False - address: Optional[OrganizationAddressCreateSchema] = [] - attorney_address: Optional[OrganizationAttorneyAddressCreateSchema] = [] + address: Optional[OrganizationAddressCreateSchema] = None + attorney_address: Optional[OrganizationAttorneyAddressCreateSchema] = None # Update schema for non-BCeID organization types with relaxed validation @@ -282,11 +282,11 @@ class OrganizationResponseSchema(BaseSchema): company_acting_as_aggregator: Optional[str] = None company_additional_notes: Optional[str] = None organization_type_id: Optional[int] = None - org_status: Optional[OrganizationStatusSchema] = [] - org_type: Optional[OrganizationTypeSchema] = [] + org_status: Optional[OrganizationStatusSchema] = None + org_type: Optional[OrganizationTypeSchema] = None records_address: Optional[str] = None - org_address: Optional[OrganizationAddressSchema] = [] - org_attorney_address: Optional[OrganizationAttorneyAddressSchema] = [] + org_address: Optional[OrganizationAddressSchema] = None + org_attorney_address: Optional[OrganizationAttorneyAddressSchema] = None class OrganizationSummaryResponseSchema(BaseSchema): diff --git a/backend/lcfs/web/api/other_uses/services.py b/backend/lcfs/web/api/other_uses/services.py index d756924de..ccfaa602a 100644 --- a/backend/lcfs/web/api/other_uses/services.py +++ b/backend/lcfs/web/api/other_uses/services.py @@ -102,14 +102,18 @@ def model_to_schema(self, model: OtherUses): quantity_supplied=model.quantity_supplied, rationale=model.rationale, units=model.units, - fuel_type=model.fuel_type.fuel_type, - fuel_category=model.fuel_category.category, - provision_of_the_act=model.provision_of_the_act.name, + fuel_type=(model.fuel_type.fuel_type if model.fuel_type else None), + fuel_category=( + model.fuel_category.category if model.fuel_category else None + ), + provision_of_the_act=( + model.provision_of_the_act.name if model.provision_of_the_act else None + ), fuel_code=model.fuel_code.fuel_code if model.fuel_code else None, is_canada_produced=model.is_canada_produced, is_q1_supplied=model.is_q1_supplied, ci_of_fuel=model.ci_of_fuel, - expected_use=model.expected_use.name, + expected_use=(model.expected_use.name if model.expected_use else None), group_uuid=model.group_uuid, version=model.version, action_type=model.action_type, diff --git a/backend/lcfs/web/application.py b/backend/lcfs/web/application.py index 49f193460..6c0072153 100644 --- a/backend/lcfs/web/application.py +++ b/backend/lcfs/web/application.py @@ -28,6 +28,7 @@ from lcfs.web.exception.exception_handler import ( validation_error_exception_handler_no_details, validation_exception_handler, + global_exception_handler, ) from lcfs.web.lifetime import register_shutdown_event, register_startup_event @@ -41,6 +42,13 @@ "https://lowcarbonfuels.gov.bc.ca", ] +# Regex pattern to match dev environment PR subdomains (e.g., lcfs-dev-3006) +import re + +dev_origin_pattern = re.compile( + r"^https://lcfs-dev-\d+\.apps\.silver\.devops\.gov\.bc\.ca$" +) + class MiddlewareExceptionWrapper(BaseHTTPMiddleware): """ @@ -56,9 +64,11 @@ async def dispatch(self, request: Request, call_next): content={"status": exc.status_code, "detail": exc.detail}, ) - # Check if the request origin is in the allowed origins + # Check if the request origin is in the allowed origins or matches dev pattern request_origin = request.headers.get("origin") - if request_origin in origins: + if request_origin in origins or ( + request_origin and dev_origin_pattern.match(request_origin) + ): response.headers["Access-Control-Allow-Origin"] = request_origin return response @@ -150,7 +160,8 @@ def get_app() -> FastAPI: # Set up CORS middleware options app.add_middleware( CORSMiddleware, - allow_origins=origins, # Allows all origins from localhost:3000 + allow_origins=origins, # Allows specific origins + allow_origin_regex=r"https://lcfs-dev-\d+\.apps\.silver\.devops\.gov\.bc\.ca", # Allows dev PR subdomains allow_credentials=True, allow_methods=["*"], # Allows all methods allow_headers=["*"], # Allows all headers @@ -183,21 +194,3 @@ def get_app() -> FastAPI: app.include_router(router=api_router, prefix="/api") return app - - -async def global_exception_handler(request: Request, exc: Exception): - """Handle uncaught exceptions.""" - logger = structlog.get_logger(__name__) - logger.error( - "Unhandled exception", - error=str(exc), - exc_info=True, - request_url=str(request.url), - method=request.method, - headers=dict(request.headers), - correlation_id=correlation_id_var.get(), - ) - return JSONResponse( - status_code=500, - content={"detail": "Internal Server Error"}, - ) diff --git a/backend/lcfs/web/exception/exception_handler.py b/backend/lcfs/web/exception/exception_handler.py index 46f7e3096..9f40499e9 100644 --- a/backend/lcfs/web/exception/exception_handler.py +++ b/backend/lcfs/web/exception/exception_handler.py @@ -1,8 +1,10 @@ +import structlog from fastapi import HTTPException from fastapi.exceptions import RequestValidationError from lcfs.web.exception.exceptions import ValidationErrorException from starlette.requests import Request from starlette.responses import JSONResponse +from lcfs.logging_config import correlation_id_var async def validation_exception_handler(request: Request, exc: RequestValidationError): @@ -20,9 +22,27 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE }, ) -async def validation_error_exception_handler_no_details(request: Request, exc: ValidationErrorException): + +async def validation_error_exception_handler_no_details( + request: Request, exc: ValidationErrorException +): """Handler for ValidationErrorException that returns content without 'detail' wrapping""" + return JSONResponse(status_code=422, content=exc.errors) + + +async def global_exception_handler(request: Request, exc: Exception): + """Handle uncaught exceptions.""" + logger = structlog.get_logger(__name__) + logger.error( + "Unhandled exception", + error=str(exc), + exc_info=True, + request_url=str(request.url), + method=request.method, + headers=dict(request.headers), + correlation_id=correlation_id_var.get(), + ) return JSONResponse( - status_code=422, - content=exc.errors + status_code=500, + content={"detail": "Internal Server Error"}, ) diff --git a/backend/lcfs/web/utils/calculations.py b/backend/lcfs/web/utils/calculations.py index 06fcee7aa..ed3fee123 100644 --- a/backend/lcfs/web/utils/calculations.py +++ b/backend/lcfs/web/utils/calculations.py @@ -1,19 +1,26 @@ def calculate_compliance_units( - TCI: float, EER: float, RCI: float, UCI: float, Q: float, ED: float + TCI: float, + EER: float, + RCI: float, + UCI: float, + Q: float, + ED: float, + is_historical: bool = False, ) -> float: """ - Calculate the compliance units using the fuel supply formula. + Calculate the compliance units using the appropriate formula based on the compliance period. Parameters: - - TCI: Target Carbon Intensity + - TCI: Target Carbon Intensity (ci_limit in TFRS) - EER: Energy Efficiency Ratio - - RCI: Recorded Carbon Intensity - - UCI: Additional Carbon Intensity Attributable to Use + - RCI: Recorded Carbon Intensity (effective_carbon_intensity in TFRS) + - UCI: Additional Carbon Intensity Attributable to Use (only used in new calculation) - Q: Quantity of Fuel Supplied - ED: Energy Density + - is_historical: Whether to use pre-2024 calculation method Returns: - - The calculated compliance units as a rounded integer. + - The calculated compliance units as a rounded float. """ # Ensure all inputs are floats TCI = float(TCI) @@ -23,12 +30,19 @@ def calculate_compliance_units( Q = float(Q) ED = float(ED) - # Perform the calculation - compliance_units = (TCI * EER - (RCI + UCI)) * ((Q * ED) / 1_000_000) + if is_historical: + # Pre-2024 calculation (TFRS method) + # Credit or Debit = (CI class × EER fuel – CI fuel) × EC fuel/1 000 000 + energy_content = Q * ED + compliance_units = (TCI * EER - RCI) * (energy_content / 1_000_000) + else: + # Post-2024 calculation (LCFS method) + compliance_units = (TCI * EER - (RCI + UCI)) * ((Q * ED) / 1_000_000) - # Return the rounded integer + # Return the rounded float return round(compliance_units, 5) + def calculate_legacy_compliance_units( TCI: float, EER: float, RCI: float, Q: float, ED: float ) -> float: @@ -40,7 +54,7 @@ def calculate_legacy_compliance_units( - EER: Energy Efficiency Ratio - RCI: Carbon Intensity of the fuel - Q: Quantity of Fuel Supplied -- ED: Energy Density + - ED: Energy Density Returns: - The calculated compliance units as a rounded integer. diff --git a/etl/data-transfer-enhanced.sh b/etl/data-transfer-enhanced.sh new file mode 100755 index 000000000..5a674e1b4 --- /dev/null +++ b/etl/data-transfer-enhanced.sh @@ -0,0 +1,451 @@ +#!/bin/bash +set -e + +# Enhanced PostgreSQL data transfer script with verification and reliability features +# +# Expected parameters: +# $1 = 'tfrs' or 'lcfs' (application) +# $2 = 'test', 'prod', 'dev', or custom pod name (environment/pod) +# $3 = 'import' or 'export' (direction of data transfer) +# $4 = local container name or id +# $5 = (optional) table name to dump (e.g., compliance_report_history) +# +# Example commands: +# . data-transfer-enhanced.sh lcfs dev export 398cd4661173 compliance_report_history +# . data-transfer-enhanced.sh lcfs lcfs-postgres-dev-2983-postgresql-0 export 398cd4661173 +# . data-transfer-enhanced.sh tfrs prod import 398cd4661173 + +if [ "$#" -lt 4 ] || [ "$#" -gt 5 ]; then + echo "Passed $# parameters. Expected 4 or 5." + echo "Usage: $0 []" + echo "Where:" + echo " is 'tfrs' or 'lcfs'" + echo " is 'test', 'prod', 'dev', or a specific pod name" + echo " is 'import' or 'export'" + echo " is the name or id of your local Docker container" + echo "
(optional) is the table name to dump (e.g., compliance_report_history)" + exit 1 +fi + +application=$1 +env_or_pod=$2 +direction=$3 +local_container=$4 + +# Optional parameter: table name +table="" +if [ "$#" -eq 5 ]; then + table=$5 +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Function to print colored messages +print_status() { + echo -e "${GREEN}** $1${NC}" +} + +print_error() { + echo -e "${RED}ERROR: $1${NC}" +} + +print_warning() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +# Validate direction +if [ "$direction" != "import" ] && [ "$direction" != "export" ]; then + print_error "Invalid direction. Use 'import' or 'export'." + exit 1 +fi + +# Check if the operation is supported +if [ "$application" = "tfrs" ] && [ "$direction" = "export" ]; then + print_error "Export operation is not supported for the TFRS application." + exit 1 +fi + +# Check if you are logged in to OpenShift +print_status "Checking Openshift creds" +oc whoami +echo "logged in" +echo + +# Check if this is a custom pod name (contains hyphens and doesn't match standard env names) +if [[ "$env_or_pod" =~ ^[a-zA-Z0-9\-]+$ ]] && [[ "$env_or_pod" != "test" ]] && [[ "$env_or_pod" != "prod" ]] && [[ "$env_or_pod" != "dev" ]]; then + # Custom pod name provided + custom_pod_name="pod/$env_or_pod" + + # Determine project from current context or pod name + if [[ "$env_or_pod" == *"prod"* ]]; then + env="prod" + elif [[ "$env_or_pod" == *"dev"* ]]; then + env="dev" + else + env="test" + fi +else + # Standard environment name + env="$env_or_pod" + custom_pod_name="" +fi + +# Set project, app label, database name, and credentials +case $application in + "tfrs") + project_name="0ab226-$env" + if [ "$env" = "prod" ]; then + app_label="tfrs-crunchy-prod-tfrs" + else + app_label="tfrs-spilo" + fi + db_name="tfrs" + remote_db_user="postgres" + local_db_user="tfrs" + ;; + "lcfs") + project_name="d2bd59-$env" + app_label="lcfs-crunchy-$env-lcfs" + db_name="lcfs" + if [ "$env" = "prod" ]; then + remote_db_user="postgres" + local_db_user="lcfs" + else + remote_db_user="postgres" + local_db_user="lcfs" + fi + ;; + *) + print_error "Invalid application. Use 'tfrs' or 'lcfs'." + exit 1 + ;; +esac + +print_status "Setting project $project_name" +oc project $project_name +echo + +# Function to get the leader pod for Crunchy Data PostgreSQL clusters +get_leader_pod() { + local project=$1 + local app_label=$2 + + # Get all pods with the given app label + pods=$(oc get pods -n $project_name -o name | grep "$app_label") + + # Loop through pods to find the leader (using remote_db_user) + for pod in $pods; do + is_leader=$(oc exec -n $project $pod -- bash -c "psql -U $remote_db_user -tAc \"SELECT pg_is_in_recovery()\"") + if [ "$is_leader" = "f" ]; then + echo $pod + return + fi + done + + echo "No leader pod found" + exit 1 +} + +# Get the appropriate pod +if [ -n "$custom_pod_name" ]; then + pod_name="$custom_pod_name" + print_status "Using custom pod: $pod_name" + + # Verify the custom pod exists and is accessible + if ! oc get $pod_name >/dev/null 2>&1; then + print_error "Custom pod $pod_name not found or not accessible" + exit 1 + fi + + # Auto-detect database credentials for custom pods + print_status "Auto-detecting database credentials..." + pod_env=$(oc exec $pod_name -- env 2>/dev/null | grep -E "POSTGRES_USER|POSTGRES_PASSWORD|POSTGRES_DATABASE" || true) + + if echo "$pod_env" | grep -q "POSTGRES_PASSWORD="; then + detected_password=$(echo "$pod_env" | grep "POSTGRES_PASSWORD=" | cut -d'=' -f2 || echo "") + detected_database=$(echo "$pod_env" | grep "POSTGRES_DATABASE=" | cut -d'=' -f2 || echo "$db_name") + + # For custom pods, use 'postgres' as the default superuser with detected password + if [ -n "$detected_password" ]; then + detected_user=$(echo "$pod_env" | grep "POSTGRES_USER=" | cut -d'=' -f2 || echo "postgres") + remote_db_user="$detected_user" # Use the detected user instead of hardcoded "postgres" + # remote_db_user="postgres" + remote_db_password="$detected_password" + db_name="$detected_database" + print_status "Detected credentials - User: $remote_db_user, Database: $db_name, Password: [HIDDEN]" + fi + fi +else + pod_name=$(get_leader_pod $project_name $app_label) + if [ -z "$pod_name" ]; then + print_error "No leader pod identified." + exit 1 + fi + print_status "Leader pod identified: $pod_name" +fi + +# Set up table option for pg_dump if a table name is provided. +table_option="" +file_suffix="$db_name" +if [ -n "$table" ]; then + table_option="-t $table" + file_suffix="${db_name}_${table}" +fi + +# Function to verify tar file integrity +verify_tar_file() { + local tar_file=$1 + local expected_min_size=${2:-1000000} # Default 1MB minimum + + if [ ! -f "$tar_file" ]; then + print_error "Tar file $tar_file does not exist" + return 1 + fi + + local file_size=$(stat -f%z "$tar_file" 2>/dev/null || stat -c%s "$tar_file" 2>/dev/null) + + if [ "$file_size" -lt "$expected_min_size" ]; then + print_warning "Tar file seems too small: $file_size bytes" + return 1 + fi + + # Check if tar file is valid + if ! tar -tf "$tar_file" >/dev/null 2>&1; then + print_error "Tar file is corrupted or invalid" + return 1 + fi + + print_status "Tar file verified: $file_size bytes" + return 0 +} + +# Function to get database size estimate +get_db_size_estimate() { + local size_query="SELECT pg_database_size('$db_name')" + if [ -n "$table" ]; then + size_query="SELECT pg_total_relation_size('$table')" + fi + + local size_bytes=$(oc exec $pod_name -- bash -c "psql -U $remote_db_user -tAc \"$size_query\" $db_name" 2>/dev/null || echo "0") + echo $size_bytes +} + +# Function to perform database dump with retry logic +dump_with_retry() { + local max_retries=3 + local retry_count=0 + local dump_file="${file_suffix}.tar" + + # Get size estimate + local expected_size=$(get_db_size_estimate) + local min_expected_size=$((expected_size / 10)) # Allow for compression, expect at least 10% of original + + print_status "Expected database/table size: $(numfmt --to=iec-i --suffix=B $expected_size 2>/dev/null || echo "$expected_size bytes")" + + while [ $retry_count -lt $max_retries ]; do + retry_count=$((retry_count + 1)) + print_status "Dump attempt $retry_count of $max_retries" + + # Method 1: Direct streaming with progress monitoring + if [ $retry_count -eq 1 ]; then + print_status "Using direct streaming method..." + oc exec $pod_name -- bash -c "pg_dump -U $remote_db_user $table_option -F t --no-privileges --no-owner -c -d $db_name" > "$dump_file" + + # Method 2: Dump to pod first, then download + elif [ $retry_count -eq 2 ]; then + print_status "Using two-stage method (dump to pod, then download)..." + oc exec $pod_name -- bash -c "pg_dump -U $remote_db_user $table_option -F t --no-privileges --no-owner -c -d $db_name > /tmp/${file_suffix}.tar" + + # Check file size on pod + remote_size=$(oc exec $pod_name -- bash -c "stat -c%s /tmp/${file_suffix}.tar 2>/dev/null || echo 0") + print_status "Remote file size: $(numfmt --to=iec-i --suffix=B $remote_size 2>/dev/null || echo "$remote_size bytes")" + + # Download with oc exec cat + oc exec $pod_name -- cat /tmp/${file_suffix}.tar > "$dump_file" + + # Clean up remote file + oc exec $pod_name -- rm -f /tmp/${file_suffix}.tar + + # Method 3: Use base64 encoding for more reliable transfer + else + print_status "Using base64 encoded transfer method..." + oc exec $pod_name -- bash -c "pg_dump -U $remote_db_user $table_option -F t --no-privileges --no-owner -c -d $db_name | base64 -w0" | base64 -d > "$dump_file" + fi + + # Verify the downloaded file + if verify_tar_file "$dump_file" "$min_expected_size"; then + print_status "Download successful!" + return 0 + else + print_warning "Download verification failed, retrying..." + rm -f "$dump_file" + fi + done + + print_error "Failed to download database dump after $max_retries attempts" + return 1 +} + +# Function to verify restoration success +verify_restoration() { + local container=$1 + local user=$2 + local database=$3 + + print_status "Verifying database restoration..." + + # Count tables + local table_count=$(docker exec $container psql -U $user -d $database -tAc "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'") + print_status "Tables restored: $table_count" + + # Count total rows across major tables + local total_rows=$(docker exec $container psql -U $user -d $database -tAc " + SELECT SUM(n_live_tup) + FROM pg_stat_user_tables + WHERE schemaname = 'public' + " 2>/dev/null || echo "0") + + print_status "Total rows across all tables: $(numfmt --to=si $total_rows 2>/dev/null || echo "$total_rows")" + + # Check for specific important tables + for important_table in compliance_report organization user_profile; do + if docker exec $container psql -U $user -d $database -tAc "SELECT 1 FROM information_schema.tables WHERE table_name = '$important_table'" | grep -q 1; then + local row_count=$(docker exec $container psql -U $user -d $database -tAc "SELECT COUNT(*) FROM $important_table") + print_status "Table $important_table: $row_count rows" + fi + done +} + +if [ "$direction" = "import" ]; then + print_status "Starting database import process..." + + # Perform dump with retry logic + if ! dump_with_retry; then + exit 1 + fi + + print_status "Copying .tar to local database container $local_container" + docker cp ${file_suffix}.tar $local_container:/tmp/${file_suffix}.tar + + print_status "Starting database restoration (this may take several minutes)..." + print_warning "You may see some errors during restoration - this is normal when importing into an existing database" + + # Run restore with detailed error logging + docker exec $local_container bash -c " + pg_restore -U $local_db_user --dbname=$db_name --no-owner --clean --if-exists --verbose /tmp/${file_suffix}.tar 2>&1 | + tee /tmp/restore_log.txt | + grep -E '(ERROR|WARNING|restored|processing|creating)' || true + " || true + + # Check for critical errors + if docker exec $local_container grep -q "ERROR: could not access file" /tmp/restore_log.txt 2>/dev/null; then + print_error "Critical errors detected during restoration" + fi + + # Verify restoration + verify_restoration $local_container $local_db_user $db_name + + print_status "Cleaning up temporary files..." + docker exec $local_container rm -f /tmp/${file_suffix}.tar /tmp/restore_log.txt + rm -f ${file_suffix}.tar + +elif [ "$direction" = "export" ]; then + print_status "Starting database export process..." + + # Show export details and ask for confirmation + echo + echo "==========================================" + echo "EXPORT CONFIRMATION" + echo "==========================================" + echo "Source (Local):" + echo " Container: $local_container" + echo " Database: $db_name" + echo " User: $local_db_user" + if [ -n "$table" ]; then + echo " Table: $table" + else + echo " Scope: Full database" + fi + echo + echo "Destination (OpenShift):" + echo " Project: $project_name" + echo " Pod: $pod_name" + echo " Database: $db_name" + echo " User: $remote_db_user" + echo + print_warning "This will OVERWRITE the destination database!" + echo + read -p "Do you want to proceed with this export? (yes/no): " confirm + + if [[ "$confirm" != "yes" ]] && [[ "$confirm" != "y" ]] && [[ "$confirm" != "Y" ]]; then + print_status "Export cancelled by user" + exit 0 + fi + + print_status "Export confirmed. Proceeding..." + echo + + print_status "Creating pg_dump on local container (using local user $local_db_user)" + docker exec $local_container bash -c "pg_dump -U $local_db_user $table_option -F t --no-privileges --no-owner -c -d $db_name > /tmp/${file_suffix}.tar" + + print_status "Copying .tar file from local container" + docker cp $local_container:/tmp/${file_suffix}.tar ./ + + # Verify the exported file + if ! verify_tar_file "${file_suffix}.tar"; then + print_error "Export verification failed" + exit 1 + fi + + print_status "Preparing .tar file for OpenShift pod" + mkdir -p tmp_transfer + mv ${file_suffix}.tar tmp_transfer/ + + print_status "Uploading .tar file to OpenShift pod" + # First create the directory on the pod + oc exec $pod_name -- mkdir -p /tmp/tmp_transfer + + # Upload the file + oc cp ./tmp_transfer/${file_suffix}.tar ${pod_name#pod/}:/tmp/tmp_transfer/${file_suffix}.tar + + print_status "Restoring database on OpenShift pod (using remote user $remote_db_user)" + # Use password if detected + if [ -n "$remote_db_password" ]; then + print_status "Using detected password for authentication" + oc exec ${pod_name#pod/} -- bash -c "PGPASSWORD='$remote_db_password' pg_restore -h localhost -p 5432 -U '$remote_db_user' --dbname='$db_name' --no-owner --clean --if-exists --verbose '/tmp/tmp_transfer/${file_suffix}.tar'" || true + else + oc exec ${pod_name#pod/} -- pg_restore -U "$remote_db_user" --dbname="$db_name" --no-owner --clean --if-exists --verbose "/tmp/tmp_transfer/${file_suffix}.tar" || true + fi + + print_status "Cleaning up temporary files on OpenShift pod" + oc exec ${pod_name#pod/} -- bash -c "rm -rf /tmp/tmp_transfer" + + print_status "Cleaning up dump file from local container" + docker exec $local_container bash -c "rm -f /tmp/${file_suffix}.tar" || true + + print_status "Cleaning up local temporary directory" + rm -rf tmp_transfer +fi + +print_status "Data transfer completed successfully!" +print_status "Summary:" +print_status " Application: $application" +print_status " Environment: $env" +print_status " Direction: $direction" +print_status " Database: $db_name" +if [ -n "$table" ]; then + print_status " Table: $table" +fi + +# Provide helpful next steps +if [ "$direction" = "import" ]; then + echo + print_status "Next steps:" + echo "1. Connect to your database: docker exec -it $local_container psql -U $local_db_user -d $db_name" + echo "2. Verify your data: \\dt (list tables), SELECT COUNT(*) FROM ;" + echo "3. Check for any missing data or errors in the restoration" +fi \ No newline at end of file diff --git a/etl/data-transfer.sh b/etl/data-transfer.sh index b8f55c3c8..3c1ab6a33 100755 --- a/etl/data-transfer.sh +++ b/etl/data-transfer.sh @@ -128,12 +128,8 @@ if [ -n "$table" ]; then fi if [ "$direction" = "import" ]; then - echo "** Starting pg_dump on OpenShift pod (using remote user $remote_db_user)" - oc exec $pod_name -- bash -c "pg_dump -U $remote_db_user $table_option -F t --no-privileges --no-owner -c -d $db_name > /tmp/${file_suffix}.tar" - echo - - echo "** Downloading .tar file from OpenShift pod" - oc rsync $pod_name:/tmp/${file_suffix}.tar ./ + echo "** Starting pg_dump on OpenShift pod and streaming directly to local file" + oc exec $pod_name -- bash -c "pg_dump -U $remote_db_user $table_option -F t --no-privileges --no-owner -c -d $db_name" > ${file_suffix}.tar echo echo "** Copying .tar to local database container $local_container" @@ -144,10 +140,6 @@ if [ "$direction" = "import" ]; then docker exec $local_container bash -c "pg_restore -U $local_db_user --dbname=$db_name --no-owner --clean --if-exists --verbose /tmp/${file_suffix}.tar" || true echo - echo "** Cleaning up dump file from OpenShift pod" - oc exec $pod_name -- bash -c "rm /tmp/${file_suffix}.tar" - echo - echo "** Cleaning up local dump file" rm ${file_suffix}.tar @@ -166,7 +158,7 @@ elif [ "$direction" = "export" ]; then echo echo "** Uploading .tar file to OpenShift pod" - oc rsync ./tmp_transfer $pod_name:/tmp/ + oc cp ./tmp_transfer/${file_suffix}.tar ${pod_name#pod/}:/tmp/tmp_transfer/${file_suffix}.tar echo echo "** Restoring database on OpenShift pod (using remote user $remote_db_user)" diff --git a/etl/database/nifi-registry-primary.mv.db b/etl/database/nifi-registry-primary.mv.db index 62fc677f5..0f071adb6 100644 Binary files a/etl/database/nifi-registry-primary.mv.db and b/etl/database/nifi-registry-primary.mv.db differ diff --git a/etl/manual-deploy-to-dev.sh b/etl/manual-deploy-to-dev.sh new file mode 100755 index 000000000..56b7d18ca --- /dev/null +++ b/etl/manual-deploy-to-dev.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# Manual deployment script for LCFS dev database +# This script contains the exact manual commands that work for deploying to a dev pod + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +print_status() { + echo -e "${GREEN}** $1${NC}" +} + +print_error() { + echo -e "${RED}ERROR: $1${NC}" +} + +print_warning() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +# Configuration +LOCAL_CONTAINER="f7091c69420b" +DEV_POD="lcfs-postgres-dev-3006-postgresql-0" +DB_NAME="lcfs" +LOCAL_USER="lcfs" +REMOTE_USER="postgres" +REMOTE_PASSWORD="test" +DUMP_FILE="lcfs-manual-deploy.tar" + +print_status "Manual LCFS Database Deployment to Dev Environment" +echo +echo "Configuration:" +echo " Local Container: $LOCAL_CONTAINER" +echo " Dev Pod: $DEV_POD" +echo " Database: $DB_NAME" +echo " Local User: $LOCAL_USER" +echo " Remote User: $REMOTE_USER" +echo + +# Step 1: Verify local container is running +print_status "Step 1: Verifying local container is running..." +if ! docker ps | grep -q "$LOCAL_CONTAINER"; then + print_error "Local container $LOCAL_CONTAINER is not running" + exit 1 +fi +print_status "Local container is running" + +# Step 2: Verify OpenShift connection and pod access +print_status "Step 2: Verifying OpenShift connection and pod access..." +if ! oc whoami >/dev/null 2>&1; then + print_error "Not logged into OpenShift" + exit 1 +fi + +if ! oc get pod "$DEV_POD" >/dev/null 2>&1; then + print_error "Cannot access pod $DEV_POD" + exit 1 +fi +print_status "OpenShift connection and pod access verified" + +# Step 3: Create database dump from local container +print_status "Step 3: Creating database dump from local container..." +docker exec "$LOCAL_CONTAINER" bash -c "pg_dump -U $LOCAL_USER -F t --no-privileges --no-owner -c -d $DB_NAME > /tmp/$DUMP_FILE" +print_status "Database dump created successfully" + +# Step 4: Copy dump file from local container to host +print_status "Step 4: Copying dump file from container to host..." +docker cp "$LOCAL_CONTAINER:/tmp/$DUMP_FILE" "./$DUMP_FILE" + +# Verify file was copied and get size +if [ ! -f "./$DUMP_FILE" ]; then + print_error "Failed to copy dump file from container" + exit 1 +fi + +file_size=$(stat -f%z "./$DUMP_FILE" 2>/dev/null || stat -c%s "./$DUMP_FILE" 2>/dev/null) +print_status "Dump file copied successfully (Size: $(numfmt --to=iec-i --suffix=B $file_size 2>/dev/null || echo "$file_size bytes"))" + +# Step 5: Upload dump file to OpenShift pod +print_status "Step 5: Uploading dump file to OpenShift pod..." +oc cp "./$DUMP_FILE" "$DEV_POD:/tmp/$DUMP_FILE" +print_status "Dump file uploaded to pod" + +# Step 6: Verify file exists on pod +print_status "Step 6: Verifying file exists on pod..." +if ! oc exec "$DEV_POD" -- ls -la "/tmp/$DUMP_FILE" >/dev/null 2>&1; then + print_error "Dump file not found on pod" + exit 1 +fi +remote_size=$(oc exec "$DEV_POD" -- stat -c%s "/tmp/$DUMP_FILE" 2>/dev/null || echo "unknown") +print_status "File verified on pod (Remote size: $remote_size bytes)" + +# Step 7: Test database connection +print_status "Step 7: Testing database connection..." +if ! oc exec "$DEV_POD" -- bash -c "PGPASSWORD='$REMOTE_PASSWORD' psql -U $REMOTE_USER -d $DB_NAME -c 'SELECT version();'" >/dev/null 2>&1; then + print_error "Cannot connect to database on pod" + exit 1 +fi +print_status "Database connection test successful" + +# Step 8: Show confirmation and get user approval +echo +print_warning "FINAL CONFIRMATION" +echo "==========================================" +echo "About to restore database with the following settings:" +echo " Source: Local container $LOCAL_CONTAINER" +echo " Destination: Pod $DEV_POD" +echo " Database: $DB_NAME" +echo " User: $REMOTE_USER" +echo " File: $DUMP_FILE ($file_size bytes)" +echo +print_warning "This will COMPLETELY REPLACE the database in the dev environment!" +echo +read -p "Type 'DEPLOY' to proceed with the restore: " confirm + +if [ "$confirm" != "DEPLOY" ]; then + print_status "Deployment cancelled by user" + # Cleanup + oc exec "$DEV_POD" -- rm -f "/tmp/$DUMP_FILE" 2>/dev/null || true + rm -f "./$DUMP_FILE" + exit 0 +fi + +# Step 9: Perform database restore +print_status "Step 9: Performing database restore..." +print_status "This may take several minutes and you will see some expected errors..." + +# Run the restore command +oc exec "$DEV_POD" -- bash -c "PGPASSWORD='$REMOTE_PASSWORD' pg_restore -U $REMOTE_USER --dbname=$DB_NAME --no-owner --clean --if-exists --verbose /tmp/$DUMP_FILE" || { + print_warning "Restore completed with some errors (this is normal)" +} + +# Step 10: Verify restore success +print_status "Step 10: Verifying restore success..." + +# Check table count +table_count=$(oc exec "$DEV_POD" -- bash -c "PGPASSWORD='$REMOTE_PASSWORD' psql -U $REMOTE_USER -d $DB_NAME -tAc \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'\"") +print_status "Tables in database: $table_count" + +# Check key table row counts +print_status "Checking key table row counts..." +for table in compliance_report organization transfer_history user_profile fuel_code; do + row_count=$(oc exec "$DEV_POD" -- bash -c "PGPASSWORD='$REMOTE_PASSWORD' psql -U $REMOTE_USER -d $DB_NAME -tAc \"SELECT COUNT(*) FROM $table\" 2>/dev/null" || echo "N/A") + echo " $table: $row_count rows" +done + +# Step 11: Cleanup files +print_status "Step 11: Cleaning up temporary files..." +oc exec "$DEV_POD" -- rm -f "/tmp/$DUMP_FILE" +docker exec "$LOCAL_CONTAINER" -- rm -f "/tmp/$DUMP_FILE" +rm -f "./$DUMP_FILE" +print_status "Cleanup completed" + +# Step 12: Final summary +echo +print_status "DEPLOYMENT COMPLETED SUCCESSFULLY!" +echo "==========================================" +echo "Summary:" +echo " Local database exported from: $LOCAL_CONTAINER" +echo " Deployed to dev pod: $DEV_POD" +echo " Database: $DB_NAME" +echo " Tables restored: $table_count" +echo +print_status "Your local changes are now live in the dev environment!" +echo +print_status "Next steps:" +echo "1. Test your application in the dev environment" +echo "2. Verify your changes are working correctly" +echo "3. Monitor application logs for any issues" + +# Show how to access logs +echo +echo "Useful commands:" +echo " Check pod logs: oc logs $DEV_POD -f" +echo " Connect to database: oc exec $DEV_POD -- bash -c \"PGPASSWORD='$REMOTE_PASSWORD' psql -U $REMOTE_USER -d $DB_NAME\"" +echo " Check pod status: oc get pod $DEV_POD" \ No newline at end of file diff --git a/etl/nifi/conf/flow.json.gz b/etl/nifi/conf/flow.json.gz index be04c1bb1..9ce7957f6 100644 Binary files a/etl/nifi/conf/flow.json.gz and b/etl/nifi/conf/flow.json.gz differ diff --git a/etl/nifi/conf/flow.xml.gz b/etl/nifi/conf/flow.xml.gz index 66613c0f0..1158bfe9b 100644 Binary files a/etl/nifi/conf/flow.xml.gz and b/etl/nifi/conf/flow.xml.gz differ diff --git a/etl/nifi_scripts/allocation_agreement.groovy b/etl/nifi_scripts/allocation_agreement.groovy index cdacd256b..da9427f6e 100644 --- a/etl/nifi_scripts/allocation_agreement.groovy +++ b/etl/nifi_scripts/allocation_agreement.groovy @@ -1,13 +1,12 @@ /* Migrate Allocation Agreements from TFRS to LCFS -1. Finds all LCFS compliance reports having a TFRS legacy_id. -2. For each TFRS compliance report, determines its chain (root_report_id). -3. Retrieves allocation agreement records for each version in the chain. -4. Computes a diff (CREATE / UPDATE) between consecutive versions. -5. Inserts rows in allocation_agreement with a stable, random group_uuid (UUID) per agreement record. -6. Versions these allocation_agreement entries so that subsequent changes increment the version. -7. Logs the actions taken for easier traceability. +Overview: +1. Retrieve LCFS compliance reports that have a TFRS legacy_id. +2. For each LCFS report, use its legacy_id to query the source for allocation agreement records. +3. For each allocation record found, insert a new row into LCFS's allocation_agreement table. +4. A stable group_uuid is generated (or reused) per allocation record, and version is computed. +5. Actions are logged for traceability. */ import groovy.transform.Field @@ -16,412 +15,322 @@ import java.sql.PreparedStatement import java.sql.ResultSet import java.util.UUID -@Field -Map < Integer, String > recordUuidMap = [: ] - -@Field -String SELECT_LCFS_IMPORTED_REPORTS_QUERY = ''' -SELECT compliance_report_id, legacy_id -FROM compliance_report -WHERE legacy_id IS NOT NULL -''' - -@Field -String SELECT_ROOT_REPORT_ID_QUERY = ''' -SELECT root_report_id -FROM compliance_report -WHERE id = ? -''' - -@Field -String SELECT_REPORT_CHAIN_QUERY = ''' -SELECT -c.id AS tfrs_report_id, - c.traversal -FROM compliance_report c -WHERE c.root_report_id = ? - ORDER BY c.traversal, c.id ''' - -@Field -String SELECT_ALLOCATION_AGREEMENTS_QUERY = """ -SELECT -crear.id AS agreement_record_id, - case when tt.the_type = 'Purchased' -then 'Allocated from' -else 'Allocated to' -end as responsibility, -aft.name as fuel_type, - aft.id as fuel_type_id, - crear.transaction_partner, - crear.postal_address, - crear.quantity, - uom.name as units, - crear.quantity_not_sold, - tt.id as transaction_type_id -FROM compliance_report cr -INNER JOIN compliance_report_exclusion_agreement crea -ON cr.exclusion_agreement_id = crea.id -INNER JOIN compliance_report_exclusion_agreement_record crear -ON crear.exclusion_agreement_id = crea.id -INNER JOIN transaction_type tt -ON crear.transaction_type_id = tt.id -INNER JOIN approved_fuel_type aft -ON crear.fuel_type_id = aft.id -INNER JOIN unit_of_measure uom -ON aft.unit_of_measure_id = uom.id -WHERE cr.id = ? - AND cr.exclusion_agreement_id is not null -ORDER BY crear.id """ - -@Field -String SELECT_LCFS_COMPLIANCE_REPORT_BY_TFRSID_QUERY = ''' -SELECT compliance_report_id -FROM compliance_report -WHERE legacy_id = ? ''' - -@Field -String SELECT_CURRENT_VERSION_QUERY = ''' -SELECT version -FROM allocation_agreement -WHERE group_uuid = ? - ORDER BY version DESC -LIMIT 1 ''' - -@Field -String INSERT_ALLOCATION_AGREEMENT_SQL = ''' -INSERT INTO allocation_agreement( - compliance_report_id, - transaction_partner, - postal_address, - quantity, - quantity_not_sold, - units, - allocation_transaction_type_id, - fuel_type_id, - fuel_category_id, - group_uuid, - version, - action_type, - user_type -) VALUES( ? - , ? - , ? - , ? - , ? - , ? - , ? - , ? - , ? - , ? - , ? - , ?::actiontypeenum - , 'SUPPLIER' -) -''' -@Field -Map responsibilityToTransactionTypeCache = [:] - -@Field -Map fuelTypeNameToIdCache = [:] - -@Field -String SELECT_TRANSACTION_TYPE_ID_QUERY = """ +// ------------------------- +// Controller Service Lookups +// ------------------------- +def sourceDbcpService = context.controllerServiceLookup.getControllerService('3245b078-0192-1000-ffff-ffffba20c1eb') +def destinationDbcpService = context.controllerServiceLookup.getControllerService('3244bf63-0192-1000-ffff-ffffc8ec6d93') + +// ------------------------- +// Global Field Declarations & Query Strings +// ------------------------- +@Field Map recordUuidMap = [:] // Maps TFRS allocation agreement record ID to a stable group UUID + +@Field String SELECT_LCFS_IMPORTED_REPORTS_QUERY = """ + SELECT compliance_report_id, legacy_id + FROM compliance_report + WHERE legacy_id IS NOT NULL +""" + +// This query gets allocation agreement records for a given TFRS compliance report (foreign key: cr.id) +@Field String SELECT_ALLOCATION_AGREEMENTS_QUERY = """ + SELECT + crear.id AS agreement_record_id, + CASE WHEN tt.the_type = 'Purchased' THEN 'Allocated from' ELSE 'Allocated to' END AS responsibility, + aft.name AS fuel_type, + aft.id AS fuel_type_id, + crear.transaction_partner, + crear.postal_address, + crear.quantity, + uom.name AS units, + crear.quantity_not_sold, + tt.id AS transaction_type_id + FROM compliance_report legacy_cr -- Alias for the report identified by the legacy_id passed in + -- Find the related report within the same org/period that has the exclusion agreement + INNER JOIN compliance_report exclusion_cr + ON legacy_cr.organization_id = exclusion_cr.organization_id + AND legacy_cr.compliance_period_id = exclusion_cr.compliance_period_id + AND exclusion_cr.exclusion_agreement_id IS NOT NULL + -- Now join from the exclusion report to the agreement tables + INNER JOIN compliance_report_exclusion_agreement crea + ON exclusion_cr.exclusion_agreement_id = crea.id + INNER JOIN compliance_report_exclusion_agreement_record crear + ON crea.id = crear.exclusion_agreement_id -- Corrected join condition here + -- Standard joins for details + INNER JOIN transaction_type tt + ON crear.transaction_type_id = tt.id + INNER JOIN approved_fuel_type aft + ON crear.fuel_type_id = aft.id + INNER JOIN unit_of_measure uom + ON aft.unit_of_measure_id = uom.id + WHERE + legacy_cr.id = ? -- The parameter is the ID of the TFRS report that has the legacy_id + ORDER BY + crear.id; +""" + +// Use this query to get the LCFS compliance report by a legacy (TFRS) ID. +@Field String SELECT_LCFS_COMPLIANCE_REPORT_BY_TFRSID_QUERY = """ + SELECT compliance_report_id + FROM compliance_report + WHERE legacy_id = ? +""" + +// To support versioning we check the current highest version for a given group_uuid. +@Field String SELECT_CURRENT_VERSION_QUERY = """ + SELECT version + FROM allocation_agreement + WHERE group_uuid = ? + ORDER BY version DESC + LIMIT 1 +""" + +// INSERT statement for allocation_agreement. +@Field String INSERT_ALLOCATION_AGREEMENT_SQL = """ + INSERT INTO allocation_agreement( + compliance_report_id, + transaction_partner, + postal_address, + quantity, + quantity_not_sold, + units, + allocation_transaction_type_id, + fuel_type_id, + fuel_category_id, + group_uuid, + version, + action_type, + create_user, + update_user + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::actiontypeenum, ?, ?) +""" + +// ------------------------- +// Reference Lookup Queries & Caches +// ------------------------- +@Field Integer GASOLINE_CATEGORY_ID = 1 +@Field Integer DIESEL_CATEGORY_ID = 2 + +@Field Map responsibilityToTransactionTypeCache = [:] +@Field String SELECT_TRANSACTION_TYPE_ID_QUERY = """ SELECT allocation_transaction_type_id FROM allocation_transaction_type WHERE type = ? """ -@Field -String SELECT_FUEL_TYPE_ID_QUERY = """ +@Field Map fuelTypeNameToIdCache = [:] +@Field String SELECT_FUEL_TYPE_ID_QUERY = """ SELECT fuel_type_id FROM fuel_type WHERE fuel_type = ? """ -// ========================================= -// NiFi Controller Services -// ========================================= -def sourceDbcpService = context.controllerServiceLookup.getControllerService('3245b078-0192-1000-ffff-ffffba20c1eb') -def destinationDbcpService = context.controllerServiceLookup.getControllerService('3244bf63-0192-1000-ffff-ffffc8ec6d93') - -// ========================================= -// Helper Functions -// ========================================= - -/** - * Checks if any relevant fields in an allocation agreement record differ between old and new. - */ -def isRecordChanged(Map oldRow, Map newRow) { - if (oldRow == null || newRow == null) return true - - if (oldRow.quantity?.compareTo(newRow.quantity) != 0) return true - if (oldRow.quantity_not_sold?.compareTo(newRow.quantity_not_sold) != 0) return true - if (oldRow.transaction_type_id != newRow.transaction_type_id) return true - if (oldRow.fuel_type_id != newRow.fuel_type_id) return true - if (oldRow.transaction_partner != newRow.transaction_partner) return true - if (oldRow.postal_address != newRow.postal_address) return true - if (oldRow.units != newRow.units) return true - - return false -} /** - * Gets allocation transaction type ID from cache or database if not cached + * Returns the allocation transaction type ID for a given responsibility. + * Uses an in-memory cache. */ def getTransactionTypeId(Connection destConn, String responsibility) { - // Check if we already have this value cached if (responsibilityToTransactionTypeCache.containsKey(responsibility)) { return responsibilityToTransactionTypeCache[responsibility] } - - // Not in cache, query the database - PreparedStatement stmt = null - ResultSet rs = null + PreparedStatement stmt = destConn.prepareStatement(SELECT_TRANSACTION_TYPE_ID_QUERY) try { - stmt = destConn.prepareStatement(SELECT_TRANSACTION_TYPE_ID_QUERY) stmt.setString(1, responsibility) - rs = stmt.executeQuery() - + ResultSet rs = stmt.executeQuery() if (rs.next()) { int typeId = rs.getInt("allocation_transaction_type_id") - // Cache for future use responsibilityToTransactionTypeCache[responsibility] = typeId return typeId } else { - log.warn("No transaction type found for responsibility: ${responsibility}") - return 1 // Default value + log.warn("No transaction type found for responsibility: ${responsibility}; using default 1.") + return 1 } } finally { - if (rs != null) rs.close() - if (stmt != null) stmt.close() + stmt.close() } } /** - * Gets fuel type ID from cache or database if not cached + * Returns the fuel type ID for a given fuel type string. + * Uses an in-memory cache. */ def getFuelTypeId(Connection destConn, String fuelType) { - // Check if we already have this value cached if (fuelTypeNameToIdCache.containsKey(fuelType)) { return fuelTypeNameToIdCache[fuelType] } - - // Not in cache, query the database - PreparedStatement stmt = null - ResultSet rs = null + PreparedStatement stmt = destConn.prepareStatement(SELECT_FUEL_TYPE_ID_QUERY) try { - stmt = destConn.prepareStatement(SELECT_FUEL_TYPE_ID_QUERY) stmt.setString(1, fuelType) - rs = stmt.executeQuery() - + ResultSet rs = stmt.executeQuery() if (rs.next()) { int typeId = rs.getInt("fuel_type_id") - // Cache for future use fuelTypeNameToIdCache[fuelType] = typeId return typeId } else { - log.warn("No fuel type found for: ${fuelType}") - return 1 // Default value + log.warn("No fuel type found for: ${fuelType}; using default 1.") + return 1 } } finally { - if (rs != null) rs.close() - if (stmt != null) stmt.close() + stmt.close() } } + /** - * Inserts a new row in allocation_agreement with action=CREATE/UPDATE - * We always do version = oldVersion + 1 or 0 if none yet. + * Inserts a new row into allocation_agreement with proper versioning. + * A stable group_uuid is generated (or reused) based on the TFRS agreement record id. */ def insertVersionRow(Connection destConn, Integer lcfsCRid, Map rowData, String action) { - def recordId = rowData.agreement_record_id - - // Retrieve or generate the stable random group uuid for this record - def groupUuid = recordUuidMap[recordId] - if (!groupUuid) { - groupUuid = UUID.randomUUID().toString() - recordUuidMap[recordId] = groupUuid - } - - // Find current highest version in allocation_agreement for that group_uuid - def currentVer = -1 - PreparedStatement verStmt = destConn.prepareStatement(SELECT_CURRENT_VERSION_QUERY) - verStmt.setString(1, groupUuid) - ResultSet verRS = verStmt.executeQuery() - if (verRS.next()) { - currentVer = verRS.getInt('version') - } - verRS.close() - verStmt.close() - - def nextVer = (currentVer < 0) ? 0 : currentVer + 1 - - // Map TFRS fields => LCFS fields - def allocTransactionTypeId = getTransactionTypeId(destConn, rowData.responsibility) - def fuelTypeId = getFuelTypeId(destConn, rowData.fuel_type) - def quantity = rowData.quantity ?: 0 - def quantityNotSold = rowData.quantity_not_sold ?: 0 - def transactionPartner = rowData.transaction_partner ?: '' - def postalAddress = rowData.postal_address ?: '' - def units = rowData.units ?: '' - - // Insert the new row - PreparedStatement insStmt = destConn.prepareStatement(INSERT_ALLOCATION_AGREEMENT_SQL) - insStmt.setInt(1, lcfsCRid) - insStmt.setString(2, transactionPartner) - insStmt.setString(3, postalAddress) - insStmt.setInt(4, quantity) - insStmt.setInt(5, quantityNotSold) - insStmt.setString(6, units) - insStmt.setInt(7, allocTransactionTypeId) - insStmt.setInt(8, fuelTypeId) - insStmt.setNull(9, java.sql.Types.INTEGER) - insStmt.setString(10, groupUuid) - insStmt.setInt(11, nextVer) - insStmt.setString(12, action) - insStmt.executeUpdate() - insStmt.close() - - log.info(" -> allocation_agreement row: recordId=${recordId}, action=${action}, groupUuid=${groupUuid}, version=${nextVer}") + def recordId = rowData.agreement_record_id + + // Retrieve or create a stable group_uuid. + def groupUuid = recordUuidMap[recordId] + if (!groupUuid) { + groupUuid = UUID.randomUUID().toString() + recordUuidMap[recordId] = groupUuid + } + + // Retrieve current highest version for this group_uuid. + int currentVer = -1 + PreparedStatement verStmt = destConn.prepareStatement(SELECT_CURRENT_VERSION_QUERY) + verStmt.setString(1, groupUuid) + ResultSet verRS = verStmt.executeQuery() + if (verRS.next()) { + currentVer = verRS.getInt("version") + } + verRS.close() + verStmt.close() + + int nextVer = (currentVer < 0) ? 0 : currentVer + 1 + + // Map source fields to destination fields. + int allocTransactionTypeId = getTransactionTypeId(destConn, rowData.responsibility) + int fuelTypeId = getFuelTypeId(destConn, rowData.fuel_type) + int quantity = rowData.quantity ?: 0 + int quantityNotSold = rowData.quantity_not_sold ?: 0 + String transactionPartner = rowData.transaction_partner ?: "" + String postalAddress = rowData.postal_address ?: "" + String units = rowData.units ?: "" + String fuelTypeString = rowData.fuel_type // Get the fuel type string + + // ---- START: Determine Fuel Category ID ---- + Integer fuelCategoryId = null // Default to null + // Adjust these string checks based on your actual source fuel_type names + if (fuelTypeString?.toLowerCase().contains('gasoline')) { + fuelCategoryId = GASOLINE_CATEGORY_ID + } else if (fuelTypeString?.toLowerCase().contains('diesel')) { + fuelCategoryId = DIESEL_CATEGORY_ID + } else { + // Optional: Log a warning if a type doesn't match known categories + log.warn("Could not determine fuel category for fuel type: ${fuelTypeString}. Setting fuel_category_id to NULL.") + } + // ---- END: Determine Fuel Category ID ---- + + PreparedStatement insStmt = destConn.prepareStatement(INSERT_ALLOCATION_AGREEMENT_SQL) + insStmt.setInt(1, lcfsCRid) + insStmt.setString(2, transactionPartner) + insStmt.setString(3, postalAddress) + insStmt.setInt(4, quantity) + insStmt.setInt(5, quantityNotSold) + insStmt.setString(6, units) + insStmt.setInt(7, allocTransactionTypeId) + insStmt.setInt(8, fuelTypeId) + if (fuelCategoryId != null) { + insStmt.setInt(9, fuelCategoryId) + } else { + insStmt.setNull(9, java.sql.Types.INTEGER) + } + insStmt.setString(10, groupUuid) + insStmt.setInt(11, nextVer) + insStmt.setString(12, action) + insStmt.setString(13, 'ETL') + insStmt.setString(14, 'ETL') + insStmt.executeUpdate() + insStmt.close() + + log.info("Inserted allocation_agreement row: recordId=${recordId}, action=${action}, groupUuid=${groupUuid}, version=${nextVer}") } -// ========================================= +// ------------------------- // Main Execution -// ========================================= - -log.warn('**** BEGIN ALLOCATION AGREEMENT MIGRATION ****') +// ------------------------- +log.warn("**** BEGIN ALLOCATION AGREEMENT MIGRATION ****") Connection sourceConn = null Connection destinationConn = null try { - sourceConn = sourceDbcpService.getConnection() - destinationConn = destinationDbcpService.getConnection() - - // 1) Find all LCFS compliance reports that have TFRS legacy_id - log.info('Retrieving LCFS compliance_report with legacy_id != null') - PreparedStatement lcfsStmt = destinationConn.prepareStatement(SELECT_LCFS_IMPORTED_REPORTS_QUERY) - ResultSet lcfsRS = lcfsStmt.executeQuery() - - def tfrsIds = [] - while (lcfsRS.next()) { - def tfrsId = lcfsRS.getInt('legacy_id') - tfrsIds << tfrsId - } - lcfsRS.close() - lcfsStmt.close() - - // For each TFRS compliance_report ID, follow the chain approach - tfrsIds.each { - tfrsId -> - log.info("Processing TFRS compliance_report.id = ${tfrsId}") - - // 2) Find the root_report_id - PreparedStatement rootStmt = sourceConn.prepareStatement(SELECT_ROOT_REPORT_ID_QUERY) - rootStmt.setInt(1, tfrsId) - def rootRS = rootStmt.executeQuery() - def rootId = null - if (rootRS.next()) { - rootId = rootRS.getInt('root_report_id') - } - rootRS.close() - rootStmt.close() - - if (!rootId) { - log.warn("No root_report_id found for TFRS #${tfrsId}; skipping.") - return + // Establish connections. + sourceConn = sourceDbcpService.getConnection() + destinationConn = destinationDbcpService.getConnection() + + // 1) Retrieve all LCFS compliance reports with a non-null legacy_id. + log.info("Retrieving LCFS compliance reports with legacy_id != NULL") + PreparedStatement lcfsStmt = destinationConn.prepareStatement(SELECT_LCFS_IMPORTED_REPORTS_QUERY) + ResultSet lcfsRS = lcfsStmt.executeQuery() + + // Build a list of TFRS IDs using legacy_id. + def tfrsIds = [] + while (lcfsRS.next()) { + tfrsIds << lcfsRS.getInt("legacy_id") } - - // 3) Gather the chain in ascending order - PreparedStatement chainStmt = sourceConn.prepareStatement(SELECT_REPORT_CHAIN_QUERY) - chainStmt.setInt(1, rootId) - def chainRS = chainStmt.executeQuery() - - def chainIds = [] - while (chainRS.next()) { - chainIds << chainRS.getInt('tfrs_report_id') - } - chainRS.close() - chainStmt.close() - - if (chainIds.isEmpty()) { - log.warn("Chain empty for root=${rootId}? skipping.") - return - } - - // Keep the old version's allocation agreement data in memory so we can do diffs - Map < Integer, Map > previousRecords = [: ] - - chainIds.eachWithIndex { - chainTfrsId, - idx -> log.info("TFRS #${chainTfrsId} (chain idx=${idx})") - - // 4) Fetch current TFRS allocation agreement records - Map < Integer, - Map > currentRecords = [: ] - PreparedStatement alocStmt = sourceConn.prepareStatement(SELECT_ALLOCATION_AGREEMENTS_QUERY) - alocStmt.setInt(1, chainTfrsId) - ResultSet alocRS = alocStmt.executeQuery() - while (alocRS.next()) { - def recId = alocRS.getInt('agreement_record_id') - currentRecords[recId] = [ - agreement_record_id: recId, - responsibility: alocRS.getString('responsibility'), - fuel_type: alocRS.getString('fuel_type'), - transaction_partner: alocRS.getString('transaction_partner'), - postal_address: alocRS.getString('postal_address'), - quantity: alocRS.getInt('quantity'), - units: alocRS.getString('units'), - quantity_not_sold: alocRS.getInt('quantity_not_sold'), - transaction_type_id: alocRS.getInt('transaction_type_id') - ] - } - alocRS.close() - alocStmt.close() - - // 5) Find the matching LCFS compliance_report - Integer lcfsCRid = null - PreparedStatement findCRstmt = destinationConn.prepareStatement(SELECT_LCFS_COMPLIANCE_REPORT_BY_TFRSID_QUERY) - findCRstmt.setInt(1, chainTfrsId) - ResultSet findCRrs = findCRstmt.executeQuery() - if (findCRrs.next()) { - lcfsCRid = findCRrs.getInt('compliance_report_id') - } - findCRrs.close() - findCRstmt.close() - - if (!lcfsCRid) { - log.warn("TFRS #${chainTfrsId} not found in LCFS? Skipping diff, just storing previousRecords.") - previousRecords = currentRecords - return - } - - // Compare old vs new - - // A) For each record in currentRecords - currentRecords.each { - recId, - newData -> - if (!previousRecords.containsKey(recId)) { - // wasn't in old => CREATE - insertVersionRow(destinationConn, lcfsCRid, newData, 'CREATE') - } else { - // existed => check if changed - def oldData = previousRecords[recId] - if (isRecordChanged(oldData, newData)) { - insertVersionRow(destinationConn, lcfsCRid, newData, 'UPDATE') - } + lcfsRS.close() + lcfsStmt.close() + + // Process each LCFS compliance report. + tfrsIds.each { tfrsId -> + log.warn("Processing TFRS compliance_report.id = ${tfrsId}") + + // Look up the original LCFS compliance_report record by legacy_id. + PreparedStatement crStmt = destinationConn.prepareStatement(SELECT_LCFS_COMPLIANCE_REPORT_BY_TFRSID_QUERY) + crStmt.setInt(1, tfrsId) + ResultSet crRS = crStmt.executeQuery() + def lcfsCRid = crRS.next() ? crRS.getInt("compliance_report_id") : null + crRS.close() + crStmt.close() + + if (!lcfsCRid) { + log.warn("No LCFS compliance_report found for TFRS legacy id ${tfrsId}; skipping allocation agreement processing.") + return } - } - // Update previousRecords for the next version - previousRecords = currentRecords - } // end chain loop - } // end each tfrsId + // 2) Retrieve allocation agreement records from source for the given TFRS report. + PreparedStatement alocStmt = sourceConn.prepareStatement(SELECT_ALLOCATION_AGREEMENTS_QUERY) + alocStmt.setInt(1, tfrsId) + ResultSet alocRS = alocStmt.executeQuery() + + boolean foundAllocationRecords = false + // Process each allocation agreement record. + while (alocRS.next()) { + foundAllocationRecords = true // Mark that we entered the loop + int recId = alocRS.getInt("agreement_record_id") + log.info("Found source allocation record ID: ${recId} for TFRS report ID: ${tfrsId}. Preparing for LCFS insert.") + def recordData = [ + agreement_record_id : recId, + responsibility : alocRS.getString("responsibility"), + fuel_type : alocRS.getString("fuel_type"), + fuel_type_id : alocRS.getInt("fuel_type_id"), + transaction_partner : alocRS.getString("transaction_partner"), + postal_address : alocRS.getString("postal_address"), + quantity : alocRS.getInt("quantity"), + units : alocRS.getString("units"), + quantity_not_sold : alocRS.getInt("quantity_not_sold"), + transaction_type_id : alocRS.getInt("transaction_type_id") + ] + // Insert each allocation agreement record. Versioning is handled via a stable group UUID. + insertVersionRow(destinationConn, lcfsCRid, recordData, 'CREATE') + } + if (!foundAllocationRecords) { + log.warn("No allocation agreement records found in source for TFRS report ID: ${tfrsId} (or cr.exclusion_agreement_id was NULL).") + } + alocRS.close() + alocStmt.close() + } } catch (Exception e) { - log.error('Error running allocation agreement migration', e) - throw e + log.error("Error running allocation agreement migration", e) + throw e } finally { - if (sourceConn != null) sourceConn.close() - if (destinationConn != null) destinationConn.close() + if (sourceConn != null) { sourceConn.close() } + if (destinationConn != null) { destinationConn.close() } } -log.warn('**** DONE: ALLOCATION AGREEMENT MIGRATION ****') +log.warn("**** DONE: ALLOCATION AGREEMENT MIGRATION ****") diff --git a/etl/nifi_scripts/compliance_report.groovy b/etl/nifi_scripts/compliance_report.groovy index 99bdfef78..8239cd180 100644 --- a/etl/nifi_scripts/compliance_report.groovy +++ b/etl/nifi_scripts/compliance_report.groovy @@ -243,6 +243,9 @@ try { int totalHistoryInserted = 0 int totalTransactionsInserted = 0 + // Create a prepared statement for checking legacy_id existence in LCFS. + def legacyCheckStmt = destinationConn.prepareStatement("SELECT compliance_report_id FROM compliance_report WHERE legacy_id = ?") + // Execute the consolidated compliance reports query PreparedStatement consolidatedStmt = sourceConn.prepareStatement(SOURCE_CONSOLIDATED_QUERY) ResultSet rs = consolidatedStmt.executeQuery() @@ -290,6 +293,16 @@ try { fair_market_value_per_credit : rs.getBigDecimal('fair_market_value_per_credit') ] + // Check if a compliance report with this legacy_id (from source compliance_report_id) already exists in LCFS. + legacyCheckStmt.setInt(1, record.compliance_report_id) + ResultSet legacyRs = legacyCheckStmt.executeQuery() + if (legacyRs.next()) { + log.warn("Skipping compliance_report_id: ${record.compliance_report_id} because legacy record already exists in LCFS.") + legacyRs.close() + continue + } + legacyRs.close() + // Map TFRS compliance period ID to LCFS compliance period ID record.compliance_period_id = mapCompliancePeriodId(record.compliance_period_id) @@ -507,6 +520,8 @@ try { rs.close() consolidatedStmt.close() + legacyCheckStmt.close() // Close the legacy check statement after processing + // Commit all changes from the original compliance report processing destinationConn.commit() @@ -923,6 +938,7 @@ def prepareStatements(Connection conn) { update_user, update_date ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (compliance_report_id, status_id) DO NOTHING """ def INSERT_TRANSACTION_SQL = ''' diff --git a/etl/nifi_scripts/compliance_summary.groovy b/etl/nifi_scripts/compliance_summary.groovy index a32bce2c7..dbbb2c594 100644 --- a/etl/nifi_scripts/compliance_summary.groovy +++ b/etl/nifi_scripts/compliance_summary.groovy @@ -35,6 +35,22 @@ try { fetchMappingStmt.close() log.info("Fetched ${legacyToLcfsIdMap.size()} legacy_id to LCFS compliance_report_id mappings from destination.") + // ========================================= + // Preload Existing Summary Records from Destination + // ========================================= + log.info("Fetching existing compliance_report_summary records from destination database.") + def existingComplianceReportIdSet = new HashSet() // LCFS compliance_report_id for which a summary exists + def existingSummaryIdSet = new HashSet() // All summary_id values in the destination + def fetchSummaryStmt = destinationConn.prepareStatement("SELECT summary_id, compliance_report_id FROM public.compliance_report_summary") + ResultSet summaryRs = fetchSummaryStmt.executeQuery() + while (summaryRs.next()) { + existingSummaryIdSet.add(summaryRs.getInt("summary_id")) + existingComplianceReportIdSet.add(summaryRs.getInt("compliance_report_id")) + } + summaryRs.close() + fetchSummaryStmt.close() + log.info("Fetched ${existingComplianceReportIdSet.size()} compliance_report_summary records (by compliance_report_id) and ${existingSummaryIdSet.size()} summary_id values from destination.") + // ========================================= // Fetch Data from Source Table // ========================================= @@ -51,10 +67,7 @@ try { crs.diesel_class_obligation, crs.diesel_class_previously_retained, crs.gasoline_class_obligation, - crs.gasoline_class_previously_retained, - crs.credits_offset_a, - crs.credits_offset_b, - crs.credits_offset_c + crs.gasoline_class_previously_retained FROM public.compliance_report cr JOIN @@ -72,10 +85,11 @@ try { // ========================================= // Prepare Destination Insert Statement // ========================================= - + // Note: Ensure that the column list and the corresponding parameters exactly match + // your destination table schema. Adjust as necessary so that there are, for example, + // 55 columns if that is what your table contains. def INSERT_DESTINATION_SUMMARY_SQL = """ INSERT INTO public.compliance_report_summary ( - summary_id, compliance_report_id, quarter, is_locked, @@ -129,15 +143,12 @@ try { line_11_fossil_derived_base_fuel_total, line_21_non_compliance_penalty_payable, total_non_compliance_penalty_payable, - credits_offset_a, - credits_offset_b, - credits_offset_c, create_date, update_date, create_user, update_user ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) """ @@ -168,82 +179,87 @@ try { def dieselClassPreviouslyRetained = rs.getBigDecimal('diesel_class_previously_retained') def gasolineClassObligation = rs.getBigDecimal('gasoline_class_obligation') def gasolineClassPreviouslyRetained = rs.getBigDecimal('gasoline_class_previously_retained') - def creditsOffsetA = rs.getInt('credits_offset_a') - def creditsOffsetB = rs.getInt('credits_offset_b') - def creditsOffsetC = rs.getInt('credits_offset_c') // Map source compliance_report_id (legacy_id) to LCFS compliance_report_id def lcfsComplianceReportId = legacyToLcfsIdMap[sourceComplianceReportLegacyId] - if (lcfsComplianceReportId == null) { - log.warn("No LCFS compliance_report found with legacy_id ${sourceComplianceReportLegacyId}. Skipping summary_id ${summaryId}.") + log.warn("No LCFS compliance_report found with legacy_id ${sourceComplianceReportLegacyId}.") totalSkipped++ - continue // Skip to the next record + continue // Skip this record } - // Create a summaryRecord map for the destination table + // Check if a summary record already exists + if (existingComplianceReportIdSet.contains(lcfsComplianceReportId)) { + log.warn("A compliance_report_summary record already exists for LCFS compliance_report_id ${lcfsComplianceReportId}.") + totalSkipped++ + continue + } + // Additionally, check if this source summary_id already exists in the destination. + // if (existingSummaryIdSet.contains(summaryId)) { + // log.warn("A compliance_report_summary record with summary_id ${summaryId} already exists. Skipping.") + // totalSkipped++ + // continue + // } + + // Build the record data map for insertion. def summaryRecord = [ - summary_id : summaryId, compliance_report_id : lcfsComplianceReportId, // Use LCFS ID quarter : null, is_locked : true, - line_1_fossil_derived_base_fuel_gasoline : null, // No direct mapping - line_1_fossil_derived_base_fuel_diesel : null, // No direct mapping - line_1_fossil_derived_base_fuel_jet_fuel : null, // No direct mapping - line_2_eligible_renewable_fuel_supplied_gasoline : null, // No direct mapping - line_2_eligible_renewable_fuel_supplied_diesel : null, // No direct mapping - line_2_eligible_renewable_fuel_supplied_jet_fuel : null, // No direct mapping - line_3_total_tracked_fuel_supplied_gasoline : null, // No direct mapping - line_3_total_tracked_fuel_supplied_diesel : null, // No direct mapping - line_3_total_tracked_fuel_supplied_jet_fuel : null, // No direct mapping - line_4_eligible_renewable_fuel_required_gasoline : null, // No direct mapping - line_4_eligible_renewable_fuel_required_diesel : null, // No direct mapping - line_4_eligible_renewable_fuel_required_jet_fuel : null, // No direct mapping - line_5_net_notionally_transferred_gasoline : null, // No direct mapping - line_5_net_notionally_transferred_diesel : null, // No direct mapping - line_5_net_notionally_transferred_jet_fuel : null, // No direct mapping + line_1_fossil_derived_base_fuel_gasoline : null, + line_1_fossil_derived_base_fuel_diesel : null, + line_1_fossil_derived_base_fuel_jet_fuel : null, + line_2_eligible_renewable_fuel_supplied_gasoline : null, + line_2_eligible_renewable_fuel_supplied_diesel : null, + line_2_eligible_renewable_fuel_supplied_jet_fuel : null, + line_3_total_tracked_fuel_supplied_gasoline : null, + line_3_total_tracked_fuel_supplied_diesel : null, + line_3_total_tracked_fuel_supplied_jet_fuel : null, + line_4_eligible_renewable_fuel_required_gasoline : null, + line_4_eligible_renewable_fuel_required_diesel : null, + line_4_eligible_renewable_fuel_required_jet_fuel : null, + line_5_net_notionally_transferred_gasoline : null, + line_5_net_notionally_transferred_diesel : null, + line_5_net_notionally_transferred_jet_fuel : null, line_6_renewable_fuel_retained_gasoline : gasolineClassRetained, line_6_renewable_fuel_retained_diesel : dieselClassRetained, - line_6_renewable_fuel_retained_jet_fuel : null, // No direct mapping + line_6_renewable_fuel_retained_jet_fuel : null, line_7_previously_retained_gasoline : gasolineClassPreviouslyRetained, line_7_previously_retained_diesel : dieselClassPreviouslyRetained, - line_7_previously_retained_jet_fuel : null, // No direct mapping - line_8_obligation_deferred_gasoline : gasolineClassDeferred, - line_8_obligation_deferred_diesel : dieselClassDeferred, - line_8_obligation_deferred_jet_fuel : null, // No direct mapping - line_9_obligation_added_gasoline : gasolineClassObligation, - line_9_obligation_added_diesel : dieselClassObligation, - line_9_obligation_added_jet_fuel : null, // No direct mapping - line_10_net_renewable_fuel_supplied_gasoline : null, // No direct mapping - line_10_net_renewable_fuel_supplied_diesel : null, // No direct mapping - line_10_net_renewable_fuel_supplied_jet_fuel : null, // No direct mapping - line_11_non_compliance_penalty_gasoline : null, // No direct mapping - line_11_non_compliance_penalty_diesel : null, // No direct mapping - line_11_non_compliance_penalty_jet_fuel : null, // No direct mapping - line_12_low_carbon_fuel_required : null, // No direct mapping - line_13_low_carbon_fuel_supplied : null, // No direct mapping - line_14_low_carbon_fuel_surplus : null, // No direct mapping - line_15_banked_units_used : null, // No direct mapping - line_16_banked_units_remaining : null, // No direct mapping - line_17_non_banked_units_used : null, // No direct mapping - line_18_units_to_be_banked : null, // No direct mapping - line_19_units_to_be_exported : null, // No direct mapping - line_20_surplus_deficit_units : null, // No direct mapping - line_21_surplus_deficit_ratio : null, // No direct mapping - line_22_compliance_units_issued : creditsOffset, + line_7_previously_retained_jet_fuel : null, + line_8_obligation_deferred_gasoline : gasolineClassDeferred, + line_8_obligation_deferred_diesel : dieselClassDeferred, + line_8_obligation_deferred_jet_fuel : null, + line_9_obligation_added_gasoline : gasolineClassObligation, + line_9_obligation_added_diesel : dieselClassObligation, + line_9_obligation_added_jet_fuel : null, + line_10_net_renewable_fuel_supplied_gasoline : null, + line_10_net_renewable_fuel_supplied_diesel : null, + line_10_net_renewable_fuel_supplied_jet_fuel : null, + line_11_non_compliance_penalty_gasoline : null, + line_11_non_compliance_penalty_diesel : null, + line_11_non_compliance_penalty_jet_fuel : null, + line_12_low_carbon_fuel_required : null, + line_13_low_carbon_fuel_supplied : null, + line_14_low_carbon_fuel_surplus : null, + line_15_banked_units_used : null, + line_16_banked_units_remaining : null, + line_17_non_banked_units_used : null, + line_18_units_to_be_banked : null, + line_19_units_to_be_exported : null, + line_20_surplus_deficit_units : null, + line_21_surplus_deficit_ratio : null, + line_22_compliance_units_issued : creditsOffset, line_11_fossil_derived_base_fuel_gasoline : null, // No direct mapping line_11_fossil_derived_base_fuel_diesel : null, // No direct mapping line_11_fossil_derived_base_fuel_jet_fuel : null, // No direct mapping line_11_fossil_derived_base_fuel_total : null, // No direct mapping line_21_non_compliance_penalty_payable : null, // No direct mapping total_non_compliance_penalty_payable : null, // No direct mapping - credits_offset_a : creditsOffsetA, // Direct mapping - credits_offset_b : creditsOffsetB, // Direct mapping - credits_offset_c : creditsOffsetC, // Direct mapping - create_date : new Timestamp(System.currentTimeMillis()), - update_date : new Timestamp(System.currentTimeMillis()), - create_user : "etl_user", // Replace with actual user or mapping - update_user : "etl_user" // Replace with actual user or mapping + create_date : new Timestamp(System.currentTimeMillis()), + update_date : new Timestamp(System.currentTimeMillis()), + create_user : "etl_user", + update_user : "etl_user" ] // ========================================= @@ -251,241 +267,99 @@ try { // ========================================= try { - // 1. summary_id (int4) - destinationStmt.setInt(1, summaryRecord.summary_id) - - // 2. compliance_report_id (int4) - destinationStmt.setInt(2, summaryRecord.compliance_report_id) - - // 3. quarter (int4) + // Bind parameters in the same order as specified in the INSERT statement. + destinationStmt.setInt(1, summaryRecord.compliance_report_id) if (summaryRecord.quarter != null) { - destinationStmt.setInt(3, summaryRecord.quarter) - } else { - destinationStmt.setNull(3, java.sql.Types.INTEGER) - } - - // 4. is_locked (bool) - destinationStmt.setBoolean(4, summaryRecord.is_locked) - - // 5. line_1_fossil_derived_base_fuel_gasoline (float8) NOT NULL - if (summaryRecord.line_1_fossil_derived_base_fuel_gasoline != null) { - destinationStmt.setDouble(5, summaryRecord.line_1_fossil_derived_base_fuel_gasoline.doubleValue()) - } else { - destinationStmt.setDouble(5, 0.0) // Default value or handle as per business logic - } - - // 6. line_1_fossil_derived_base_fuel_diesel (float8) NOT NULL - if (summaryRecord.line_1_fossil_derived_base_fuel_diesel != null) { - destinationStmt.setDouble(6, summaryRecord.line_1_fossil_derived_base_fuel_diesel.doubleValue()) - } else { - destinationStmt.setDouble(6, 0.0) - } - - // 7. line_1_fossil_derived_base_fuel_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(7, 0.0) // No mapping - - // 8. line_2_eligible_renewable_fuel_supplied_gasoline (float8) NOT NULL - destinationStmt.setDouble(8, 0.0) // No mapping - - // 9. line_2_eligible_renewable_fuel_supplied_diesel (float8) NOT NULL - destinationStmt.setDouble(9, 0.0) // No mapping - - // 10. line_2_eligible_renewable_fuel_supplied_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(10, 0.0) // No mapping - - // 11. line_3_total_tracked_fuel_supplied_gasoline (float8) NOT NULL - destinationStmt.setDouble(11, 0.0) // No mapping - - // 12. line_3_total_tracked_fuel_supplied_diesel (float8) NOT NULL - destinationStmt.setDouble(12, 0.0) // No mapping - - // 13. line_3_total_tracked_fuel_supplied_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(13, 0.0) // No mapping - - // 14. line_4_eligible_renewable_fuel_required_gasoline (float8) NOT NULL - destinationStmt.setDouble(14, 0.0) // No mapping - - // 15. line_4_eligible_renewable_fuel_required_diesel (float8) NOT NULL - destinationStmt.setDouble(15, 0.0) // No mapping - - // 16. line_4_eligible_renewable_fuel_required_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(16, 0.0) // No mapping - - // 17. line_5_net_notionally_transferred_gasoline (float8) NOT NULL - destinationStmt.setDouble(17, 0.0) // No mapping - - // 18. line_5_net_notionally_transferred_diesel (float8) NOT NULL - destinationStmt.setDouble(18, 0.0) // No mapping - - // 19. line_5_net_notionally_transferred_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(19, 0.0) // No mapping - - // 20. line_6_renewable_fuel_retained_gasoline (float8) NOT NULL - if (summaryRecord.line_6_renewable_fuel_retained_gasoline != null) { - destinationStmt.setDouble(20, summaryRecord.line_6_renewable_fuel_retained_gasoline.doubleValue()) + destinationStmt.setInt(2, summaryRecord.quarter) } else { - destinationStmt.setDouble(20, 0.0) + destinationStmt.setNull(2, java.sql.Types.INTEGER) } - - // 21. line_6_renewable_fuel_retained_diesel (float8) NOT NULL - if (summaryRecord.line_6_renewable_fuel_retained_diesel != null) { - destinationStmt.setDouble(21, summaryRecord.line_6_renewable_fuel_retained_diesel.doubleValue()) - } else { - destinationStmt.setDouble(21, 0.0) - } - - // 22. line_6_renewable_fuel_retained_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(22, 0.0) // No mapping - - // 23. line_7_previously_retained_gasoline (float8) NOT NULL - if (summaryRecord.line_7_previously_retained_gasoline != null) { - destinationStmt.setDouble(23, summaryRecord.line_7_previously_retained_gasoline.doubleValue()) - } else { - destinationStmt.setDouble(23, 0.0) - } - - // 24. line_7_previously_retained_diesel (float8) NOT NULL - if (summaryRecord.line_7_previously_retained_diesel != null) { - destinationStmt.setDouble(24, summaryRecord.line_7_previously_retained_diesel.doubleValue()) - } else { - destinationStmt.setDouble(24, 0.0) - } - - // 25. line_7_previously_retained_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(25, 0.0) // No mapping - - // 26. line_8_obligation_deferred_gasoline (float8) NOT NULL - if (summaryRecord.line_8_obligation_deferred_gasoline != null) { - destinationStmt.setDouble(26, summaryRecord.line_8_obligation_deferred_gasoline.doubleValue()) - } else { - destinationStmt.setDouble(26, 0.0) - } - - // 27. line_8_obligation_deferred_diesel (float8) NOT NULL - if (summaryRecord.line_8_obligation_deferred_diesel != null) { - destinationStmt.setDouble(27, summaryRecord.line_8_obligation_deferred_diesel.doubleValue()) - } else { - destinationStmt.setDouble(27, 0.0) - } - - // 28. line_8_obligation_deferred_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(28, 0.0) // No mapping - - // 29. line_9_obligation_added_gasoline (float8) NOT NULL - if (summaryRecord.line_9_obligation_added_gasoline != null) { - destinationStmt.setDouble(29, summaryRecord.line_9_obligation_added_gasoline.doubleValue()) - } else { - destinationStmt.setDouble(29, 0.0) - } - - // 30. line_9_obligation_added_diesel (float8) NOT NULL - if (summaryRecord.line_9_obligation_added_diesel != null) { - destinationStmt.setDouble(30, summaryRecord.line_9_obligation_added_diesel.doubleValue()) - } else { - destinationStmt.setDouble(30, 0.0) - } - - // 31. line_9_obligation_added_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(31, 0.0) // No mapping - - // 32. line_10_net_renewable_fuel_supplied_gasoline (float8) NOT NULL - destinationStmt.setDouble(32, 0.0) // No mapping - - // 33. line_10_net_renewable_fuel_supplied_diesel (float8) NOT NULL - destinationStmt.setDouble(33, 0.0) // No mapping - - // 34. line_10_net_renewable_fuel_supplied_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(34, 0.0) // No mapping - - // 35. line_11_non_compliance_penalty_gasoline (float8) - destinationStmt.setNull(35, java.sql.Types.FLOAT) // No mapping - - // 36. line_11_non_compliance_penalty_diesel (float8) - destinationStmt.setNull(36, java.sql.Types.FLOAT) // No mapping - - // 37. line_11_non_compliance_penalty_jet_fuel (float8) - destinationStmt.setNull(37, java.sql.Types.FLOAT) // No mapping - - // 38. line_12_low_carbon_fuel_required (float8) NOT NULL - destinationStmt.setDouble(38, 0.0) // No mapping - - // 39. line_13_low_carbon_fuel_supplied (float8) NOT NULL - destinationStmt.setDouble(39, 0.0) // No mapping - - // 40. line_14_low_carbon_fuel_surplus (float8) NOT NULL - destinationStmt.setDouble(40, 0.0) // No mapping - - // 41. line_15_banked_units_used (float8) NOT NULL - destinationStmt.setDouble(41, 0.0) // No mapping - - // 42. line_16_banked_units_remaining (float8) NOT NULL - destinationStmt.setDouble(42, 0.0) // No mapping - - // 43. line_17_non_banked_units_used (float8) NOT NULL - destinationStmt.setDouble(43, 0.0) // No mapping - - // 44. line_18_units_to_be_banked (float8) NOT NULL - destinationStmt.setDouble(44, 0.0) // No mapping - - // 45. line_19_units_to_be_exported (float8) NOT NULL - destinationStmt.setDouble(45, 0.0) // No mapping - - // 46. line_20_surplus_deficit_units (float8) NOT NULL - destinationStmt.setDouble(46, 0.0) // No mapping - - // 47. line_21_surplus_deficit_ratio (float8) NOT NULL - destinationStmt.setDouble(47, 0.0) // No mapping - - // 48. line_22_compliance_units_issued (float8) NOT NULL - destinationStmt.setDouble(48, summaryRecord.line_22_compliance_units_issued) - + destinationStmt.setBoolean(3, summaryRecord.is_locked) + // For columns with no mapping, default to 0.0 (or null if that fits your business logic) + destinationStmt.setDouble(4, (summaryRecord.line_1_fossil_derived_base_fuel_gasoline != null) ? summaryRecord.line_1_fossil_derived_base_fuel_gasoline.doubleValue() : 0.0) + destinationStmt.setDouble(5, (summaryRecord.line_1_fossil_derived_base_fuel_diesel != null) ? summaryRecord.line_1_fossil_derived_base_fuel_diesel.doubleValue() : 0.0) + destinationStmt.setDouble(6, 0.0) + destinationStmt.setDouble(7, 0.0) + destinationStmt.setDouble(8, 0.0) + destinationStmt.setDouble(9, 0.0) + destinationStmt.setDouble(10, 0.0) + destinationStmt.setDouble(11, 0.0) + destinationStmt.setDouble(12, 0.0) + destinationStmt.setDouble(13, 0.0) + destinationStmt.setDouble(14, 0.0) + destinationStmt.setDouble(15, 0.0) + destinationStmt.setDouble(16, 0.0) + destinationStmt.setDouble(17, 0.0) + destinationStmt.setDouble(18, 0.0) + destinationStmt.setDouble(19, (summaryRecord.line_6_renewable_fuel_retained_gasoline != null) ? summaryRecord.line_6_renewable_fuel_retained_gasoline.doubleValue() : 0.0) + destinationStmt.setDouble(20, (summaryRecord.line_6_renewable_fuel_retained_diesel != null) ? summaryRecord.line_6_renewable_fuel_retained_diesel.doubleValue() : 0.0) + destinationStmt.setDouble(21, 0.0) + destinationStmt.setDouble(22, (summaryRecord.line_7_previously_retained_gasoline != null) ? summaryRecord.line_7_previously_retained_gasoline.doubleValue() : 0.0) + destinationStmt.setDouble(23, (summaryRecord.line_7_previously_retained_diesel != null) ? summaryRecord.line_7_previously_retained_diesel.doubleValue() : 0.0) + destinationStmt.setDouble(24, 0.0) + destinationStmt.setDouble(25, (summaryRecord.line_8_obligation_deferred_gasoline != null) ? summaryRecord.line_8_obligation_deferred_gasoline.doubleValue() : 0.0) + destinationStmt.setDouble(26, (summaryRecord.line_8_obligation_deferred_diesel != null) ? summaryRecord.line_8_obligation_deferred_diesel.doubleValue() : 0.0) + destinationStmt.setDouble(27, 0.0) + destinationStmt.setDouble(28, (summaryRecord.line_9_obligation_added_gasoline != null) ? summaryRecord.line_9_obligation_added_gasoline.doubleValue() : 0.0) + destinationStmt.setDouble(29, (summaryRecord.line_9_obligation_added_diesel != null) ? summaryRecord.line_9_obligation_added_diesel.doubleValue() : 0.0) + destinationStmt.setDouble(30, 0.0) + destinationStmt.setDouble(31, 0.0) + destinationStmt.setDouble(32, 0.0) + destinationStmt.setDouble(33, 0.0) + destinationStmt.setNull(34, java.sql.Types.FLOAT) + destinationStmt.setNull(35, java.sql.Types.FLOAT) + destinationStmt.setNull(36, java.sql.Types.FLOAT) + destinationStmt.setDouble(37, 0.0) + destinationStmt.setDouble(38, 0.0) + destinationStmt.setDouble(39, 0.0) + destinationStmt.setDouble(40, 0.0) + destinationStmt.setDouble(41, 0.0) + destinationStmt.setDouble(42, 0.0) + destinationStmt.setDouble(43, 0.0) + destinationStmt.setDouble(44, 0.0) + destinationStmt.setDouble(45, 0.0) + destinationStmt.setDouble(46, 0.0) + destinationStmt.setDouble(47, summaryRecord.line_22_compliance_units_issued) + // 49. line_11_fossil_derived_base_fuel_gasoline (float8) NOT NULL - destinationStmt.setDouble(49, 0.0) // No mapping + destinationStmt.setDouble(48, 0.0) // No mapping // 50. line_11_fossil_derived_base_fuel_diesel (float8) NOT NULL - destinationStmt.setDouble(50, 0.0) // No mapping + destinationStmt.setDouble(49, 0.0) // No mapping // 51. line_11_fossil_derived_base_fuel_jet_fuel (float8) NOT NULL - destinationStmt.setDouble(51, 0.0) // No mapping + destinationStmt.setDouble(50, 0.0) // No mapping // 52. line_11_fossil_derived_base_fuel_total (float8) NOT NULL - destinationStmt.setDouble(52, 0.0) // No mapping + destinationStmt.setDouble(51, 0.0) // No mapping // 53. line_21_non_compliance_penalty_payable (float8) NOT NULL - destinationStmt.setDouble(53, 0.0) // No mapping + destinationStmt.setDouble(52, 0.0) // No mapping // 54. total_non_compliance_penalty_payable (float8) NOT NULL - destinationStmt.setDouble(54, 0.0) // No mapping - - // 55. credits_offset_a (int4) - destinationStmt.setInt(55, summaryRecord.credits_offset_a) - - // 56. credits_offset_b (int4) - destinationStmt.setInt(56, summaryRecord.credits_offset_b) - - // 57. credits_offset_c (int4) - destinationStmt.setInt(57, summaryRecord.credits_offset_c) + destinationStmt.setDouble(53, 0.0) // No mapping - // 58. create_date (timestamptz) - destinationStmt.setTimestamp(58, summaryRecord.create_date) + // 54. create_date (timestamptz) + destinationStmt.setTimestamp(54, summaryRecord.create_date) - // 59. update_date (timestamptz) - destinationStmt.setTimestamp(59, summaryRecord.update_date) + // 55. update_date (timestamptz) + destinationStmt.setTimestamp(55, summaryRecord.update_date) - // 60. create_user (varchar) - destinationStmt.setString(60, summaryRecord.create_user) + // 56. create_user (varchar) + destinationStmt.setString(56, summaryRecord.create_user) - // 61. update_user (varchar) - destinationStmt.setString(61, summaryRecord.update_user) + // 57. update_user (varchar) + destinationStmt.setString(57, summaryRecord.update_user) // Add to batch destinationStmt.addBatch() totalInserted++ - + // Also add the LCFS compliance_report_id and summary_id to our existing sets. + existingComplianceReportIdSet.add(lcfsComplianceReportId) + existingSummaryIdSet.add(summaryId) } catch (Exception e) { - log.error("Failed to insert summary_record for LCFS compliance_report_id: ${summaryRecord.compliance_report_id}, summary_id: ${summaryRecord.summary_id}", e) + log.error("Failed to insert summary_record for LCFS compliance_report_id: ${summaryRecord.compliance_report_id}", e) totalSkipped++ - // Continue processing other records continue } } @@ -499,7 +373,7 @@ try { destinationConn.commit() log.info("Successfully inserted ${totalInserted} records into destination compliance_report_summary.") if (totalSkipped > 0) { - log.warn("Skipped ${totalSkipped} records due to missing LCFS compliance_report_id in destination or insertion errors.") + log.warn("Skipped ${totalSkipped} records due to missing LCFS compliance_report_id, existing summary records, or insertion errors.") } } catch (Exception e) { log.error("Batch insertion failed. Rolling back.", e) @@ -517,7 +391,6 @@ try { } catch (Exception e) { log.error("An error occurred during the ETL process.", e) - // Ensure connections are closed in case of unexpected errors if (sourceConn != null && !sourceConn.isClosed()) sourceConn.close() if (destinationConn != null && !destinationConn.isClosed()) destinationConn.close() throw e diff --git a/etl/nifi_scripts/compliance_summary_update.groovy b/etl/nifi_scripts/compliance_summary_update.groovy index baaa75292..a31258638 100644 --- a/etl/nifi_scripts/compliance_summary_update.groovy +++ b/etl/nifi_scripts/compliance_summary_update.groovy @@ -108,9 +108,7 @@ try { line_11_fossil_derived_base_fuel_total = ?, line_21_non_compliance_penalty_payable = ?, total_non_compliance_penalty_payable = ?, - credits_offset_a = ?, - credits_offset_b = ?, - credits_offset_c = ? + historical_snapshot = ?::jsonb WHERE compliance_report_id = ? """ PreparedStatement updateStmt = destinationConn.prepareStatement(UPDATE_SQL) @@ -130,6 +128,7 @@ try { skipCount++ continue } + log.warn("Processing source record with legacy_id: ${legacyComplianceReportId}") def snapshotJson = rs.getString("snapshot") def summaryJson = jsonSlurper.parseText(snapshotJson) @@ -185,13 +184,6 @@ try { def line28NonCompliance = new BigDecimal(summaryJson.summary.lines."28") // Part 3 penalty def totalPayable = new BigDecimal(summaryJson.summary.total_payable) // Total payable from snapshot - // ------------------------------ - // Credits Offset Fields - // ------------------------------ - def creditsOffsetA = summaryJson.summary.lines."26A" as Integer ?: null - def creditsOffsetB = summaryJson.summary.lines."26B" as Integer ?: null - def creditsOffsetC = summaryJson.summary.lines."26C" as Integer ?: null - // Set parameters using a running index int idx = 1 // Gasoline Class @@ -250,14 +242,13 @@ try { // Non-compliance Penalty Fields updateStmt.setBigDecimal(idx++, line28NonCompliance) // line_21_non_compliance_penalty_payable updateStmt.setBigDecimal(idx++, totalPayable) // total_non_compliance_penalty_payable (from snapshot total_payable) - // Credits Offset Fields - updateStmt.setObject(idx++, creditsOffsetA) // credits_offset_a - updateStmt.setObject(idx++, creditsOffsetB) // credits_offset_b - updateStmt.setObject(idx++, creditsOffsetC) // credits_offset_c + // Historical Snapshot + updateStmt.setString(idx++, snapshotJson) // Store the entire snapshot JSON // WHERE clause: compliance_report_id updateStmt.setInt(idx++, lcfsComplianceReportId) updateStmt.addBatch() + log.info("Successfully processed legacy id ${legacyComplianceReportId} (LCFS ID: ${lcfsComplianceReportId}), adding to batch.") updateCount++ } catch (Exception e) { log.error("Error processing legacy compliance_report_id ${legacyComplianceReportId}", e) diff --git a/etl/nifi_scripts/fuel_supply.groovy b/etl/nifi_scripts/fuel_supply.groovy index de02306e0..aeafffad5 100644 --- a/etl/nifi_scripts/fuel_supply.groovy +++ b/etl/nifi_scripts/fuel_supply.groovy @@ -14,8 +14,9 @@ def destinationDbcpService = context.controllerServiceLookup.getControllerServic log.warn('**** BEGIN FUEL SUPPLY MIGRATION ****') // Declare connection variables at script level so they're visible in finally block -Connection sourceConn = null -Connection destinationConn = null +// Connections will be managed per-report inside the loop +// Connection sourceConn = null +// Connection destinationConn = null // Track records with null quantity issues def failedRecords = [] @@ -56,6 +57,39 @@ try { throw new Exception("Table 'compliance_report_schedule_b_record' does not exist in source database") } + // --- Pre-fetch Base Versions for Action Type Determination --- + def groupBaseVersions = [:] + Connection baseVersionConn = null + try { + baseVersionConn = destinationDbcpService.getConnection() + if (baseVersionConn == null || baseVersionConn.isClosed()) { + throw new Exception("Failed to get DESTINATION connection for fetching base versions") + } + log.warn("Fetching base versions from compliance_report table...") + def baseVersionSql = ''' + SELECT compliance_report_group_uuid, MIN(version) as base_version + FROM compliance_report + WHERE compliance_report_group_uuid IS NOT NULL + GROUP BY compliance_report_group_uuid + ''' + def baseVersionStmt = baseVersionConn.prepareStatement(baseVersionSql) + def baseVersionRS = baseVersionStmt.executeQuery() + while (baseVersionRS.next()) { + groupBaseVersions[baseVersionRS.getString('compliance_report_group_uuid')] = baseVersionRS.getInt('base_version') + } + baseVersionRS.close() + baseVersionStmt.close() + log.warn("Finished fetching ${groupBaseVersions.size()} base versions.") + } catch (Exception e) { + log.error("Error fetching base versions: ${e.getMessage()}") + throw e // Re-throw to stop the script if base versions can't be fetched + } finally { + if (baseVersionConn != null) { + try { baseVersionConn.close() } catch (Exception e) { log.error("Error closing base version connection: ${e.getMessage()}") } + } + } + // --- End Pre-fetch Base Versions --- + // Unit mapping dictionary def unitMapping = [ 'L': 'Litres', @@ -77,638 +111,604 @@ try { def groupUuid = complianceReports.getString('compliance_report_group_uuid') def version = complianceReports.getInt('version') - // Fetch the corresponding snapshot record from compliance_report_snapshot in TFRS - def snapshotStmt = sourceConn.prepareStatement(''' - SELECT snapshot - FROM compliance_report_snapshot - WHERE compliance_report_id = ? - ''') - snapshotStmt.setInt(1, legacyId) - def snapshotResult = snapshotStmt.executeQuery() - - def useSnapshot = false - def scheduleBRecords = [] - - if (snapshotResult.next()) { - // Parse JSON snapshot - def snapshotJson = new JsonSlurper().parseText(snapshotResult.getString('snapshot')) - if (snapshotJson.schedule_b && snapshotJson.schedule_b.records) { - scheduleBRecords = snapshotJson.schedule_b.records - useSnapshot = true + // --- Manage Connections Per Report --- + Connection sourceConn = null // Define here + Connection destinationConn = null // Define here + try { + // Get fresh connections for this report + sourceConn = sourceDbcpService.getConnection() + destinationConn = destinationDbcpService.getConnection() + + // Validate connections + if (sourceConn == null || sourceConn.isClosed()) { + throw new Exception("Failed to get valid SOURCE connection for source CR ID ${legacyId}") } - } else { - // Fallback: Retrieve fuel supply data from TFRS if snapshot is missing - def fuelSupplyStmt = sourceConn.prepareStatement(""" - WITH schedule_b AS ( - SELECT crsbr.id as fuel_supply_id, - cr.id as cr_legacy_id, - crsbr.quantity, - uom.name as unit_of_measure, - (SELECT cil.density - FROM carbon_intensity_limit cil - WHERE cil.fuel_class_id = crsbr.fuel_class_id - AND cil.effective_date <= cp.effective_date - AND cil.expiration_date > cp.effective_date - ORDER BY cil.effective_date DESC, cil.update_timestamp DESC - LIMIT 1) as ci_limit, - CASE - WHEN dt.the_type = 'Alternative' THEN crsbr.intensity - WHEN dt.the_type = 'GHGenius' THEN crsbr.intensity -- TODO fix intensity to extract from Schedule-D sheets - WHEN dt.the_type = 'Fuel Code' THEN fc1.carbon_intensity - WHEN dt.the_type IN ('Default Carbon Intensity', 'Carbon Intensity') - THEN (SELECT dci.density - FROM default_carbon_intensity dci - JOIN default_carbon_intensity_category dcic - ON dcic.id = aft.default_carbon_intensity_category_id - WHERE dci.effective_date <= cp.effective_date - AND dci.expiration_date > cp.effective_date - ORDER BY dci.effective_date DESC, dci.update_timestamp DESC - LIMIT 1) - ELSE NULL - END as ci_of_fuel, - (SELECT ed.density - FROM energy_density ed - JOIN energy_density_category edc - ON edc.id = aft.energy_density_category_id - WHERE ed.effective_date <= cp.effective_date - AND ed.expiration_date > cp.effective_date - ORDER BY ed.effective_date DESC, ed.update_timestamp DESC - LIMIT 1) as energy_density, - (SELECT eer.ratio - FROM energy_effectiveness_ratio eer - JOIN energy_effectiveness_ratio_category eerc - ON eerc.id = aft.energy_effectiveness_ratio_category_id - WHERE eer.effective_date <= cp.effective_date - AND eer.expiration_date > cp.effective_date - ORDER BY eer.effective_date DESC, eer.update_timestamp DESC - LIMIT 1) as eer, - fc.fuel_class as fuel_category, - fc1.fuel_code as fuel_code_prefix, - CAST(CONCAT(fc1.fuel_code_version, '.', fc1.fuel_code_version_minor) AS CHAR) as fuel_code_suffix, - aft.name as fuel_type, - CONCAT(TRIM(pa.description), ' - ', TRIM(pa.provision)) as provision_act, - cr.create_timestamp as create_date, - cr.update_timestamp as update_date, - 'ETL' as create_user, - 'ETL' as update_user, - 'SUPPLIER' as user_type, - 'CREATE' as action_type - FROM compliance_report_schedule_b_record crsbr - INNER JOIN fuel_class fc ON fc.id = crsbr.fuel_class_id - INNER JOIN approved_fuel_type aft ON aft.id = crsbr.fuel_type_id - INNER JOIN provision_act pa ON pa.id = crsbr.provision_of_the_act_id - LEFT JOIN carbon_intensity_fuel_determination cifd - ON cifd.fuel_id = aft.id AND cifd.provision_act_id = pa.id - LEFT JOIN determination_type dt ON dt.id = cifd.determination_type_id - INNER JOIN compliance_report cr ON cr.schedule_b_id = crsbr.schedule_id - INNER JOIN compliance_period cp ON cp.id = cr.compliance_period_id - LEFT JOIN fuel_code fc1 ON fc1.id = crsbr.fuel_code_id - LEFT JOIN unit_of_measure uom ON uom.id = aft.unit_of_measure_id - WHERE cr.id = ? - ) - SELECT b.*, (b.energy_density * b.quantity) AS energy_content, - ((((b.ci_limit * b.eer) - b.ci_of_fuel) * (b.energy_density * b.quantity)) / 1000000) AS compliance_units - FROM schedule_b b - """) - fuelSupplyStmt.setInt(1, legacyId) - scheduleBRecords = fuelSupplyStmt.executeQuery().collect { it } - } - - log.warn("Processing ${scheduleBRecords.size()} schedule B records") - - // Add validation to ensure we have valid records before iterating - if (scheduleBRecords == null) { - log.warn("scheduleBRecords is null - skipping processing") - continue - } - - if (scheduleBRecords.isEmpty()) { - log.warn("No schedule B records found for compliance report ${legacyId}") - continue - } - - // For debugging special cases - if (legacyId == 7) { - log.warn("DEBUG: Examining scheduleBRecords for CR ID 7:") - scheduleBRecords.eachWithIndex { rec, idx -> - log.warn("Record #${idx} type: ${rec.getClass().name}") - if (rec instanceof Map) { - log.warn("Keys available: ${rec.keySet()}") - // Print first few values to see structure - def count = 0 - rec.each { k, v -> - if (count < 5) { - log.warn(" $k = $v (${v?.getClass()?.name ?: 'null'})") - count++ - } - } - } else { - log.warn("Not a map: ${rec}") - } + if (destinationConn == null || destinationConn.isClosed()) { + throw new Exception("Failed to get valid DESTINATION connection for source CR ID ${legacyId}") } - } - - try { - scheduleBRecords.each { record -> - // Skip null records - if (record == null) { - log.warn("Skipping null record") - return + log.warn("Acquired connections for source CR ID ${legacyId}") + + // --- Start processing logic for this report --- + + // Fetch the corresponding snapshot record from compliance_report_snapshot in TFRS + def snapshotStmt = sourceConn.prepareStatement(''' + SELECT snapshot + FROM compliance_report_snapshot + WHERE compliance_report_id = ? + ''') + snapshotStmt.setInt(1, legacyId) + def snapshotResult = snapshotStmt.executeQuery() + + def useSnapshot = false + def scheduleBRecords = [] + + if (snapshotResult.next()) { + // Parse JSON snapshot + def snapshotJson = new JsonSlurper().parseText(snapshotResult.getString('snapshot')) + if (snapshotJson.schedule_b && snapshotJson.schedule_b.records) { + scheduleBRecords = snapshotJson.schedule_b.records + useSnapshot = true } + } else { + log.warn("No snapshot found for source CR ID ${legacyId}. Using direct SQL query fallback.") + useSnapshot = false + // Fallback: Retrieve fuel supply data from TFRS using Sql.eachRow + def fuelSupplySQL = """ + WITH schedule_b AS ( + SELECT crsbr.id as fuel_supply_id, + cr.id as cr_legacy_id, + crsbr.quantity, + uom.name as unit_of_measure, + (SELECT cil.density + FROM carbon_intensity_limit cil + WHERE cil.fuel_class_id = crsbr.fuel_class_id + AND cil.effective_date <= cp.effective_date + AND cil.expiration_date > cp.effective_date + ORDER BY cil.effective_date DESC, cil.update_timestamp DESC + LIMIT 1) as ci_limit, + CASE + WHEN dt.the_type = 'Alternative' THEN crsbr.intensity + WHEN dt.the_type = 'GHGenius' THEN ( + -- Calculate sum of intensities from relevant Schedule D outputs + SELECT SUM(sdo.intensity) + FROM public.compliance_report_schedule_d crsd + JOIN public.compliance_report_schedule_d_sheet sds ON sds.schedule_id = crsd.id + JOIN public.compliance_report_schedule_d_sheet_output sdo ON sdo.sheet_id = sds.id + WHERE crsd.id = cr.schedule_d_id -- Join back to the main report's schedule D + AND sds.fuel_type_id = crsbr.fuel_type_id -- Filter sheet by fuel type + AND sds.fuel_class_id = crsbr.fuel_class_id -- Filter sheet by fuel class + ) + WHEN dt.the_type = 'Fuel Code' THEN fc1.carbon_intensity + WHEN dt.the_type IN ('Default Carbon Intensity', 'Carbon Intensity') + THEN (SELECT dci.density + FROM default_carbon_intensity dci + JOIN default_carbon_intensity_category dcic + ON dcic.id = dci.category_id -- Assuming category_id links dci to dcic + JOIN approved_fuel_type aft_sub -- Alias to avoid conflict with outer aft + ON aft_sub.default_carbon_intensity_category_id = dcic.id + WHERE aft_sub.id = aft.id -- Match the correct approved fuel type + AND dci.effective_date <= cp.effective_date + AND (dci.expiration_date > cp.effective_date OR dci.expiration_date IS NULL) -- Handle NULL expiration + ORDER BY dci.effective_date DESC, dci.update_timestamp DESC -- Keep ordering + LIMIT 1) + ELSE NULL + END as ci_of_fuel, + (SELECT ed.density + FROM energy_density ed + JOIN energy_density_category edc + ON edc.id = aft.energy_density_category_id + WHERE ed.effective_date <= cp.effective_date + AND ed.expiration_date > cp.effective_date + ORDER BY ed.effective_date DESC, ed.update_timestamp DESC + LIMIT 1) as energy_density, + (SELECT eer.ratio + FROM energy_effectiveness_ratio eer + JOIN energy_effectiveness_ratio_category eerc + ON eerc.id = aft.energy_effectiveness_ratio_category_id + WHERE eer.effective_date <= cp.effective_date + AND eer.expiration_date > cp.effective_date + ORDER BY eer.effective_date DESC, eer.update_timestamp DESC + LIMIT 1) as eer, + fc.fuel_class as fuel_category, + fc1.fuel_code as fuel_code_prefix, + CAST(CONCAT(fc1.fuel_code_version, '.', fc1.fuel_code_version_minor) AS CHAR) as fuel_code_suffix, + aft.name as fuel_type, + CONCAT(TRIM(pa.description), ' - ', TRIM(pa.provision)) as provision_act, + cr.create_timestamp as create_date, + cr.update_timestamp as update_date, + 'ETL' as create_user, + 'ETL' as update_user, + 'SUPPLIER' as user_type, + 'CREATE' as action_type + FROM compliance_report_schedule_b_record crsbr + INNER JOIN fuel_class fc ON fc.id = crsbr.fuel_class_id + INNER JOIN approved_fuel_type aft ON aft.id = crsbr.fuel_type_id + INNER JOIN provision_act pa ON pa.id = crsbr.provision_of_the_act_id + LEFT JOIN carbon_intensity_fuel_determination cifd + ON cifd.fuel_id = aft.id AND cifd.provision_act_id = pa.id + LEFT JOIN determination_type dt ON dt.id = cifd.determination_type_id + INNER JOIN compliance_report cr ON cr.schedule_b_id = crsbr.schedule_id + INNER JOIN compliance_period cp ON cp.id = cr.compliance_period_id + LEFT JOIN fuel_code fc1 ON fc1.id = crsbr.fuel_code_id + LEFT JOIN unit_of_measure uom ON uom.id = aft.unit_of_measure_id + WHERE cr.id = ? + ) + SELECT b.*, (b.energy_density * b.quantity) AS energy_content, + ((((b.ci_limit * b.eer) - b.ci_of_fuel) * (b.energy_density * b.quantity)) / 1000000) AS compliance_units + FROM schedule_b b + """ - log.warn("Processing record of type: ${record.getClass().name}") - - // Safely access unit_of_measure without defaults - def unitOfMeasure = null + // Use Groovy Sql for robust row iteration + def sql = new groovy.sql.Sql(sourceConn) try { - if (useSnapshot) { - unitOfMeasure = record.unit_of_measure - } else { - // For ResultSet objects, try to get the value safely - try { - unitOfMeasure = record.getString("unit_of_measure") - } catch (Exception e1) { - // Try direct property access as fallback - try { - unitOfMeasure = record.unit_of_measure - } catch (Exception e2) { - log.warn("Cannot access unit_of_measure: " + e2.getMessage()) - } - } - } - } catch (Exception e) { - log.warn("Error accessing unit_of_measure: " + e.getMessage()) - } - - // Map unit values - keep original if not in mapping - def unitFullForm = unitOfMeasure != null ? unitMapping.get(unitOfMeasure, unitOfMeasure) : null - - // Lookup provision_of_the_act_id - safely access properties - def provisionLookupValue = null - try { - if (useSnapshot) { - def description = null - def provision = null - try { - description = record.provision_of_the_act_description - provision = record.provision_of_the_act - if (description != null && provision != null) { - provisionLookupValue = "${description} - ${provision}" - } - } catch (Exception e) { - log.warn("Error accessing provision data from snapshot: " + e.getMessage()) - } - } else { - // Try to get provision_act safely from ResultSet - try { - provisionLookupValue = record.getString("provision_act") - } catch (Exception e1) { - try { - provisionLookupValue = record.provision_act - } catch (Exception e2) { - log.warn("Cannot access provision_act: " + e2.getMessage()) - } - } + sql.eachRow(fuelSupplySQL, [legacyId]) { record -> + // --- Process Each SQL Record --- + // The 'record' object here is a GroovyRowResult, supports .get() + log.warn("Processing SQL fallback record for source CR ID ${legacyId}") + // Determine Action Type for SQL fallback + def baseVersion = groupBaseVersions.get(groupUuid) + def actionType = (baseVersion == null || version == baseVersion) ? 'CREATE' : 'UPDATE' + processScheduleBRecord(record, useSnapshot, complianceReportId, legacyId, groupUuid, version, unitMapping, destinationConn, failedRecords, log, actionType) + } // end sql.eachRow + } finally { + sql.close() // Close the groovy.sql.Sql instance } - } catch (Exception e) { - log.warn("Error determining provision value: " + e.getMessage()) - } - - def provisionId = null - if (provisionLookupValue != null) { - def provisionStmt = destinationConn.prepareStatement(''' - SELECT provision_of_the_act_id FROM provision_of_the_act WHERE name = ? - ''') - provisionStmt.setString(1, provisionLookupValue) - def provisionResult = provisionStmt.executeQuery() - provisionId = provisionResult.next() ? provisionResult.getInt('provision_of_the_act_id') : null } - // Lookup fuel_category_id - safely access properties - def fuelCategoryLookupValue = null - try { - if (useSnapshot) { - try { - fuelCategoryLookupValue = record.fuel_class - } catch (Exception e) { - log.warn("Cannot access fuel_class from snapshot: " + e.getMessage()) - } - } else { - // Try to get fuel_category safely from ResultSet - try { - fuelCategoryLookupValue = record.getString("fuel_category") - } catch (Exception e1) { - try { - fuelCategoryLookupValue = record.fuel_category - } catch (Exception e2) { - log.warn("Cannot access fuel_category: " + e2.getMessage()) - } - } + // --- Process Snapshot Records (if applicable) --- + if (useSnapshot) { + log.warn("Processing ${scheduleBRecords.size()} records from snapshot for source CR ID ${legacyId}") + // Add validation to ensure we have valid records before iterating + if (scheduleBRecords == null || scheduleBRecords.isEmpty()) { + log.warn("Snapshot record list is null or empty for source CR ID ${legacyId} - skipping.") + continue // Skip to the next report } - } catch (Exception e) { - log.warn("Error determining fuel category: " + e.getMessage()) - } - - def fuelCategoryId = null - if (fuelCategoryLookupValue != null) { - def fuelCategoryStmt = destinationConn.prepareStatement(''' - SELECT fuel_category_id FROM fuel_category WHERE category = ?::fuel_category_enum - ''') - fuelCategoryStmt.setString(1, fuelCategoryLookupValue) - def fuelCategoryResult = fuelCategoryStmt.executeQuery() - fuelCategoryId = fuelCategoryResult.next() ? fuelCategoryResult.getInt('fuel_category_id') : null - } - // Lookup fuel_code_id - def fuelCodeStmt = destinationConn.prepareStatement(''' - select * from fuel_code fc, fuel_code_prefix fcp where fcp.fuel_code_prefix_id = fc.prefix_id and fcp.prefix = ? and fc.fuel_suffix = ? - ''') - def fuelCodeId = null - def fuelCode = null - - // Safely access fuel_code or fuel_code_prefix - try { - if (useSnapshot) { - try { - fuelCode = record.fuel_code - } catch (Exception e) { - log.warn("Cannot access fuel_code from snapshot: " + e.getMessage()) - } - } else { - // For SQL data, try to get fuel_code_prefix - try { - fuelCode = record.getString("fuel_code_prefix") - } catch (Exception e1) { - try { - fuelCode = record.fuel_code_prefix - } catch (Exception e2) { - log.warn("Cannot access fuel_code_prefix: " + e2.getMessage()) - } - } + // Determine Action Type for Snapshot processing + def baseVersion = groupBaseVersions.get(groupUuid) + def actionType = (baseVersion == null || version == baseVersion) ? 'CREATE' : 'UPDATE' + + scheduleBRecords.each { record -> + // The 'record' object here is a LazyMap + processScheduleBRecord(record, useSnapshot, complianceReportId, legacyId, groupUuid, version, unitMapping, destinationConn, failedRecords, log, actionType) } - } catch (Exception e) { - log.warn("Error determining fuel code: " + e.getMessage()) } - - if (fuelCode != null) { + + // --- End processing logic for this report --- + + } catch (Exception reportEx) { + // Log errors specific to processing this single report + log.error("Error processing source compliance report ID ${legacyId}: ${reportEx.getMessage()}") + failedRecords << [ + crId: legacyId, + complianceReportId: complianceReportId, + recordType: 'Report Level', + reason: "Exception during report processing: ${reportEx.getMessage()}", + recordData: "N/A" + ] + // Continue to the next report + } finally { + // --- Ensure Connections are Closed for this Report --- + if (sourceConn != null) { try { - // Convert to string if it's not already a string - String fuelCodeStr = fuelCode.toString() - - def prefix = fuelCodeStr.length() >= 4 ? fuelCodeStr.substring(0, 4) : fuelCodeStr - def suffix = fuelCodeStr.length() > 4 ? fuelCodeStr.substring(4) : "" - - fuelCodeStmt.setString(1, prefix) - try { - // Try to parse suffix as BigDecimal - fuelCodeStmt.setBigDecimal(2, new BigDecimal(suffix)) - } catch (Exception e) { - // If parsing fails, use the string as is - fuelCodeStmt.setString(2, suffix) - } - - def fuelCodeResult = fuelCodeStmt.executeQuery() - fuelCodeId = fuelCodeResult.next() ? fuelCodeResult.getInt('fuel_code_id') : null - log.warn("fuelCodeId: " + fuelCodeId) + sourceConn.close() + // log.warn("Closed source connection for source CR ID ${legacyId}") // Optional: reduces log noise } catch (Exception e) { - log.warn("Error processing fuel code: " + e.getMessage()) + log.error("Error closing source connection for source CR ID ${legacyId}: " + e.getMessage()) } } - // Lookup fuel_type_id - safely access properties - def fuelTypeValue = null - try { - if (useSnapshot) { - try { - fuelTypeValue = record.fuel_type - } catch (Exception e) { - log.warn("Cannot access fuel_type from snapshot: " + e.getMessage()) - } - } else { - // Try to get fuel_type safely from ResultSet - try { - fuelTypeValue = record.getString("fuel_type") - } catch (Exception e1) { - try { - fuelTypeValue = record.fuel_type - } catch (Exception e2) { - log.warn("Cannot access fuel_type: " + e2.getMessage()) - } - } + if (destinationConn != null) { + try { + destinationConn.close() + // log.warn("Closed destination connection for source CR ID ${legacyId}") // Optional: reduces log noise + } catch (Exception e) { + log.error("Error closing destination connection for source CR ID ${legacyId}: " + e.getMessage()) } - } catch (Exception e) { - log.warn("Error determining fuel type: " + e.getMessage()) - } - - def fuelTypeId = null - if (fuelTypeValue != null) { - def fuelTypeStmt = destinationConn.prepareStatement(''' - SELECT fuel_type_id FROM fuel_type WHERE fuel_type = ? - ''') - fuelTypeStmt.setString(1, fuelTypeValue) - def fuelTypeResult = fuelTypeStmt.executeQuery() - fuelTypeId = fuelTypeResult.next() ? fuelTypeResult.getInt('fuel_type_id') : null - log.warn("fuelTypeId: " + fuelTypeId) } + } + // --- End Connection Management for this Report --- - // Insert records into fuel_supply table in LCFS (destination) - def fuelSupplyInsertStmt = destinationConn.prepareStatement(''' - INSERT INTO public.fuel_supply ( - compliance_report_id, quantity, units, compliance_units, target_ci, ci_of_fuel, - energy_density, eer, energy, fuel_type_other, fuel_category_id, fuel_code_id, - fuel_type_id, provision_of_the_act_id, end_use_id, create_date, update_date, - create_user, update_user, group_uuid, version, action_type - ) VALUES (?, ?, ?::quantityunitsenum, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::actiontypeenum) - ''') - log.warn("record.complianceReportId: " + (complianceReportId ?: "NULL")) + } // end while (complianceReports.next()) + complianceReports.close() + +} catch (Exception e) { + log.error('Error running Fuel Supply migration: ' + e.getMessage()) + log.error('Stack trace: ' + e.getStackTrace().join("\n")) + throw e +} + +// --- Helper Function to Process Schedule B Record (Snapshot Map or SQL GroovyRowResult) --- +def processScheduleBRecord(record, useSnapshot, complianceReportId, legacyId, groupUuid, version, unitMapping, destinationConn, failedRecords, log, actionType) { + try { + // Skip null records (paranoid check) + if (record == null) { + log.warn("Skipping null record passed to processScheduleBRecord for source CR ID ${legacyId}") + return + } - // Safe function to get numeric values - improved for snapshot handling - def safeGetNumber = { fieldName -> + log.warn("Processing record of type: ${record.getClass().name} for source CR ID ${legacyId}") + + // Safe function to get numeric values - handles snapshot maps and SQL result objects + def safeGetNumber = { String fieldName -> def result = null def val = null try { if (useSnapshot) { - // For JSON snapshot records, access the key directly. - val = record[fieldName] + // Use .get() for snapshot maps + val = record.get(fieldName) } else { - // For SQL ResultSet objects, try multiple variants. - try { - val = record.getString(fieldName) - } catch (Exception e1) { - // Try direct property access. - try { - val = record."$fieldName" - } catch (Exception e2) { - // Try alternative key formats. - def altFieldNames = [ - fieldName, - fieldName.toUpperCase(), - fieldName.toLowerCase(), - fieldName.replaceAll(/([A-Z])/, '_$1').toLowerCase(), - fieldName.replaceAll(/_([a-z])/) { _, c -> c.toUpperCase() } - ] - for (alt in altFieldNames) { - try { - val = record.getString(alt) - if (val != null) { - log.warn("Using alternative field name '${alt}' for ${fieldName}: ${val}") - break - } - } catch (Exception ignore) { - // Continue trying other alternatives. - } - } - } - } + // Use ['key'] syntax for SQL proxy objects + val = record[fieldName] } - + // Handle explicit "null" string values. - if (val == "null") { + if (val instanceof String && val.equalsIgnoreCase("null")) { val = null } - + // If the field is missing or null, record the issue for quantity. if (val == null) { if (fieldName == "quantity") { - def errorMsg = "ERROR: Found null quantity in source data for field '${fieldName}'" + def errorMsg = "ERROR: Found null quantity in source data for field '${fieldName}' for CR ID ${legacyId}" log.error(errorMsg) failedRecords << [ crId: legacyId, complianceReportId: complianceReportId, recordType: record.getClass().name, reason: "Null quantity value", - recordData: useSnapshot ? - record.toString().take(200) + "..." : - "SQL ResultSet (details unavailable)" + recordData: "Record data preview unavailable in helper function" ] } - return new BigDecimal(0) + return null } - + // Convert to BigDecimal. if (val instanceof String) { val = val.trim().replaceAll(",", "") if (val.matches("-?\\d+(\\.\\d+)?")) { result = new BigDecimal(val) } else { - log.warn("String value '${val}' for ${fieldName} is not a valid number") + log.warn("String value '${val}' for ${fieldName} (CR ID ${legacyId}) is not a valid number") if (fieldName == "quantity") { - log.error("ERROR: Invalid quantity value '${val}' in source data.") + log.error("ERROR: Invalid quantity value '${val}' in source data for CR ID ${legacyId}.") failedRecords << [ crId: legacyId, complianceReportId: complianceReportId, recordType: record.getClass().name, reason: "Non-numeric quantity value: '${val}'", - recordData: useSnapshot ? - record.toString().take(200) + "..." : - "SQL ResultSet (details unavailable)" + recordData: "Record data preview unavailable in helper function" ] - return new BigDecimal(0) + return null } result = null } } else if (val instanceof Number) { result = new BigDecimal(val.toString()) } else { - log.warn("Unknown type for ${fieldName}: ${val.getClass().name}") + log.warn("Unknown type for ${fieldName} (CR ID ${legacyId}): ${val.getClass().name}") if (fieldName == "quantity") { - log.error("ERROR: Quantity has unexpected type: ${val.getClass().name}. Cannot convert to numeric value.") + log.error("ERROR: Quantity has unexpected type: ${val.getClass().name} for CR ID ${legacyId}. Cannot convert.") failedRecords << [ crId: legacyId, complianceReportId: complianceReportId, recordType: record.getClass().name, reason: "Quantity has unexpected type: ${val.getClass().name}", - recordData: useSnapshot ? - record.toString().take(200) + "..." : - "SQL ResultSet (details unavailable)" + recordData: "Record data preview unavailable in helper function" ] - return new BigDecimal(0) + return null } result = null } } catch (Exception e) { - log.warn("Error converting ${fieldName}: " + e.getMessage()) + log.warn("Error accessing/converting ${fieldName} for CR ID ${legacyId}: " + e.getMessage()) if (fieldName == "quantity") { - log.error("ERROR: Exception while processing quantity value: ${e.getMessage()}") + log.error("ERROR: Exception while processing quantity value for CR ID ${legacyId}: ${e.getMessage()}") failedRecords << [ crId: legacyId, complianceReportId: complianceReportId, recordType: record.getClass().name, reason: "Exception while processing quantity: ${e.getMessage()}", - recordData: useSnapshot ? - record.toString().take(200) + "..." : - "SQL ResultSet (details unavailable)" + recordData: "Record data preview unavailable in helper function" ] - return new BigDecimal(0) + return null } } return result } - - // Get numeric values safely - def quantity = safeGetNumber("quantity") - - // Determine compliance units - def complianceUnits = null + // --- Lookups using appropriate access method --- + // Unit of Measure + def unitOfMeasure = null + def provisionActDescription = null // Variable to store provision description + try { if (useSnapshot) { - def credits = safeGetNumber("credits") - def debits = safeGetNumber("debits") - - if (credits != null) { - complianceUnits = credits.negate() - } else if (debits != null) { - complianceUnits = debits - } - } else { - complianceUnits = safeGetNumber("compliance_units") - } - - // Get other numeric values safely - def ciLimit = safeGetNumber("ci_limit") - def ciOfFuel = safeGetNumber("ci_of_fuel") - def energyDensity = safeGetNumber("energy_density") - def eer = safeGetNumber("eer") - def energyContent = safeGetNumber("energy_content") - - // Must use setNull for potentially null values - fuelSupplyInsertStmt.setInt(1, complianceReportId) - - // Handle numeric values safely - if (quantity != null) { - fuelSupplyInsertStmt.setBigDecimal(2, quantity) - } else { - fuelSupplyInsertStmt.setNull(2, java.sql.Types.NUMERIC) - } - - if (unitFullForm != null) { - fuelSupplyInsertStmt.setString(3, unitFullForm) - } else { - fuelSupplyInsertStmt.setNull(3, java.sql.Types.VARCHAR) - } - - if (complianceUnits != null) { - fuelSupplyInsertStmt.setBigDecimal(4, complianceUnits) - } else { - fuelSupplyInsertStmt.setNull(4, java.sql.Types.NUMERIC) - } - - if (ciLimit != null) { - fuelSupplyInsertStmt.setBigDecimal(5, ciLimit) + unitOfMeasure = record.get('unit_of_measure') + provisionActDescription = record.get('provision_of_the_act_description') // Get description from snapshot } else { - fuelSupplyInsertStmt.setNull(5, java.sql.Types.NUMERIC) + unitOfMeasure = record['unit_of_measure'] + // Description not directly available in fallback SQL, would need join if required here } - - if (ciOfFuel != null) { - fuelSupplyInsertStmt.setBigDecimal(6, ciOfFuel) + } catch (Exception e) { + log.warn("Error accessing unit_of_measure or provision_description for CR ID ${legacyId}: " + e.getMessage()) + } + def unitFullForm = unitOfMeasure != null ? unitMapping.get(unitOfMeasure, unitOfMeasure) : null + + // Provision of the Act + def provisionLookupValue = null + try { + if (useSnapshot) { + def description = record.get('provision_of_the_act_description') + def provision = record.get('provision_of_the_act') + if (description != null && provision != null) { provisionLookupValue = "${description} - ${provision}" } } else { - fuelSupplyInsertStmt.setNull(6, java.sql.Types.NUMERIC) + provisionLookupValue = record["provision_act"] } - - if (energyDensity != null) { - fuelSupplyInsertStmt.setBigDecimal(7, energyDensity) - } else { - fuelSupplyInsertStmt.setNull(7, java.sql.Types.NUMERIC) + } catch (Exception e) { + log.warn("Error determining provision value for CR ID ${legacyId}: " + e.getMessage()) + } + def provisionId = null + if (provisionLookupValue != null) { + // Check connection before preparing statement + if (destinationConn == null || destinationConn.isClosed()) { throw new Exception("Destination connection is null or closed before provision lookup!") } + def provisionStmt = destinationConn.prepareStatement('''SELECT provision_of_the_act_id FROM provision_of_the_act WHERE name = ?''') + try { + provisionStmt.setString(1, provisionLookupValue) + def provisionResult = provisionStmt.executeQuery() + provisionId = provisionResult.next() ? provisionResult.getInt('provision_of_the_act_id') : null + provisionResult.close() + } finally { + provisionStmt.close() } - - if (eer != null) { - fuelSupplyInsertStmt.setBigDecimal(8, eer) - } else { - fuelSupplyInsertStmt.setNull(8, java.sql.Types.NUMERIC) + } else { + log.warn("Provision lookup value is null for source CR ID ${legacyId}") + } + + // Fuel Category + def fuelCategoryLookupValue = null + try { + fuelCategoryLookupValue = useSnapshot ? record.get('fuel_class') : record['fuel_category'] + } catch (Exception e) { + log.warn("Error determining fuel category for CR ID ${legacyId}: " + e.getMessage()) + } + def fuelCategoryId = null + if (fuelCategoryLookupValue != null) { + // Check connection before preparing statement + if (destinationConn == null || destinationConn.isClosed()) { throw new Exception("Destination connection is null or closed before fuel category lookup!") } + def fuelCategoryStmt = destinationConn.prepareStatement('''SELECT fuel_category_id FROM fuel_category WHERE category = ?::fuel_category_enum''') + try { + fuelCategoryStmt.setString(1, fuelCategoryLookupValue) + def fuelCategoryResult = fuelCategoryStmt.executeQuery() + fuelCategoryId = fuelCategoryResult.next() ? fuelCategoryResult.getInt('fuel_category_id') : null + fuelCategoryResult.close() + } finally { + fuelCategoryStmt.close() } - - if (energyContent != null) { - fuelSupplyInsertStmt.setBigDecimal(9, energyContent) + } else { + log.warn("Fuel category lookup value is null for CR ID ${legacyId}") + } + + // Fuel Code + def fuelCodeLookupValue = null + def fuelCodeSuffixValue = null + def fuelCodeId = null + try { + if (useSnapshot) { + def fuelCodeDesc = record.get('fuel_code_description') // e.g., "BCLCF185.1" + if (fuelCodeDesc != null && fuelCodeDesc instanceof String && !fuelCodeDesc.isEmpty()) { + // Find the index of the first digit to split prefix/suffix + int firstDigitIndex = -1 + for (int i = 0; i < fuelCodeDesc.length(); i++) { + if (Character.isDigit(fuelCodeDesc.charAt(i))) { + firstDigitIndex = i + break + } + } + + if (firstDigitIndex != -1) { + fuelCodeLookupValue = fuelCodeDesc.substring(0, firstDigitIndex) + fuelCodeSuffixValue = fuelCodeDesc.substring(firstDigitIndex) + log.debug("Parsed snapshot fuel code: Prefix='${fuelCodeLookupValue}', Suffix='${fuelCodeSuffixValue}' from Desc='${fuelCodeDesc}'") + } else { + // Handle cases where description might only be a prefix or has unexpected format + fuelCodeLookupValue = fuelCodeDesc // Assume whole string is prefix if no digits found + fuelCodeSuffixValue = "" + log.warn("Could not split fuel code description '${fuelCodeDesc}' into prefix/suffix based on first digit for CR ID ${legacyId}. Using full string as prefix.") + } + } else { + log.warn("Fuel code description is null or invalid in snapshot for CR ID ${legacyId}") + } } else { - fuelSupplyInsertStmt.setNull(9, java.sql.Types.NUMERIC) + // SQL fallback - assumes query provides correct columns + fuelCodeLookupValue = record["fuel_code_prefix"] + fuelCodeSuffixValue = record["fuel_code_suffix"] } - - // Always null for fuel_type_other - fuelSupplyInsertStmt.setNull(10, java.sql.Types.VARCHAR) - - // Handle integer IDs safely - if (fuelCategoryId != null) { - fuelSupplyInsertStmt.setInt(11, fuelCategoryId) - } else { - fuelSupplyInsertStmt.setNull(11, java.sql.Types.INTEGER) + } catch (Exception e) { + log.warn("Error determining fuel code components for CR ID ${legacyId}: " + e.getMessage()) + } + + // --- Perform Destination Lookup --- + if (fuelCodeLookupValue != null) { + // Check connection before preparing statement + if (destinationConn == null || destinationConn.isClosed()) { throw new Exception("Destination connection is null or closed before fuel code lookup!") } + // Ensure table/column names match LCFS schema EXACTLY + def fuelCodeStmt = destinationConn.prepareStatement('''SELECT fuel_code_id FROM fuel_code fc JOIN fuel_code_prefix fcp ON fcp.fuel_code_prefix_id = fc.prefix_id WHERE fcp.prefix = ? AND fc.fuel_suffix = ?''') + try { + fuelCodeStmt.setString(1, fuelCodeLookupValue) + fuelCodeStmt.setString(2, fuelCodeSuffixValue ?: "") + def fuelCodeResult = fuelCodeStmt.executeQuery() + fuelCodeId = fuelCodeResult.next() ? fuelCodeResult.getInt('fuel_code_id') : null + fuelCodeResult.close() + } finally { + fuelCodeStmt.close() } - - if (fuelCodeId != null) { - fuelSupplyInsertStmt.setInt(12, fuelCodeId) - } else { - fuelSupplyInsertStmt.setNull(12, java.sql.Types.INTEGER) + } else { + log.warn("Fuel code prefix is null for CR ID ${legacyId}") + } + + // Fuel Type + def fuelTypeLookupValue = null + try { + fuelTypeLookupValue = useSnapshot ? record.get('fuel_type') : record['fuel_type'] + } catch (Exception e) { + log.warn("Error determining fuel type for CR ID ${legacyId}: " + e.getMessage()) + } + def fuelTypeId = null + if (fuelTypeLookupValue != null) { + // Check connection before preparing statement + if (destinationConn == null || destinationConn.isClosed()) { throw new Exception("Destination connection is null or closed before fuel type lookup!") } + def fuelTypeStmt = destinationConn.prepareStatement('''SELECT fuel_type_id FROM fuel_type WHERE fuel_type = ?''') + try { + fuelTypeStmt.setString(1, fuelTypeLookupValue) + def fuelTypeResult = fuelTypeStmt.executeQuery() + fuelTypeId = fuelTypeResult.next() ? fuelTypeResult.getInt('fuel_type_id') : null + fuelTypeResult.close() + } finally { + fuelTypeStmt.close() } - - if (fuelTypeId != null) { - fuelSupplyInsertStmt.setInt(13, fuelTypeId) + } else { + log.warn("Fuel type lookup value is null for CR ID ${legacyId}") + } + + // --- Get Numeric Values --- + def quantity = safeGetNumber("quantity") + def complianceUnits = null + def ciOfFuel = null // Initialize ciOfFuel + + if (useSnapshot) { + // Prioritize snapshot values + def credits = safeGetNumber("credits") + def debits = safeGetNumber("debits") + if (credits != null) { complianceUnits = credits.negate() } + else if (debits != null) { complianceUnits = debits } + + // --- Get ciOfFuel from Snapshot --- + // Check if it's a 'Prescribed' value first + if (provisionActDescription == "Prescribed carbon intensity") { + log.warn("Using effective_carbon_intensity from snapshot for Prescribed CI (CR ID: ${legacyId})") + ciOfFuel = safeGetNumber("effective_carbon_intensity") + if (ciOfFuel == null) { + log.error("ERROR: effective_carbon_intensity is NULL in snapshot for Prescribed CI record (CR ID: ${legacyId})") + } + } else if (provisionActDescription == "Approved fuel code") { + log.warn("Using effective_carbon_intensity from snapshot for Approved Fuel Code (CR ID: ${legacyId})") + ciOfFuel = safeGetNumber("effective_carbon_intensity") + if (ciOfFuel == null) { + log.error("ERROR: effective_carbon_intensity is NULL in snapshot for Approved fuel code record (CR ID: ${legacyId})") + } + } else if (provisionActDescription == "Default Carbon Intensity Value") { + log.warn("Using effective_carbon_intensity from snapshot for Default CI Value (CR ID: ${legacyId})") + ciOfFuel = safeGetNumber("effective_carbon_intensity") + if (ciOfFuel == null) { + log.error("ERROR: effective_carbon_intensity is NULL in snapshot for Default CI Value record (CR ID: ${legacyId})") + } + } else if (provisionActDescription == "GHGenius modelled") { // Handle GHGenius from snapshot + log.warn("Using effective_carbon_intensity from snapshot for GHGenius modelled (CR ID: ${legacyId})") + ciOfFuel = safeGetNumber("effective_carbon_intensity") + if (ciOfFuel == null) { + log.error("ERROR: effective_carbon_intensity is NULL in snapshot for GHGenius modelled record (CR ID: ${legacyId})") + } } else { - fuelSupplyInsertStmt.setNull(13, java.sql.Types.INTEGER) + // For non-prescribed, non-fuel-code types (e.g., Alternative, Default CI from snapshot), try the 'intensity' field + ciOfFuel = safeGetNumber("intensity") + // If intensity is null in snapshot, we might need more complex logic here + // based on determination type from snapshot, but ideally snapshot has final value. + if (ciOfFuel == null) { + log.warn("Snapshot field 'intensity' is NULL for non-prescribed/non-fuel-code record (CR ID: ${legacyId}, Provision: ${provisionActDescription}). ci_of_fuel will be null.") + } } - - if (provisionId != null) { - fuelSupplyInsertStmt.setInt(14, provisionId) - } else { - fuelSupplyInsertStmt.setNull(14, java.sql.Types.INTEGER) + // --- End Get ciOfFuel from Snapshot --- + + } else { + // SQL Fallback logic + complianceUnits = safeGetNumber("compliance_units") + ciOfFuel = safeGetNumber("ci_of_fuel") // Get value calculated by the SQL CASE statement + } + + def ciLimit = safeGetNumber("ci_limit") + def energyDensity = safeGetNumber("energy_density") + def eer = safeGetNumber("eer") + def energyContent = safeGetNumber("energy_content") + + // --- PRE-INSERT VALIDATION --- + boolean canInsert = true + List validationErrors = [] + if (quantity == null) { + canInsert = false + validationErrors.add("Quantity is NULL or invalid") + } + if (unitFullForm == null) { + canInsert = false + validationErrors.add("Units (unit_of_measure) is NULL or could not be determined") + } + + if (!canInsert) { + log.error("Skipping insert for CR ID ${legacyId} (LCFS ID: ${complianceReportId}) due to validation errors: ${validationErrors.join(', ')}") + if (!validationErrors.contains("Quantity is NULL or invalid")) { + failedRecords << [ + crId: legacyId, + complianceReportId: complianceReportId, + recordType: record.getClass().name, + reason: "Validation failed: ${validationErrors.join(', ')}", + recordData: "Record data preview unavailable in helper function" + ] } - - // Always null fields + return // Skip this record + } + + // --- Insert Record --- + // Check connection before preparing final insert statement + if (destinationConn == null || destinationConn.isClosed()) { throw new Exception("Destination connection is null or closed before final insert!") } + def fuelSupplyInsertStmt = destinationConn.prepareStatement(''' + INSERT INTO public.fuel_supply ( + compliance_report_id, quantity, units, compliance_units, target_ci, ci_of_fuel, + energy_density, eer, energy, fuel_type_other, fuel_category_id, fuel_code_id, + fuel_type_id, provision_of_the_act_id, end_use_id, create_date, update_date, + create_user, update_user, group_uuid, version, action_type + ) VALUES (?, ?, ?::quantityunitsenum, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::actiontypeenum) + ''') + try { + // Bind parameters + fuelSupplyInsertStmt.setInt(1, complianceReportId) + fuelSupplyInsertStmt.setBigDecimal(2, quantity) + fuelSupplyInsertStmt.setString(3, unitFullForm) + // ... (Bind remaining parameters 4-22 using setNull appropriately) ... + if (complianceUnits != null) { fuelSupplyInsertStmt.setBigDecimal(4, complianceUnits) } else { fuelSupplyInsertStmt.setNull(4, java.sql.Types.NUMERIC) } + if (ciLimit != null) { fuelSupplyInsertStmt.setBigDecimal(5, ciLimit) } else { fuelSupplyInsertStmt.setNull(5, java.sql.Types.NUMERIC) } + if (ciOfFuel != null) { fuelSupplyInsertStmt.setBigDecimal(6, ciOfFuel) } else { fuelSupplyInsertStmt.setNull(6, java.sql.Types.NUMERIC) } + if (energyDensity != null) { fuelSupplyInsertStmt.setBigDecimal(7, energyDensity) } else { fuelSupplyInsertStmt.setNull(7, java.sql.Types.NUMERIC) } + if (eer != null) { fuelSupplyInsertStmt.setBigDecimal(8, eer) } else { fuelSupplyInsertStmt.setNull(8, java.sql.Types.NUMERIC) } + if (energyContent != null) { fuelSupplyInsertStmt.setBigDecimal(9, energyContent) } else { fuelSupplyInsertStmt.setNull(9, java.sql.Types.NUMERIC) } + fuelSupplyInsertStmt.setNull(10, java.sql.Types.VARCHAR) // fuel_type_other + if (fuelCategoryId != null) { fuelSupplyInsertStmt.setInt(11, fuelCategoryId) } else { fuelSupplyInsertStmt.setNull(11, java.sql.Types.INTEGER) } + if (fuelCodeId != null) { fuelSupplyInsertStmt.setInt(12, fuelCodeId) } else { fuelSupplyInsertStmt.setNull(12, java.sql.Types.INTEGER) } + if (fuelTypeId != null) { fuelSupplyInsertStmt.setInt(13, fuelTypeId) } else { fuelSupplyInsertStmt.setNull(13, java.sql.Types.INTEGER) } + if (provisionId != null) { fuelSupplyInsertStmt.setInt(14, provisionId) } else { fuelSupplyInsertStmt.setNull(14, java.sql.Types.INTEGER) } fuelSupplyInsertStmt.setNull(15, java.sql.Types.INTEGER) // end_use_id fuelSupplyInsertStmt.setNull(16, java.sql.Types.TIMESTAMP) // create_date fuelSupplyInsertStmt.setNull(17, java.sql.Types.TIMESTAMP) // update_date - - // Non-null string values fuelSupplyInsertStmt.setString(18, 'ETL') fuelSupplyInsertStmt.setString(19, 'ETL') fuelSupplyInsertStmt.setString(20, groupUuid) fuelSupplyInsertStmt.setInt(21, version) - fuelSupplyInsertStmt.setString(22, 'SUPPLIER') - fuelSupplyInsertStmt.setString(23, 'CREATE') + fuelSupplyInsertStmt.setString(22, actionType) // Use the passed actionType + fuelSupplyInsertStmt.executeUpdate() - } // end of record loop - } catch (Exception e) { - log.error("Error processing individual record: " + e.getMessage()) - log.error("Stack trace: " + e.getStackTrace().join("\n")) - // Continue processing other records - } - } // end of scheduleBRecords.each - } catch (Exception e) { - log.error('Error running Fuel Supply migration: ' + e.getMessage()) - log.error('Stack trace: ' + e.getStackTrace().join("\n")) - throw e -} finally { - // Safely close connections - if (sourceConn != null) { - try { - sourceConn.close() - log.warn("Source connection closed") - } catch (Exception e) { - log.error("Error closing source connection: " + e.getMessage()) - } - } - - if (destinationConn != null) { - try { - destinationConn.close() - log.warn("Destination connection closed") - } catch (Exception e) { - log.error("Error closing destination connection: " + e.getMessage()) - } - } - - // Output statistics on records that failed due to null quantity - log.warn("**** SCHEDULE B MIGRATION ISSUES SUMMARY ****") - log.warn("Total records with issues: ${failedRecords.size()}") - - if (failedRecords.size() > 0) { - log.warn("\nList of records with issues:") - failedRecords.eachWithIndex { record, index -> - log.warn("${index + 1}. Compliance Report ID: ${record.crId} (LCFS ID: ${record.complianceReportId})") - log.warn(" Reason: ${record.reason}") - log.warn(" Record Type: ${record.recordType}") - log.warn(" Data Preview: ${record.recordData}") - log.warn(" -----------------------------") + } finally { + fuelSupplyInsertStmt.close() } - } -} + } catch (Exception e) { + log.error("Error processing individual record (CR ID ${legacyId}, LCFS ID: ${complianceReportId}): " + e.getMessage()) + log.error("Stack trace: " + e.getStackTrace().join("\n")) + // Add to failed records + failedRecords << [ + crId: legacyId, + complianceReportId: complianceReportId, + recordType: record?.getClass()?.name ?: 'Unknown', + reason: "Exception during processing: ${e.getMessage()}", + recordData: "Record data preview unavailable in helper function after error" + ] + } +} // end of processScheduleBRecord -log.warn('**** DONE: FUEL SUPPLY MIGRATION ****') +log.warn('**** DONE: FUEL SUPPLY MIGRATION ****') \ No newline at end of file diff --git a/etl/nifi_scripts/notional_transfer.groovy b/etl/nifi_scripts/notional_transfer.groovy index c4d2e63fe..6a87f9f37 100644 --- a/etl/nifi_scripts/notional_transfer.groovy +++ b/etl/nifi_scripts/notional_transfer.groovy @@ -86,8 +86,9 @@ String INSERT_NOTIONAL_TRANSFER_SQL = """ received_or_transferred, group_uuid, version, - user_type, - action_type + action_type, + create_user, + update_user ) VALUES ( ?, ?, @@ -97,8 +98,9 @@ String INSERT_NOTIONAL_TRANSFER_SQL = """ ?::receivedOrTransferredEnum, ?, ?, - 'SUPPLIER', - ?::actiontypeenum + ?::actiontypeenum, + ?, + ? ) """ @@ -193,6 +195,8 @@ def insertVersionRow(Connection destConn, Integer lcfsCRid, Map rowData, String insStmt.setString(7, groupUuid) insStmt.setInt(8, nextVer) insStmt.setString(9, action) + insStmt.setString(10, 'ETL') + insStmt.setString(11, 'ETL') insStmt.executeUpdate() insStmt.close() diff --git a/etl/nifi_scripts/orphaned_allocation_agreement.groovy b/etl/nifi_scripts/orphaned_allocation_agreement.groovy new file mode 100644 index 000000000..b439b2d3a --- /dev/null +++ b/etl/nifi_scripts/orphaned_allocation_agreement.groovy @@ -0,0 +1,692 @@ +/* +Migrate Orphaned Allocation Agreements from TFRS to LCFS + +Overview: +1. Identify TFRS compliance reports marked as 'exclusion reports' that do not have a + corresponding 'main' compliance report in the same compliance period/organization. +2. For each orphaned TFRS exclusion report: + a. Check if an LCFS compliance report with legacy_id = TFRS report ID already exists. + b. If not, create a new minimal LCFS compliance report (type='Supplemental', status='Draft'). + c. Fetch the allocation agreement records linked to the TFRS exclusion_agreement_id. + d. Insert these records into LCFS allocation_agreement, linked to the new LCFS report. +*/ + +import groovy.transform.Field +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.util.UUID + +// ------------------------- +// Controller Service Lookups +// ------------------------- +def sourceDbcpService = context.controllerServiceLookup.getControllerService('3245b078-0192-1000-ffff-ffffba20c1eb') +def destinationDbcpService = context.controllerServiceLookup.getControllerService('3244bf63-0192-1000-ffff-ffffc8ec6d93') + +// ------------------------- +// Global Field Declarations & Query Strings +// ------------------------- +@Field Map recordUuidMap = [:] // Maps TFRS allocation agreement record ID to a stable group UUID + +// Query to find TFRS exclusion reports without a sibling report, including director status +@Field String SELECT_ORPHANED_EXCLUSION_REPORTS_QUERY = ''' + SELECT + cr_excl.id AS tfrs_exclusion_report_id, + cr_excl.organization_id AS tfrs_organization_id, + cr_excl.compliance_period_id AS tfrs_compliance_period_id, + cr_excl.exclusion_agreement_id, + ws.director_status_id AS tfrs_director_status -- Get the director status + FROM compliance_report cr_excl + JOIN compliance_report_workflow_state ws ON cr_excl.status_id = ws.id -- Join to get status + WHERE cr_excl.exclusion_agreement_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM compliance_report cr_main + WHERE cr_main.organization_id = cr_excl.organization_id + AND cr_main.compliance_period_id = cr_excl.compliance_period_id + AND cr_main.id != cr_excl.id + ); +''' + +// Query to check if an LCFS report with a specific legacy_id exists +@Field String CHECK_LCFS_REPORT_EXISTS_QUERY = ''' + SELECT 1 FROM compliance_report WHERE legacy_id = ? LIMIT 1 +''' + +// Query to get TFRS organization name based on TFRS organization ID +@Field String SELECT_TFRS_ORG_NAME_QUERY = ''' + SELECT name FROM organization WHERE id = ? LIMIT 1 +''' + +// Query to get LCFS organization ID based on organization name +@Field String SELECT_LCFS_ORG_ID_QUERY = ''' + SELECT organization_id FROM organization WHERE name = ? LIMIT 1 +''' + +// Query to get TFRS compliance period description based on TFRS period ID +@Field String SELECT_TFRS_PERIOD_DESC_QUERY = ''' + SELECT description FROM compliance_period WHERE id = ? LIMIT 1 +''' + +// Query to get LCFS compliance period ID based on description +@Field String SELECT_LCFS_PERIOD_ID_QUERY = ''' + SELECT compliance_period_id FROM compliance_period WHERE description = ? LIMIT 1 +''' + +// Query to get LCFS compliance report status ID (e.g., for 'Draft') +@Field String SELECT_LCFS_REPORT_STATUS_ID_QUERY = ''' + SELECT compliance_report_status_id FROM compliance_report_status WHERE status = ?::compliancereportstatusenum LIMIT 1 +''' + +// Insert statement for the new minimal LCFS compliance report +// *** Placeholder: Adjust column names as needed *** +@Field String INSERT_LCFS_COMPLIANCE_REPORT_SQL = ''' + INSERT INTO compliance_report ( + organization_id, compliance_period_id, current_status_id, reporting_frequency, + compliance_report_group_uuid, version, legacy_id, create_user, update_user, + nickname + ) VALUES (?, ?, ?, ?::reportingfrequency, ?, ?, ?, 'ETL', 'ETL', ?) + RETURNING compliance_report_id; +''' + +// Query to get allocation agreement records directly via exclusion_agreement_id +@Field String SELECT_ALLOCATION_RECORDS_BY_AGREEMENT_ID_QUERY = ''' + SELECT + crear.id AS agreement_record_id, + CASE WHEN tt.the_type = 'Purchased' THEN 'Allocated from' ELSE 'Allocated to' END AS responsibility, + aft.name AS fuel_type, + aft.id AS tfrs_fuel_type_id, -- Note: TFRS fuel type ID + crear.transaction_partner, + crear.postal_address, + crear.quantity, + uom.name AS units, + crear.quantity_not_sold, + tt.id AS transaction_type_id + FROM compliance_report_exclusion_agreement_record crear + INNER JOIN transaction_type tt ON crear.transaction_type_id = tt.id + INNER JOIN approved_fuel_type aft ON crear.fuel_type_id = aft.id + INNER JOIN unit_of_measure uom ON aft.unit_of_measure_id = uom.id + WHERE crear.exclusion_agreement_id = ? + ORDER BY crear.id; +''' + +// --- Reusing logic from allocation_agreement.groovy --- +// To support versioning we check the current highest version for a given group_uuid. +@Field String SELECT_CURRENT_ALLOCATION_VERSION_QUERY = ''' + SELECT version FROM allocation_agreement WHERE group_uuid = ? ORDER BY version DESC LIMIT 1 +''' + +// INSERT statement for allocation_agreement. +@Field String INSERT_ALLOCATION_AGREEMENT_SQL = ''' + INSERT INTO allocation_agreement( + compliance_report_id, + transaction_partner, + postal_address, + quantity, + quantity_not_sold, + units, + allocation_transaction_type_id, + fuel_type_id, -- LCFS Fuel Type ID + fuel_category_id, -- LCFS Fuel Category ID + -- ci_of_fuel, -- Not available from source + -- provision_of_the_act_id, -- Not available from source + -- fuel_code_id, -- Not available from source + group_uuid, + version, + action_type, + create_user, + update_user + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::actiontypeenum, ?, ?) -- Use placeholders for ETL users +''' + +// --- Lookups --- +@Field Map responsibilityToTransactionTypeCache = [:] +@Field String SELECT_LCFS_TRANSACTION_TYPE_ID_QUERY = ''' + SELECT allocation_transaction_type_id FROM allocation_transaction_type WHERE type = ? +''' + +// @Field Map tfrsFuelTypeToLcfsFuelTypeCache = [:] // Removed TFRS ID based cache +@Field Map tfrsFuelNameToLcfsFuelTypeCache = [:] // Cache based on TFRS Fuel Name -> LCFS ID +@Field String SELECT_LCFS_FUEL_TYPE_ID_BY_NAME_QUERY = ''' + SELECT fuel_type_id FROM fuel_type WHERE fuel_type = ? -- Lookup by LCFS fuel_type name +''' + +// Insert statement for default summary record (Corrected column/value count) +@Field String INSERT_LCFS_SUMMARY_SQL = ''' + INSERT INTO compliance_report_summary ( + compliance_report_id, quarter, is_locked, + line_1_fossil_derived_base_fuel_gasoline, line_1_fossil_derived_base_fuel_diesel, line_1_fossil_derived_base_fuel_jet_fuel, + line_2_eligible_renewable_fuel_supplied_gasoline, line_2_eligible_renewable_fuel_supplied_diesel, line_2_eligible_renewable_fuel_supplied_jet_fuel, + line_3_total_tracked_fuel_supplied_gasoline, line_3_total_tracked_fuel_supplied_diesel, line_3_total_tracked_fuel_supplied_jet_fuel, + line_4_eligible_renewable_fuel_required_gasoline, line_4_eligible_renewable_fuel_required_diesel, line_4_eligible_renewable_fuel_required_jet_fuel, + line_5_net_notionally_transferred_gasoline, line_5_net_notionally_transferred_diesel, line_5_net_notionally_transferred_jet_fuel, + line_6_renewable_fuel_retained_gasoline, line_6_renewable_fuel_retained_diesel, line_6_renewable_fuel_retained_jet_fuel, + line_7_previously_retained_gasoline, line_7_previously_retained_diesel, line_7_previously_retained_jet_fuel, + line_8_obligation_deferred_gasoline, line_8_obligation_deferred_diesel, line_8_obligation_deferred_jet_fuel, + line_9_obligation_added_gasoline, line_9_obligation_added_diesel, line_9_obligation_added_jet_fuel, + line_10_net_renewable_fuel_supplied_gasoline, line_10_net_renewable_fuel_supplied_diesel, line_10_net_renewable_fuel_supplied_jet_fuel, + line_11_non_compliance_penalty_gasoline, line_11_non_compliance_penalty_diesel, line_11_non_compliance_penalty_jet_fuel, -- Nullable penalties included + line_12_low_carbon_fuel_required, line_13_low_carbon_fuel_supplied, line_14_low_carbon_fuel_surplus, + line_15_banked_units_used, line_16_banked_units_remaining, line_17_non_banked_units_used, + line_18_units_to_be_banked, line_19_units_to_be_exported, line_20_surplus_deficit_units, line_21_surplus_deficit_ratio, + line_22_compliance_units_issued, + line_11_fossil_derived_base_fuel_gasoline, line_11_fossil_derived_base_fuel_diesel, line_11_fossil_derived_base_fuel_jet_fuel, line_11_fossil_derived_base_fuel_total, + line_21_non_compliance_penalty_payable, total_non_compliance_penalty_payable, + create_user, update_user, + early_issuance_credits_q1, early_issuance_credits_q2, early_issuance_credits_q3, early_issuance_credits_q4, historical_snapshot + ) VALUES ( + ?, null, false, -- compliance_report_id, quarter, is_locked (3) + -- Lines 1-10 (3*10 = 30 values) + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + null, null, null, -- Line 11 nullable penalties (3) + 0.0, 0.0, 0.0, -- Lines 12-14 (3) + 0.0, 0.0, 0.0, -- Lines 15-17 (3) + 0.0, 0.0, 0.0, 0.0, -- Lines 18-21 ratio (4) + 0.0, -- Line 22 (1) + 0.0, 0.0, 0.0, 0.0, -- Line 11 fossil derived (renamed in schema?) (4) + 0.0, 0.0, -- Line 21 penalty payable, total (2) + 'ETL', 'ETL', -- users (2) + null, null, null, null, null -- early issuance, historical snapshot (5) + ) -- Total values: 3 + 30 + 3 + 3 + 3 + 4 + 1 + 4 + 2 + 2 + 5 = 60 +''' + +// Query to get LCFS organization details for snapshot +@Field String SELECT_LCFS_ORG_DETAILS_QUERY = ''' + SELECT + org.name, + org.operating_name, + org.email, + org.phone, + org.records_address, + addr.street_address, + addr.address_other, + addr.city, + addr.province_state, + addr.country, + addr."postalCode_zipCode" -- Corrected camelCase column name (quoted) + FROM organization org + LEFT JOIN organization_address addr ON org.organization_address_id = addr.organization_address_id + WHERE org.organization_id = ? +''' + +// Insert statement for organization snapshot +@Field String INSERT_LCFS_ORG_SNAPSHOT_SQL = ''' + INSERT INTO compliance_report_organization_snapshot ( + compliance_report_id, name, operating_name, email, phone, + service_address, head_office_address, records_address, is_edited, + create_user, update_user + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'ETL', 'ETL') +''' + +@Field Integer GASOLINE_CATEGORY_ID = 1 +@Field Integer DIESEL_CATEGORY_ID = 2 +// ------------------------- +// Helper Functions +// ------------------------- + +/** + * Gets LCFS Org ID from TFRS Org Name. + */ +def getLcfsOrgId(Connection destConn, String tfrsOrgName) { + if (!tfrsOrgName) { + log.error("Cannot map LCFS organization ID from null TFRS organization name.") + return null + } + PreparedStatement stmt = destConn.prepareStatement(SELECT_LCFS_ORG_ID_QUERY) + stmt.setString(1, tfrsOrgName) + ResultSet rs = stmt.executeQuery() + Integer lcfsId = rs.next() ? rs.getInt("organization_id") : null + rs.close() + stmt.close() + if (!lcfsId) { + log.error("Could not find LCFS organization mapped to TFRS organization name: ${tfrsOrgName}") + } + return lcfsId +} + +/** + * Gets LCFS Period ID from TFRS Period Description. + */ +def getLcfsPeriodId(Connection destConn, String tfrsPeriodDesc) { + if (!tfrsPeriodDesc) { + log.error("Cannot map LCFS compliance period ID from null TFRS period description.") + return null + } + PreparedStatement stmt = destConn.prepareStatement(SELECT_LCFS_PERIOD_ID_QUERY) + stmt.setString(1, tfrsPeriodDesc) + ResultSet rs = stmt.executeQuery() + Integer lcfsId = rs.next() ? rs.getInt("compliance_period_id") : null + rs.close() + stmt.close() + if (!lcfsId) { + log.error("Could not find LCFS compliance period mapped to TFRS description: ${tfrsPeriodDesc}") + } + return lcfsId +} + +/** + * Gets LCFS Report Status ID by name. + */ +def getLcfsReportStatusId(Connection destConn, String statusName) { + // Caching recommended for performance + PreparedStatement stmt = destConn.prepareStatement(SELECT_LCFS_REPORT_STATUS_ID_QUERY) + stmt.setString(1, statusName) + ResultSet rs = stmt.executeQuery() + Integer statusId = rs.next() ? rs.getInt("compliance_report_status_id") : null + rs.close() + stmt.close() + if (!statusId) { + log.error("Could not find LCFS compliance report status ID for status: ${statusName}") + } + return statusId +} + +/** + * Gets LCFS Transaction Type ID from TFRS Responsibility string. + */ +def getLcfsTransactionTypeId(Connection destConn, String responsibility) { + if (responsibilityToTransactionTypeCache.containsKey(responsibility)) { + return responsibilityToTransactionTypeCache[responsibility] + } + PreparedStatement stmt = destConn.prepareStatement(SELECT_LCFS_TRANSACTION_TYPE_ID_QUERY) + stmt.setString(1, responsibility) + ResultSet rs = stmt.executeQuery() + Integer typeId = rs.next() ? rs.getInt("allocation_transaction_type_id") : null + rs.close() + stmt.close() + if (typeId != null) { + responsibilityToTransactionTypeCache[responsibility] = typeId + } else { + log.warn("No LCFS transaction type found for responsibility: ${responsibility}; returning null.") + } + return typeId +} + +/** + * Gets LCFS Fuel Type ID from TFRS Fuel Type Name. + */ +def getLcfsFuelTypeIdByName(Connection destConn, String tfrsFuelTypeName) { + if (!tfrsFuelTypeName) { + log.error("Cannot map LCFS fuel type ID from null TFRS fuel type name.") + return null + } + if (tfrsFuelNameToLcfsFuelTypeCache.containsKey(tfrsFuelTypeName)) { + return tfrsFuelNameToLcfsFuelTypeCache[tfrsFuelTypeName] + } + // Assumes TFRS fuel name (e.g., 'Biodiesel') matches LCFS fuel_type column value + PreparedStatement stmt = destConn.prepareStatement(SELECT_LCFS_FUEL_TYPE_ID_BY_NAME_QUERY) + stmt.setString(1, tfrsFuelTypeName) + ResultSet rs = stmt.executeQuery() + Integer lcfsId = rs.next() ? rs.getInt("fuel_type_id") : null + rs.close() + stmt.close() + if (lcfsId != null) { + tfrsFuelNameToLcfsFuelTypeCache[tfrsFuelTypeName] = lcfsId + } else { + log.warn("No LCFS fuel type found mapped for TFRS fuel type name: ${tfrsFuelTypeName}; returning null.") + } + return lcfsId +} + +/** + * Creates a minimal LCFS Compliance Report record, default summary, and org snapshot. + */ +def createLcfsPlaceholderReport(Connection destConn, Integer lcfsOrgId, Integer lcfsPeriodId, Integer statusId, String reportingFrequency, Integer tfrsLegacyId) { + String groupUuid = UUID.randomUUID().toString() + Integer version = 0 // Initial version + Integer newLcfsReportId = null + + // 1. Create Compliance Report + try { + PreparedStatement stmt = destConn.prepareStatement(INSERT_LCFS_COMPLIANCE_REPORT_SQL) + stmt.setInt(1, lcfsOrgId) + stmt.setInt(2, lcfsPeriodId) + stmt.setInt(3, statusId) + stmt.setString(4, reportingFrequency) + stmt.setString(5, groupUuid) + stmt.setInt(6, version) + stmt.setInt(7, tfrsLegacyId) + stmt.setString(8, "Original Report") // Set nickname + ResultSet rs = stmt.executeQuery() + if (rs.next()) { + newLcfsReportId = rs.getInt("compliance_report_id") + } + rs.close() + stmt.close() + + if (newLcfsReportId) { + log.info("Created placeholder LCFS compliance report ID: ${newLcfsReportId} for TFRS legacy ID: ${tfrsLegacyId}") + } else { + log.error("Failed to create placeholder LCFS compliance report for TFRS legacy ID: ${tfrsLegacyId}") + return null // Stop if report creation failed + } + } catch (Exception e) { + log.error("Exception creating placeholder LCFS compliance report for TFRS legacy ID: ${tfrsLegacyId}", e) + return null + } + + // 2. Create Default Summary Record + try { + PreparedStatement summaryStmt = destConn.prepareStatement(INSERT_LCFS_SUMMARY_SQL) + summaryStmt.setInt(1, newLcfsReportId) + summaryStmt.executeUpdate() + summaryStmt.close() + log.info(" -> Created default summary record for LCFS report ID: ${newLcfsReportId}") + } catch (Exception e) { + log.error("Exception creating default summary for LCFS report ID: ${newLcfsReportId}", e) + // Decide if failure here is critical. Maybe log and continue? + } + + // 3. Create Organization Snapshot + try { + // Fetch org details + PreparedStatement orgDetailsStmt = destConn.prepareStatement(SELECT_LCFS_ORG_DETAILS_QUERY) + orgDetailsStmt.setInt(1, lcfsOrgId) + ResultSet orgDetailsRS = orgDetailsStmt.executeQuery() + + if (orgDetailsRS.next()) { + String name = orgDetailsRS.getString("name") + String operatingName = orgDetailsRS.getString("operating_name") + String email = orgDetailsRS.getString("email") + String phone = orgDetailsRS.getString("phone") + String recordsAddr = orgDetailsRS.getString("records_address") + String street = orgDetailsRS.getString("street_address") + String other = orgDetailsRS.getString("address_other") + String city = orgDetailsRS.getString("city") + String province = orgDetailsRS.getString("province_state") + String country = orgDetailsRS.getString("country") + String postal = orgDetailsRS.getString("postalCode_zipCode") + + // Construct addresses (adjust formatting as needed) + String fullAddress = [street, other, city, province, country, postal].findAll { it != null && !it.isEmpty() }.join(', ') + // Assuming service and head office address are the same from organization_address for snapshot + String serviceAddress = fullAddress + String headOfficeAddress = fullAddress + + // Insert snapshot + PreparedStatement snapshotStmt = destConn.prepareStatement(INSERT_LCFS_ORG_SNAPSHOT_SQL) + snapshotStmt.setInt(1, newLcfsReportId) + snapshotStmt.setString(2, name ?: '') // Use fetched values or defaults + snapshotStmt.setString(3, operatingName ?: '') + snapshotStmt.setString(4, email ?: '') + snapshotStmt.setString(5, phone ?: '') + snapshotStmt.setString(6, serviceAddress ?: '') + snapshotStmt.setString(7, headOfficeAddress ?: '') + snapshotStmt.setString(8, recordsAddr ?: '') + snapshotStmt.setBoolean(9, false) // is_edited = false + snapshotStmt.executeUpdate() + snapshotStmt.close() + log.info(" -> Created organization snapshot for LCFS report ID: ${newLcfsReportId}") + + } else { + log.warn(" -> Could not find LCFS organization details for ID: ${lcfsOrgId} to create snapshot.") + } + orgDetailsRS.close() + orgDetailsStmt.close() + + } catch (Exception e) { + log.error("Exception creating organization snapshot for LCFS report ID: ${newLcfsReportId}", e) + // Decide if failure here is critical. + } + + return newLcfsReportId +} + + +/** + * Inserts a new row into LCFS allocation_agreement with proper versioning. + * Adapted from allocation_agreement.groovy + */ +def insertAllocationAgreementVersionRow(Connection destConn, Integer lcfsCRid, Map rowData, String action) { + def recordId = rowData.agreement_record_id // TFRS agreement record ID + + // Retrieve or create a stable group_uuid based on TFRS record ID. + def groupUuid = recordUuidMap[recordId] + if (!groupUuid) { + groupUuid = UUID.randomUUID().toString() + recordUuidMap[recordId] = groupUuid + } + + // Retrieve current highest version for this group_uuid in LCFS. + int currentVer = -1 + PreparedStatement verStmt = destConn.prepareStatement(SELECT_CURRENT_ALLOCATION_VERSION_QUERY) + verStmt.setString(1, groupUuid) + ResultSet verRS = verStmt.executeQuery() + if (verRS.next()) { + currentVer = verRS.getInt("version") + } + verRS.close() + verStmt.close() + + int nextVer = (currentVer < 0) ? 0 : currentVer + 1 + + // --- Map source fields to LCFS fields --- + int lcfsAllocTransactionTypeId = getLcfsTransactionTypeId(destConn, rowData.responsibility) + // Map Fuel Type ID (TFRS Name -> LCFS ID) + int lcfsFuelTypeId = getLcfsFuelTypeIdByName(destConn, rowData.fuel_type) + int quantity = rowData.quantity ?: 0 + int quantityNotSold = rowData.quantity_not_sold ?: 0 + String transactionPartner = rowData.transaction_partner ?: "" + String postalAddress = rowData.postal_address ?: "" + String units = rowData.units ?: "" + String fuelTypeString = rowData.fuel_type // TFRS fuel type name + + // Determine LCFS Fuel Category ID based on TFRS fuel type name + Integer fuelCategoryId = null + if (fuelTypeString?.toLowerCase().contains('gasoline')) { + fuelCategoryId = GASOLINE_CATEGORY_ID + } else if (fuelTypeString?.toLowerCase().contains('diesel')) { + fuelCategoryId = DIESEL_CATEGORY_ID + } else { + log.warn("Could not determine LCFS fuel category for TFRS fuel type: ${fuelTypeString}. Setting fuel_category_id to NULL.") + } + // --- End Mapping --- + + // --- Validation --- + if (lcfsAllocTransactionTypeId == null || lcfsFuelTypeId == null) { + log.error("Skipping insert for TFRS record ID ${recordId} due to missing LCFS mapping (TransactionType: ${lcfsAllocTransactionTypeId}, FuelType: ${lcfsFuelTypeId})") + return // Skip insert if essential foreign keys are missing + } + // --- End Validation --- + + + PreparedStatement insStmt = destConn.prepareStatement(INSERT_ALLOCATION_AGREEMENT_SQL) + insStmt.setInt(1, lcfsCRid) + insStmt.setString(2, transactionPartner) + insStmt.setString(3, postalAddress) + insStmt.setInt(4, quantity) + insStmt.setInt(5, quantityNotSold) + insStmt.setString(6, units) + insStmt.setInt(7, lcfsAllocTransactionTypeId) + insStmt.setInt(8, lcfsFuelTypeId) // LCFS Fuel Type ID + if (fuelCategoryId != null) { + insStmt.setInt(9, fuelCategoryId) // LCFS Fuel Category ID + } else { + insStmt.setNull(9, java.sql.Types.INTEGER) + } + insStmt.setString(10, groupUuid) + insStmt.setInt(11, nextVer) + insStmt.setString(12, action) // Should always be 'CREATE' in this script's context + insStmt.setString(13, 'ETL') // Bind create_user + insStmt.setString(14, 'ETL') // Bind update_user + insStmt.executeUpdate() + insStmt.close() + + log.info(" -> Inserted LCFS allocation_agreement row: TFRS_recordId=${recordId}, LCFS_CR_ID=${lcfsCRid}, action=${action}, groupUuid=${groupUuid}, version=${nextVer}") +} + + +// ------------------------- +// Main Execution +// ------------------------- +log.warn("**** BEGIN ORPHANED ALLOCATION AGREEMENT MIGRATION ****") + +Connection sourceConn = null +Connection destinationConn = null + +try { + sourceConn = sourceDbcpService.getConnection() + destinationConn = destinationDbcpService.getConnection() + + // --- Pre-fetch necessary LCFS IDs --- + // Integer lcfsDraftStatusId = getLcfsReportStatusId(destinationConn, 'Draft') + String defaultReportingFrequency = 'ANNUAL' // Corrected enum value + + // if (lcfsDraftStatusId == null) { // Adjusted check -- REMOVED + // throw new Exception("Failed to retrieve necessary LCFS Status ID. Aborting.") + // } + log.info("Using default Reporting Frequency: ${defaultReportingFrequency}") + // --- End Pre-fetch --- + + + // 1) Find orphaned TFRS exclusion reports + log.info("Querying TFRS for orphaned exclusion reports...") + PreparedStatement orphanStmt = sourceConn.prepareStatement(SELECT_ORPHANED_EXCLUSION_REPORTS_QUERY) + ResultSet orphanRS = orphanStmt.executeQuery() + + int orphanedCount = 0 + int processedCount = 0 + int skippedCount = 0 + + while (orphanRS.next()) { + orphanedCount++ + int tfrsExclusionReportId = orphanRS.getInt("tfrs_exclusion_report_id") + int tfrsOrgId = orphanRS.getInt("tfrs_organization_id") + int tfrsPeriodId = orphanRS.getInt("tfrs_compliance_period_id") + int tfrsExclusionAgreementId = orphanRS.getInt("exclusion_agreement_id") + String tfrsDirectorStatus = orphanRS.getString("tfrs_director_status") + + log.warn("Found orphaned TFRS exclusion report ID: ${tfrsExclusionReportId} (Org: ${tfrsOrgId}, Period: ${tfrsPeriodId}, Agreement: ${tfrsExclusionAgreementId}, DirectorStatus: ${tfrsDirectorStatus})") + + // 2a) Check if already migrated (LCFS report exists with this legacy_id) + PreparedStatement checkStmt = destinationConn.prepareStatement(CHECK_LCFS_REPORT_EXISTS_QUERY) + checkStmt.setInt(1, tfrsExclusionReportId) + ResultSet checkRS = checkStmt.executeQuery() + boolean alreadyExists = checkRS.next() + checkRS.close() + checkStmt.close() + + if (alreadyExists) { + log.warn(" -> LCFS report with legacy_id ${tfrsExclusionReportId} already exists. Skipping.") + skippedCount++ + continue // Move to the next orphaned report + } + + // --- Get TFRS Org Name for Mapping --- + String tfrsOrgName = null + PreparedStatement orgNameStmt = sourceConn.prepareStatement(SELECT_TFRS_ORG_NAME_QUERY) + orgNameStmt.setInt(1, tfrsOrgId) + ResultSet orgNameRS = orgNameStmt.executeQuery() + if (orgNameRS.next()) { + tfrsOrgName = orgNameRS.getString("name") + } + orgNameRS.close() + orgNameStmt.close() + // --- End Get TFRS Org Name --- + + // --- Get TFRS Period Description for Mapping --- + String tfrsPeriodDesc = null + PreparedStatement periodDescStmt = sourceConn.prepareStatement(SELECT_TFRS_PERIOD_DESC_QUERY) + periodDescStmt.setInt(1, tfrsPeriodId) + ResultSet periodDescRS = periodDescStmt.executeQuery() + if (periodDescRS.next()) { + tfrsPeriodDesc = periodDescRS.getString("description") + } + periodDescRS.close() + periodDescStmt.close() + // --- End Get TFRS Period Description --- + + // --- Determine Target LCFS Status --- + String targetLcfsStatusName = 'Draft' // Default to Draft + if (tfrsDirectorStatus == 'Accepted') { + targetLcfsStatusName = 'Assessed' + } else if (tfrsDirectorStatus == 'Rejected') { + targetLcfsStatusName = 'Rejected' + } + log.info(" -> Mapping TFRS Director Status '${tfrsDirectorStatus}' to LCFS Status '${targetLcfsStatusName}'") + Integer lcfsStatusId = getLcfsReportStatusId(destinationConn, targetLcfsStatusName) + if (lcfsStatusId == null) { + log.error(" -> Failed to find LCFS Status ID for '${targetLcfsStatusName}'. Skipping creation.") + skippedCount++ + continue + } + // --- End Determine Target LCFS Status --- + + // 2b) Create placeholder LCFS report + log.info(" -> Creating placeholder LCFS report with Status ID: ${lcfsStatusId}...") + Integer lcfsOrgId = getLcfsOrgId(destinationConn, tfrsOrgName) + Integer lcfsPeriodId = getLcfsPeriodId(destinationConn, tfrsPeriodDesc) + + if (lcfsOrgId == null || lcfsPeriodId == null) { + log.error(" -> Failed to map TFRS Org/Period IDs for TFRS report ${tfrsExclusionReportId}. Skipping creation and associated records.") + skippedCount++ + continue + } + + Integer newLcfsReportId = createLcfsPlaceholderReport( + destinationConn, + lcfsOrgId, + lcfsPeriodId, + lcfsStatusId, // Pass the dynamically determined status ID + defaultReportingFrequency, + tfrsExclusionReportId + ) + + if (newLcfsReportId == null) { + log.error(" -> Failed to create placeholder LCFS report for TFRS ID ${tfrsExclusionReportId}. Skipping associated records.") + skippedCount++ + continue + } + + // 3) Fetch associated allocation records from TFRS + log.info(" -> Fetching allocation records from TFRS for agreement ID: ${tfrsExclusionAgreementId}") + PreparedStatement allocStmt = sourceConn.prepareStatement(SELECT_ALLOCATION_RECORDS_BY_AGREEMENT_ID_QUERY) + allocStmt.setInt(1, tfrsExclusionAgreementId) + ResultSet allocRS = allocStmt.executeQuery() + + int agreementRecordsFound = 0 + while (allocRS.next()) { + agreementRecordsFound++ + def recordData = [ + agreement_record_id : allocRS.getInt("agreement_record_id"), + responsibility : allocRS.getString("responsibility"), + fuel_type : allocRS.getString("fuel_type"), // TFRS Fuel Type Name + tfrs_fuel_type_id : allocRS.getInt("tfrs_fuel_type_id"), // TFRS Fuel Type ID + transaction_partner : allocRS.getString("transaction_partner"), + postal_address : allocRS.getString("postal_address"), + quantity : allocRS.getInt("quantity"), + units : allocRS.getString("units"), + quantity_not_sold : allocRS.getInt("quantity_not_sold"), + transaction_type_id : allocRS.getInt("transaction_type_id") // TFRS Transaction Type ID (used for responsibility mapping) + ] + + // 4) Insert into LCFS allocation_agreement table + insertAllocationAgreementVersionRow(destinationConn, newLcfsReportId, recordData, 'CREATE') + } + allocRS.close() + allocStmt.close() + + if (agreementRecordsFound == 0) { + log.warn(" -> No allocation records found in TFRS for agreement ID: ${tfrsExclusionAgreementId}") + } + processedCount++ + + } // End while loop for orphaned reports + + orphanRS.close() + orphanStmt.close() + + log.warn("Finished processing. Found ${orphanedCount} orphaned TFRS reports. Processed: ${processedCount}. Skipped: ${skippedCount}.") + + +} catch (Exception e) { + log.error("Error running Orphaned Allocation Agreement migration", e) + // Potentially re-throw depending on NiFi error handling requirements + throw e +} finally { + if (sourceConn != null) { try { sourceConn.close() } catch (Exception e) { log.error("Error closing source connection: ${e.message}")} } + if (destinationConn != null) { try { destinationConn.close() } catch (Exception e) { log.error("Error closing destination connection: ${e.message}")} } +} + +log.warn("**** DONE: ORPHANED ALLOCATION AGREEMENT MIGRATION ****") \ No newline at end of file diff --git a/etl/nifi_scripts/other_uses.groovy b/etl/nifi_scripts/other_uses.groovy index b0707711c..134232c9f 100644 --- a/etl/nifi_scripts/other_uses.groovy +++ b/etl/nifi_scripts/other_uses.groovy @@ -38,11 +38,13 @@ Map recordUuidMap = [:] scr.expected_use_id, scr.rationale, cr.id AS compliance_report_id, - uom.name AS unit_of_measure + uom.name AS unit_of_measure, + dci.density AS default_ci_of_fuel FROM compliance_report_schedule_c_record scr JOIN compliance_report_schedule_c sc ON sc.id = scr.schedule_id JOIN compliance_report cr ON cr.schedule_c_id = sc.id LEFT JOIN approved_fuel_type aft ON aft.id = scr.fuel_type_id + LEFT JOIN default_carbon_intensity dci ON dci.category_id = aft.default_carbon_intensity_category_id LEFT JOIN unit_of_measure uom ON uom.id = aft.unit_of_measure_id WHERE cr.id = ? """ @@ -74,11 +76,10 @@ Map recordUuidMap = [:] rationale, group_uuid, version, - user_type, action_type, create_user, update_user - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'SUPPLIER', ?::actiontypeenum, 'ETL', 'ETL') + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::actiontypeenum, 'ETL', 'ETL') """; def sourceDbcpService = context.controllerServiceLookup.getControllerService("3245b078-0192-1000-ffff-ffffba20c1eb") @@ -174,6 +175,7 @@ def insertVersionRow(Connection destConn, Integer lcfsCRid, Map rowData, String def quantity = rowData.quantity?: 0 def rationale= rowData.rationale?: "" def units = rowData.unit_of_measure?: "" + def ci_of_fuel = rowData.ci_of_fuel?: 0 // Map and insert the record @@ -182,8 +184,8 @@ def insertVersionRow(Connection destConn, Integer lcfsCRid, Map rowData, String setInt(1, lcfsCRid) setInt(2, fuelTypeId) setInt(3, fuelCatId) - setNull(4, Types.INTEGER) // provision_of_the_act_id - setNull(5, Types.NUMERIC) // ci_of_fuel + setInt(4, 7) // provision_of_the_act_id + setBigDecimal(5, ci_of_fuel) setBigDecimal(6, quantity) setString(7,units) setInt(8, expectedUseId) @@ -279,6 +281,7 @@ try { expected_use_id: scrRS.getInt("expected_use_id"), rationale: scrRS.getString("rationale"), unit_of_measure: scrRS.getString("unit_of_measure"), + ci_of_fuel: scrRS.getBigDecimal("default_ci_of_fuel") ] } scrRS.close() diff --git a/etl/python_migration/.env.example b/etl/python_migration/.env.example new file mode 100644 index 000000000..00dccb445 --- /dev/null +++ b/etl/python_migration/.env.example @@ -0,0 +1,12 @@ +# Database Configuration +TFRS_DB_HOST=localhost +TFRS_DB_PORT=5435 +TFRS_DB_NAME=tfrs +TFRS_DB_USER=tfrs +TFRS_DB_PASSWORD=development_only + +LCFS_DB_HOST=localhost +LCFS_DB_PORT=5432 +LCFS_DB_NAME=lcfs +LCFS_DB_USER=lcfs +LCFS_DB_PASSWORD=development_only \ No newline at end of file diff --git a/etl/python_migration/.gitignore b/etl/python_migration/.gitignore new file mode 100644 index 000000000..b01787e80 --- /dev/null +++ b/etl/python_migration/.gitignore @@ -0,0 +1,35 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Logs +*.log +logs/ + +# Database dumps - preserve locally for easy reset +dumps/ +*.tar + +# Environment files +.env +*.env.local +*.env.*.local + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/etl/python_migration/Makefile b/etl/python_migration/Makefile new file mode 100644 index 000000000..b6b3d64bc --- /dev/null +++ b/etl/python_migration/Makefile @@ -0,0 +1,246 @@ +# TFRS to LCFS Migration Makefile +# +# Prerequisites: +# - Docker and Docker Compose installed +# - OpenShift CLI (oc) installed +# - Python 3.8+ with requirements installed +# - User must be logged into OpenShift: oc login +# +# SECURITY NOTE: This Makefile only supports IMPORT operations. +# NO EXPORT to production databases is allowed. + +.PHONY: help setup setup-prod setup-dev quick-start migrate validate check clean stop status docker-start docker-stop + +# Default target +help: + @echo "TFRS to LCFS Migration Commands" + @echo "===============================" + @echo "" + @echo "🚀 Quick Start:" + @echo " make quick-start - Complete migration with auto Docker setup (dev data)" + @echo " make setup-prod - Setup databases with PRODUCTION data (requires oc login)" + @echo " make setup-dev - Setup databases with DEV data (requires oc login)" + @echo "" + @echo "🐳 Docker Management:" + @echo " make docker-start - Start TFRS container and LCFS environment" + @echo " make docker-stop - Stop TFRS container" + @echo " make status - Show running containers and database status" + @echo "" + @echo "🔧 Migration Operations:" + @echo " make migrate - Run migrations only (assumes databases ready)" + @echo " make validate - Run validation only (assumes migration complete)" + @echo " make check - Check if databases are ready for migration" + @echo "" + @echo "🔄 Database Reset:" + @echo " make reset - Reset LCFS database from existing .tar file" + @echo " make reset-lcfs - Reset LCFS database from existing .tar file" + @echo " make reset-tfrs - Reset TFRS database from existing .tar file" + @echo " make reset-all - Reset both TFRS and LCFS databases" + @echo "" + @echo "🧹 Cleanup:" + @echo " make clean - Stop containers and clean up volumes" + @echo " make stop - Stop all migration containers" + @echo "" + @echo "📋 Prerequisites:" + @echo " - Correct project structure: ./setup-paths.sh" + @echo " - Docker running" + @echo " - OpenShift login: oc login (for data import)" + @echo " - Python requirements: pip install -r requirements.txt" + +# Quick start - complete migration with dev data +quick-start: + @echo "🚀 Starting complete migration with automatic setup..." + @echo "📋 Using DEV environment data" + @./quick-start.sh --env dev + +# Setup databases with production data (IMPORT ONLY) +setup-prod: + @echo "⚠️ Setting up databases with PRODUCTION data" + @echo "🔒 SECURITY: Only IMPORT from production is allowed" + @echo "📋 Checking OpenShift login..." + @oc whoami || (echo "❌ Please login to OpenShift first: oc login" && exit 1) + @echo "🐳 Starting Docker containers and importing PRODUCTION data..." + python setup/migration_orchestrator.py auto-setup --env prod + +# Setup databases with dev data +setup-dev: + @echo "🔧 Setting up databases with DEV data" + @echo "📋 Checking OpenShift login..." + @oc whoami || (echo "❌ Please login to OpenShift first: oc login" && exit 1) + @echo "🐳 Starting Docker containers and importing DEV data..." + python setup/migration_orchestrator.py auto-setup --env dev + +# Setup databases with test data +setup-test: + @echo "🧪 Setting up databases with TEST data" + @echo "📋 Checking OpenShift login..." + @oc whoami || (echo "❌ Please login to OpenShift first: oc login" && exit 1) + @echo "🐳 Starting Docker containers and importing TEST data..." + python setup/migration_orchestrator.py auto-setup --env test + +# Start Docker containers without data import +docker-start: + @echo "🐳 Starting Docker containers..." + @docker compose up -d tfrs + @echo "🔄 Starting LCFS environment..." + @if [ -f "../../../docker-compose.yml" ]; then \ + cd ../../../ && docker compose up -d; \ + else \ + echo "⚠️ LCFS docker-compose.yml not found at ../../../docker-compose.yml"; \ + echo "Please start LCFS environment manually"; \ + fi + @echo "✅ Docker containers started" + @make status + +# Stop TFRS container +docker-stop: + @echo "🛑 Stopping TFRS container..." + @docker compose down tfrs + @echo "✅ TFRS container stopped" + +# Run migrations only (assumes setup complete) +migrate: + @echo "🚀 Running data migrations..." + @echo "📋 Checking migration readiness..." + python setup/migration_orchestrator.py check || (echo "❌ Databases not ready for migration" && exit 1) + @echo "▶️ Running all migration scripts..." + python setup/migration_orchestrator.py migrate + +# Run validation only (assumes migration complete) +validate: + @echo "🔎 Running migration validation..." + python setup/migration_orchestrator.py validate + +# Check if databases are ready for migration +check: + @echo "🔍 Checking migration readiness..." + python setup/migration_orchestrator.py check + +# Show status of containers and databases +status: + @echo "🐳 Docker Container Status:" + @echo "==========================" + @docker ps --format "table {{.ID}}\\t{{.Names}}\\t{{.Status}}\\t{{.Ports}}" | grep -E "(CONTAINER|tfrs|lcfs|postgres)" || echo "No migration containers running" + @echo "" + @echo "💾 Database Status:" + @echo "==================" + @python setup/database_manager.py verify-tfrs 2>/dev/null || echo "TFRS: Not accessible" + @python setup/database_manager.py verify-lcfs 2>/dev/null || echo "LCFS: Not accessible" + @echo "" + @echo "📊 Migration Readiness:" + @echo "======================" + @python setup/migration_orchestrator.py check 2>/dev/null && echo "✅ Ready for migration" || echo "❌ Not ready for migration" + +# Stop all containers +stop: + @echo "🛑 Stopping all migration containers..." + @docker compose down + @echo "ℹ️ Note: LCFS environment containers are left running" + @echo " To stop LCFS: cd ../../../ && docker compose down" + +# Clean up everything +clean: stop + @echo "🧹 Cleaning up Docker volumes..." + @docker compose down -v + @docker volume prune -f + @echo "✅ Cleanup complete" + +# Development helpers +install: + @echo "📦 Installing Python requirements..." + pip install -r requirements.txt + +lint: + @echo "🔍 Running Python linting..." + @python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics || echo "⚠️ Linting issues found (not blocking)" + +test-docker: + @echo "🐳 Testing Docker setup..." + @docker --version + @docker compose version + @echo "✅ Docker is ready" + +test-openshift: + @echo "🔐 Testing OpenShift connection..." + @oc whoami && echo "✅ OpenShift connection ready" || echo "❌ Please login: oc login" + +# Production data import with extra confirmation +import-prod: + @echo "⚠️ WARNING: This will import PRODUCTION data to local containers!" + @echo "🔒 SECURITY: This is a READ-ONLY operation from production" + @echo "📋 Production data will OVERWRITE existing local data" + @read -p "Are you sure you want to proceed? (yes/no): " answer && \ + if [ "$$answer" = "yes" ]; then \ + make setup-prod; \ + else \ + echo "❌ Production import cancelled"; \ + fi + +# Show environment info +info: + @echo "🔧 Environment Information:" + @echo "==========================" + @echo "Python: $$(python --version 2>&1)" + @echo "Docker: $$(docker --version)" + @echo "Docker Compose: $$(docker compose version --short 2>/dev/null || docker-compose --version)" + @echo "OpenShift CLI: $$(oc version --client --short 2>/dev/null || echo 'Not installed')" + @echo "OpenShift User: $$(oc whoami 2>/dev/null || echo 'Not logged in')" + @echo "" + @echo "📁 Project Structure:" + @echo "====================" + @ls -la | grep -E "(core|migrations|setup|validation|docker-compose)" + +# Complete workflow examples +workflow-dev: + @echo "🔄 Complete DEV workflow..." + @make setup-dev + @make migrate + @make validate + +workflow-prod: + @echo "🔄 Complete PRODUCTION workflow..." + @make import-prod + @make migrate + @make validate + +# Individual container operations +start-tfrs: + @echo "🐳 Starting TFRS container only..." + @docker compose up -d tfrs + @echo "✅ TFRS container started" + +start-lcfs: + @echo "🐳 Starting LCFS environment..." + @if [ -f "../../../docker-compose.yml" ]; then \ + cd ../../../ && docker compose up -d; \ + echo "✅ LCFS environment started"; \ + else \ + echo "❌ LCFS docker-compose.yml not found at ../../../docker-compose.yml"; \ + fi + +# Database operations +reset-tfrs: + @echo "🔄 Resetting TFRS database from existing .tar file..." + @python setup/database_manager.py reset tfrs tfrs_migration + +reset-lcfs: + @echo "🔄 Resetting LCFS database from existing .tar file..." + @python setup/database_manager.py reset lcfs db + @echo "🔧 Running Alembic database migrations..." + @cd ../../backend && poetry run alembic upgrade head + @echo "✅ Database schema updated to latest version" + +reset: reset-lcfs + @echo "✅ LCFS database has been reset from the existing dump file" + +reset-all: reset-tfrs reset-lcfs + @echo "✅ Both TFRS and LCFS databases have been reset from existing dump files" + +# Logs +logs-tfrs: + @echo "📋 TFRS container logs:" + @docker compose logs tfrs + +logs-all: + @echo "📋 All container logs:" + @docker compose logs \ No newline at end of file diff --git a/etl/python_migration/README.md b/etl/python_migration/README.md new file mode 100644 index 000000000..d220ee880 --- /dev/null +++ b/etl/python_migration/README.md @@ -0,0 +1,309 @@ +# TFRS to LCFS Migration Framework + +A comprehensive Python-based framework for migrating data from the legacy TFRS (Transportation Fuels Reporting System) to the new LCFS (Low Carbon Fuel Standard) system. + +## 📁 Project Structure + +``` +python_migration/ +├── core/ # Core infrastructure +│ ├── config.py # Database and environment configuration +│ ├── database.py # Database connection utilities +│ └── utils.py # Common utility functions +│ +├── migrations/ # Migration scripts +│ ├── migrate_compliance_summaries.py +│ ├── migrate_compliance_summary_updates.py +│ ├── migrate_fuel_supply.py +│ ├── migrate_notional_transfers.py +│ ├── migrate_other_uses.py +│ ├── migrate_allocation_agreements.py +│ ├── migrate_orphaned_allocation_agreements.py +│ ├── migrate_compliance_report_history.py +│ └── run_all_migrations.py # Migration orchestrator +│ +├── setup/ # Setup and orchestration +│ ├── database_manager.py # Database setup and data transfer +│ ├── migration_orchestrator.py # Complete migration workflow +│ └── validation_runner.py # Validation orchestrator +│ +├── validation/ # Validation scripts +│ ├── validation_base.py # Base validation class +│ ├── validate_allocation_agreements.py +│ ├── validate_fuel_supply.py +│ ├── validate_notional_transfers.py +│ ├── validate_other_uses.py +│ └── README.md # Validation documentation +│ +├── docs/ # Documentation +│ ├── README_SETUP.md # Setup and usage guide +│ └── VERSIONING_REVIEW.md # Group UUID versioning analysis +│ +├── requirements.txt # Python dependencies +└── README.md # This file +``` + +## 🚀 Quick Start + +### 1. Simple Makefile Commands (Recommended) + +**New!** Simple make commands for common operations: + +```bash +# Complete migration with dev data (one command!) +make quick-start + +# Setup with production data (requires: oc login) +make setup-prod + +# Setup with dev data +make setup-dev + +# Show all available commands +make help + +# Check status of containers and databases +make status +``` + +### 2. Automatic Migration Scripts + +Fully automated setup with Docker containers managed automatically: + +```bash +# One-command complete migration (starts containers, imports data, migrates, validates) +./quick-start.sh + +# With options +./quick-start.sh --env dev --skip-validation + +# Setup Docker environment only +./quick-start.sh --setup-only + +# Using Python directly +python setup/migration_orchestrator.py auto-complete --env dev +python setup/migration_orchestrator.py auto-setup --env dev +``` + +### 3. Manual Migration Process + +Run the entire migration with manual container management: + +```bash +# Complete migration with database setup +python setup/migration_orchestrator.py complete \ + --tfrs-container my-tfrs-container \ + --lcfs-container my-lcfs-container \ + --env dev + +# Migration using existing databases +python setup/migration_orchestrator.py complete --skip-setup +``` + +### 4. Individual Components + +Set up databases only: +```bash +# Development/test data +python setup/migration_orchestrator.py setup my-tfrs-container my-lcfs-container --env dev + +# Production data (with confirmation) +python setup/migration_orchestrator.py setup-prod my-tfrs-container my-lcfs-container +``` + +Run migrations only: +```bash +python setup/migration_orchestrator.py migrate +``` + +Run validations only: +```bash +python setup/migration_orchestrator.py validate +``` + +### 3. Database Management + +```bash +# Setup test environment +python setup/database_manager.py setup my-tfrs-container my-lcfs-container dev + +# Setup with production data +python setup/database_manager.py setup-prod my-tfrs-container my-lcfs-container + +# Check migration readiness +python setup/database_manager.py check-readiness + +# Verify database population +python setup/database_manager.py verify-tfrs +python setup/database_manager.py verify-lcfs +``` + +## 📋 Prerequisites + +1. **Project Structure** + ```bash + # Run path setup helper + ./setup-paths.sh + + # Expected structure (relative to main LCFS project): + lcfs/ + ├── docker-compose.yml # Main LCFS environment + └── etl/ + └── python_migration/ + ├── docker-compose.yml # TFRS database only + └── Makefile + ``` + +2. **Python Environment** + ```bash + pip install -r requirements.txt + cp env.example .env # Update with your settings + ``` + +3. **Docker Setup** + - Docker and Docker Compose installed + - LCFS environment accessible (see path setup above) + +4. **OpenShift Access** (for database setup) + - OpenShift CLI (`oc`) installed and configured + - Access to TFRS and LCFS projects + - Login required: `oc login` + +## ⚙️ Configuration + +Create a `.env` file with database connection details: + +```bash +# TFRS (Source) Database +TFRS_DB_HOST=localhost +TFRS_DB_PORT=5432 +TFRS_DB_NAME=tfrs +TFRS_DB_USER=tfrs +TFRS_DB_PASSWORD=tfrs + +# LCFS (Destination) Database +LCFS_DB_HOST=localhost +LCFS_DB_PORT=5433 +LCFS_DB_NAME=lcfs +LCFS_DB_USER=lcfs +LCFS_DB_PASSWORD=lcfs +``` + +## 🔄 Migration Components + +### Core Infrastructure (`core/`) +- **config.py**: Environment and database configuration management +- **database.py**: Database connection pooling and transaction management +- **utils.py**: Common utilities for type conversion and logging + +### Migration Scripts (`migrations/`) +Each script handles a specific data type: +- **Compliance Summaries**: Report summary data and JSON snapshots +- **Fuel Supply**: Schedule B (fuel supply) records with GHGenius processing +- **Notional Transfers**: Schedule A (notional transfer) records +- **Other Uses**: Schedule C (other uses) records +- **Allocation Agreements**: Allocation agreement and exclusion records +- **Compliance Report History**: Historical compliance reports + +### Setup Tools (`setup/`) +- **database_manager.py**: Database setup and data transfer from OpenShift +- **migration_orchestrator.py**: Complete workflow orchestration +- **validation_runner.py**: Validation suite runner + +### Validation (`validation/`) +Comprehensive validation suite that verifies: +- Record count accuracy +- Data mapping integrity +- Version chain consistency +- Calculation accuracy +- Data quality checks + +## 🎯 Key Features + +✅ **Automated Database Setup**: Import data from OpenShift environments (dev/test/prod) +✅ **Production Data Support**: Safe import of production data with confirmations +✅ **Security First**: NO EXPORT to production allowed - import only +✅ **Simple Make Commands**: Easy-to-use Makefile for common operations +✅ **Complete Migration Suite**: All TFRS data types supported +✅ **Group UUID Versioning**: Proper version chain management +✅ **Comprehensive Validation**: Extensive data integrity checks +✅ **Error Handling**: Robust error detection and recovery +✅ **Modular Design**: Run complete process or individual components +✅ **Progress Tracking**: Detailed logging and progress reporting +✅ **CI/CD Ready**: Proper exit codes and automation support + +## 📊 Migration Process + +1. **Database Setup** (Optional) + - Download data from OpenShift environments + - Restore to local PostgreSQL containers + - Verify data population + +2. **Migration Execution** + - Run migrations in dependency order + - Handle group UUID versioning + - Process chain-based supplemental reports + - Track progress and errors + +3. **Validation** (Optional) + - Compare source vs destination counts + - Validate sample records + - Check version chain integrity + - Verify calculations and mappings + +## 🔧 Development + +### Running Individual Migrations + +```bash +cd migrations/ +python migrate_fuel_supply.py +python migrate_notional_transfers.py +``` + +### Running Individual Validations + +```bash +cd validation/ +python validate_fuel_supply.py +python validate_allocation_agreements.py +``` + +### Adding New Migrations + +1. Create new migration script in `migrations/` +2. Inherit from appropriate base class +3. Implement required methods +4. Add to `run_all_migrations.py` +5. Create corresponding validation script + +## 📖 Documentation + +- **[Setup Guide](docs/README_SETUP.md)**: Detailed setup and usage instructions +- **[Validation Guide](validation/README.md)**: Validation framework documentation +- **[Versioning Review](docs/VERSIONING_REVIEW.md)**: Group UUID versioning analysis + +## 🚨 Important Notes + +### Group UUID Versioning +The migration scripts implement a critical group UUID versioning system. See `docs/VERSIONING_REVIEW.md` for detailed analysis of this system and known issues. + +### GHGenius Processing +Fuel supply records with GHGenius determination require special processing to calculate carbon intensity from Schedule D data. + +### Chain Processing +Some reports are processed in chains where supplemental reports reference base reports. + +## 🤝 Contributing + +1. Follow the established project structure +2. Add tests for new functionality +3. Update documentation +4. Run validation suite before submitting changes + +## 📞 Support + +For issues and questions: +1. Check existing documentation +2. Review validation reports for data issues +3. Check logs for detailed error information \ No newline at end of file diff --git a/etl/python_migration/__init__.py b/etl/python_migration/__init__.py new file mode 100644 index 000000000..c86076db8 --- /dev/null +++ b/etl/python_migration/__init__.py @@ -0,0 +1 @@ +# TFRS to LCFS Migration Package \ No newline at end of file diff --git a/etl/python_migration/core/__init__.py b/etl/python_migration/core/__init__.py new file mode 100644 index 000000000..1b27f4536 --- /dev/null +++ b/etl/python_migration/core/__init__.py @@ -0,0 +1 @@ +# Core infrastructure modules \ No newline at end of file diff --git a/etl/python_migration/core/config.py b/etl/python_migration/core/config.py new file mode 100644 index 000000000..829763d18 --- /dev/null +++ b/etl/python_migration/core/config.py @@ -0,0 +1,26 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +class DatabaseConfig: + def __init__(self): + # TFRS (source) database configuration + self.tfrs_config = { + 'host': os.getenv('TFRS_DB_HOST', 'localhost'), + 'port': int(os.getenv('TFRS_DB_PORT', 5432)), + 'database': os.getenv('TFRS_DB_NAME', 'tfrs'), + 'user': os.getenv('TFRS_DB_USER', 'tfrs_user'), + 'password': os.getenv('TFRS_DB_PASSWORD', 'tfrs_password') + } + + # LCFS (destination) database configuration + self.lcfs_config = { + 'host': os.getenv('LCFS_DB_HOST', 'localhost'), + 'port': int(os.getenv('LCFS_DB_PORT', 5432)), + 'database': os.getenv('LCFS_DB_NAME', 'lcfs'), + 'user': os.getenv('LCFS_DB_USER', 'lcfs_user'), + 'password': os.getenv('LCFS_DB_PASSWORD', 'lcfs_password') + } + +db_config = DatabaseConfig() \ No newline at end of file diff --git a/etl/python_migration/core/database.py b/etl/python_migration/core/database.py new file mode 100644 index 000000000..4d11c56bf --- /dev/null +++ b/etl/python_migration/core/database.py @@ -0,0 +1,48 @@ +import psycopg2 +import logging +from contextlib import contextmanager +from .config import db_config + +logger = logging.getLogger(__name__) + + +class DatabaseConnection: + def __init__(self, config): + self.config = config + + @contextmanager + def get_connection(self): + conn = None + try: + conn = psycopg2.connect(**self.config) + conn.autocommit = False + yield conn + except Exception as e: + if conn: + conn.rollback() + logger.error(f"Database connection error: {e}") + raise + finally: + if conn: + conn.close() + + def test_connection(self): + try: + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT 1") + result = cursor.fetchone() + cursor.close() + return result[0] == 1 + except Exception as e: + logger.error(f"Connection test failed: {e}") + return False + + +# Database connection instances +tfrs_db = DatabaseConnection(db_config.tfrs_config) +lcfs_db = DatabaseConnection(db_config.lcfs_config) + +# Connection instances +get_source_connection = DatabaseConnection(db_config.tfrs_config).get_connection +get_destination_connection = DatabaseConnection(db_config.lcfs_config).get_connection diff --git a/etl/python_migration/core/utils.py b/etl/python_migration/core/utils.py new file mode 100644 index 000000000..27a9808c1 --- /dev/null +++ b/etl/python_migration/core/utils.py @@ -0,0 +1,59 @@ +import logging +from datetime import datetime +from decimal import Decimal +from typing import Dict, Any, Optional + +def setup_logging(log_level=logging.INFO): + logging.basicConfig( + level=log_level, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(), + logging.FileHandler(f'migration_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log') + ] + ) + +def safe_decimal(value: Any, default: Decimal = Decimal('0.0')) -> Decimal: + if value is None: + return default + try: + return Decimal(str(value)) + except (ValueError, TypeError): + return default + +def safe_int(value: Any, default: int = 0) -> int: + if value is None: + return default + try: + return int(value) + except (ValueError, TypeError): + return default + +def safe_str(value: Any, default: str = '') -> str: + if value is None: + return default + return str(value) + +def execute_query_with_retry(cursor, query: str, params: tuple = None, max_retries: int = 3) -> bool: + for attempt in range(max_retries): + try: + if params: + cursor.execute(query, params) + else: + cursor.execute(query) + return True + except Exception as e: + logging.warning(f"Query attempt {attempt + 1} failed: {e}") + if attempt == max_retries - 1: + logging.error(f"Query failed after {max_retries} attempts: {e}") + raise + return False + +def build_legacy_mapping(cursor, table_name: str = "compliance_report") -> Dict[int, int]: + query = f"SELECT compliance_report_id, legacy_id FROM {table_name} WHERE legacy_id IS NOT NULL" + cursor.execute(query) + mapping = {} + for row in cursor.fetchall(): + lcfs_id, legacy_id = row + mapping[legacy_id] = lcfs_id + return mapping \ No newline at end of file diff --git a/etl/python_migration/docker-compose.yml b/etl/python_migration/docker-compose.yml new file mode 100644 index 000000000..96269483b --- /dev/null +++ b/etl/python_migration/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3.8' + +services: + # TFRS database for migration testing + tfrs: + image: postgres:17 + container_name: tfrs-migration + environment: + POSTGRES_USER: tfrs + POSTGRES_PASSWORD: development_only + POSTGRES_DB: tfrs + ports: + - "5435:5432" + volumes: + - tfrs_migration_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U tfrs -d tfrs"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - migration_network + +volumes: + tfrs_migration_data: + +networks: + migration_network: + driver: bridge \ No newline at end of file diff --git a/etl/python_migration/documentation/README.md b/etl/python_migration/documentation/README.md new file mode 100644 index 000000000..877755fc8 --- /dev/null +++ b/etl/python_migration/documentation/README.md @@ -0,0 +1,257 @@ +# TFRS to LCFS Data Migration - Python Scripts + +This directory contains Python migration scripts that replace the Groovy scripts used for migrating data from the TFRS (Transportation Fuels Reporting System) to the LCFS (Low Carbon Fuel Standard) system. + +## Overview + +These scripts migrate historical compliance data from TFRS to LCFS, handling schema differences between the two systems. The migration includes compliance summaries, history records, fuel supply data, allocation agreements, and other uses data. + +## Project Structure + +``` +python_migration/ +├── README.md # This file +├── requirements.txt # Python dependencies +├── .env.example # Environment configuration template +├── config.py # Database configuration +├── database.py # Database connection utilities +├── utils.py # Common utility functions +├── compliance_summary.py # Compliance summary migration +├── compliance_summary_update.py # Compliance summary updates +├── compliance_report_history.py # Compliance report history migration +├── allocation_agreement.py # Allocation agreement migration +├── other_uses.py # Other uses (Schedule C) migration +├── notional_transfer.py # Notional transfer (Schedule A) migration +├── fuel_supply.py # Fuel supply (Schedule B) migration +├── orphaned_allocation_agreement.py # Orphaned allocation agreement migration +├── run_all_migrations.py # Script to run all migrations in order +└── VERSIONING_REVIEW.md # Group UUID versioning system review +``` + +## Setup + +1. **Install Dependencies** + ```bash + pip install -r requirements.txt + ``` + +2. **Configure Environment** + ```bash + cp .env.example .env + # Edit .env with your database credentials + ``` + +3. **Database Configuration** + + Update the `.env` file with your database connection details: + ``` + # TFRS (source) database + TFRS_DB_HOST=localhost + TFRS_DB_PORT=5432 + TFRS_DB_NAME=tfrs + TFRS_DB_USER=tfrs_user + TFRS_DB_PASSWORD=tfrs_password + + # LCFS (destination) database + LCFS_DB_HOST=localhost + LCFS_DB_PORT=5432 + LCFS_DB_NAME=lcfs + LCFS_DB_USER=lcfs_user + LCFS_DB_PASSWORD=lcfs_password + ``` + +## Migration Scripts + +### 1. compliance_summary.py +Migrates compliance report summary data from TFRS to LCFS. + +**What it does:** +- Fetches compliance summary records from TFRS +- Maps legacy compliance report IDs to LCFS IDs +- Inserts summary records with proper field mapping +- Handles gasoline/diesel class data mapping + +**Usage:** +```bash +python compliance_summary.py +``` + +### 2. compliance_summary_update.py +Updates existing compliance summary records with snapshot data from TFRS. + +**What it does:** +- Parses JSON snapshot data from TFRS +- Extracts line-by-line compliance data +- Updates LCFS summary records with calculated values +- Stores historical snapshot data + +**Usage:** +```bash +python compliance_summary_update.py +``` + +### 3. compliance_report_history.py +Migrates compliance report workflow history. + +**What it does:** +- Maps TFRS workflow states to LCFS status IDs +- Handles status progression logic +- Excludes draft and supplemental records +- Maintains audit trail of report changes + +**Usage:** +```bash +python compliance_report_history.py +``` + +### 4. allocation_agreement.py +Migrates allocation agreement data between organizations. + +**What it does:** +- Processes exclusion agreement records +- Maps fuel types and transaction types +- Implements versioning for agreement changes +- Handles partner and quantity information + +**Usage:** +```bash +python allocation_agreement.py +``` + +### 5. other_uses.py +Migrates Schedule C (other uses) data. + +**What it does:** +- Processes fuel use records for non-transportation purposes +- Handles supplemental report chains +- Maps expected use types and fuel categories +- Implements change detection and versioning + +**Usage:** +```bash +python other_uses.py +``` + +## Common Features + +All migration scripts include: + +- **Error Handling**: Comprehensive exception handling with detailed logging +- **Data Validation**: Input validation and safe type conversion +- **Logging**: Detailed logging to files and console +- **Transaction Management**: Proper database transaction handling +- **Mapping Logic**: Handles schema differences between TFRS and LCFS +- **Progress Tracking**: Reports on successful/skipped records + +## Running Migrations + +1. **Test Database Connections** + ```python + from database import tfrs_db, lcfs_db + print("TFRS:", tfrs_db.test_connection()) + print("LCFS:", lcfs_db.test_connection()) + ``` + +2. **Run Individual Scripts** + ```bash + # Run specific migration + python compliance_summary.py + + # Run with debug logging + python -c " + import logging + logging.basicConfig(level=logging.DEBUG) + exec(open('compliance_summary.py').read()) + " + ``` + +3. **Monitor Progress** + - Check console output for real-time progress + - Review log files for detailed information + - Monitor database for inserted records + +## Key Differences from Groovy Scripts + +- **Database Connections**: Uses connection pooling with context managers +- **Error Handling**: More granular exception handling +- **Type Safety**: Explicit type conversion and validation +- **Logging**: Structured logging with multiple output formats +- **Configuration**: Environment-based configuration management +- **Testing**: Built-in connection testing capabilities + +## Troubleshooting + +### Common Issues + +1. **Connection Errors** + - Verify database credentials in `.env` + - Check network connectivity + - Ensure databases are running + +2. **Permission Errors** + - Verify user has read access to TFRS database + - Verify user has write access to LCFS database + - Check table-level permissions + +3. **Data Issues** + - Review log files for validation errors + - Check for missing reference data + - Verify legacy_id mappings exist + +4. **Performance Issues** + - Monitor database connections + - Consider batch size adjustments + - Check database indexes + +### Log Analysis + +Log files are created with timestamps: +``` +migration_YYYYMMDD_HHMMSS.log +``` + +Look for: +- `ERROR` level messages for failures +- `WARNING` level messages for data issues +- `INFO` level messages for progress updates + +## Migration Order + +Recommended order for running migrations: + +1. `compliance_summary.py` - Base summary data +2. `compliance_summary_update.py` - Enhanced summary data +3. `compliance_report_history.py` - Workflow history +4. `allocation_agreement.py` - Agreement data +5. `other_uses.py` - Schedule C data + +## Data Mapping Notes + +### Status Mapping +- TFRS workflow states → LCFS status IDs +- Handles multi-stage approval process +- Excludes draft and supplemental states + +### Fuel Type Mapping +- TFRS fuel types → LCFS fuel types +- Includes category classification +- Handles legacy type conversions + +### Unit Conversion +- Standardizes unit representations +- Maps abbreviated units to full names +- Validates unit compatibility + +## Performance Considerations + +- Scripts process records in batches +- Database connections are managed per operation +- Memory usage is optimized for large datasets +- Progress is logged for monitoring + +## Maintenance + +- Review and update mappings as schemas evolve +- Monitor log files for data quality issues +- Test scripts against sample data before production runs +- Keep configuration files secure and backed up \ No newline at end of file diff --git a/etl/python_migration/documentation/README_SETUP.md b/etl/python_migration/documentation/README_SETUP.md new file mode 100644 index 000000000..5020d2584 --- /dev/null +++ b/etl/python_migration/documentation/README_SETUP.md @@ -0,0 +1,243 @@ +# Database Setup and Complete Migration Guide + +This document explains how to use the new Python database setup and migration orchestration tools. + +## Overview + +The Python migration framework now includes automated database setup and orchestration capabilities that replace and enhance the original `data-transfer.sh` script. + +## Key Components + +### 1. `database_setup.py` +Python equivalent of `data-transfer.sh` with additional features: +- Automated database setup from OpenShift environments +- Database population verification +- Migration readiness checks +- Support for both TFRS and LCFS systems + +### 2. `run_complete_migration.py` +Complete migration orchestrator that handles: +- Database setup (optional) +- Migration readiness verification +- Running all migration scripts +- Running all validation scripts +- Comprehensive reporting + +## Usage Examples + +### Complete Migration Process + +Run the entire migration process from start to finish: + +```bash +# Complete migration with database setup +python run_complete_migration.py complete \ + --tfrs-container my-tfrs-container \ + --lcfs-container my-lcfs-container \ + --env dev + +# Complete migration using existing databases +python run_complete_migration.py complete --skip-setup + +# Migration without validation (faster for testing) +python run_complete_migration.py complete \ + --tfrs-container my-tfrs-container \ + --lcfs-container my-lcfs-container \ + --skip-validation +``` + +### Individual Components + +Set up databases only: +```bash +python run_complete_migration.py setup my-tfrs-container my-lcfs-container --env dev +``` + +Run migration only (assumes databases are ready): +```bash +python run_complete_migration.py migrate +``` + +Run validation only (assumes migration is complete): +```bash +python run_complete_migration.py validate +``` + +Check if databases are ready for migration: +```bash +python run_complete_migration.py check +``` + +### Database Setup Utilities + +```bash +# Set up test environment with both databases +python database_setup.py setup my-tfrs-container my-lcfs-container dev + +# Transfer specific data +python database_setup.py transfer tfrs dev import my-tfrs-container + +# Verify database population +python database_setup.py verify-tfrs +python database_setup.py verify-lcfs + +# Check migration readiness +python database_setup.py check-readiness +``` + +## Prerequisites + +### 1. OpenShift CLI Setup +- Install and configure `oc` CLI +- Login to OpenShift: `oc login` +- Ensure you have access to both TFRS and LCFS projects + +### 2. Docker Setup +- Ensure Docker is running +- Have local PostgreSQL containers for TFRS and LCFS +- Containers should be running and accessible + +### 3. Environment Configuration +- Set up `.env` file with database connection details +- Ensure Python dependencies are installed: `pip install -r requirements.txt` + +## Database Container Setup + +Example Docker commands to create local PostgreSQL containers: + +```bash +# TFRS container +docker run -d --name my-tfrs-container \ + -e POSTGRES_DB=tfrs \ + -e POSTGRES_USER=tfrs \ + -e POSTGRES_PASSWORD=tfrs \ + -p 5432:5432 \ + postgres:13 + +# LCFS container +docker run -d --name my-lcfs-container \ + -e POSTGRES_DB=lcfs \ + -e POSTGRES_USER=lcfs \ + -e POSTGRES_PASSWORD=lcfs \ + -p 5433:5432 \ + postgres:13 +``` + +## Environment Variables + +Required in `.env` file: + +```bash +# TFRS (Source) Database +TFRS_DB_HOST=localhost +TFRS_DB_PORT=5432 +TFRS_DB_NAME=tfrs +TFRS_DB_USER=tfrs +TFRS_DB_PASSWORD=tfrs + +# LCFS (Destination) Database +LCFS_DB_HOST=localhost +LCFS_DB_PORT=5433 +LCFS_DB_NAME=lcfs +LCFS_DB_USER=lcfs +LCFS_DB_PASSWORD=lcfs +``` + +## Migration Process Flow + +1. **Database Setup** (Optional) + - Downloads data from OpenShift environments + - Restores to local PostgreSQL containers + - Verifies data population + +2. **Readiness Check** + - Verifies source database has required tables and data + - Confirms destination database is accessible + - Checks for minimum data requirements + +3. **Migration Execution** + - Runs all migration scripts in correct order + - Tracks progress and errors + - Provides detailed logging + +4. **Validation** (Optional) + - Runs comprehensive validation suite + - Compares source vs destination data + - Generates detailed reports + +## Output and Logging + +The orchestrator produces: +- **Console output** with progress indicators and summaries +- **Detailed logs** with timestamps and component information +- **JSON result files** for validation results +- **Status codes** for integration with CI/CD systems + +## Error Handling + +- Each step validates prerequisites before proceeding +- Failed components don't prevent other components from running +- Detailed error messages help diagnose issues +- Cleanup operations ensure no leftover temporary files + +## Advantages Over Shell Script + +1. **Better Error Handling**: More robust error detection and recovery +2. **Integration**: Seamlessly integrates with Python migration scripts +3. **Validation**: Built-in verification of database state and migration results +4. **Flexibility**: Modular design allows running individual components +5. **Logging**: Comprehensive logging and reporting +6. **Cross-Platform**: Works on any platform with Python and Docker + +## Troubleshooting + +### Common Issues + +**OpenShift Connection Failed** +```bash +# Check login status +oc whoami + +# Re-login if needed +oc login --server=https://your-openshift-server +``` + +**Docker Container Not Found** +```bash +# Check running containers +docker ps + +# Start container if stopped +docker start my-tfrs-container +``` + +**Database Connection Failed** +- Verify `.env` file configuration +- Check container ports and networking +- Ensure PostgreSQL is accepting connections + +**Insufficient Data** +- Verify source environment has sufficient test data +- Check table population with `verify-tfrs` command +- Consider using a different source environment + +## Testing Workflow + +Recommended workflow for development and testing: + +```bash +# 1. Set up test environment +python run_complete_migration.py setup my-tfrs-container my-lcfs-container --env dev + +# 2. Run migration with validation +python run_complete_migration.py complete --skip-setup + +# 3. During development, run individual components +python run_complete_migration.py migrate # Quick migration testing +python run_complete_migration.py validate # Validation only + +# 4. Check readiness between iterations +python run_complete_migration.py check +``` + +This setup provides a robust, automated foundation for TFRS to LCFS migration testing and development. \ No newline at end of file diff --git a/etl/python_migration/documentation/VERSIONING_REVIEW.md b/etl/python_migration/documentation/VERSIONING_REVIEW.md new file mode 100644 index 000000000..fdbbf810d --- /dev/null +++ b/etl/python_migration/documentation/VERSIONING_REVIEW.md @@ -0,0 +1,201 @@ +# Group UUID Versioning System Review + +## Overview + +This document provides a comprehensive review of the group_uuid versioning system implemented across all Python migration scripts, identifying potential bugs and areas of concern based on analysis of the original Groovy scripts. + +## Versioning System Architecture + +The versioning system uses: +- **group_uuid**: A stable UUID that groups related records across different versions +- **version**: An integer that increments for each change to a record group +- **action_type**: Enum indicating whether this is a CREATE or UPDATE action + +## Script-by-Script Analysis + +### 1. allocation_agreement.py + +**Versioning Implementation:** +- Uses `record_uuid_map` to maintain stable group UUIDs per TFRS agreement_record_id +- Queries current max version for group_uuid before inserting +- Sets next_ver = current_ver + 1 or 0 for new groups + +**Potential Issues:** +✅ **CORRECT**: Properly maintains stable group_uuid per source record ID +✅ **CORRECT**: Version increment logic is sound +⚠️ **CONCERN**: No validation that the same group_uuid isn't used across different record types + +### 2. other_uses.py (Schedule C) + +**Versioning Implementation:** +- Uses chain-based processing with change detection +- Maintains `record_uuid_map` for schedule_c_record_id +- Implements `is_record_changed()` for diff detection + +**Potential Issues:** +✅ **CORRECT**: Proper change detection between chain versions +✅ **CORRECT**: Stable UUID generation per record ID +⚠️ **CONCERN**: Chain processing assumes sequential order - could fail if traversal is incorrect + +### 3. notional_transfer.py (Schedule A) + +**Versioning Implementation:** +- Similar chain-based approach to other_uses.py +- Uses schedule_a_record_id for group UUID mapping +- Implements change detection for diff processing + +**Potential Issues:** +✅ **CORRECT**: Change detection logic matches other_uses.py +⚠️ **ISSUE**: Mapping logic in `map_received_or_transferred()` appears inverted: +```python +def map_received_or_transferred(self, transfer_type_id: int) -> str: + if transfer_type_id == 1: + return "Received" # This seems backwards + return "Transferred" +``` +**RECOMMENDATION**: Verify this mapping with business logic - it contradicts the comment. + +### 4. fuel_supply.py + +**Versioning Implementation:** +- Uses pre-determined action_type based on base_version comparison +- Does NOT implement its own versioning - relies on compliance_report versioning +- Uses group_uuid from compliance report, not per fuel supply record + +**Potential Issues:** +🚨 **CRITICAL BUG**: No stable group_uuid per fuel supply record! +```python +# Current implementation uses compliance report's group_uuid +# This means ALL fuel supply records for a report share the same group_uuid +# This violates the versioning pattern used in other scripts +``` + +**RECOMMENDATION**: +1. Add `fuel_supply_record_uuid_map` similar to other scripts +2. Generate stable UUID per fuel supply record ID from source +3. Implement proper version tracking per fuel supply record + +### 5. orphaned_allocation_agreement.py + +**Versioning Implementation:** +- Reuses allocation_agreement versioning logic +- Creates new compliance reports with proper group_uuid +- Uses same record ID mapping as regular allocation agreements + +**Potential Issues:** +⚠️ **CONCERN**: Shares `record_uuid_map` namespace with regular allocation agreements +- Could cause UUID collisions if same TFRS record ID exists in both regular and orphaned reports +- **RECOMMENDATION**: Use separate UUID namespace or prefix + +### 6. compliance_summary.py & compliance_summary_update.py + +**Versioning Implementation:** +- These scripts don't use the group_uuid versioning system +- They work at the compliance report level, not individual records + +**Potential Issues:** +✅ **CORRECT**: No versioning needed - these are summary-level operations + +### 7. compliance_report_history.py + +**Versioning Implementation:** +- No versioning system - inserts history records directly +- Operates on compliance report level + +**Potential Issues:** +✅ **CORRECT**: History records don't need versioning + +## Critical Issues Identified + +### 1. 🚨 CRITICAL: fuel_supply.py Missing Record-Level Versioning + +**Problem**: Uses compliance report group_uuid instead of per-record versioning +**Impact**: Cannot track individual fuel supply record changes +**Fix Required**: Implement record-level group_uuid mapping + +### 2. ⚠️ MEDIUM: UUID Namespace Collisions + +**Problem**: Multiple scripts use same record ID space for UUID mapping +**Impact**: Potential UUID collisions between different record types +**Recommendation**: Use prefixed or namespaced UUIDs + +### 3. ⚠️ MEDIUM: Transfer Type Mapping Inconsistency + +**Problem**: `notional_transfer.py` mapping appears inverted +**Impact**: Data may be incorrectly categorized +**Recommendation**: Verify business logic mapping + +## Recommended Fixes + +### Fix 1: fuel_supply.py Versioning System + +```python +class FuelSupplyMigrator: + def __init__(self): + # Add missing versioning components + self.fuel_supply_record_uuid_map: Dict[int, str] = {} + + def get_fuel_supply_group_uuid(self, record_id: int) -> str: + """Get or create stable group UUID for fuel supply record""" + if record_id not in self.fuel_supply_record_uuid_map: + self.fuel_supply_record_uuid_map[record_id] = str(uuid.uuid4()) + return self.fuel_supply_record_uuid_map[record_id] + + def get_current_fuel_supply_version(self, lcfs_cursor, group_uuid: str) -> int: + """Get current version for fuel supply group""" + query = "SELECT version FROM fuel_supply WHERE group_uuid = %s ORDER BY version DESC LIMIT 1" + lcfs_cursor.execute(query, (group_uuid,)) + result = lcfs_cursor.fetchone() + return result[0] if result else -1 +``` + +### Fix 2: UUID Namespace Separation + +```python +# Use prefixed UUIDs to avoid collisions +def get_namespaced_uuid(record_type: str, record_id: int) -> str: + key = f"{record_type}:{record_id}" + if key not in uuid_map: + uuid_map[key] = str(uuid.uuid4()) + return uuid_map[key] + +# Usage: +allocation_uuid = get_namespaced_uuid("allocation", record_id) +fuel_supply_uuid = get_namespaced_uuid("fuel_supply", record_id) +``` + +### Fix 3: Verify Transfer Type Mapping + +```python +def map_received_or_transferred(self, transfer_type_id: int) -> str: + """Maps TFRS transfer_type_id to 'Received' or 'Transferred' + + VERIFY THIS MAPPING WITH BUSINESS LOGIC: + Original comment suggests: 1=Transferred, 2=Received + But code returns: 1->Received, 2->Transferred + + This needs business validation! + """ + # Current implementation - verify correctness: + if transfer_type_id == 1: + return "Received" # Is this correct? + return "Transferred" +``` + +## Testing Recommendations + +1. **Version Consistency Tests**: Verify version increments are monotonic and sequential +2. **UUID Uniqueness Tests**: Ensure no UUID collisions across record types +3. **Change Detection Tests**: Verify diff logic correctly identifies changed vs unchanged records +4. **Chain Processing Tests**: Test supplemental report chain processing with various traversal orders + +## Conclusion + +The versioning system is generally well-implemented across most scripts, but has one critical issue in `fuel_supply.py` and several medium-priority concerns around namespace collisions and mapping verification. The fixes outlined above should address these issues while maintaining the integrity of the existing versioning system. + +## Implementation Priority + +1. **HIGH**: Fix fuel_supply.py versioning system +2. **MEDIUM**: Implement UUID namespacing +3. **MEDIUM**: Verify transfer type mapping +4. **LOW**: Add comprehensive testing for edge cases \ No newline at end of file diff --git a/etl/python_migration/env.example b/etl/python_migration/env.example new file mode 100644 index 000000000..2fbed191e --- /dev/null +++ b/etl/python_migration/env.example @@ -0,0 +1,23 @@ +# TFRS to LCFS Migration Environment Configuration +# Copy this file to .env and update the values for your local setup + +# TFRS (Source) Database Configuration +TFRS_DB_HOST=localhost +TFRS_DB_PORT=5435 +TFRS_DB_NAME=tfrs +TFRS_DB_USER=tfrs +TFRS_DB_PASSWORD=development_only + +# LCFS (Destination) Database Configuration +LCFS_DB_HOST=localhost +LCFS_DB_PORT=5432 +LCFS_DB_NAME=lcfs +LCFS_DB_USER=lcfs +LCFS_DB_PASSWORD=development_only + +# Optional: Logging Configuration +LOG_LEVEL=INFO + +# Note: These settings work with the default Docker Compose setup +# TFRS container runs on port 5435 +# LCFS container typically runs on port 5432 (from main LCFS docker-compose) \ No newline at end of file diff --git a/etl/python_migration/migrations/__init__.py b/etl/python_migration/migrations/__init__.py new file mode 100644 index 000000000..6f358d698 --- /dev/null +++ b/etl/python_migration/migrations/__init__.py @@ -0,0 +1 @@ +# Migration scripts \ No newline at end of file diff --git a/etl/python_migration/migrations/migrate_allocation_agreements.py b/etl/python_migration/migrations/migrate_allocation_agreements.py new file mode 100644 index 000000000..72a7bbe66 --- /dev/null +++ b/etl/python_migration/migrations/migrate_allocation_agreements.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +""" +Allocation Agreement Migration Script + +Migrates allocation agreement data from TFRS to LCFS database. +This script replicates the functionality of allocation_agreement.groovy +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +import sys +import uuid +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import ( + setup_logging, + safe_decimal, + safe_int, + safe_str, + build_legacy_mapping, +) + +logger = logging.getLogger(__name__) + + +class AllocationAgreementMigrator: + def __init__(self): + self.record_uuid_map: Dict[int, str] = {} + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + self.responsibility_to_transaction_type_cache: Dict[str, int] = {} + self.fuel_type_name_to_id_cache: Dict[str, int] = {} + + # Constants + self.GASOLINE_CATEGORY_ID = 1 + self.DIESEL_CATEGORY_ID = 2 + + def load_mappings(self, lcfs_cursor): + """Load legacy ID to LCFS compliance_report_id mappings""" + logger.info("Loading legacy ID to LCFS compliance_report_id mappings") + self.legacy_to_lcfs_mapping = build_legacy_mapping(lcfs_cursor) + logger.info(f"Loaded {len(self.legacy_to_lcfs_mapping)} legacy mappings") + + def get_transaction_type_id(self, lcfs_cursor, responsibility: str) -> int: + """Returns the allocation transaction type ID for a given responsibility with caching""" + if responsibility in self.responsibility_to_transaction_type_cache: + return self.responsibility_to_transaction_type_cache[responsibility] + + query = """ + SELECT allocation_transaction_type_id + FROM allocation_transaction_type + WHERE type = %s + """ + + try: + lcfs_cursor.execute(query, (responsibility,)) + result = lcfs_cursor.fetchone() + if result: + type_id = result[0] + self.responsibility_to_transaction_type_cache[responsibility] = type_id + return type_id + else: + logger.warning( + f"No transaction type found for responsibility: {responsibility}; using default 1." + ) + return 1 + except Exception as e: + logger.error(f"Error looking up transaction type for {responsibility}: {e}") + return 1 + + def get_fuel_type_id(self, lcfs_cursor, fuel_type: str) -> int: + """Returns the fuel type ID for a given fuel type string with caching""" + if fuel_type in self.fuel_type_name_to_id_cache: + return self.fuel_type_name_to_id_cache[fuel_type] + + query = """ + SELECT fuel_type_id + FROM fuel_type + WHERE fuel_type = %s + """ + + try: + lcfs_cursor.execute(query, (fuel_type,)) + result = lcfs_cursor.fetchone() + if result: + type_id = result[0] + self.fuel_type_name_to_id_cache[fuel_type] = type_id + return type_id + else: + logger.warning(f"No fuel type found for: {fuel_type}; using default 1.") + return 1 + except Exception as e: + logger.error(f"Error looking up fuel type for {fuel_type}: {e}") + return 1 + + def get_current_version(self, lcfs_cursor, group_uuid: str) -> int: + """Get current highest version for a group UUID""" + query = """ + SELECT version + FROM allocation_agreement + WHERE group_uuid = %s + ORDER BY version DESC + LIMIT 1 + """ + lcfs_cursor.execute(query, (group_uuid,)) + result = lcfs_cursor.fetchone() + return result[0] if result else -1 + + def get_lcfs_reports_with_legacy_ids(self, lcfs_cursor) -> List[int]: + """Get all LCFS compliance reports with legacy IDs""" + query = """ + SELECT compliance_report_id, legacy_id + FROM compliance_report + WHERE legacy_id IS NOT NULL + """ + lcfs_cursor.execute(query) + return [row[1] for row in lcfs_cursor.fetchall()] # Return legacy_ids + + def get_lcfs_reports_with_org_period_info( + self, lcfs_cursor + ) -> List[Tuple[int, int, int]]: + """Get LCFS compliance reports with their organization and period info for proper versioning scope""" + query = """ + SELECT cr.legacy_id, cr.organization_id, cr.compliance_period_id + FROM compliance_report cr + WHERE cr.legacy_id IS NOT NULL + ORDER BY cr.organization_id, cr.compliance_period_id, cr.legacy_id + """ + + lcfs_cursor.execute(query) + return [(row[0], row[1], row[2]) for row in lcfs_cursor.fetchall()] + + def get_allocation_agreement_records(self, tfrs_cursor, tfrs_id: int) -> List[Dict]: + """Get allocation agreement records for a given TFRS compliance report""" + query = """ + SELECT + crear.id AS agreement_record_id, + CASE WHEN tt.the_type = 'Purchased' THEN 'Allocated from' ELSE 'Allocated to' END AS responsibility, + aft.name AS fuel_type, + aft.id AS fuel_type_id, + crear.transaction_partner, + crear.postal_address, + crear.quantity, + uom.name AS units, + crear.quantity_not_sold, + tt.id AS transaction_type_id + FROM compliance_report legacy_cr + -- First, try to join to the report's own exclusion agreement if it exists (combo reports) + LEFT JOIN compliance_report_exclusion_agreement crea_direct + ON legacy_cr.exclusion_agreement_id = crea_direct.id + LEFT JOIN compliance_report_exclusion_agreement_record crear_direct + ON crea_direct.id = crear_direct.exclusion_agreement_id + -- If not found, join to the latest exclusion report in the same organization/period + LEFT JOIN ( + SELECT DISTINCT ON (exclusion_cr.organization_id, exclusion_cr.compliance_period_id) + exclusion_cr.exclusion_agreement_id, + exclusion_cr.organization_id, + exclusion_cr.compliance_period_id + FROM compliance_report exclusion_cr + WHERE exclusion_cr.exclusion_agreement_id IS NOT NULL + ORDER BY exclusion_cr.organization_id, exclusion_cr.compliance_period_id, exclusion_cr.traversal DESC + ) latest_exclusion + ON latest_exclusion.organization_id = legacy_cr.organization_id + AND latest_exclusion.compliance_period_id = legacy_cr.compliance_period_id + AND crea_direct.id IS NULL -- Only use if direct exclusion not found + LEFT JOIN compliance_report_exclusion_agreement crea_latest + ON latest_exclusion.exclusion_agreement_id = crea_latest.id + LEFT JOIN compliance_report_exclusion_agreement_record crear_latest + ON crea_latest.id = crear_latest.exclusion_agreement_id + -- Coalesce the results to use direct first, then latest + CROSS JOIN ( + SELECT + COALESCE(crear_direct.id, crear_latest.id) AS id, + COALESCE(crear_direct.transaction_partner, crear_latest.transaction_partner) AS transaction_partner, + COALESCE(crear_direct.postal_address, crear_latest.postal_address) AS postal_address, + COALESCE(crear_direct.quantity, crear_latest.quantity) AS quantity, + COALESCE(crear_direct.quantity_not_sold, crear_latest.quantity_not_sold) AS quantity_not_sold, + COALESCE(crear_direct.transaction_type_id, crear_latest.transaction_type_id) AS transaction_type_id, + COALESCE(crear_direct.fuel_type_id, crear_latest.fuel_type_id) AS fuel_type_id + ) crear + -- Standard joins for details + INNER JOIN transaction_type tt + ON crear.transaction_type_id = tt.id + INNER JOIN approved_fuel_type aft + ON crear.fuel_type_id = aft.id + INNER JOIN unit_of_measure uom + ON aft.unit_of_measure_id = uom.id + WHERE + legacy_cr.id = %s + AND crear.id IS NOT NULL -- Ensure we have allocation data + ORDER BY + crear.id; + """ + + tfrs_cursor.execute(query, (tfrs_id,)) + records = [] + + for row in tfrs_cursor.fetchall(): + records.append( + { + "agreement_record_id": row[0], + "responsibility": row[1], + "fuel_type": row[2], + "fuel_type_id": row[3], + "transaction_partner": row[4], + "postal_address": row[5], + "quantity": row[6], + "units": row[7], + "quantity_not_sold": row[8], + "transaction_type_id": row[9], + } + ) + + return records + + def determine_fuel_category_id(self, fuel_type_string: str) -> Optional[int]: + """Determine fuel category ID based on fuel type string""" + if not fuel_type_string: + return None + + fuel_type_lower = fuel_type_string.lower() + if "gasoline" in fuel_type_lower: + return self.GASOLINE_CATEGORY_ID + elif "diesel" in fuel_type_lower: + return self.DIESEL_CATEGORY_ID + else: + logger.warning( + f"Could not determine fuel category for fuel type: {fuel_type_string}. Setting fuel_category_id to NULL." + ) + return None + + def generate_logical_record_key(self, record_data: Dict) -> str: + """Generate a logical key for allocation agreement versioning based on business data""" + # Use key business fields that define a unique logical allocation agreement + # NOTE: Quantity is excluded so that quantity changes create new versions, not new records + transaction_partner = record_data.get("transaction_partner", "").strip() + responsibility = record_data.get("responsibility", "").strip() + fuel_type = record_data.get("fuel_type", "").strip() + + # Create a logical key from the business identifiers (excluding quantity) + logical_key = f"{transaction_partner}|{responsibility}|{fuel_type}" + return logical_key + + def insert_version_row( + self, lcfs_cursor, lcfs_cr_id: int, row_data: Dict, action: str + ) -> bool: + """Inserts a new row into allocation_agreement with proper versioning""" + try: + record_id = row_data["agreement_record_id"] + + # Generate logical key for this allocation agreement + logical_key = self.generate_logical_record_key(row_data) + + # Retrieve or create a stable group_uuid based on logical key + group_uuid = self.record_uuid_map.get(logical_key) + if not group_uuid: + group_uuid = str(uuid.uuid4()) + self.record_uuid_map[logical_key] = group_uuid + logger.debug( + f"Created new group_uuid {group_uuid} for logical key: {logical_key}" + ) + + # Retrieve current highest version for this group_uuid + current_ver = self.get_current_version(lcfs_cursor, group_uuid) + next_ver = 0 if current_ver < 0 else current_ver + 1 + + # Map source fields to destination fields + alloc_transaction_type_id = self.get_transaction_type_id( + lcfs_cursor, row_data["responsibility"] + ) + fuel_type_id = self.get_fuel_type_id(lcfs_cursor, row_data["fuel_type"]) + quantity = safe_int(row_data.get("quantity", 0)) + quantity_not_sold = safe_int(row_data.get("quantity_not_sold", 0)) + transaction_partner = safe_str(row_data.get("transaction_partner", "")) + postal_address = safe_str(row_data.get("postal_address", "")) + units = safe_str(row_data.get("units", "")) + fuel_type_string = row_data.get("fuel_type", "") + + # Determine Fuel Category ID + fuel_category_id = self.determine_fuel_category_id(fuel_type_string) + + # Insert the record + insert_sql = """ + INSERT INTO allocation_agreement( + compliance_report_id, + transaction_partner, + postal_address, + quantity, + quantity_not_sold, + units, + allocation_transaction_type_id, + fuel_type_id, + fuel_category_id, + group_uuid, + version, + action_type, + create_user, + update_user + ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::actiontypeenum, %s, %s) + """ + + params = [ + lcfs_cr_id, + transaction_partner, + postal_address, + quantity, + quantity_not_sold, + units, + alloc_transaction_type_id, + fuel_type_id, + fuel_category_id, + group_uuid, + next_ver, + action, + "ETL", + "ETL", + ] + + lcfs_cursor.execute(insert_sql, params) + logger.info( + f"Inserted allocation_agreement row: recordId={record_id}, action={action}, groupUuid={group_uuid}, version={next_ver}" + ) + return True + + except Exception as e: + logger.error(f"Failed to insert allocation agreement record: {e}") + return False + + def migrate(self) -> Tuple[int, int]: + """Main migration logic""" + total_inserted = 0 + total_skipped = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Load mappings + self.load_mappings(lcfs_cursor) + + # Get all LCFS compliance reports with legacy IDs and their org/period info + logger.info( + "Retrieving LCFS compliance reports with legacy_id != NULL" + ) + reports_data = self.get_lcfs_reports_with_org_period_info( + lcfs_cursor + ) + logger.info(f"Found {len(reports_data)} reports to process") + + # Group reports by organization + period to ensure proper versioning scope + report_groups = {} + for report_data in reports_data: + tfrs_id, org_id, period_id = report_data + group_key = (org_id, period_id) + if group_key not in report_groups: + report_groups[group_key] = [] + report_groups[group_key].append(tfrs_id) + + # Process each organization/period group with its own record_uuid_map + for (org_id, period_id), tfrs_ids in report_groups.items(): + self.record_uuid_map = ( + {} + ) # Reset for each organization + period combination + logger.info( + f"Processing organization {org_id}, period {period_id} with {len(tfrs_ids)} reports" + ) + + for tfrs_id in tfrs_ids: + logger.info( + f"Processing TFRS compliance_report.id = {tfrs_id}" + ) + + # Look up the original LCFS compliance_report record by legacy_id + lcfs_cr_id = self.legacy_to_lcfs_mapping.get(tfrs_id) + if not lcfs_cr_id: + logger.warning( + f"No LCFS compliance_report found for TFRS legacy id {tfrs_id}; skipping allocation agreement processing." + ) + total_skipped += 1 + continue + + # Check if this is a supplemental report that needs special handling + logger.debug( + f"Checking if TFRS report {tfrs_id} is a supplemental report that needs allocation inheritance" + ) + + # Retrieve allocation agreement records from source for the given TFRS report + allocation_records = self.get_allocation_agreement_records( + tfrs_cursor, tfrs_id + ) + + if not allocation_records: + logger.warning( + f"No allocation agreement records found in source for TFRS report ID: {tfrs_id} (or cr.exclusion_agreement_id was NULL)." + ) + continue + + # Process each allocation agreement record + for record_data in allocation_records: + rec_id = record_data["agreement_record_id"] + logger.info( + f"Found source allocation record ID: {rec_id} for TFRS report ID: {tfrs_id}. Preparing for LCFS insert." + ) + + # Insert each allocation agreement record with versioning + if self.insert_version_row( + lcfs_cursor, lcfs_cr_id, record_data, "CREATE" + ): + total_inserted += 1 + else: + total_skipped += 1 + + # Commit all changes + lcfs_conn.commit() + logger.info( + f"Successfully committed {total_inserted} allocation agreement records" + ) + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + return total_inserted, total_skipped + + +def main(): + setup_logging() + logger.info("Starting Allocation Agreement Migration") + + migrator = AllocationAgreementMigrator() + + try: + inserted, skipped = migrator.migrate() + logger.info( + f"Migration completed successfully. Inserted: {inserted}, Skipped: {skipped}" + ) + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/migrate_compliance_report_history.py b/etl/python_migration/migrations/migrate_compliance_report_history.py new file mode 100644 index 000000000..cd1934e4f --- /dev/null +++ b/etl/python_migration/migrations/migrate_compliance_report_history.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +""" +Compliance Report History Migration Script + +Migrates compliance report history data from TFRS to LCFS database. +This script replicates the functionality of compliance_report_history.groovy +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +import sys +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import setup_logging, build_legacy_mapping + +logger = logging.getLogger(__name__) + + +class ComplianceReportHistoryMigrator: + def __init__(self): + self.legacy_mapping: Dict[int, int] = {} + self.status_mapping: Dict[str, int] = {} + self.status_id_to_name: Dict[int, str] = {} + + def load_reference_data(self, lcfs_cursor): + """Load reference data for status mapping""" + logger.info("Loading reference data for status mapping") + + # Load legacy ID mappings + self.legacy_mapping = build_legacy_mapping(lcfs_cursor, "compliance_report") + logger.info(f"Loaded {len(self.legacy_mapping)} legacy mappings") + + # Load status mappings + query = ( + "SELECT compliance_report_status_id, status FROM compliance_report_status" + ) + lcfs_cursor.execute(query) + + for row in lcfs_cursor.fetchall(): + status_id, status_name = row + status_lower = status_name.lower() + self.status_mapping[status_lower] = status_id + self.status_id_to_name[status_id] = status_lower + + logger.info(f"Loaded {len(self.status_mapping)} status mappings") + + def map_final_status( + self, + fuel_status: str, + analyst_status: str, + manager_status: str, + director_status: str, + ) -> Optional[int]: + """Map workflow statuses to a final status ID""" + + # Normalize all statuses to lower-case + fuel_status = fuel_status.lower() if fuel_status else "" + analyst_status = analyst_status.lower() if analyst_status else "" + manager_status = manager_status.lower() if manager_status else "" + director_status = director_status.lower() if director_status else "" + + # Exclude records if any field contains "requested supplemental" + if ( + "requested supplemental" in fuel_status + or "requested supplemental" in analyst_status + or "requested supplemental" in manager_status + or "requested supplemental" in director_status + ): + logger.debug( + "Record marked as 'Requested Supplemental'; skipping history record." + ) + return None + + # Exclude records with a draft status + if fuel_status == "draft": + logger.debug("Record marked as 'Draft'; skipping history record.") + return None + + # Compute the stage for this history record + # The intended order is: + # Stage 1: Submitted + # Stage 2: Recommended by Analyst + # Stage 3: Recommended by Manager + # Stage 4: Accepted by Director (Assessed) + computed_stage = 1 # default to Submitted + + if analyst_status == "recommended": + computed_stage = max(computed_stage, 2) + if manager_status == "recommended": + computed_stage = max(computed_stage, 3) + if director_status == "accepted": + computed_stage = max(computed_stage, 4) + + # Map the computed stage to a status name + status_name_map = { + 1: "submitted", + 2: "recommended_by_analyst", + 3: "recommended_by_manager", + 4: "assessed", + } + + status_name = status_name_map.get(computed_stage) + if not status_name: + return None + + # Return the corresponding status ID from the reference data + return self.status_mapping.get(status_name) + + def truncate_destination_table(self, lcfs_cursor): + """Truncate the destination compliance_report_history table for clean reload""" + logger.info("Truncating destination compliance_report_history table") + lcfs_cursor.execute("TRUNCATE TABLE compliance_report_history CASCADE") + + def fetch_source_history_records(self, tfrs_cursor) -> List[Dict]: + """Fetch history records from source database""" + source_query = """ + SELECT + crh.id AS history_id, + crh.compliance_report_id, + crh.create_timestamp, + crh.update_timestamp, + crh.create_user_id, + crh.status_id AS original_status_id, + crh.update_user_id, + cws.analyst_status_id, + cws.director_status_id, + cws.fuel_supplier_status_id, + cws.manager_status_id + FROM compliance_report_history crh + JOIN compliance_report_workflow_state cws ON crh.status_id = cws.id + ORDER BY crh.compliance_report_id, crh.create_timestamp; + """ + + logger.info("Fetching source history records from TFRS") + tfrs_cursor.execute(source_query) + records = [] + + for row in tfrs_cursor.fetchall(): + records.append( + { + "history_id": row[0], + "compliance_report_id": row[1], + "create_timestamp": row[2], + "update_timestamp": row[3], + "create_user_id": row[4], + "original_status_id": row[5], + "update_user_id": row[6], + "analyst_status_id": row[7], + "director_status_id": row[8], + "fuel_supplier_status_id": row[9], + "manager_status_id": row[10], + } + ) + + logger.info(f"Fetched {len(records)} source history records") + return records + + def insert_history_record(self, lcfs_cursor, record_data: Dict) -> bool: + """Insert a single history record into destination""" + insert_sql = """ + INSERT INTO compliance_report_history ( + compliance_report_id, + status_id, + user_profile_id, + create_user, + create_date, + update_user, + update_date + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """ + + try: + params = [ + record_data["destination_report_id"], + record_data["final_status_id"], + record_data["create_user_id"], + str(record_data["create_user_id"]), + record_data["create_timestamp"], + ( + str(record_data["update_user_id"]) + if record_data["update_user_id"] + else None + ), + record_data["update_timestamp"], + ] + + lcfs_cursor.execute(insert_sql, params) + return True + + except Exception as e: + logger.error(f"Failed to insert history record: {e}") + return False + + def migrate(self) -> Tuple[int, int]: + """Main migration logic""" + records_processed = 0 + records_skipped = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Load reference data + self.load_reference_data(lcfs_cursor) + + # Truncate destination table for clean reload + self.truncate_destination_table(lcfs_cursor) + + # Fetch source records + source_records = self.fetch_source_history_records(tfrs_cursor) + + logger.info("Starting history migration process") + for record in source_records: + # Get the TFRS compliance report id from the source history record + legacy_id = record["compliance_report_id"] + + # Look up the corresponding LCFS compliance_report_id using the legacy mapping + destination_report_id = self.legacy_mapping.get(legacy_id) + if destination_report_id is None: + logger.warning( + f"No matching LCFS compliance report found for legacy id: {legacy_id}" + ) + records_skipped += 1 + continue + + # Retrieve the workflow status values + analyst_status = record["analyst_status_id"] or "" + director_status = record["director_status_id"] or "" + fuel_status = record["fuel_supplier_status_id"] or "" + manager_status = record["manager_status_id"] or "" + + # Recalculate the final status using the updated mapping function + final_status_id = self.map_final_status( + fuel_status, analyst_status, manager_status, director_status + ) + if final_status_id is None: + logger.debug( + f"Skipping history record for legacy id: {legacy_id} due to unmapped status" + ) + records_skipped += 1 + continue + + # Prepare record for insertion + record_data = { + "destination_report_id": destination_report_id, + "final_status_id": final_status_id, + "create_user_id": record["create_user_id"], + "update_user_id": record["update_user_id"], + "create_timestamp": record["create_timestamp"], + "update_timestamp": record["update_timestamp"], + } + + if self.insert_history_record(lcfs_cursor, record_data): + records_processed += 1 + else: + records_skipped += 1 + + # Commit all changes + lcfs_conn.commit() + logger.info( + f"Successfully committed {records_processed} history records" + ) + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + return records_processed, records_skipped + + +def main(): + setup_logging() + logger.info("Starting Compliance Report History Migration") + + migrator = ComplianceReportHistoryMigrator() + + try: + processed, skipped = migrator.migrate() + logger.info( + f"Migration completed successfully. Processed: {processed}, Skipped: {skipped}" + ) + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/migrate_compliance_summaries.py b/etl/python_migration/migrations/migrate_compliance_summaries.py new file mode 100644 index 000000000..5ba86122a --- /dev/null +++ b/etl/python_migration/migrations/migrate_compliance_summaries.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +""" +Compliance Summary Migration Script + +Migrates compliance report summary data from TFRS to LCFS database. +This script replicates the functionality of compliance_summary.groovy +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +import sys +from datetime import datetime +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import ( + setup_logging, + safe_decimal, + safe_int, + safe_str, + build_legacy_mapping, +) + +logger = logging.getLogger(__name__) + + +class ComplianceSummaryMigrator: + def __init__(self): + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + self.existing_summary_ids: set = set() + self.existing_compliance_report_ids: set = set() + + def load_mappings(self, lcfs_cursor): + logger.info("Loading legacy ID to LCFS compliance_report_id mappings") + self.legacy_to_lcfs_mapping = build_legacy_mapping(lcfs_cursor) + logger.info(f"Loaded {len(self.legacy_to_lcfs_mapping)} legacy mappings") + + logger.info("Loading existing compliance_report_summary records") + lcfs_cursor.execute( + "SELECT summary_id, compliance_report_id FROM compliance_report_summary" + ) + for row in lcfs_cursor.fetchall(): + summary_id, compliance_report_id = row + self.existing_summary_ids.add(summary_id) + self.existing_compliance_report_ids.add(compliance_report_id) + + logger.info( + f"Found {len(self.existing_compliance_report_ids)} existing summary records" + ) + + def fetch_source_data(self, tfrs_cursor) -> List[Dict]: + source_query = """ + SELECT + cr.summary_id, + cr.id AS compliance_report_id, + crs.gasoline_class_retained, + crs.gasoline_class_deferred, + crs.diesel_class_retained, + crs.diesel_class_deferred, + crs.credits_offset, + crs.diesel_class_obligation, + crs.diesel_class_previously_retained, + crs.gasoline_class_obligation, + crs.gasoline_class_previously_retained + FROM + public.compliance_report cr + JOIN + public.compliance_report_summary crs + ON cr.summary_id = crs.id + WHERE + cr.summary_id IS NOT NULL + ORDER BY + cr.id; + """ + + logger.info("Fetching source data from TFRS") + tfrs_cursor.execute(source_query) + records = [] + + for row in tfrs_cursor.fetchall(): + records.append( + { + "summary_id": row[0], + "compliance_report_id": row[1], + "gasoline_class_retained": row[2], + "gasoline_class_deferred": row[3], + "diesel_class_retained": row[4], + "diesel_class_deferred": row[5], + "credits_offset": row[6], + "diesel_class_obligation": row[7], + "diesel_class_previously_retained": row[8], + "gasoline_class_obligation": row[9], + "gasoline_class_previously_retained": row[10], + } + ) + + logger.info(f"Fetched {len(records)} source records") + return records + + def build_summary_record(self, source_record: Dict) -> Dict: + source_compliance_report_legacy_id = source_record["compliance_report_id"] + lcfs_compliance_report_id = self.legacy_to_lcfs_mapping.get( + source_compliance_report_legacy_id + ) + + if lcfs_compliance_report_id is None: + return None + + if lcfs_compliance_report_id in self.existing_compliance_report_ids: + logger.warning( + f"Summary already exists for LCFS compliance_report_id {lcfs_compliance_report_id}" + ) + return None + + current_timestamp = datetime.now() + + return { + "compliance_report_id": lcfs_compliance_report_id, + "quarter": None, + "is_locked": True, + "line_1_fossil_derived_base_fuel_gasoline": None, + "line_1_fossil_derived_base_fuel_diesel": None, + "line_1_fossil_derived_base_fuel_jet_fuel": None, + "line_2_eligible_renewable_fuel_supplied_gasoline": None, + "line_2_eligible_renewable_fuel_supplied_diesel": None, + "line_2_eligible_renewable_fuel_supplied_jet_fuel": None, + "line_3_total_tracked_fuel_supplied_gasoline": None, + "line_3_total_tracked_fuel_supplied_diesel": None, + "line_3_total_tracked_fuel_supplied_jet_fuel": None, + "line_4_eligible_renewable_fuel_required_gasoline": None, + "line_4_eligible_renewable_fuel_required_diesel": None, + "line_4_eligible_renewable_fuel_required_jet_fuel": None, + "line_5_net_notionally_transferred_gasoline": None, + "line_5_net_notionally_transferred_diesel": None, + "line_5_net_notionally_transferred_jet_fuel": None, + "line_6_renewable_fuel_retained_gasoline": safe_decimal( + source_record["gasoline_class_retained"] + ), + "line_6_renewable_fuel_retained_diesel": safe_decimal( + source_record["diesel_class_retained"] + ), + "line_6_renewable_fuel_retained_jet_fuel": None, + "line_7_previously_retained_gasoline": safe_decimal( + source_record["gasoline_class_previously_retained"] + ), + "line_7_previously_retained_diesel": safe_decimal( + source_record["diesel_class_previously_retained"] + ), + "line_7_previously_retained_jet_fuel": None, + "line_8_obligation_deferred_gasoline": safe_decimal( + source_record["gasoline_class_deferred"] + ), + "line_8_obligation_deferred_diesel": safe_decimal( + source_record["diesel_class_deferred"] + ), + "line_8_obligation_deferred_jet_fuel": None, + "line_9_obligation_added_gasoline": safe_decimal( + source_record["gasoline_class_obligation"] + ), + "line_9_obligation_added_diesel": safe_decimal( + source_record["diesel_class_obligation"] + ), + "line_9_obligation_added_jet_fuel": None, + "line_10_net_renewable_fuel_supplied_gasoline": None, + "line_10_net_renewable_fuel_supplied_diesel": None, + "line_10_net_renewable_fuel_supplied_jet_fuel": None, + "line_11_non_compliance_penalty_gasoline": None, + "line_11_non_compliance_penalty_diesel": None, + "line_11_non_compliance_penalty_jet_fuel": None, + "line_12_low_carbon_fuel_required": None, + "line_13_low_carbon_fuel_supplied": None, + "line_14_low_carbon_fuel_surplus": None, + "line_15_banked_units_used": None, + "line_16_banked_units_remaining": None, + "line_17_non_banked_units_used": None, + "line_18_units_to_be_banked": None, + "line_19_units_to_be_exported": None, + "line_20_surplus_deficit_units": None, + "line_21_surplus_deficit_ratio": None, + # Do not map credits_offset here; leave line 22 empty for updater to populate from snapshot (end-of-period balance) + "line_22_compliance_units_issued": None, + "line_11_fossil_derived_base_fuel_gasoline": None, + "line_11_fossil_derived_base_fuel_diesel": None, + "line_11_fossil_derived_base_fuel_jet_fuel": None, + "line_11_fossil_derived_base_fuel_total": None, + "line_21_non_compliance_penalty_payable": None, + "total_non_compliance_penalty_payable": None, + "create_date": current_timestamp, + "update_date": current_timestamp, + "create_user": "etl_user", + "update_user": "etl_user", + } + + def insert_summary_record(self, lcfs_cursor, record: Dict) -> bool: + insert_sql = """ + INSERT INTO compliance_report_summary ( + compliance_report_id, + quarter, + is_locked, + line_1_fossil_derived_base_fuel_gasoline, + line_1_fossil_derived_base_fuel_diesel, + line_1_fossil_derived_base_fuel_jet_fuel, + line_2_eligible_renewable_fuel_supplied_gasoline, + line_2_eligible_renewable_fuel_supplied_diesel, + line_2_eligible_renewable_fuel_supplied_jet_fuel, + line_3_total_tracked_fuel_supplied_gasoline, + line_3_total_tracked_fuel_supplied_diesel, + line_3_total_tracked_fuel_supplied_jet_fuel, + line_4_eligible_renewable_fuel_required_gasoline, + line_4_eligible_renewable_fuel_required_diesel, + line_4_eligible_renewable_fuel_required_jet_fuel, + line_5_net_notionally_transferred_gasoline, + line_5_net_notionally_transferred_diesel, + line_5_net_notionally_transferred_jet_fuel, + line_6_renewable_fuel_retained_gasoline, + line_6_renewable_fuel_retained_diesel, + line_6_renewable_fuel_retained_jet_fuel, + line_7_previously_retained_gasoline, + line_7_previously_retained_diesel, + line_7_previously_retained_jet_fuel, + line_8_obligation_deferred_gasoline, + line_8_obligation_deferred_diesel, + line_8_obligation_deferred_jet_fuel, + line_9_obligation_added_gasoline, + line_9_obligation_added_diesel, + line_9_obligation_added_jet_fuel, + line_10_net_renewable_fuel_supplied_gasoline, + line_10_net_renewable_fuel_supplied_diesel, + line_10_net_renewable_fuel_supplied_jet_fuel, + line_11_non_compliance_penalty_gasoline, + line_11_non_compliance_penalty_diesel, + line_11_non_compliance_penalty_jet_fuel, + line_12_low_carbon_fuel_required, + line_13_low_carbon_fuel_supplied, + line_14_low_carbon_fuel_surplus, + line_15_banked_units_used, + line_16_banked_units_remaining, + line_17_non_banked_units_used, + line_18_units_to_be_banked, + line_19_units_to_be_exported, + line_20_surplus_deficit_units, + line_21_surplus_deficit_ratio, + line_22_compliance_units_issued, + line_11_fossil_derived_base_fuel_gasoline, + line_11_fossil_derived_base_fuel_diesel, + line_11_fossil_derived_base_fuel_jet_fuel, + line_11_fossil_derived_base_fuel_total, + line_21_non_compliance_penalty_payable, + total_non_compliance_penalty_payable, + create_date, + update_date, + create_user, + update_user + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + """ + + try: + # Build parameters list in the same order as the INSERT statement + params = [ + record["compliance_report_id"], + record["quarter"], + record["is_locked"], + ( + float(record["line_1_fossil_derived_base_fuel_gasoline"]) + if record["line_1_fossil_derived_base_fuel_gasoline"] is not None + else 0.0 + ), + ( + float(record["line_1_fossil_derived_base_fuel_diesel"]) + if record["line_1_fossil_derived_base_fuel_diesel"] is not None + else 0.0 + ), + 0.0, # jet_fuel + 0.0, # gasoline supplied + 0.0, # diesel supplied + 0.0, # jet fuel supplied + 0.0, # gasoline tracked + 0.0, # diesel tracked + 0.0, # jet fuel tracked + 0.0, # gasoline required + 0.0, # diesel required + 0.0, # jet fuel required + 0.0, # gasoline transferred + 0.0, # diesel transferred + 0.0, # jet fuel transferred + ( + float(record["line_6_renewable_fuel_retained_gasoline"]) + if record["line_6_renewable_fuel_retained_gasoline"] is not None + else 0.0 + ), + ( + float(record["line_6_renewable_fuel_retained_diesel"]) + if record["line_6_renewable_fuel_retained_diesel"] is not None + else 0.0 + ), + 0.0, # jet fuel retained + ( + float(record["line_7_previously_retained_gasoline"]) + if record["line_7_previously_retained_gasoline"] is not None + else 0.0 + ), + ( + float(record["line_7_previously_retained_diesel"]) + if record["line_7_previously_retained_diesel"] is not None + else 0.0 + ), + 0.0, # jet fuel previously retained + ( + float(record["line_8_obligation_deferred_gasoline"]) + if record["line_8_obligation_deferred_gasoline"] is not None + else 0.0 + ), + ( + float(record["line_8_obligation_deferred_diesel"]) + if record["line_8_obligation_deferred_diesel"] is not None + else 0.0 + ), + 0.0, # jet fuel deferred + ( + float(record["line_9_obligation_added_gasoline"]) + if record["line_9_obligation_added_gasoline"] is not None + else 0.0 + ), + ( + float(record["line_9_obligation_added_diesel"]) + if record["line_9_obligation_added_diesel"] is not None + else 0.0 + ), + 0.0, # jet fuel added + 0.0, # gasoline net supplied + 0.0, # diesel net supplied + 0.0, # jet fuel net supplied + None, # gasoline penalty + None, # diesel penalty + None, # jet fuel penalty + 0.0, # low carbon fuel required + 0.0, # low carbon fuel supplied + 0.0, # low carbon fuel surplus + 0.0, # banked units used + 0.0, # banked units remaining + 0.0, # non-banked units used + 0.0, # units to be banked + 0.0, # units to be exported + 0.0, # surplus deficit units + 0.0, # surplus deficit ratio + ( + float(record["line_22_compliance_units_issued"]) + if record["line_22_compliance_units_issued"] is not None + else 0.0 + ), + 0.0, # fossil gasoline (repeat) + 0.0, # fossil diesel (repeat) + 0.0, # fossil jet fuel (repeat) + 0.0, # fossil total (repeat) + 0.0, # penalty payable + 0.0, # total penalty payable + record["create_date"], + record["update_date"], + record["create_user"], + record["update_user"], + ] + + lcfs_cursor.execute(insert_sql, params) + return True + + except Exception as e: + logger.error( + f"Failed to insert summary record for compliance_report_id {record['compliance_report_id']}: {e}" + ) + return False + + def migrate(self) -> Tuple[int, int]: + total_inserted = 0 + total_skipped = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Load mappings and existing data + self.load_mappings(lcfs_cursor) + + # Fetch source data + source_records = self.fetch_source_data(tfrs_cursor) + + logger.info("Starting migration process") + for source_record in source_records: + summary_record = self.build_summary_record(source_record) + + if summary_record is None: + total_skipped += 1 + continue + + if self.insert_summary_record(lcfs_cursor, summary_record): + total_inserted += 1 + # Track that we've inserted this compliance_report_id + self.existing_compliance_report_ids.add( + summary_record["compliance_report_id"] + ) + else: + total_skipped += 1 + + # Commit all changes + lcfs_conn.commit() + logger.info(f"Successfully committed {total_inserted} records") + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + return total_inserted, total_skipped + + +def main(): + setup_logging() + logger.info("Starting Compliance Summary Migration") + + migrator = ComplianceSummaryMigrator() + + try: + inserted, skipped = migrator.migrate() + logger.info( + f"Migration completed successfully. Inserted: {inserted}, Skipped: {skipped}" + ) + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/migrate_compliance_summary_updates.py b/etl/python_migration/migrations/migrate_compliance_summary_updates.py new file mode 100644 index 000000000..28db4ec14 --- /dev/null +++ b/etl/python_migration/migrations/migrate_compliance_summary_updates.py @@ -0,0 +1,704 @@ +#!/usr/bin/env python3 +""" +Compliance Summary Update Script + +Updates existing compliance report summary records with data from TFRS snapshots. +This script replicates the functionality of compliance_summary_update.groovy +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import json +import logging +import sys +import zoneinfo +from datetime import datetime +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import setup_logging, safe_decimal, build_legacy_mapping + +logger = logging.getLogger(__name__) + +MIGRATION_USER = "ETL_COMPLIANCE_SUMMARY" + + +class ComplianceSummaryUpdater: + def __init__(self): + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + + # Statistics for reporting + self.stats = { + "tfrs_snapshots_found": 0, + "snapshots_processed": 0, + "snapshots_skipped_no_mapping": 0, + "snapshots_skipped_parse_error": 0, + "updates_successful": 0, + "updates_failed": 0, + "line_17_calculations": 0, + "batch_commits": 0, + } + + def load_mappings(self, lcfs_cursor): + logger.info("Loading legacy ID to LCFS compliance_report_id mappings") + self.legacy_to_lcfs_mapping = build_legacy_mapping(lcfs_cursor) + logger.info(f"Loaded {len(self.legacy_to_lcfs_mapping)} legacy mappings") + + def calculate_line_17_balance( + self, lcfs_cursor, organization_id: int, compliance_period: int + ) -> float: + """Calculate Line 17 available balance using raw SQL""" + try: + # Calculate compliance period end date + vancouver_timezone = zoneinfo.ZoneInfo("America/Vancouver") + compliance_period_end = datetime.strptime( + f"{str(compliance_period + 1)}-03-31", "%Y-%m-%d" + ) + compliance_period_end_local = compliance_period_end.replace( + hour=23, + minute=59, + second=59, + microsecond=999999, + tzinfo=vancouver_timezone, + ) + + line_17_query = """ + WITH assessment_balance AS ( + SELECT COALESCE(SUM(t.compliance_units), 0) as balance + FROM transaction t + JOIN compliance_report cr ON t.transaction_id = cr.transaction_id + JOIN compliance_report_status crs ON cr.current_status_id = crs.compliance_report_status_id + WHERE t.organization_id = %s + AND crs.status IN ('Assessed') + AND t.create_date <= %s + AND t.transaction_action = 'Adjustment' + ), + transfer_purchases AS ( + SELECT COALESCE(SUM(quantity), 0) as balance + FROM transfer + WHERE to_organization_id = %s + AND current_status_id = 6 + AND transaction_effective_date <= %s + ), + transfer_sales AS ( + SELECT COALESCE(SUM(quantity), 0) as balance + FROM transfer + WHERE from_organization_id = %s + AND current_status_id = 6 + AND transaction_effective_date <= %s + ), + initiative_agreements AS ( + SELECT COALESCE(SUM(compliance_units), 0) as balance + FROM initiative_agreement + WHERE to_organization_id = %s + AND current_status_id = 3 + AND transaction_effective_date <= %s + ), + admin_adjustments AS ( + SELECT COALESCE(SUM(compliance_units), 0) as balance + FROM admin_adjustment + WHERE to_organization_id = %s + AND current_status_id = 3 + AND transaction_effective_date <= %s + ), + future_transfer_debits AS ( + SELECT COALESCE(SUM(quantity), 0) as balance + FROM transfer + WHERE from_organization_id = %s + AND current_status_id = 6 + AND transaction_effective_date > %s + ), + future_negative_transactions AS ( + SELECT COALESCE(SUM(ABS(compliance_units)), 0) as balance + FROM transaction + WHERE organization_id = %s + AND create_date > %s + AND compliance_units < 0 + AND transaction_action != 'Released' + ) + SELECT GREATEST( + (SELECT balance FROM assessment_balance) + + (SELECT balance FROM transfer_purchases) - + (SELECT balance FROM transfer_sales) + + (SELECT balance FROM initiative_agreements) + + (SELECT balance FROM admin_adjustments) - + (SELECT balance FROM future_transfer_debits) - + (SELECT balance FROM future_negative_transactions), + 0 + ) AS available_balance + """ + + params = [ + organization_id, # assessment_balance + compliance_period_end_local, + organization_id, # transfer_purchases + compliance_period_end_local.date(), + organization_id, # transfer_sales + compliance_period_end_local.date(), + organization_id, # initiative_agreements + compliance_period_end_local.date(), + organization_id, # admin_adjustments + compliance_period_end_local.date(), + organization_id, # future_transfer_debits + compliance_period_end_local.date(), + organization_id, # future_negative_transactions + compliance_period_end_local, + ] + + lcfs_cursor.execute(line_17_query, params) + result = lcfs_cursor.fetchone() + + self.stats["line_17_calculations"] += 1 + return float(result[0] if result and result[0] is not None else 0) + + except Exception as e: + logger.error( + f"Error calculating line 17 balance for org {organization_id}: {e}" + ) + return 0.0 + + def get_compliance_units_received( + self, + lcfs_cursor, + organization_id: int, + compliance_period_start: str, + compliance_period_end: str, + ) -> float: + """Get compliance units received through transfers""" + query = """ + SELECT COALESCE(SUM(quantity), 0) AS total_transferred_out + FROM transfer + WHERE agreement_date BETWEEN %s AND %s + AND to_organization_id = %s + AND current_status_id = 6 -- Recorded + """ + + lcfs_cursor.execute( + query, (compliance_period_start, compliance_period_end, organization_id) + ) + result = lcfs_cursor.fetchone() + return float(result[0] if result and result[0] is not None else 0.0) + + def get_transferred_out_compliance_units( + self, + lcfs_cursor, + organization_id: int, + compliance_period_start: str, + compliance_period_end: str, + ) -> float: + """Get compliance units transferred away""" + query = """ + SELECT COALESCE(SUM(quantity), 0) AS total_transferred_out + FROM transfer + WHERE agreement_date BETWEEN %s AND %s + AND from_organization_id = %s + AND current_status_id = 6 -- Recorded + """ + + lcfs_cursor.execute( + query, (compliance_period_start, compliance_period_end, organization_id) + ) + result = lcfs_cursor.fetchone() + return float(result[0] if result and result[0] is not None else 0.0) + + def get_issued_compliance_units( + self, + lcfs_cursor, + organization_id: int, + compliance_period_start: str, + compliance_period_end: str, + ) -> float: + """Get compliance units issued under initiative agreements""" + query = """ + SELECT COALESCE(SUM(compliance_units), 0) AS total_compliance_units + FROM initiative_agreement + WHERE transaction_effective_date BETWEEN %s AND %s + AND to_organization_id = %s + AND current_status_id = 3; -- Approved + """ + + lcfs_cursor.execute( + query, (compliance_period_start, compliance_period_end, organization_id) + ) + result = lcfs_cursor.fetchone() + return float(result[0] if result and result[0] is not None else 0.0) + + def fetch_snapshot_data(self, tfrs_cursor) -> List[Dict]: + """Fetch snapshot data from LCFS compliance reports, filtering for unprocessed records""" + source_query = """ + SELECT compliance_report_id, snapshot + FROM public.compliance_report_snapshot + WHERE snapshot IS NOT NULL + """ + + logger.info("Fetching snapshot data from TFRS") + tfrs_cursor.execute(source_query) + records = [] + + for row in tfrs_cursor.fetchall(): + try: + # Check if data is already a dict (from psycopg2's JSON handling) + if isinstance(row[1], dict): + snapshot_data = row[1] + snapshot_json = json.dumps(row[1]) + else: + # It's a string, parse it + snapshot_data = json.loads(row[1]) + snapshot_json = row[1] + + records.append( + { + "compliance_report_id": row[0], + "snapshot": snapshot_data, + "snapshot_json": snapshot_json, + } + ) + except (json.JSONDecodeError, TypeError) as e: + logger.warning( + f"Failed to parse JSON for compliance_report_id {row[0]}: {e}" + ) + self.stats["snapshots_skipped_parse_error"] += 1 + continue + + self.stats["tfrs_snapshots_found"] = len(records) + logger.info(f"Fetched {len(records)} snapshot records") + return records + + def parse_summary_data(self, snapshot: Dict, lcfs_cursor) -> Optional[Dict]: + """Enhanced parsing with Alembic migration logic""" + try: + # Extract organization and compliance period info + organization_id = snapshot.get("organization", {}).get("id", 0) + compliance_period_start = snapshot.get("compliance_period", {}).get( + "effective_date", "9999-12-31" + ) + compliance_period_end = snapshot.get("compliance_period", {}).get( + "expiration_date", "9999-12-31" + ) + + # Extract year for line 17 calculation + compliance_period_year = ( + int(compliance_period_start[:4]) + if compliance_period_start != "9999-12-31" + else 2020 + ) + + summary = snapshot.get("summary", {}) + summary_lines = summary.get("lines", {}) + + # Calculate line 17 balance using enhanced logic + line_17 = self.calculate_line_17_balance( + lcfs_cursor, organization_id, compliance_period_year + ) + + # Get dynamic compliance unit calculations + compliance_units_received = Decimal(str(self.get_compliance_units_received( + lcfs_cursor, + organization_id, + compliance_period_start, + compliance_period_end, + ))) + transferred_out_units = Decimal(str(self.get_transferred_out_compliance_units( + lcfs_cursor, + organization_id, + compliance_period_start, + compliance_period_end, + ))) + issued_compliance_units = Decimal(str(self.get_issued_compliance_units( + lcfs_cursor, + organization_id, + compliance_period_start, + compliance_period_end, + ))) + + # Extract gasoline class mappings (lines 1-11) + line1_gas = safe_decimal(summary_lines.get("1", 0)) + line2_gas = safe_decimal(summary_lines.get("2", 0)) + line3_gas = safe_decimal(summary_lines.get("3", 0)) + line4_gas = safe_decimal(summary_lines.get("4", 0)) + line5_gas = safe_decimal(summary_lines.get("5", 0)) + line6_gas = safe_decimal(summary_lines.get("6", 0)) + line7_gas = safe_decimal(summary_lines.get("7", 0)) + line8_gas = safe_decimal(summary_lines.get("8", 0)) + line9_gas = safe_decimal(summary_lines.get("9", 0)) + line10_gas = safe_decimal(summary_lines.get("10", 0)) + line11_gas = safe_decimal(summary_lines.get("11", 0)) + + # Extract diesel class mappings (lines 12-22) + line1_diesel = safe_decimal(summary_lines.get("12", 0)) + line2_diesel = safe_decimal(summary_lines.get("13", 0)) + line3_diesel = safe_decimal(summary_lines.get("14", 0)) + line4_diesel = safe_decimal(summary_lines.get("15", 0)) + line5_diesel = safe_decimal(summary_lines.get("16", 0)) + line6_diesel = safe_decimal(summary_lines.get("17", 0)) + line7_diesel = safe_decimal(summary_lines.get("18", 0)) + line8_diesel = safe_decimal(summary_lines.get("19", 0)) + line9_diesel = safe_decimal(summary_lines.get("20", 0)) + line10_diesel = safe_decimal(summary_lines.get("21", 0)) + line11_diesel = safe_decimal(summary_lines.get("22", 0)) + + # Extract other summary data with enhanced calculations + compliance_units_issued = safe_decimal(summary_lines.get("25", 0)) + banked_used = safe_decimal(summary.get("credits_offset", 0)) + line28_non_compliance = safe_decimal(summary_lines.get("28", 0)) + + # Calculate fossil fuel totals + fossil_gas = line1_gas + fossil_diesel = line1_diesel + fossil_total = fossil_gas + fossil_diesel + + # Enhanced calculations from Alembic migration + # Convert line_17 (float) to Decimal for calculations + line_17_decimal = Decimal(str(line_17)) + balance_chg_from_assessment = compliance_units_issued - banked_used + # Available compliance unit balance at the end of the compliance date for the period + # This should be the available balance at period end, not credits issued + available_balance_at_period_end = line_17_decimal # This is the correct Line 22 value + total_payable = line11_gas + line11_diesel + line28_non_compliance + + return { + # Gasoline class data + "line_1_fossil_derived_base_fuel_gasoline": line1_gas, + "line_2_eligible_renewable_fuel_supplied_gasoline": line2_gas, + "line_3_total_tracked_fuel_supplied_gasoline": line3_gas, + "line_4_eligible_renewable_fuel_required_gasoline": line4_gas, + "line_5_net_notionally_transferred_gasoline": line5_gas, + "line_6_renewable_fuel_retained_gasoline": line6_gas, + "line_7_previously_retained_gasoline": line7_gas, + "line_8_obligation_deferred_gasoline": line8_gas, + "line_9_obligation_added_gasoline": line9_gas, + "line_10_net_renewable_fuel_supplied_gasoline": line10_gas, + "line_11_non_compliance_penalty_gasoline": line11_gas, + # Diesel class data + "line_1_fossil_derived_base_fuel_diesel": line1_diesel, + "line_2_eligible_renewable_fuel_supplied_diesel": line2_diesel, + "line_3_total_tracked_fuel_supplied_diesel": line3_diesel, + "line_4_eligible_renewable_fuel_required_diesel": line4_diesel, + "line_5_net_notionally_transferred_diesel": line5_diesel, + "line_6_renewable_fuel_retained_diesel": line6_diesel, + "line_7_previously_retained_diesel": line7_diesel, + "line_8_obligation_deferred_diesel": line8_diesel, + "line_9_obligation_added_diesel": line9_diesel, + "line_10_net_renewable_fuel_supplied_diesel": line10_diesel, + "line_11_non_compliance_penalty_diesel": line11_diesel, + # Jet fuel (all zeros since no TFRS data) + "line_1_fossil_derived_base_fuel_jet_fuel": Decimal("0.0"), + "line_2_eligible_renewable_fuel_supplied_jet_fuel": Decimal("0.0"), + "line_3_total_tracked_fuel_supplied_jet_fuel": Decimal("0.0"), + "line_4_eligible_renewable_fuel_required_jet_fuel": Decimal("0.0"), + "line_5_net_notionally_transferred_jet_fuel": Decimal("0.0"), + "line_6_renewable_fuel_retained_jet_fuel": Decimal("0.0"), + "line_7_previously_retained_jet_fuel": Decimal("0.0"), + "line_8_obligation_deferred_jet_fuel": Decimal("0.0"), + "line_9_obligation_added_jet_fuel": Decimal("0.0"), + "line_10_net_renewable_fuel_supplied_jet_fuel": Decimal("0.0"), + "line_11_non_compliance_penalty_jet_fuel": Decimal("0.0"), + # Low carbon fuel requirement summary - Enhanced with dynamic calculations + # Compliance units transferred away + "line_12_low_carbon_fuel_required": transferred_out_units, + # Compliance units received through transfers + "line_13_low_carbon_fuel_supplied": compliance_units_received, + # Compliance units issued under initiative agreements + "line_14_low_carbon_fuel_surplus": issued_compliance_units, + "line_15_banked_units_used": banked_used, + "line_16_banked_units_remaining": Decimal("0.0"), # Not tracked in TFRS + "line_17_non_banked_units_used": line_17_decimal, + "line_18_units_to_be_banked": issued_compliance_units, + "line_19_units_to_be_exported": Decimal("0.0"), # Not tracked in TFRS + "line_20_surplus_deficit_units": balance_chg_from_assessment, + "line_21_surplus_deficit_ratio": line28_non_compliance, + "line_22_compliance_units_issued": available_balance_at_period_end, + # Fossil derived base fuel (aggregate) + "line_11_fossil_derived_base_fuel_gasoline": fossil_gas, + "line_11_fossil_derived_base_fuel_diesel": fossil_diesel, + "line_11_fossil_derived_base_fuel_jet_fuel": Decimal("0.0"), + "line_11_fossil_derived_base_fuel_total": fossil_total, + # Non-compliance penalty fields + "line_21_non_compliance_penalty_payable": line28_non_compliance, + "total_non_compliance_penalty_payable": total_payable, + } + + except Exception as e: + logger.error(f"Failed to parse summary data from snapshot: {e}") + return None + + def update_summary_record( + self, + lcfs_cursor, + lcfs_compliance_report_id: int, + summary_data: Dict, + snapshot_json: str, + ) -> bool: + """Update summary record with enhanced field mapping""" + update_sql = """ + UPDATE public.compliance_report_summary + SET + update_user = %s, + update_date = NOW(), + line_1_fossil_derived_base_fuel_gasoline = %s, + line_2_eligible_renewable_fuel_supplied_gasoline = %s, + line_3_total_tracked_fuel_supplied_gasoline = %s, + line_4_eligible_renewable_fuel_required_gasoline = %s, + line_5_net_notionally_transferred_gasoline = %s, + line_6_renewable_fuel_retained_gasoline = %s, + line_7_previously_retained_gasoline = %s, + line_8_obligation_deferred_gasoline = %s, + line_9_obligation_added_gasoline = %s, + line_10_net_renewable_fuel_supplied_gasoline = %s, + line_11_non_compliance_penalty_gasoline = %s, + line_1_fossil_derived_base_fuel_diesel = %s, + line_2_eligible_renewable_fuel_supplied_diesel = %s, + line_3_total_tracked_fuel_supplied_diesel = %s, + line_4_eligible_renewable_fuel_required_diesel = %s, + line_5_net_notionally_transferred_diesel = %s, + line_6_renewable_fuel_retained_diesel = %s, + line_7_previously_retained_diesel = %s, + line_8_obligation_deferred_diesel = %s, + line_9_obligation_added_diesel = %s, + line_10_net_renewable_fuel_supplied_diesel = %s, + line_11_non_compliance_penalty_diesel = %s, + line_1_fossil_derived_base_fuel_jet_fuel = %s, + line_2_eligible_renewable_fuel_supplied_jet_fuel = %s, + line_3_total_tracked_fuel_supplied_jet_fuel = %s, + line_4_eligible_renewable_fuel_required_jet_fuel = %s, + line_5_net_notionally_transferred_jet_fuel = %s, + line_6_renewable_fuel_retained_jet_fuel = %s, + line_7_previously_retained_jet_fuel = %s, + line_8_obligation_deferred_jet_fuel = %s, + line_9_obligation_added_jet_fuel = %s, + line_10_net_renewable_fuel_supplied_jet_fuel = %s, + line_11_non_compliance_penalty_jet_fuel = %s, + line_12_low_carbon_fuel_required = %s, + line_13_low_carbon_fuel_supplied = %s, + line_14_low_carbon_fuel_surplus = %s, + line_15_banked_units_used = %s, + line_16_banked_units_remaining = %s, + line_17_non_banked_units_used = %s, + line_18_units_to_be_banked = %s, + line_19_units_to_be_exported = %s, + line_20_surplus_deficit_units = %s, + line_21_surplus_deficit_ratio = %s, + line_22_compliance_units_issued = %s, + line_11_fossil_derived_base_fuel_gasoline = %s, + line_11_fossil_derived_base_fuel_diesel = %s, + line_11_fossil_derived_base_fuel_jet_fuel = %s, + line_11_fossil_derived_base_fuel_total = %s, + line_21_non_compliance_penalty_payable = %s, + total_non_compliance_penalty_payable = %s, + historical_snapshot = %s::jsonb + WHERE compliance_report_id = %s + """ + + try: + params = [ + MIGRATION_USER, # update_user + # Gasoline class + float(summary_data["line_1_fossil_derived_base_fuel_gasoline"]), + float(summary_data["line_2_eligible_renewable_fuel_supplied_gasoline"]), + float(summary_data["line_3_total_tracked_fuel_supplied_gasoline"]), + float(summary_data["line_4_eligible_renewable_fuel_required_gasoline"]), + float(summary_data["line_5_net_notionally_transferred_gasoline"]), + float(summary_data["line_6_renewable_fuel_retained_gasoline"]), + float(summary_data["line_7_previously_retained_gasoline"]), + float(summary_data["line_8_obligation_deferred_gasoline"]), + float(summary_data["line_9_obligation_added_gasoline"]), + float(summary_data["line_10_net_renewable_fuel_supplied_gasoline"]), + float(summary_data["line_11_non_compliance_penalty_gasoline"]), + # Diesel class + float(summary_data["line_1_fossil_derived_base_fuel_diesel"]), + float(summary_data["line_2_eligible_renewable_fuel_supplied_diesel"]), + float(summary_data["line_3_total_tracked_fuel_supplied_diesel"]), + float(summary_data["line_4_eligible_renewable_fuel_required_diesel"]), + float(summary_data["line_5_net_notionally_transferred_diesel"]), + float(summary_data["line_6_renewable_fuel_retained_diesel"]), + float(summary_data["line_7_previously_retained_diesel"]), + float(summary_data["line_8_obligation_deferred_diesel"]), + float(summary_data["line_9_obligation_added_diesel"]), + float(summary_data["line_10_net_renewable_fuel_supplied_diesel"]), + float(summary_data["line_11_non_compliance_penalty_diesel"]), + # Jet fuel (all zeros) + float(summary_data["line_1_fossil_derived_base_fuel_jet_fuel"]), + float(summary_data["line_2_eligible_renewable_fuel_supplied_jet_fuel"]), + float(summary_data["line_3_total_tracked_fuel_supplied_jet_fuel"]), + float(summary_data["line_4_eligible_renewable_fuel_required_jet_fuel"]), + float(summary_data["line_5_net_notionally_transferred_jet_fuel"]), + float(summary_data["line_6_renewable_fuel_retained_jet_fuel"]), + float(summary_data["line_7_previously_retained_jet_fuel"]), + float(summary_data["line_8_obligation_deferred_jet_fuel"]), + float(summary_data["line_9_obligation_added_jet_fuel"]), + float(summary_data["line_10_net_renewable_fuel_supplied_jet_fuel"]), + float(summary_data["line_11_non_compliance_penalty_jet_fuel"]), + # Low carbon fuel requirement summary + float(summary_data["line_12_low_carbon_fuel_required"]), + float(summary_data["line_13_low_carbon_fuel_supplied"]), + float(summary_data["line_14_low_carbon_fuel_surplus"]), + float(summary_data["line_15_banked_units_used"]), + float(summary_data["line_16_banked_units_remaining"]), + float(summary_data["line_17_non_banked_units_used"]), + float(summary_data["line_18_units_to_be_banked"]), + float(summary_data["line_19_units_to_be_exported"]), + float(summary_data["line_20_surplus_deficit_units"]), + float(summary_data["line_21_surplus_deficit_ratio"]), + float(summary_data["line_22_compliance_units_issued"]), + # Fossil derived base fuel (aggregate) + float(summary_data["line_11_fossil_derived_base_fuel_gasoline"]), + float(summary_data["line_11_fossil_derived_base_fuel_diesel"]), + float(summary_data["line_11_fossil_derived_base_fuel_jet_fuel"]), + float(summary_data["line_11_fossil_derived_base_fuel_total"]), + # Non-compliance penalty fields + float(summary_data["line_21_non_compliance_penalty_payable"]), + float(summary_data["total_non_compliance_penalty_payable"]), + # Historical snapshot + snapshot_json, + # WHERE clause + lcfs_compliance_report_id, + ] + + lcfs_cursor.execute(update_sql, params) + return lcfs_cursor.rowcount > 0 + + except Exception as e: + logger.error( + f"Failed to update summary for compliance_report_id {lcfs_compliance_report_id}: {e}" + ) + return False + + def update_summaries(self) -> Tuple[int, int]: + """Main update logic with enhanced batch processing""" + update_count = 0 + skip_count = 0 + BATCH_SIZE = 10 # Commit every 10 records + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Load mappings + self.load_mappings(lcfs_cursor) + + # Fetch snapshot data + snapshot_records = self.fetch_snapshot_data(tfrs_cursor) + + logger.info("Starting summary update process") + for record in snapshot_records: + legacy_compliance_report_id = record["compliance_report_id"] + lcfs_compliance_report_id = self.legacy_to_lcfs_mapping.get( + legacy_compliance_report_id + ) + + if lcfs_compliance_report_id is None: + logger.warning( + f"No LCFS compliance report found for legacy id {legacy_compliance_report_id}" + ) + skip_count += 1 + continue + + logger.info( + f"Processing legacy id {legacy_compliance_report_id} (LCFS ID: {lcfs_compliance_report_id})" + ) + + # Parse summary data with enhanced logic + summary_data = self.parse_summary_data( + record["snapshot"], lcfs_cursor + ) + if summary_data is None: + logger.error( + f"Failed to parse summary data for legacy id {legacy_compliance_report_id}" + ) + self.stats["snapshots_skipped_parse_error"] += 1 + skip_count += 1 + continue + + # Update the record + if self.update_summary_record( + lcfs_cursor, + lcfs_compliance_report_id, + summary_data, + record["snapshot_json"], + ): + update_count += 1 + self.stats["updates_successful"] += 1 + logger.info( + f"Successfully updated legacy id {legacy_compliance_report_id} (LCFS ID: {lcfs_compliance_report_id})" + ) + # Commit every batch_size records + if update_count % BATCH_SIZE == 0: + lcfs_conn.commit() + self.stats["batch_commits"] += 1 + logger.info( + f"Committed batch. Total processed: {update_count}" + ) + else: + self.stats["updates_failed"] += 1 + skip_count += 1 + + # Commit all changes + lcfs_conn.commit() + logger.info(f"Successfully committed {update_count} updates") + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Update process failed: {e}") + raise + + return update_count, skip_count + + def print_statistics(self): + """Print comprehensive migration statistics""" + logger.info("=" * 60) + logger.info("COMPLIANCE SUMMARY UPDATE STATISTICS") + logger.info("=" * 60) + + logger.info(f"📊 Source Data:") + logger.info(f" • Snapshot records found: {self.stats['tfrs_snapshots_found']}") + logger.info(f" • Snapshots processed: {self.stats['snapshots_processed']}") + + logger.info(f"🔄 Processing Results:") + logger.info(f" • Successful updates: {self.stats['updates_successful']}") + logger.info(f" • Failed updates: {self.stats['updates_failed']}") + logger.info(f" • Parse errors: {self.stats['snapshots_skipped_parse_error']}") + logger.info(f" • Line 17 calculations: {self.stats['line_17_calculations']}") + logger.info(f" • Batch commits: {self.stats['batch_commits']}") + + total_processed = ( + self.stats["updates_successful"] + self.stats["updates_failed"] + ) + success_rate = ( + (self.stats["updates_successful"] / total_processed * 100) + if total_processed > 0 + else 0 + ) + logger.info(f" • Success rate: {success_rate:.1f}%") + + logger.info("=" * 60) + + +def main(): + setup_logging() + logger.info("Starting Compliance Summary Update") + + updater = ComplianceSummaryUpdater() + + try: + updated, skipped = updater.update_summaries() + # Print statistics + updater.print_statistics() + logger.info( + f"Update completed successfully. Updated: {updated}, Skipped: {skipped}" + ) + except Exception as e: + logger.error(f"Update failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/migrate_data_cleanup.py b/etl/python_migration/migrations/migrate_data_cleanup.py new file mode 100644 index 000000000..14905f1c7 --- /dev/null +++ b/etl/python_migration/migrations/migrate_data_cleanup.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +""" +Data Cleanup Migration Script + +Pre-migration script that prepares the LCFS database with necessary reference data +and fallback entries to handle all possible TFRS data scenarios. This script: + +1. Creates missing fuel types and reference data in LCFS +2. Adds fallback/default entries for unmappable TFRS data +3. Ensures LCFS has all necessary lookup tables populated +4. DOES NOT modify any TFRS data (source data is sacrosanct) + +This script should be run FIRST before all other migrations. +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import setup_logging + +logger = logging.getLogger(__name__) + + +class DataCleanupMigrator: + def __init__(self): + self.cleanup_results = { + "fuel_types_created": 0, + "fallback_entries_created": 0, + "reference_data_validated": 0, + "total_preparations": 0, + } + + def create_fallback_fuel_types(self, lcfs_cursor) -> int: + """Create fallback fuel types in LCFS to handle unmappable TFRS data""" + logger.info("🔧 Creating fallback fuel types in LCFS...") + + # Fallback fuel types for unmappable TFRS data + fallback_fuel_types = [ + ("Unknown", "Fallback for unmappable TFRS fuel types", 0.0, "L", True), + ("Legacy TFRS", "Legacy fuel type from TFRS migration", 0.0, "L", True), + ("Unmapped", "TFRS fuel type with no LCFS equivalent", 0.0, "L", True), + ] + + created_count = 0 + for fuel_type, description, ci, units, unrecognized in fallback_fuel_types: + try: + # Check if it already exists + lcfs_cursor.execute( + "SELECT fuel_type_id FROM fuel_type WHERE fuel_type = %s", + (fuel_type,), + ) + if not lcfs_cursor.fetchone(): + # Create it using the actual LCFS schema + lcfs_cursor.execute( + """ + INSERT INTO fuel_type + (fuel_type, default_carbon_intensity, units, unrecognized, + fossil_derived, other_uses_fossil_derived, renewable, is_legacy, + create_user, update_user) + VALUES (%s, %s, %s::quantityunitsenum, %s, FALSE, FALSE, FALSE, TRUE, + 'ETL_CLEANUP', 'ETL_CLEANUP') + """, + (fuel_type, ci, units, unrecognized), + ) + created_count += 1 + logger.info(f"✅ Created fallback fuel type: {fuel_type}") + except Exception as e: + logger.warning(f"⚠️ Could not create fuel type {fuel_type}: {e}") + + return created_count + + def create_fallback_provisions(self, lcfs_cursor) -> int: + """Create fallback provision entries for unmappable TFRS data""" + logger.info("🔧 Creating fallback provisions in LCFS...") + + created_count = 0 + + # Default provision for unmappable entries + try: + lcfs_cursor.execute( + "SELECT provision_of_the_act_id FROM provision_of_the_act WHERE name = %s", + ("Unknown/Legacy",), + ) + if not lcfs_cursor.fetchone(): + lcfs_cursor.execute( + """ + INSERT INTO provision_of_the_act (name, description, display_order, create_user, update_user) + VALUES (%s, %s, 999, 'ETL_CLEANUP', 'ETL_CLEANUP') + """, + ( + "Unknown/Legacy", + "Fallback provision for unmappable TFRS entries", + ), + ) + created_count += 1 + logger.info("✅ Created fallback provision") + except Exception as e: + logger.warning(f"⚠️ Could not create fallback provision: {e}") + + return created_count + + def ensure_default_fuel_categories(self, lcfs_cursor) -> int: + """Ensure basic fuel categories exist for TFRS mapping""" + logger.info("🔧 Validating fuel categories...") + + # Check that basic categories exist + required_categories = ["Gasoline", "Diesel", "Jet fuel"] + missing_count = 0 + + for category in required_categories: + try: + lcfs_cursor.execute( + "SELECT fuel_category_id FROM fuel_category WHERE category = %s", + (category,), + ) + if not lcfs_cursor.fetchone(): + logger.warning(f"⚠️ Missing fuel category: {category}") + missing_count += 1 + else: + logger.info(f"✅ Found fuel category: {category}") + except Exception as e: + logger.error(f"❌ Error checking fuel category {category}: {e}") + + return len(required_categories) - missing_count + + def create_cleanup_log_table(self, lcfs_cursor): + """Create a table to log cleanup activities""" + try: + lcfs_cursor.execute( + """ + CREATE TABLE IF NOT EXISTS migration_cleanup_log ( + id SERIAL PRIMARY KEY, + cleanup_type VARCHAR(100), + description TEXT, + status VARCHAR(50), + details JSONB, + cleanup_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + create_user VARCHAR(100) DEFAULT 'ETL_CLEANUP' + ) + """ + ) + logger.info("✅ Created cleanup log table") + except Exception as e: + logger.warning(f"⚠️ Could not create cleanup log table: {e}") + + def log_cleanup_action( + self, + lcfs_cursor, + cleanup_type: str, + description: str, + status: str, + details: Dict = None, + ): + """Log a cleanup action to the database""" + try: + lcfs_cursor.execute( + """ + INSERT INTO migration_cleanup_log + (cleanup_type, description, status, details) + VALUES (%s, %s, %s, %s) + """, + (cleanup_type, description, status, str(details) if details else None), + ) + except Exception as e: + logger.warning(f"⚠️ Could not log cleanup action: {e}") + + def validate_lcfs_reference_data(self, lcfs_cursor) -> Dict: + """Validate that essential reference data exists in LCFS""" + logger.info("🔍 Validating LCFS reference data...") + + validation_results = {} + + # Check essential tables have data + essential_tables = [ + ("fuel_type", "fuel_type_id"), + ("fuel_category", "fuel_category_id"), + ("provision_of_the_act", "provision_of_the_act_id"), + ("compliance_period", "compliance_period_id"), + ] + + for table, id_field in essential_tables: + try: + lcfs_cursor.execute(f"SELECT COUNT(*) FROM {table}") + count = lcfs_cursor.fetchone()[0] + validation_results[table] = count + + if count == 0: + logger.error( + f"❌ {table} is empty - this will cause migration failures" + ) + else: + logger.info(f"✅ {table}: {count} records") + + except Exception as e: + logger.error(f"❌ Could not validate {table}: {e}") + validation_results[table] = -1 + + return validation_results + + def analyze_tfrs_data_patterns(self, tfrs_cursor) -> Dict: + """Analyze TFRS data to understand what we need to support (READ ONLY)""" + logger.info("🔍 Analyzing TFRS data patterns (read-only)...") + + analysis_results = {} + + try: + # Check for problematic fuel types (but don't fix them) + tfrs_cursor.execute( + """ + SELECT COUNT(*) as total_records, + COUNT(CASE WHEN aft.name IS NULL OR aft.name = '' THEN 1 END) as null_names, + COUNT(DISTINCT aft.name) as unique_fuel_types + FROM compliance_report_schedule_b_record crsbr + LEFT JOIN approved_fuel_type aft ON aft.id = crsbr.fuel_type_id + """ + ) + result = tfrs_cursor.fetchone() + analysis_results["fuel_supply_records"] = { + "total": result[0], + "null_fuel_types": result[1], + "unique_fuel_types": result[2], + } + + # Check for null quantities (but don't fix them) + tfrs_cursor.execute( + """ + SELECT COUNT(*) as null_quantities + FROM compliance_report_schedule_b_record + WHERE quantity IS NULL OR quantity = 0 + """ + ) + analysis_results["null_quantities"] = tfrs_cursor.fetchone()[0] + + # Get sample of unique fuel type names + tfrs_cursor.execute( + """ + SELECT DISTINCT aft.name + FROM approved_fuel_type aft + WHERE aft.name IS NOT NULL AND aft.name != '' + ORDER BY aft.name + LIMIT 20 + """ + ) + fuel_types = [row[0] for row in tfrs_cursor.fetchall()] + analysis_results["sample_fuel_types"] = fuel_types + + logger.info(f"📊 TFRS Analysis Results:") + logger.info( + f" - Total fuel supply records: {analysis_results['fuel_supply_records']['total']}" + ) + logger.info( + f" - Records with null fuel types: {analysis_results['fuel_supply_records']['null_fuel_types']}" + ) + logger.info( + f" - Records with null quantities: {analysis_results['null_quantities']}" + ) + logger.info(f" - Sample fuel types: {', '.join(fuel_types[:10])}...") + + except Exception as e: + logger.error(f"❌ Error analyzing TFRS data: {e}") + + return analysis_results + + def migrate(self) -> Tuple[int, int]: + """Run the complete data cleanup process (main interface for migration runner)""" + total_preparations, _ = self.run_cleanup() + return ( + total_preparations, + 0, + ) # Return (processed, total) format expected by runner + + def run_cleanup(self) -> Tuple[int, Dict]: + """Run the complete data cleanup process""" + logger.info("🧹 Starting LCFS database preparation for TFRS migration") + logger.info( + "📋 NOTE: TFRS data will NOT be modified (source data is sacrosanct)" + ) + logger.info("=" * 70) + + total_preparations = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Step 1: Create cleanup log table + self.create_cleanup_log_table(lcfs_cursor) + + # Step 2: Validate LCFS reference data + validation_results = self.validate_lcfs_reference_data(lcfs_cursor) + self.cleanup_results["reference_data_validated"] = len( + [v for v in validation_results.values() if v > 0] + ) + + # Step 3: Analyze TFRS data patterns (read-only) + tfrs_analysis = self.analyze_tfrs_data_patterns(tfrs_cursor) + self.log_cleanup_action( + lcfs_cursor, + "TFRS_ANALYSIS", + "Analyzed TFRS data patterns", + "COMPLETED", + tfrs_analysis, + ) + + # Step 4: Create fallback fuel types in LCFS + fuel_types_created = self.create_fallback_fuel_types(lcfs_cursor) + self.cleanup_results["fuel_types_created"] = fuel_types_created + total_preparations += fuel_types_created + + # Step 5: Create fallback provisions in LCFS + provisions_created = self.create_fallback_provisions(lcfs_cursor) + fallback_categories = self.ensure_default_fuel_categories( + lcfs_cursor + ) + fallback_total = provisions_created + fallback_categories + self.cleanup_results["fallback_entries_created"] = fallback_total + total_preparations += fallback_total + + # Step 6: Log final summary + self.cleanup_results["total_preparations"] = total_preparations + self.log_cleanup_action( + lcfs_cursor, + "PREPARATION_COMPLETE", + "LCFS database prepared for migration", + "SUCCESS", + self.cleanup_results, + ) + + # Commit LCFS changes only (TFRS is read-only) + lcfs_conn.commit() + + logger.info("=" * 70) + logger.info(f"🎉 LCFS database preparation completed!") + logger.info(f"📊 Total preparations made: {total_preparations}") + logger.info(f"📋 Summary:") + for key, value in self.cleanup_results.items(): + logger.info(f" - {key}: {value}") + + if tfrs_analysis.get("null_quantities", 0) > 0: + logger.warning( + f"⚠️ TFRS has {tfrs_analysis['null_quantities']} records with null quantities" + ) + logger.warning( + " Migration scripts will handle these with fallback values" + ) + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"❌ LCFS database preparation failed: {e}") + raise + + return total_preparations, self.cleanup_results + + +def main(): + """Main function to run data cleanup""" + setup_logging() + + migrator = DataCleanupMigrator() + + try: + total_preparations, results = migrator.run_cleanup() + + if total_preparations > 0: + print(f"\n✅ LCFS database preparation completed successfully!") + print(f"📊 Made {total_preparations} preparations") + print("🚀 Ready to run main migrations") + return 0 + else: + print("\n✅ LCFS database already prepared - ready for migration!") + return 0 + + except Exception as e: + print(f"\n❌ LCFS database preparation failed: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/etl/python_migration/migrations/migrate_fuel_supply.py b/etl/python_migration/migrations/migrate_fuel_supply.py new file mode 100644 index 000000000..bb663e007 --- /dev/null +++ b/etl/python_migration/migrations/migrate_fuel_supply.py @@ -0,0 +1,1256 @@ +#!/usr/bin/env python3 +""" +Fixed Fuel Supply Migration Script + +Migrates fuel supply (Schedule B) data from TFRS to LCFS database with proper record-level versioning. +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import json +import logging +import sys +import uuid +from decimal import Decimal, InvalidOperation +from typing import Dict, List, Optional, Tuple, Any +from datetime import datetime + +from core.database import get_source_connection, get_destination_connection +from core.utils import ( + setup_logging, + safe_decimal, + safe_int, + safe_str, + build_legacy_mapping, +) + +logger = logging.getLogger(__name__) + + +class FuelSupplyMigrator: + def __init__(self): + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + self.group_base_versions: Dict[str, int] = {} + self.unit_mapping = { + # Map TFRS units to LCFS enum values + "m³": "Cubic_metres", + "L": "Litres", + "kg": "Kilograms", + "kWh": "Kilowatt_hour", + # Add other common unit variations + "Litres": "Litres", + "Kilograms": "Kilograms", + "Kilowatt_hour": "Kilowatt_hour", + "Cubic_metres": "Cubic_metres", + } + + # Map TFRS provision names to LCFS provision names + self.provision_mapping = { + "Default Carbon Intensity Value": "Default Carbon Intensity Value - Section 6 (5) (d) (i)", + "Prescribed carbon intensity": "Prescribed carbon intensity - section 19 (a)", + "Approved fuel code": "Approved fuel code - Section 6 (5) (c)", + "GHGenius modelled": "GHGenius modelled - Section 6 (5) (d) (ii) (A)", + "Alternative Method": "Alternative Method - Section 6 (5) (d) (ii) (B)", + "Fuel code": "Fuel code - section 19 (b) (i)", + "Default carbon intensity": "Default carbon intensity - section 19 (b) (ii)", + } + + self.legacy_to_lcfs_mapping = {} + + def load_mappings(self, lcfs_cursor): + """Load reference data mappings from LCFS database""" + # Initialize mapping dictionaries + self.unit_mapping = { + # Map TFRS units to LCFS enum values + "m³": "Cubic_metres", + "L": "Litres", + "kg": "Kilograms", + "kWh": "Kilowatt_hour", + # Add other common unit variations + "Litres": "Litres", + "Kilograms": "Kilograms", + "Kilowatt_hour": "Kilowatt_hour", + "Cubic_metres": "Cubic_metres", + } + self.legacy_to_lcfs_mapping = {} + + try: + # Load fuel category mappings + lcfs_cursor.execute("SELECT fuel_category_id, category FROM fuel_category") + for row in lcfs_cursor.fetchall(): + self.legacy_to_lcfs_mapping[row[1]] = row[0] + + # Load fuel type mappings + lcfs_cursor.execute("SELECT fuel_type_id, fuel_type FROM fuel_type") + for row in lcfs_cursor.fetchall(): + self.legacy_to_lcfs_mapping[row[1]] = row[0] + + # Load provision mappings + lcfs_cursor.execute( + "SELECT provision_of_the_act_id, name FROM provision_of_the_act" + ) + for row in lcfs_cursor.fetchall(): + self.legacy_to_lcfs_mapping[row[1]] = row[0] + + logger.info("Loaded reference data mappings") + + except Exception as e: + logger.error(f"Failed to load mappings: {e}") + raise + + def fetch_base_versions(self, lcfs_cursor): + """Pre-fetch base versions for action type determination""" + logger.info("Fetching base versions from compliance_report table...") + query = """ + SELECT compliance_report_group_uuid, MIN(version) as base_version + FROM compliance_report + WHERE compliance_report_group_uuid IS NOT NULL + GROUP BY compliance_report_group_uuid + """ + lcfs_cursor.execute(query) + + for row in lcfs_cursor.fetchall(): + self.group_base_versions[row[0]] = row[1] + + logger.info(f"Finished fetching {len(self.group_base_versions)} base versions.") + + def get_compliance_reports_with_legacy_ids(self, lcfs_cursor) -> List[Tuple]: + """Get all compliance reports with non-null legacy_id from LCFS""" + query = """ + SELECT compliance_report_id, legacy_id, compliance_report_group_uuid, version + FROM compliance_report + WHERE legacy_id IS NOT NULL + """ + lcfs_cursor.execute(query) + return lcfs_cursor.fetchall() + + def fetch_snapshot_data(self, tfrs_cursor, legacy_id: int) -> Optional[Dict]: + """Fetch the snapshot record from compliance_report_snapshot in TFRS""" + query = """ + SELECT snapshot + FROM compliance_report_snapshot + WHERE compliance_report_id = %s + """ + tfrs_cursor.execute(query, (legacy_id,)) + result = tfrs_cursor.fetchone() + + if result and result[0]: + try: + # Handle both string and already-parsed dict data + snapshot_data = result[0] + if isinstance(snapshot_data, str): + return json.loads(snapshot_data) + elif isinstance(snapshot_data, dict): + return snapshot_data + else: + logger.error( + f"Unexpected snapshot data type for legacy_id {legacy_id}: {type(snapshot_data)}" + ) + return None + except json.JSONDecodeError as e: + logger.error( + f"Failed to parse JSON snapshot for legacy_id {legacy_id}: {e}" + ) + return None + return None + + def lookup_fuel_category_id(self, lcfs_cursor, fuel_category: str) -> Optional[int]: + """Look up fuel category ID by name""" + if not fuel_category: + logger.info("fuel_category is empty/None, returning None") + return None + query = "SELECT fuel_category_id FROM fuel_category WHERE category = %s" + lcfs_cursor.execute(query, (fuel_category,)) + result = lcfs_cursor.fetchone() + + if not result: + # Debug: Show available categories when lookup fails + logger.info( + f"Failed to find fuel_category '{fuel_category}', checking available categories..." + ) + lcfs_cursor.execute("SELECT category FROM fuel_category ORDER BY category") + available_categories = [row[0] for row in lcfs_cursor.fetchall()] + logger.info(f"Available fuel categories: {available_categories}") + return None + + return result[0] + + def lookup_fuel_type_id(self, lcfs_cursor, fuel_type: str) -> Optional[int]: + """Look up fuel type ID by name""" + if not fuel_type: + return None + query = "SELECT fuel_type_id FROM fuel_type WHERE fuel_type = %s" + lcfs_cursor.execute(query, (fuel_type,)) + result = lcfs_cursor.fetchone() + return result[0] if result else None + + def lookup_provision_id(self, lcfs_cursor, provision_name: str) -> Optional[int]: + """Look up provision of the act ID by name""" + if not provision_name: + logger.info("provision_name is empty/None, returning None") + return None + + # Try direct lookup first + query = ( + "SELECT provision_of_the_act_id FROM provision_of_the_act WHERE name = %s" + ) + lcfs_cursor.execute(query, (provision_name,)) + result = lcfs_cursor.fetchone() + + if result: + return result[0] + + # If direct lookup fails, try the mapping + mapped_provision = self.provision_mapping.get(provision_name) + if mapped_provision: + logger.info( + f"Using provision mapping: '{provision_name}' -> '{mapped_provision}'" + ) + lcfs_cursor.execute(query, (mapped_provision,)) + result = lcfs_cursor.fetchone() + if result: + return result[0] + + # Debug: Show available provisions when lookup fails + logger.info( + f"Failed to find provision '{provision_name}' (mapped: '{mapped_provision}'), checking available provisions..." + ) + lcfs_cursor.execute("SELECT name FROM provision_of_the_act ORDER BY name") + available_provisions = [row[0] for row in lcfs_cursor.fetchall()] + logger.info(f"Available provisions: {available_provisions}") + return None + + def lookup_fuel_code_id( + self, lcfs_cursor, fuel_code_prefix: str, fuel_code_suffix: str + ) -> Optional[int]: + """Look up LCFS fuel code ID by prefix and suffix""" + if not fuel_code_prefix: + return None + + try: + # Try to find exact match for prefix and suffix + lcfs_cursor.execute( + """ + SELECT fc.fuel_code_id + FROM fuel_code fc + JOIN fuel_code_prefix fcp ON fc.prefix_id = fcp.fuel_code_prefix_id + WHERE fcp.prefix = %s AND fc.fuel_suffix = %s + AND fc.fuel_status_id != 3 -- Exclude deleted fuel codes + """, + (fuel_code_prefix, fuel_code_suffix), + ) + result = lcfs_cursor.fetchone() + if result: + logger.debug( + f"Found LCFS fuel code ID {result[0]} for prefix={fuel_code_prefix}, suffix={fuel_code_suffix}" + ) + return result[0] + + logger.warning( + f"Could not find LCFS fuel code for prefix={fuel_code_prefix}, suffix={fuel_code_suffix}" + ) + return None + + except Exception as e: + logger.error(f"Error looking up fuel code by prefix/suffix: {e}") + return None + + def lookup_fuel_code_by_full_code( + self, lcfs_cursor, full_fuel_code: str + ) -> Optional[int]: + """Look up LCFS fuel code ID by full fuel code string (e.g., BCLCF236.1)""" + if not full_fuel_code: + return None + + try: + # Try to find an exact match first + lcfs_cursor.execute( + """ + SELECT fc.fuel_code_id + FROM fuel_code fc + JOIN fuel_code_prefix fcp ON fc.prefix_id = fcp.fuel_code_prefix_id + WHERE CONCAT(fcp.prefix, fc.fuel_suffix) = %s + AND fc.fuel_status_id != 3 -- Exclude deleted fuel codes + """, + (full_fuel_code,), + ) + result = lcfs_cursor.fetchone() + if result: + logger.debug( + f"Found exact LCFS fuel code match for {full_fuel_code}: {result[0]}" + ) + return result[0] + + # If exact match fails, try with "C-" prefix (TFRS vs LCFS fuel code format difference) + if full_fuel_code.startswith("BCLCF"): + prefixed_fuel_code = f"C-{full_fuel_code}" + lcfs_cursor.execute( + """ + SELECT fc.fuel_code_id + FROM fuel_code fc + JOIN fuel_code_prefix fcp ON fc.prefix_id = fcp.fuel_code_prefix_id + WHERE CONCAT(fcp.prefix, fc.fuel_suffix) = %s + AND fc.fuel_status_id != 3 -- Exclude deleted fuel codes + """, + (prefixed_fuel_code,), + ) + result = lcfs_cursor.fetchone() + if result: + logger.debug( + f"Found LCFS fuel code with C- prefix for {full_fuel_code} -> {prefixed_fuel_code}: {result[0]}" + ) + return result[0] + + # If no exact match, try parsing the fuel code to extract prefix and suffix + # Look for patterns like BCLCF236.1 where BCLCF is prefix and 236.1 is suffix + import re + + match = re.match(r"^([A-Z]+)(.+)$", full_fuel_code) + if match: + prefix_part = match.group(1) + suffix_part = match.group(2) + + # Try with the parsed prefix and suffix + lcfs_cursor.execute( + """ + SELECT fc.fuel_code_id + FROM fuel_code fc + JOIN fuel_code_prefix fcp ON fc.prefix_id = fcp.fuel_code_prefix_id + WHERE fcp.prefix = %s AND fc.fuel_suffix = %s + AND fc.fuel_status_id != 3 -- Exclude deleted fuel codes + """, + (prefix_part, suffix_part), + ) + result = lcfs_cursor.fetchone() + if result: + logger.debug( + f"Found LCFS fuel code by parsing {full_fuel_code} -> prefix={prefix_part}, suffix={suffix_part}: {result[0]}" + ) + return result[0] + + logger.debug(f"No LCFS fuel code found for {full_fuel_code}") + return None + + except Exception as e: + logger.error(f"Error looking up LCFS fuel code for {full_fuel_code}: {e}") + return None + + def lookup_end_use_id(self, lcfs_cursor, end_use: str) -> Optional[int]: + """Look up end use type ID by name""" + if not end_use: + return None + query = "SELECT end_use_type_id FROM end_use_type WHERE name = %s" + lcfs_cursor.execute(query, (end_use,)) + result = lcfs_cursor.fetchone() + return result[0] if result else None + + def safe_get_number( + self, record: Dict, field: str, use_snapshot: bool, legacy_id: int + ) -> Optional[Decimal]: + """Safely get and convert a number from record""" + try: + val = record.get(field) + if val is None or val == "": + return None + elif isinstance(val, (int, float, Decimal)): + return Decimal(str(val)) + else: + # Try to convert string to Decimal + val_str = str(val).strip() + if val_str == "": + return None + return Decimal(val_str) + except (ValueError, TypeError, InvalidOperation) as e: + logger.warning( + f"Could not convert {field}='{val}' to Decimal for legacy_id {legacy_id}: {e}" + ) + return None + + def get_standardized_fuel_data( + self, + lcfs_cursor, + fuel_type_id: int, + fuel_category_id: int, + end_use_id: int, + compliance_period: str, + fuel_code_id: Optional[int] = None, + provision_id: Optional[int] = None, + ) -> Dict[str, Optional[float]]: + """ + Fetch standardized fuel data similar to LCFS backend logic. + Returns calculated values for RCI, TCI, EER, energy_density, UCI. + """ + result = { + "effective_carbon_intensity": None, # RCI + "target_ci": None, # TCI + "eer": 1.0, # EER (default to 1.0) + "energy_density": None, # Energy Density + "uci": None, # UCI + } + + try: + # Get compliance period ID + lcfs_cursor.execute( + "SELECT compliance_period_id FROM compliance_period WHERE description = %s", + (compliance_period,), + ) + compliance_period_row = lcfs_cursor.fetchone() + if not compliance_period_row: + logger.warning(f"No compliance period found for '{compliance_period}'") + return result + compliance_period_id = compliance_period_row[0] + + # Get fuel type details + lcfs_cursor.execute( + "SELECT unrecognized, default_carbon_intensity FROM fuel_type WHERE fuel_type_id = %s", + (fuel_type_id,), + ) + fuel_type_row = lcfs_cursor.fetchone() + if not fuel_type_row: + logger.warning(f"No fuel type found for ID {fuel_type_id}") + return result + is_unrecognized, default_ci = fuel_type_row + + # Get energy density for this fuel type and compliance period + lcfs_cursor.execute( + """ + SELECT density FROM energy_density + WHERE fuel_type_id = %s AND compliance_period_id <= %s + ORDER BY compliance_period_id DESC LIMIT 1 + """, + (fuel_type_id, compliance_period_id), + ) + energy_density_row = lcfs_cursor.fetchone() + if energy_density_row: + result["energy_density"] = float(energy_density_row[0]) + + # Get effective carbon intensity (RCI) + if fuel_code_id: + # Use fuel code carbon intensity if available + lcfs_cursor.execute( + "SELECT carbon_intensity FROM fuel_code WHERE fuel_code_id = %s", + (fuel_code_id,), + ) + fuel_code_row = lcfs_cursor.fetchone() + if fuel_code_row: + result["effective_carbon_intensity"] = float(fuel_code_row[0]) + + if not result["effective_carbon_intensity"]: + if is_unrecognized: + # Use category carbon intensity for unrecognized fuels + lcfs_cursor.execute( + """ + SELECT cci.category_carbon_intensity + FROM category_carbon_intensity cci + WHERE cci.fuel_category_id = %s AND cci.compliance_period_id = %s + """, + (fuel_category_id, compliance_period_id), + ) + category_ci_row = lcfs_cursor.fetchone() + if category_ci_row: + result["effective_carbon_intensity"] = float(category_ci_row[0]) + else: + # Use default carbon intensity from fuel type + if default_ci: + result["effective_carbon_intensity"] = float(default_ci) + + # Get target carbon intensity (TCI) + lcfs_cursor.execute( + """ + SELECT target_carbon_intensity + FROM target_carbon_intensity + WHERE fuel_category_id = %s AND compliance_period_id = %s + """, + (fuel_category_id, compliance_period_id), + ) + tci_row = lcfs_cursor.fetchone() + if tci_row: + result["target_ci"] = float(tci_row[0]) + + # Get energy effectiveness ratio (EER) + if end_use_id: + lcfs_cursor.execute( + """ + SELECT ratio FROM energy_effectiveness_ratio + WHERE fuel_type_id = %s AND fuel_category_id = %s + AND compliance_period_id = %s AND end_use_type_id = %s + """, + (fuel_type_id, fuel_category_id, compliance_period_id, end_use_id), + ) + eer_row = lcfs_cursor.fetchone() + if eer_row: + result["eer"] = float(eer_row[0]) + + # Get additional carbon intensity (UCI) + if end_use_id: + lcfs_cursor.execute( + """ + SELECT intensity FROM additional_carbon_intensity + WHERE fuel_type_id = %s AND end_use_type_id = %s + AND compliance_period_id = %s + """, + (fuel_type_id, end_use_id, compliance_period_id), + ) + uci_row = lcfs_cursor.fetchone() + if uci_row: + result["uci"] = float(uci_row[0]) + + except Exception as e: + logger.error(f"Error getting standardized fuel data: {e}") + + return result + + def process_schedule_b_record( + self, + record: Dict, + use_snapshot: bool, + compliance_report_id: int, + legacy_id: int, + group_uuid: str, + version: int, + lcfs_cursor, + ) -> bool: + """Process a single Schedule B record for fuel supply""" + try: + # Get unit of measure + unit_of_measure = None + if use_snapshot: + # For snapshot data, get the original unit_of_measure from _full_record + if "_full_record" in record: + unit_of_measure = record["_full_record"].get("unit_of_measure") + else: + unit_of_measure = record.get("unit_of_measure") + else: + # For other data sources, use 'units' field + unit_of_measure = record.get("units") + + unit_full_form = ( + self.unit_mapping.get(unit_of_measure, unit_of_measure) + if unit_of_measure + else None + ) + + # Get provision of the act + provision_act_description = None + if use_snapshot: + # For snapshot data, provision info is in provision_of_the_act_description field + provision_act_description = record.get( + "provision_of_the_act_description" + ) + else: + provision_act_description = record.get("provision_act") + + # Check if provision_act is empty, try from _full_record + if not provision_act_description and "_full_record" in record: + full_record_provision = record["_full_record"].get( + "provision_of_the_act_description" + ) + if full_record_provision: + provision_act_description = full_record_provision + else: + # Try the direct provision field + full_record_provision_direct = record["_full_record"].get( + "provision_of_the_act" + ) + provision_act_description = full_record_provision_direct + + provision_id = self.lookup_provision_id( + lcfs_cursor, provision_act_description + ) + + # Get end use + end_use_value = record.get("end_use") + end_use_id = self.lookup_end_use_id(lcfs_cursor, end_use_value) + + # Get fuel category + fuel_category_lookup_value = None + if use_snapshot: + # For snapshot data, fuel category info is in fuel_class field + fuel_category_lookup_value = record.get("fuel_class") + else: + fuel_category_lookup_value = record.get("fuel_category") + + # Check if fuel_category is empty, try fuel_class from _full_record as fallback + if not fuel_category_lookup_value and "_full_record" in record: + fuel_class = record["_full_record"].get("fuel_class") + fuel_category_lookup_value = fuel_class + + fuel_category_id = self.lookup_fuel_category_id( + lcfs_cursor, fuel_category_lookup_value + ) + + # Get fuel code - Updated logic to handle TFRS numeric fuel codes + fuel_code_id = None + if use_snapshot: + # For snapshot data, TFRS has numeric fuel_code that needs lookup + tfrs_fuel_code_id = record.get("fuel_code") + if tfrs_fuel_code_id: + try: + # Look up the full fuel code details from TFRS + with get_source_connection() as tfrs_conn: + tfrs_cursor = tfrs_conn.cursor() + tfrs_cursor.execute( + """ + SELECT fuel_code, fuel_code_version, fuel_code_version_minor + FROM fuel_code + WHERE id = %s + """, + (tfrs_fuel_code_id,), + ) + tfrs_fuel_result = tfrs_cursor.fetchone() + + if tfrs_fuel_result: + base_code, version, minor = tfrs_fuel_result + # Construct the full fuel code (e.g., BCLCF236.1) + full_fuel_code = f"{base_code}{version}.{minor}" + + # Now look up this fuel code in LCFS + fuel_code_id = self.lookup_fuel_code_by_full_code( + lcfs_cursor, full_fuel_code + ) + + if fuel_code_id: + logger.debug( + f"Found LCFS fuel_code_id {fuel_code_id} for TFRS fuel code {full_fuel_code}" + ) + else: + logger.warning( + f"Could not find LCFS fuel code for {full_fuel_code}" + ) + else: + logger.warning( + f"Could not find TFRS fuel code details for ID {tfrs_fuel_code_id}" + ) + except Exception as e: + logger.error( + f"Error looking up TFRS fuel code {tfrs_fuel_code_id}: {e}" + ) + + # Fallback: try the old logic for fuel_code_description + if not fuel_code_id: + fuel_code_desc = record.get("fuel_code_description", "") + if fuel_code_desc and isinstance(fuel_code_desc, str): + # Find the index of the first digit to split prefix/suffix + first_digit_index = -1 + for i, char in enumerate(fuel_code_desc): + if char.isdigit(): + first_digit_index = i + break + + if first_digit_index != -1: + fuel_code_prefix = fuel_code_desc[:first_digit_index] + fuel_code_suffix = fuel_code_desc[first_digit_index:] + else: + fuel_code_prefix = fuel_code_desc + fuel_code_suffix = "" + + fuel_code_id = self.lookup_fuel_code_id( + lcfs_cursor, fuel_code_prefix, fuel_code_suffix + ) + else: + fuel_code_prefix = record.get("fuel_code_prefix") + fuel_code_suffix = record.get("fuel_code_suffix") + if fuel_code_prefix: + fuel_code_id = self.lookup_fuel_code_id( + lcfs_cursor, fuel_code_prefix, fuel_code_suffix + ) + + # Get fuel type + fuel_type_lookup_value = record.get("fuel_type") + fuel_type_id = self.lookup_fuel_type_id(lcfs_cursor, fuel_type_lookup_value) + + # Get numeric values + quantity = self.safe_get_number(record, "quantity", use_snapshot, legacy_id) + + # Handle compliance units from TFRS data + compliance_units = None + if use_snapshot: + # Prioritize snapshot values + credits = self.safe_get_number( + record, "credits", use_snapshot, legacy_id + ) + debits = self.safe_get_number(record, "debits", use_snapshot, legacy_id) + if credits is not None: + compliance_units = credits # Credits are positive compliance units + elif debits is not None: + compliance_units = -debits # Debits are negative compliance units + else: + # SQL fallback + compliance_units = self.safe_get_number( + record, "compliance_units", use_snapshot, legacy_id + ) + + # Get standardized fuel data from LCFS fuel code lookup system + # This will populate the derived fields (RCI, TCI, EER, energy density, UCI) + standardized_data = None + if fuel_type_id and fuel_category_id: + # Get compliance period from the record context + compliance_period = "2023" # Default, should be determined from report + try: + # Try to get the compliance period from the compliance report + lcfs_cursor.execute( + """ + SELECT cp.description + FROM compliance_report cr + JOIN compliance_period cp ON cr.compliance_period_id = cp.compliance_period_id + WHERE cr.compliance_report_id = %s + """, + (compliance_report_id,), + ) + period_row = lcfs_cursor.fetchone() + if period_row: + compliance_period = period_row[0] + except Exception as e: + logger.warning(f"Could not get compliance period: {e}") + + standardized_data = self.get_standardized_fuel_data( + lcfs_cursor=lcfs_cursor, + fuel_type_id=fuel_type_id, + fuel_category_id=fuel_category_id, + end_use_id=end_use_id, + compliance_period=compliance_period, + fuel_code_id=fuel_code_id, + provision_id=provision_id, + ) + + # Use standardized data if available, otherwise fallback to TFRS values + if standardized_data: + ci_of_fuel = standardized_data.get("effective_carbon_intensity") + target_ci = standardized_data.get("target_ci") + eer = standardized_data.get("eer") or 1.0 + energy_density = standardized_data.get("energy_density") + uci = standardized_data.get("uci") + + # Calculate energy content if we have energy density and quantity + energy_content = None + if energy_density and quantity: + energy_content = float(energy_density) * float(quantity) + else: + # Fallback to TFRS values if standardized lookup fails + logger.warning(f"Using TFRS fallback values for legacy_id {legacy_id}") + if use_snapshot: + # For snapshot data, use effective_carbon_intensity for all provision types + ci_of_fuel = self.safe_get_number( + record, "effective_carbon_intensity", use_snapshot, legacy_id + ) + else: + ci_of_fuel = self.safe_get_number( + record, "ci_of_fuel", use_snapshot, legacy_id + ) + + target_ci = self.safe_get_number( + record, "target_ci", use_snapshot, legacy_id + ) + energy_density = self.safe_get_number( + record, "energy_density", use_snapshot, legacy_id + ) + eer = self.safe_get_number(record, "eer", use_snapshot, legacy_id) + energy_content = self.safe_get_number( + record, "energy_content", use_snapshot, legacy_id + ) + uci = None # Not available in TFRS data + + # Pre-insert validation + validation_errors = [] + if quantity is None: + validation_errors.append("Quantity is NULL or invalid") + if unit_full_form is None: + validation_errors.append( + "Units (unit_of_measure) is NULL or could not be determined" + ) + + if validation_errors: + logger.error( + f"Validation failed for record in CR {legacy_id}: {', '.join(validation_errors)}" + ) + return False + + # Insert into fuel_supply table + insert_query = """ + INSERT INTO fuel_supply ( + compliance_report_id, fuel_category_id, fuel_type_id, provision_of_the_act_id, + fuel_code_id, end_use_id, fuel_type_other, + quantity, units, ci_of_fuel, energy_density, + eer, uci, energy, compliance_units, target_ci, create_date, update_date, + create_user, update_user, group_uuid, version + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + """ + + # Extract "other" fields - only use fuel_type_other since that's the only one that exists + fuel_type_other = record.get("fuel_type_other") + + lcfs_cursor.execute( + insert_query, + ( + compliance_report_id, + fuel_category_id, + fuel_type_id, + provision_id, + fuel_code_id, + end_use_id, + fuel_type_other, + quantity, + unit_full_form, + ci_of_fuel, + energy_density, + eer, + uci, + energy_content, + compliance_units, + target_ci, + None, # create_date + None, # update_date + "ETL", # create_user + "ETL", # update_user + group_uuid, + version, + ), + ) + + return True + + except Exception as e: + logger.error(f"Error inserting fuel supply record for CR {legacy_id}: {e}") + return False + + def get_fuel_supply_records_for_report( + self, tfrs_cursor, legacy_id: int + ) -> List[Dict]: + """Get all fuel supply records for a specific TFRS compliance report""" + records = [] + + # Try to get snapshot data first + snapshot_data = self.fetch_snapshot_data(tfrs_cursor, legacy_id) + + if ( + snapshot_data + and snapshot_data is not None + and "schedule_b" in snapshot_data + and snapshot_data["schedule_b"] is not None + and "records" in snapshot_data["schedule_b"] + ): + # Use snapshot data + raw_records = snapshot_data["schedule_b"]["records"] + logger.info( + f"Found {len(raw_records)} raw records in snapshot for legacy_id {legacy_id}" + ) + + for i, record in enumerate(raw_records): + logger.debug( + f"Processing record {i+1}: fuel_type={record.get('fuel_type')}, fuel_category={record.get('fuel_category')}, quantity={record.get('quantity')}" + ) + normalized_record = self.normalize_tfrs_record(record) + if normalized_record: + records.append(normalized_record) + logger.debug(f"Record {i+1} normalized successfully") + else: + logger.warning(f"Record {i+1} failed normalization and was skipped") + else: + # Fall back to SQL query + logger.info( + f"No snapshot data available for legacy_id {legacy_id}, using SQL fallback" + ) + + logger.info( + f"Final count: {len(records)} processed records for legacy_id {legacy_id}" + ) + return records + + def normalize_tfrs_record(self, record: Dict) -> Optional[Dict]: + """Normalize TFRS record fields to standard format for comparison""" + if not record: + return None + + # Handle fuel_code - preserve numeric fuel code from TFRS + fuel_code = "" + # For TFRS data, preserve the numeric fuel_code if it exists + if record.get("fuel_code"): + fuel_code = str(record.get("fuel_code")) + else: + # Fallback: try to build from prefix/suffix if available + fuel_code_prefix = record.get("fuel_code_prefix", "") or "" + fuel_code_suffix = record.get("fuel_code_suffix", "") or "" + if fuel_code_prefix and fuel_code_suffix: + fuel_code = f"{fuel_code_prefix}.{fuel_code_suffix}" + elif fuel_code_prefix: + fuel_code = fuel_code_prefix + elif fuel_code_suffix: + fuel_code = fuel_code_suffix + + return { + "fuel_type": str(record.get("fuel_type", "")).strip(), + "fuel_category": str(record.get("fuel_category", "")).strip(), + "provision_of_the_act": str(record.get("provision_act", "")).strip(), + "fuel_code": fuel_code.strip(), + "end_use": str(record.get("end_use", "")).strip(), + "ci_of_fuel": str(record.get("ci_of_fuel", "")).strip(), + "quantity": str(record.get("quantity", "")).strip(), + "units": str(record.get("unit_of_measure", "")).strip(), + "compliance_units": str(record.get("compliance_units", "")).strip(), + "target_ci": str(record.get("target_ci", "")).strip(), + "energy_density": str(record.get("energy_density", "")).strip(), + "eer": str(record.get("eer", "")).strip(), + "energy_content": str(record.get("energy_content", "")).strip(), + # Include the full record for processing + "_full_record": record, + } + + def insert_fuel_supply_record( + self, + lcfs_cursor, + record_data: Dict, + compliance_report_id: int, + group_uuid: str, + version: int, + ) -> bool: + """Insert a fuel supply record into LCFS database with proper error handling""" + try: + # Get the full record data + full_record = record_data.get("_full_record", record_data) + + # Process the record using existing logic + return self.process_schedule_b_record( + full_record, + True, # from_snapshot + compliance_report_id, + 0, # legacy_id (not used in new logic) + group_uuid, + version, + lcfs_cursor, + ) + except Exception as e: + logger.error(f"Failed to insert fuel supply record: {e}") + # Don't fail the entire migration for one record + return False + + def migrate(self) -> Tuple[int, int]: + """Migrate fuel supply data from TFRS to LCFS with proper independent record versioning""" + # Initialize tracking variables + processed_count = 0 + total_count = 0 + total_reports_processed = 0 + total_records_found = 0 + total_records_processed = 0 + total_records_skipped = 0 + total_errors = 0 + reports_with_no_records = 0 + + try: + # Step 1: Setup - Clear existing data and load mappings + self._setup_migration() + + # Step 2: Get compliance reports list + compliance_reports = self._get_compliance_reports() + logger.info( + f"Found {len(compliance_reports)} compliance reports to process" + ) + + # Step 3: Group reports by organization + period and process each group + report_groups = {} + for report_data in compliance_reports: + compliance_report_id, legacy_id, org_id, period_id, version = report_data + group_key = (org_id, period_id) + if group_key not in report_groups: + report_groups[group_key] = [] + report_groups[group_key].append(report_data) + + # Process each organization/period group with its own logical_records dictionary + for (org_id, period_id), group_reports in report_groups.items(): + logical_records = {} # Reset for each organization + period combination + logger.info(f"Processing organization {org_id}, period {period_id} with {len(group_reports)} reports") + + for report_data in group_reports: + compliance_report_id, legacy_id, org_id, period_id, version = report_data + total_reports_processed += 1 + + logger.info( + f"Processing CR {compliance_report_id} (legacy {legacy_id}), org {org_id}, period {period_id}, version {version}" + ) + + try: + # Process this compliance report + report_stats = self._process_single_compliance_report( + compliance_report_id, legacy_id, logical_records + ) + + # Update counters + total_records_found += report_stats["records_found"] + total_count += report_stats["records_found"] + processed_count += report_stats["records_processed"] + total_records_processed += report_stats["records_processed"] + total_records_skipped += report_stats["records_skipped"] + total_errors += report_stats["errors"] + + if report_stats["records_found"] == 0: + reports_with_no_records += 1 + + except Exception as e: + total_errors += 1 + logger.error( + f"Error processing compliance report {compliance_report_id}: {e}" + ) + continue + + # Enhanced summary logging + logger.info("=== MIGRATION SUMMARY ===") + logger.info(f"Compliance Reports Found: {len(compliance_reports)}") + logger.info(f"Compliance Reports Processed: {total_reports_processed}") + logger.info( + f"Compliance Reports with No Records: {reports_with_no_records}" + ) + logger.info(f"Total TFRS Records Found: {total_records_found}") + logger.info(f"Records Successfully Processed: {total_records_processed}") + logger.info(f"Records Skipped: {total_records_skipped}") + logger.info(f"Errors Encountered: {total_errors}") + logger.info(f"Migration completed for {len(report_groups)} organization/period groups") + logger.info(f"LCFS Record Versions Created: {processed_count}") + logger.info("=========================") + + return processed_count, total_count + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + def _setup_migration(self): + """Setup the migration by clearing existing data and loading mappings""" + with get_destination_connection() as lcfs_conn: + lcfs_cursor = lcfs_conn.cursor() + try: + # Load mappings first + self.load_mappings(lcfs_cursor) + # Skip clearing data - migration should be additive only + logger.info("Migration running in additive mode - no existing data will be deleted") + lcfs_conn.commit() + logger.info("Setup completed successfully") + except Exception as e: + logger.error(f"Failed during setup: {e}") + lcfs_conn.rollback() + raise + finally: + lcfs_cursor.close() + + def _get_compliance_reports(self): + """Get the list of compliance reports to process""" + with get_destination_connection() as lcfs_conn: + lcfs_cursor = lcfs_conn.cursor() + try: + return self.get_compliance_reports_chronological(lcfs_cursor) + finally: + lcfs_cursor.close() + + def _process_single_compliance_report( + self, compliance_report_id, legacy_id, logical_records + ): + """Process a single compliance report and return statistics""" + stats = { + "records_found": 0, + "records_processed": 0, + "records_skipped": 0, + "errors": 0, + } + + # Use separate connections for TFRS and LCFS + with get_source_connection() as tfrs_conn, get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + try: + # Get fuel supply records for this compliance report + report_records = self.get_fuel_supply_records_for_report( + tfrs_cursor, legacy_id + ) + stats["records_found"] = len(report_records) + logger.info( + f"Successfully fetched {len(report_records)} records for legacy_id {legacy_id}" + ) + + # Process each fuel supply record + for record in report_records: + try: + # Process individual record + if self._process_individual_record( + record, compliance_report_id, logical_records, lcfs_cursor + ): + stats["records_processed"] += 1 + else: + stats["records_skipped"] += 1 + except Exception as e: + stats["errors"] += 1 + logger.error(f"Error processing individual record: {e}") + continue + + # Commit the transaction for this compliance report + lcfs_conn.commit() + + except Exception as e: + stats["errors"] += 1 + logger.error(f"Failed to process compliance report {legacy_id}: {e}") + lcfs_conn.rollback() + raise + finally: + tfrs_cursor.close() + lcfs_cursor.close() + + return stats + + def _process_individual_record( + self, record, compliance_report_id, logical_records, lcfs_cursor + ): + """Process an individual fuel supply record""" + # Generate logical record key based on business identifiers + record_key = self.generate_logical_record_key(record) + + # Record is already normalized from get_fuel_supply_records_for_report + normalized_data = record + if not normalized_data: + logger.debug("Skipped record due to normalization failure") + return False + + if record_key not in logical_records: + # First time seeing this logical record + group_uuid = str(uuid.uuid4()) + version_num = 1 + logger.debug(f"Creating new logical record with key: {record_key}") + + # Insert version 1 of this logical record + if self.insert_fuel_supply_record( + lcfs_cursor, + normalized_data, + compliance_report_id, + group_uuid, + version_num, + ): + # Track this logical record + logical_records[record_key] = { + "group_uuid": group_uuid, + "current_version": version_num, + "last_data": normalized_data, + } + return True + else: + return False + else: + # This logical record already exists + logical_record = logical_records[record_key] + logger.debug(f"Found existing logical record with key: {record_key}") + + # Compare with previous version + if self.records_are_different(normalized_data, logical_record["last_data"]): + # Record changed - create new version + new_version = logical_record["current_version"] + 1 + logger.debug( + f"Creating new version {new_version} for logical record: {record_key}" + ) + + if self.insert_fuel_supply_record( + lcfs_cursor, + normalized_data, + compliance_report_id, + logical_record["group_uuid"], + new_version, + ): + # Update tracking + logical_record["current_version"] = new_version + logical_record["last_data"] = normalized_data + return True + else: + logger.debug( + f"Failed to insert new version for logical record: {record_key}" + ) + return False + else: + # Record unchanged + logger.debug(f"Record unchanged for logical record: {record_key}") + return False + + def get_compliance_reports_chronological(self, lcfs_cursor) -> List[Tuple]: + """Get compliance reports ordered chronologically (original first, supplementals in order)""" + lcfs_cursor.execute( + """ + SELECT + cr.compliance_report_id, + cr.legacy_id, + cr.organization_id, + cr.compliance_period_id, + cr.version + FROM compliance_report cr + WHERE cr.legacy_id IS NOT NULL + ORDER BY cr.organization_id, cr.compliance_period_id, cr.version + """ + ) + return lcfs_cursor.fetchall() + + def generate_logical_record_key(self, record: Dict) -> str: + """Generate a stable key that identifies a logical fuel supply record across compliance reports + + This key represents the business identity of a fuel supply record, independent of + which compliance report version it appears in. + """ + # Extract values from the full record if available + full_record = record.get("_full_record", record) + + # Use business identifiers that define a unique logical fuel supply record + key_parts = [ + str(full_record.get("fuel_type", "")).strip(), + str(full_record.get("fuel_class", "") or full_record.get("fuel_category", "")).strip(), + str(full_record.get("provision_of_the_act_description", "") or full_record.get("provision_act", "")).strip(), + str(full_record.get("fuel_code", "")).strip(), # This is the TFRS numeric ID + str(full_record.get("fuel_code_description", "")).strip(), # This is the actual fuel code string + str(full_record.get("end_use", "")).strip(), + # Add other business identifiers that make a record unique + str(full_record.get("fuel_type_other", "")).strip(), + ] + return "|".join(key_parts) + + def records_are_different(self, new_data: Dict, old_data: Dict) -> bool: + """Check if the data content of a logical record has changed""" + # Extract full records for comparison + new_full = new_data.get("_full_record", new_data) + old_full = old_data.get("_full_record", old_data) + + # Compare the meaningful data fields (excluding metadata) + data_fields = [ + "quantity", + "unit_of_measure", + "credits", + "debits", + "effective_carbon_intensity", + "target_ci", + "energy_density", + "eer", + "energy_content", + ] + + for field in data_fields: + new_val = str(new_full.get(field, "")).strip() + old_val = str(old_full.get(field, "")).strip() + if new_val != old_val: + logger.debug(f"Field '{field}' changed from '{old_val}' to '{new_val}'") + return True + + return False + + +def main(): + setup_logging() + logger.info("Starting Fixed Fuel Supply Migration") + + migrator = FuelSupplyMigrator() + + try: + processed, failed = migrator.migrate() + logger.info(f"Migration completed. Processed: {processed}, Failed: {failed}") + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/etl/python_migration/migrations/migrate_notional_transfers.py b/etl/python_migration/migrations/migrate_notional_transfers.py new file mode 100644 index 000000000..7bf4a4862 --- /dev/null +++ b/etl/python_migration/migrations/migrate_notional_transfers.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +""" +Notional Transfer Migration Script + +Migrates notional transfers (Schedule A) data from TFRS to LCFS database. +This script replicates the functionality of notional_transfer.groovy + +Key features: +1. Finds all LCFS compliance reports having a TFRS legacy_id +2. For each TFRS compliance report, determines its chain (root_report_id) +3. Retrieves schedule_a records for each version in the chain +4. Computes a diff (CREATE / UPDATE) between consecutive versions +5. Inserts rows in notional_transfer with a stable, random group_uuid per schedule_a_record_id +6. Versions these notional_transfer entries so that subsequent changes increment the version +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +import sys +import uuid +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import ( + setup_logging, + safe_decimal, + safe_int, + safe_str, + build_legacy_mapping, +) + +logger = logging.getLogger(__name__) + + +class NotionalTransferMigrator: + def __init__(self): + self.record_uuid_map: Dict[int, str] = {} + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + + def load_mappings(self, lcfs_cursor): + """Load legacy ID to LCFS compliance_report_id mappings""" + logger.info("Loading legacy ID to LCFS compliance_report_id mappings") + self.legacy_to_lcfs_mapping = build_legacy_mapping(lcfs_cursor) + logger.info(f"Loaded {len(self.legacy_to_lcfs_mapping)} legacy mappings") + + def map_received_or_transferred(self, transfer_type_id: int) -> str: + """Maps TFRS transfer_type_id to 'Received' or 'Transferred' + TFRS: 1=Transferred, 2=Received + + Note: This appears to be inverted in the original Groovy code. + The mapping returns "Received" for type 1, which seems counterintuitive. + Preserving original logic for consistency. + """ + if transfer_type_id == 1: + return "Received" + return "Transferred" + + def map_fuel_category_id(self, fuel_class_id: int) -> Optional[int]: + """Maps TFRS fuel_class_id to LCFS fuel_category_id""" + mapping = { + 1: 2, # Diesel + 2: 1, # Gasoline + } + return mapping.get(fuel_class_id) + + def is_record_changed(self, old_row: Optional[Dict], new_row: Dict) -> bool: + """Checks if any relevant fields in a schedule_a record differ between old and new""" + if old_row is None or new_row is None: + return True + + # Check numeric field changes + if old_row.get("quantity") != new_row.get("quantity"): + return True + + # Check other field changes + if ( + old_row.get("fuel_class_id") != new_row.get("fuel_class_id") + or old_row.get("transfer_type_id") != new_row.get("transfer_type_id") + or old_row.get("trading_partner") != new_row.get("trading_partner") + or old_row.get("postal_address") != new_row.get("postal_address") + ): + return True + + return False + + def get_current_version(self, lcfs_cursor, group_uuid: str) -> int: + """Find current highest version in notional_transfer for that group_uuid""" + query = """ + SELECT version + FROM notional_transfer + WHERE group_uuid = %s + ORDER BY version DESC + LIMIT 1 + """ + lcfs_cursor.execute(query, (group_uuid,)) + result = lcfs_cursor.fetchone() + return result[0] if result else -1 + + def get_lcfs_reports_with_legacy_ids(self, lcfs_cursor) -> List[int]: + """Find all LCFS compliance reports that have TFRS legacy_id""" + query = """ + SELECT compliance_report_id, legacy_id + FROM compliance_report + WHERE legacy_id IS NOT NULL + """ + lcfs_cursor.execute(query) + return [row[1] for row in lcfs_cursor.fetchall()] # Return legacy_ids + + def get_root_report_id(self, tfrs_cursor, tfrs_id: int) -> Optional[int]: + """Find the root_report_id for a given TFRS report""" + query = """ + SELECT root_report_id + FROM compliance_report + WHERE id = %s + """ + tfrs_cursor.execute(query, (tfrs_id,)) + result = tfrs_cursor.fetchone() + return result[0] if result else None + + def get_report_chain(self, tfrs_cursor, root_id: int) -> List[int]: + """Gather the chain of reports in ascending order""" + query = """ + SELECT + c.id AS tfrs_report_id, + c.traversal + FROM compliance_report c + WHERE c.root_report_id = %s + ORDER BY c.traversal, c.id + """ + tfrs_cursor.execute(query, (root_id,)) + return [row[0] for row in tfrs_cursor.fetchall()] + + def get_schedule_a_records( + self, tfrs_cursor, tfrs_report_id: int + ) -> Dict[int, Dict]: + """Fetch current TFRS schedule_a records for a report""" + query = """ + SELECT + sar.id AS schedule_a_record_id, + sar.quantity, + sar.trading_partner, + sar.postal_address, + sar.fuel_class_id, + sar.transfer_type_id + FROM compliance_report_schedule_a_record sar + JOIN compliance_report_schedule_a sa ON sa.id = sar.schedule_id + JOIN compliance_report c ON c.schedule_a_id = sa.id + WHERE c.id = %s + ORDER BY sar.id + """ + + tfrs_cursor.execute(query, (tfrs_report_id,)) + records = {} + + for row in tfrs_cursor.fetchall(): + rec_id = row[0] + records[rec_id] = { + "schedule_a_record_id": rec_id, + "quantity": row[1], + "trading_partner": row[2], + "postal_address": row[3], + "fuel_class_id": row[4], + "transfer_type_id": row[5], + } + + return records + + def get_lcfs_compliance_report_id(self, lcfs_cursor, tfrs_id: int) -> Optional[int]: + """Find the matching LCFS compliance_report by legacy_id""" + query = """ + SELECT compliance_report_id + FROM compliance_report + WHERE legacy_id = %s + """ + lcfs_cursor.execute(query, (tfrs_id,)) + result = lcfs_cursor.fetchone() + return result[0] if result else None + + def insert_version_row( + self, lcfs_cursor, lcfs_cr_id: int, row_data: Dict, action: str + ) -> bool: + """Inserts a new row in notional_transfer with proper versioning""" + try: + record_id = row_data["schedule_a_record_id"] + + # Retrieve or generate the stable random group uuid for this record + group_uuid = self.record_uuid_map.get(record_id) + if not group_uuid: + group_uuid = str(uuid.uuid4()) + self.record_uuid_map[record_id] = group_uuid + + # Find current highest version + current_ver = self.get_current_version(lcfs_cursor, group_uuid) + next_ver = 0 if current_ver < 0 else current_ver + 1 + + # Map TFRS fields to LCFS fields + rec_or_xfer = self.map_received_or_transferred( + row_data.get("transfer_type_id", 2) + ) + fuel_cat_id = self.map_fuel_category_id(row_data.get("fuel_class_id", 1)) + quantity = safe_decimal(row_data.get("quantity", 0)) + trade_partner = safe_str(row_data.get("trading_partner", "")) + postal_address = safe_str(row_data.get("postal_address", "")) + + # Insert the new row + insert_sql = """ + INSERT INTO notional_transfer ( + compliance_report_id, + quantity, + legal_name, + address_for_service, + fuel_category_id, + received_or_transferred, + group_uuid, + version, + action_type, + create_user, + update_user + ) VALUES ( + %s, %s, %s, %s, %s, %s::receivedOrTransferredEnum, %s, %s, %s::actiontypeenum, %s, %s + ) + """ + + params = [ + lcfs_cr_id, + float(quantity), + trade_partner, + postal_address, + fuel_cat_id, + rec_or_xfer, + group_uuid, + next_ver, + action, + "ETL", + "ETL", + ] + + lcfs_cursor.execute(insert_sql, params) + logger.info( + f"Inserted notional_transfer row: recordId={record_id}, action={action}, groupUuid={group_uuid}, version={next_ver}" + ) + return True + + except Exception as e: + logger.error(f"Failed to insert notional transfer record: {e}") + return False + + def migrate(self) -> Tuple[int, int]: + """Main migration logic""" + total_processed = 0 + total_skipped = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Load mappings + self.load_mappings(lcfs_cursor) + + # Find all LCFS compliance reports that have TFRS legacy_id + logger.info( + "Retrieving LCFS compliance_report with legacy_id != null" + ) + tfrs_ids = self.get_lcfs_reports_with_legacy_ids(lcfs_cursor) + logger.info(f"Found {len(tfrs_ids)} reports to process") + + # For each TFRS compliance_report ID, follow the chain approach + for tfrs_id in tfrs_ids: + logger.info(f"Processing TFRS compliance_report.id = {tfrs_id}") + + # Find the root_report_id + root_id = self.get_root_report_id(tfrs_cursor, tfrs_id) + if not root_id: + logger.warning( + f"No root_report_id found for TFRS #{tfrs_id}; skipping." + ) + total_skipped += 1 + continue + + # Gather the chain in ascending order + chain_ids = self.get_report_chain(tfrs_cursor, root_id) + if not chain_ids: + logger.warning(f"Chain empty for root={root_id}? skipping.") + total_skipped += 1 + continue + + # Keep the old version's schedule_a data in memory for diffs + previous_records = {} + + for idx, chain_tfrs_id in enumerate(chain_ids): + logger.info(f"TFRS #{chain_tfrs_id} (chain idx={idx})") + + # Fetch current TFRS schedule_a records + current_records = self.get_schedule_a_records( + tfrs_cursor, chain_tfrs_id + ) + + # Find the matching LCFS compliance_report + lcfs_cr_id = self.get_lcfs_compliance_report_id( + lcfs_cursor, chain_tfrs_id + ) + if not lcfs_cr_id: + logger.warning( + f"TFRS #{chain_tfrs_id} not found in LCFS? Skipping diff, just storing previousRecords." + ) + previous_records = current_records + continue + + # Compare old vs new for each record in currentRecords + for rec_id, new_data in current_records.items(): + if rec_id not in previous_records: + # Wasn't in old => CREATE + if self.insert_version_row( + lcfs_cursor, lcfs_cr_id, new_data, "CREATE" + ): + total_processed += 1 + else: + # Existed => check if changed + old_data = previous_records[rec_id] + if self.is_record_changed(old_data, new_data): + if self.insert_version_row( + lcfs_cursor, lcfs_cr_id, new_data, "UPDATE" + ): + total_processed += 1 + + # Update previousRecords for the next version + previous_records = current_records + + # Commit all changes + lcfs_conn.commit() + logger.info( + f"Successfully committed {total_processed} notional transfer records" + ) + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + return total_processed, total_skipped + + +def main(): + setup_logging() + logger.info("Starting Notional Transfer Migration") + + migrator = NotionalTransferMigrator() + + try: + processed, skipped = migrator.migrate() + logger.info( + f"Migration completed successfully. Processed: {processed}, Skipped: {skipped}" + ) + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/migrate_orphaned_allocation_agreements.py b/etl/python_migration/migrations/migrate_orphaned_allocation_agreements.py new file mode 100644 index 000000000..ac769b934 --- /dev/null +++ b/etl/python_migration/migrations/migrate_orphaned_allocation_agreements.py @@ -0,0 +1,740 @@ +#!/usr/bin/env python3 +""" +Orphaned Allocation Agreement Migration Script + +Migrates orphaned allocation agreements from TFRS to LCFS database. +This script replicates the functionality of orphaned_allocation_agreement.groovy + +Overview: +1. Identify TFRS compliance reports marked as 'exclusion reports' that do not have a + corresponding 'main' compliance report in the same compliance period/organization. +2. For each orphaned TFRS exclusion report: + a. Check if an LCFS compliance report with legacy_id = TFRS report ID already exists. + b. If not, create a new minimal LCFS compliance report (type='Supplemental', status='Draft'). + c. Fetch the allocation agreement records linked to the TFRS exclusion_agreement_id. + d. Insert these records into LCFS allocation_agreement, linked to the new LCFS report. +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +import sys +import uuid +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import setup_logging, safe_decimal, safe_int, safe_str + +logger = logging.getLogger(__name__) + + +class OrphanedAllocationAgreementMigrator: + def __init__(self): + self.record_uuid_map: Dict[int, str] = {} + self.responsibility_to_transaction_type_cache: Dict[str, int] = {} + self.tfrs_fuel_name_to_lcfs_fuel_type_cache: Dict[str, int] = {} + + # Constants + self.GASOLINE_CATEGORY_ID = 1 + self.DIESEL_CATEGORY_ID = 2 + + def find_orphaned_exclusion_reports(self, tfrs_cursor) -> List[Dict]: + """Find TFRS exclusion reports without a sibling report""" + query = """ + /* + * Find exclusion reports that are truly standalone (orphaned): + * 1. Has exclusion_agreement_id (is an exclusion report) + * 2. Does NOT have a main compliance report in the same organization/period + * 3. Excludes reports that are part of combo compliance+exclusion reports + * If multiple exclusion supplementals exist in a chain, pick the latest one. + */ + WITH exclusion_candidates AS ( + SELECT + cr_excl.id AS tfrs_exclusion_report_id, + cr_excl.root_report_id, + cr_excl.traversal, + cr_excl.organization_id AS tfrs_organization_id, + cr_excl.compliance_period_id AS tfrs_compliance_period_id, + cr_excl.exclusion_agreement_id, + ws.director_status_id AS tfrs_director_status + FROM compliance_report cr_excl + JOIN compliance_report_workflow_state ws ON cr_excl.status_id = ws.id + WHERE cr_excl.exclusion_agreement_id IS NOT NULL + -- Check if this is truly a standalone exclusion report + AND NOT EXISTS ( + SELECT 1 + FROM compliance_report cr_other + WHERE cr_other.organization_id = cr_excl.organization_id + AND cr_other.compliance_period_id = cr_excl.compliance_period_id + AND cr_other.root_report_id = cr_excl.root_report_id + AND cr_other.exclusion_agreement_id IS NULL -- Has main compliance data + ) + ), latest_exclusion_per_chain AS ( + SELECT ec.* + FROM exclusion_candidates ec + JOIN ( + SELECT root_report_id, MAX(traversal) AS max_traversal + FROM exclusion_candidates + GROUP BY root_report_id + ) mx + ON ec.root_report_id = mx.root_report_id AND ec.traversal = mx.max_traversal + ) + SELECT + tfrs_exclusion_report_id, + tfrs_organization_id, + tfrs_compliance_period_id, + exclusion_agreement_id, + tfrs_director_status + FROM latest_exclusion_per_chain; + """ + + tfrs_cursor.execute(query) + reports = [] + + for row in tfrs_cursor.fetchall(): + reports.append( + { + "tfrs_exclusion_report_id": row[0], + "tfrs_organization_id": row[1], + "tfrs_compliance_period_id": row[2], + "exclusion_agreement_id": row[3], + "tfrs_director_status": row[4], + } + ) + + return reports + + def check_lcfs_report_exists(self, lcfs_cursor, legacy_id: int) -> bool: + """Check if an LCFS report with a specific legacy_id exists""" + query = "SELECT 1 FROM compliance_report WHERE legacy_id = %s LIMIT 1" + lcfs_cursor.execute(query, (legacy_id,)) + return lcfs_cursor.fetchone() is not None + + def get_tfrs_org_name(self, tfrs_cursor, org_id: int) -> Optional[str]: + """Get TFRS organization name based on TFRS organization ID""" + query = "SELECT name FROM organization WHERE id = %s LIMIT 1" + tfrs_cursor.execute(query, (org_id,)) + result = tfrs_cursor.fetchone() + return result[0] if result else None + + def get_lcfs_org_id(self, lcfs_cursor, org_name: str) -> Optional[int]: + """Get LCFS organization ID based on organization name""" + if not org_name: + logger.error( + "Cannot map LCFS organization ID from null TFRS organization name." + ) + return None + + query = "SELECT organization_id FROM organization WHERE name = %s LIMIT 1" + lcfs_cursor.execute(query, (org_name,)) + result = lcfs_cursor.fetchone() + + if not result: + logger.error( + f"Could not find LCFS organization mapped to TFRS organization name: {org_name}" + ) + return None + + return result[0] + + def get_tfrs_period_desc(self, tfrs_cursor, period_id: int) -> Optional[str]: + """Get TFRS compliance period description based on TFRS period ID""" + query = "SELECT description FROM compliance_period WHERE id = %s LIMIT 1" + tfrs_cursor.execute(query, (period_id,)) + result = tfrs_cursor.fetchone() + return result[0] if result else None + + def get_lcfs_period_id(self, lcfs_cursor, period_desc: str) -> Optional[int]: + """Get LCFS compliance period ID based on description""" + if not period_desc: + logger.error( + "Cannot map LCFS compliance period ID from null TFRS period description." + ) + return None + + query = "SELECT compliance_period_id FROM compliance_period WHERE description = %s LIMIT 1" + lcfs_cursor.execute(query, (period_desc,)) + result = lcfs_cursor.fetchone() + + if not result: + logger.error( + f"Could not find LCFS compliance period mapped to TFRS description: {period_desc}" + ) + return None + + return result[0] + + def get_lcfs_report_status_id(self, lcfs_cursor, status_name: str) -> Optional[int]: + """Get LCFS report status ID by name""" + query = "SELECT compliance_report_status_id FROM compliance_report_status WHERE status = %s::compliancereportstatusenum LIMIT 1" + lcfs_cursor.execute(query, (status_name,)) + result = lcfs_cursor.fetchone() + + if not result: + logger.error( + f"Could not find LCFS compliance report status ID for status: {status_name}" + ) + return None + + return result[0] + + def create_lcfs_placeholder_report( + self, + lcfs_cursor, + lcfs_org_id: int, + lcfs_period_id: int, + status_id: int, + reporting_frequency: str, + tfrs_legacy_id: int, + ) -> Optional[int]: + """Create a minimal LCFS Compliance Report record, default summary, and org snapshot""" + group_uuid = str(uuid.uuid4()) + version = 0 # Initial version + + try: + # 1. Create Compliance Report + insert_report_sql = """ + INSERT INTO compliance_report ( + organization_id, compliance_period_id, current_status_id, reporting_frequency, + compliance_report_group_uuid, version, legacy_id, create_user, update_user, + nickname + ) VALUES (%s, %s, %s, %s::reportingfrequency, %s, %s, %s, 'ETL', 'ETL', %s) + RETURNING compliance_report_id; + """ + + params = [ + lcfs_org_id, + lcfs_period_id, + status_id, + reporting_frequency, + group_uuid, + version, + tfrs_legacy_id, + "Original Report", + ] + + lcfs_cursor.execute(insert_report_sql, params) + result = lcfs_cursor.fetchone() + + if not result: + logger.error( + f"Failed to create placeholder LCFS compliance report for TFRS legacy ID: {tfrs_legacy_id}" + ) + return None + + new_lcfs_report_id = result[0] + logger.info( + f"Created placeholder LCFS compliance report ID: {new_lcfs_report_id} for TFRS legacy ID: {tfrs_legacy_id}" + ) + + # 2. Create Default Summary Record + self.create_default_summary(lcfs_cursor, new_lcfs_report_id) + + # 3. Create Organization Snapshot + self.create_organization_snapshot( + lcfs_cursor, new_lcfs_report_id, lcfs_org_id + ) + + return new_lcfs_report_id + + except Exception as e: + logger.error( + f"Exception creating placeholder LCFS compliance report for TFRS legacy ID: {tfrs_legacy_id}: {e}" + ) + return None + + def create_default_summary(self, lcfs_cursor, report_id: int): + """Create default summary record for a compliance report""" + try: + # Corrected to have 60 values total as per the Groovy script + insert_summary_sql = """ + INSERT INTO compliance_report_summary ( + compliance_report_id, quarter, is_locked, + line_1_fossil_derived_base_fuel_gasoline, line_1_fossil_derived_base_fuel_diesel, line_1_fossil_derived_base_fuel_jet_fuel, + line_2_eligible_renewable_fuel_supplied_gasoline, line_2_eligible_renewable_fuel_supplied_diesel, line_2_eligible_renewable_fuel_supplied_jet_fuel, + line_3_total_tracked_fuel_supplied_gasoline, line_3_total_tracked_fuel_supplied_diesel, line_3_total_tracked_fuel_supplied_jet_fuel, + line_4_eligible_renewable_fuel_required_gasoline, line_4_eligible_renewable_fuel_required_diesel, line_4_eligible_renewable_fuel_required_jet_fuel, + line_5_net_notionally_transferred_gasoline, line_5_net_notionally_transferred_diesel, line_5_net_notionally_transferred_jet_fuel, + line_6_renewable_fuel_retained_gasoline, line_6_renewable_fuel_retained_diesel, line_6_renewable_fuel_retained_jet_fuel, + line_7_previously_retained_gasoline, line_7_previously_retained_diesel, line_7_previously_retained_jet_fuel, + line_8_obligation_deferred_gasoline, line_8_obligation_deferred_diesel, line_8_obligation_deferred_jet_fuel, + line_9_obligation_added_gasoline, line_9_obligation_added_diesel, line_9_obligation_added_jet_fuel, + line_10_net_renewable_fuel_supplied_gasoline, line_10_net_renewable_fuel_supplied_diesel, line_10_net_renewable_fuel_supplied_jet_fuel, + line_11_non_compliance_penalty_gasoline, line_11_non_compliance_penalty_diesel, line_11_non_compliance_penalty_jet_fuel, + line_12_low_carbon_fuel_required, line_13_low_carbon_fuel_supplied, line_14_low_carbon_fuel_surplus, + line_15_banked_units_used, line_16_banked_units_remaining, line_17_non_banked_units_used, + line_18_units_to_be_banked, line_19_units_to_be_exported, line_20_surplus_deficit_units, line_21_surplus_deficit_ratio, + line_22_compliance_units_issued, + line_11_fossil_derived_base_fuel_gasoline, line_11_fossil_derived_base_fuel_diesel, line_11_fossil_derived_base_fuel_jet_fuel, line_11_fossil_derived_base_fuel_total, + line_21_non_compliance_penalty_payable, total_non_compliance_penalty_payable, + create_user, update_user, + early_issuance_credits_q1, early_issuance_credits_q2, early_issuance_credits_q3, early_issuance_credits_q4 + ) VALUES ( + %s, null, false, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + null, null, null, + 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, + 0.0, + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, + 'ETL', 'ETL', + null, null, null, null + ) + """ + + lcfs_cursor.execute(insert_summary_sql, (report_id,)) + logger.info( + f"Created default summary record for LCFS report ID: {report_id}" + ) + + except Exception as e: + logger.error( + f"Exception creating default summary for LCFS report ID: {report_id}: {e}" + ) + + def create_organization_snapshot(self, lcfs_cursor, report_id: int, org_id: int): + """Create organization snapshot for a compliance report""" + try: + # Fetch org details + query = """ + SELECT + org.name, + org.operating_name, + org.email, + org.phone, + org.records_address, + addr.street_address, + addr.address_other, + addr.city, + addr.province_state, + addr.country, + addr."postalCode_zipCode" + FROM organization org + LEFT JOIN organization_address addr ON org.organization_address_id = addr.organization_address_id + WHERE org.organization_id = %s + """ + + lcfs_cursor.execute(query, (org_id,)) + result = lcfs_cursor.fetchone() + + if result: + name, operating_name, email, phone, records_addr = result[:5] + street, other, city, province, country, postal = result[5:] + + # Construct addresses + address_parts = [ + part + for part in [street, other, city, province, country, postal] + if part + ] + full_address = ", ".join(address_parts) + service_address = full_address + head_office_address = full_address + + # Insert snapshot + insert_snapshot_sql = """ + INSERT INTO compliance_report_organization_snapshot ( + compliance_report_id, name, operating_name, email, phone, + service_address, head_office_address, records_address, is_edited, + create_user, update_user + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'ETL', 'ETL') + """ + + params = [ + report_id, + name or "", + operating_name or "", + email or "", + phone or "", + service_address or "", + head_office_address or "", + records_addr or "", + False, + ] + + lcfs_cursor.execute(insert_snapshot_sql, params) + logger.info( + f"Created organization snapshot for LCFS report ID: {report_id}" + ) + else: + logger.warning( + f"Could not find LCFS organization details for ID: {org_id} to create snapshot." + ) + + except Exception as e: + logger.error( + f"Exception creating organization snapshot for LCFS report ID: {report_id}: {e}" + ) + + def get_lcfs_transaction_type_id( + self, lcfs_cursor, responsibility: str + ) -> Optional[int]: + """Get LCFS Transaction Type ID from TFRS Responsibility string""" + if responsibility in self.responsibility_to_transaction_type_cache: + return self.responsibility_to_transaction_type_cache[responsibility] + + query = "SELECT allocation_transaction_type_id FROM allocation_transaction_type WHERE type = %s" + lcfs_cursor.execute(query, (responsibility,)) + result = lcfs_cursor.fetchone() + + if result: + type_id = result[0] + self.responsibility_to_transaction_type_cache[responsibility] = type_id + return type_id + else: + logger.warning( + f"No LCFS transaction type found for responsibility: {responsibility}; returning null." + ) + return None + + def get_lcfs_fuel_type_id_by_name( + self, lcfs_cursor, tfrs_fuel_type_name: str + ) -> Optional[int]: + """Get LCFS Fuel Type ID from TFRS Fuel Type Name""" + if not tfrs_fuel_type_name: + logger.error("Cannot map LCFS fuel type ID from null TFRS fuel type name.") + return None + + if tfrs_fuel_type_name in self.tfrs_fuel_name_to_lcfs_fuel_type_cache: + return self.tfrs_fuel_name_to_lcfs_fuel_type_cache[tfrs_fuel_type_name] + + query = "SELECT fuel_type_id FROM fuel_type WHERE fuel_type = %s" + lcfs_cursor.execute(query, (tfrs_fuel_type_name,)) + result = lcfs_cursor.fetchone() + + if result: + lcfs_id = result[0] + self.tfrs_fuel_name_to_lcfs_fuel_type_cache[tfrs_fuel_type_name] = lcfs_id + return lcfs_id + else: + logger.warning( + f"No LCFS fuel type found mapped for TFRS fuel type name: {tfrs_fuel_type_name}; returning null." + ) + return None + + def get_allocation_records_by_agreement_id( + self, tfrs_cursor, agreement_id: int + ) -> List[Dict]: + """Get allocation agreement records directly via exclusion_agreement_id""" + query = """ + SELECT + crear.id AS agreement_record_id, + CASE WHEN tt.the_type = 'Purchased' THEN 'Allocated from' ELSE 'Allocated to' END AS responsibility, + aft.name AS fuel_type, + aft.id AS tfrs_fuel_type_id, + crear.transaction_partner, + crear.postal_address, + crear.quantity, + uom.name AS units, + crear.quantity_not_sold, + tt.id AS transaction_type_id + FROM compliance_report_exclusion_agreement_record crear + INNER JOIN transaction_type tt ON crear.transaction_type_id = tt.id + INNER JOIN approved_fuel_type aft ON crear.fuel_type_id = aft.id + INNER JOIN unit_of_measure uom ON aft.unit_of_measure_id = uom.id + WHERE crear.exclusion_agreement_id = %s + ORDER BY crear.id; + """ + + tfrs_cursor.execute(query, (agreement_id,)) + records = [] + + for row in tfrs_cursor.fetchall(): + records.append( + { + "agreement_record_id": row[0], + "responsibility": row[1], + "fuel_type": row[2], + "tfrs_fuel_type_id": row[3], + "transaction_partner": row[4], + "postal_address": row[5], + "quantity": row[6], + "units": row[7], + "quantity_not_sold": row[8], + "transaction_type_id": row[9], + } + ) + + return records + + def get_current_allocation_version(self, lcfs_cursor, group_uuid: str) -> int: + """Get current highest version for allocation agreement group UUID""" + query = "SELECT version FROM allocation_agreement WHERE group_uuid = %s ORDER BY version DESC LIMIT 1" + lcfs_cursor.execute(query, (group_uuid,)) + result = lcfs_cursor.fetchone() + return result[0] if result else -1 + + def insert_allocation_agreement_version_row( + self, lcfs_cursor, lcfs_cr_id: int, row_data: Dict, action: str + ) -> bool: + """Inserts a new row into LCFS allocation_agreement with proper versioning""" + try: + record_id = row_data["agreement_record_id"] + + # Retrieve or create a stable group_uuid + group_uuid = self.record_uuid_map.get(record_id) + if not group_uuid: + group_uuid = str(uuid.uuid4()) + self.record_uuid_map[record_id] = group_uuid + + # Retrieve current highest version + current_ver = self.get_current_allocation_version(lcfs_cursor, group_uuid) + next_ver = 0 if current_ver < 0 else current_ver + 1 + + # Map source fields to LCFS fields + lcfs_alloc_transaction_type_id = self.get_lcfs_transaction_type_id( + lcfs_cursor, row_data["responsibility"] + ) + lcfs_fuel_type_id = self.get_lcfs_fuel_type_id_by_name( + lcfs_cursor, row_data["fuel_type"] + ) + quantity = safe_int(row_data.get("quantity", 0)) + quantity_not_sold = safe_int(row_data.get("quantity_not_sold", 0)) + transaction_partner = safe_str(row_data.get("transaction_partner", "")) + postal_address = safe_str(row_data.get("postal_address", "")) + units = safe_str(row_data.get("units", "")) + fuel_type_string = row_data.get("fuel_type", "") + + # Determine LCFS Fuel Category ID based on TFRS fuel type name + fuel_category_id = None + if fuel_type_string: + fuel_type_lower = fuel_type_string.lower() + if "gasoline" in fuel_type_lower: + fuel_category_id = self.GASOLINE_CATEGORY_ID + elif "diesel" in fuel_type_lower: + fuel_category_id = self.DIESEL_CATEGORY_ID + else: + logger.warning( + f"Could not determine LCFS fuel category for TFRS fuel type: {fuel_type_string}. Setting fuel_category_id to NULL." + ) + + # Validation + if lcfs_alloc_transaction_type_id is None or lcfs_fuel_type_id is None: + logger.error( + f"Skipping insert for TFRS record ID {record_id} due to missing LCFS mapping (TransactionType: {lcfs_alloc_transaction_type_id}, FuelType: {lcfs_fuel_type_id})" + ) + return False + + # Insert the record + insert_sql = """ + INSERT INTO allocation_agreement( + compliance_report_id, + transaction_partner, + postal_address, + quantity, + quantity_not_sold, + units, + allocation_transaction_type_id, + fuel_type_id, + fuel_category_id, + group_uuid, + version, + action_type, + create_user, + update_user + ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::actiontypeenum, %s, %s) + """ + + params = [ + lcfs_cr_id, + transaction_partner, + postal_address, + quantity, + quantity_not_sold, + units, + lcfs_alloc_transaction_type_id, + lcfs_fuel_type_id, + fuel_category_id, + group_uuid, + next_ver, + action, + "ETL", + "ETL", + ] + + lcfs_cursor.execute(insert_sql, params) + logger.info( + f"Inserted LCFS allocation_agreement row: TFRS_recordId={record_id}, LCFS_CR_ID={lcfs_cr_id}, action={action}, groupUuid={group_uuid}, version={next_ver}" + ) + return True + + except Exception as e: + logger.error(f"Failed to insert allocation agreement record: {e}") + return False + + def migrate(self) -> Tuple[int, int, int]: + """Main migration logic""" + orphaned_count = 0 + processed_count = 0 + skipped_count = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Pre-fetch necessary data + default_reporting_frequency = "ANNUAL" + logger.info( + f"Using default Reporting Frequency: {default_reporting_frequency}" + ) + + # Find orphaned TFRS exclusion reports + logger.info("Querying TFRS for orphaned exclusion reports...") + orphaned_reports = self.find_orphaned_exclusion_reports(tfrs_cursor) + orphaned_count = len(orphaned_reports) + logger.info( + f"Found {orphaned_count} orphaned TFRS exclusion reports" + ) + + for report in orphaned_reports: + tfrs_exclusion_report_id = report["tfrs_exclusion_report_id"] + tfrs_org_id = report["tfrs_organization_id"] + tfrs_period_id = report["tfrs_compliance_period_id"] + tfrs_exclusion_agreement_id = report["exclusion_agreement_id"] + tfrs_director_status = report["tfrs_director_status"] + + logger.info( + f"Processing orphaned TFRS exclusion report ID: {tfrs_exclusion_report_id} (Org: {tfrs_org_id}, Period: {tfrs_period_id}, Agreement: {tfrs_exclusion_agreement_id}, DirectorStatus: {tfrs_director_status})" + ) + + # Check if already migrated + if self.check_lcfs_report_exists( + lcfs_cursor, tfrs_exclusion_report_id + ): + logger.warning( + f"LCFS report with legacy_id {tfrs_exclusion_report_id} already exists. Skipping." + ) + skipped_count += 1 + continue + + # Get TFRS Org Name for Mapping + tfrs_org_name = self.get_tfrs_org_name(tfrs_cursor, tfrs_org_id) + + # Get TFRS Period Description for Mapping + tfrs_period_desc = self.get_tfrs_period_desc( + tfrs_cursor, tfrs_period_id + ) + + # Determine Target LCFS Status + target_lcfs_status_name = "Draft" # Default to Draft + if tfrs_director_status == "Accepted": + target_lcfs_status_name = "Assessed" + elif tfrs_director_status == "Rejected": + target_lcfs_status_name = "Rejected" + + logger.info( + f"Mapping TFRS Director Status '{tfrs_director_status}' to LCFS Status '{target_lcfs_status_name}'" + ) + lcfs_status_id = self.get_lcfs_report_status_id( + lcfs_cursor, target_lcfs_status_name + ) + + if lcfs_status_id is None: + logger.error( + f"Failed to find LCFS Status ID for '{target_lcfs_status_name}'. Skipping creation." + ) + skipped_count += 1 + continue + + # Create placeholder LCFS report + logger.info( + f"Creating placeholder LCFS report with Status ID: {lcfs_status_id}..." + ) + lcfs_org_id = self.get_lcfs_org_id(lcfs_cursor, tfrs_org_name) + lcfs_period_id = self.get_lcfs_period_id( + lcfs_cursor, tfrs_period_desc + ) + + if lcfs_org_id is None or lcfs_period_id is None: + logger.error( + f"Failed to map TFRS Org/Period IDs for TFRS report {tfrs_exclusion_report_id}. Skipping creation and associated records." + ) + skipped_count += 1 + continue + + new_lcfs_report_id = self.create_lcfs_placeholder_report( + lcfs_cursor, + lcfs_org_id, + lcfs_period_id, + lcfs_status_id, + default_reporting_frequency, + tfrs_exclusion_report_id, + ) + + if new_lcfs_report_id is None: + logger.error( + f"Failed to create placeholder LCFS report for TFRS ID {tfrs_exclusion_report_id}. Skipping associated records." + ) + skipped_count += 1 + continue + + # Fetch associated allocation records from TFRS + logger.info( + f"Fetching allocation records from TFRS for agreement ID: {tfrs_exclusion_agreement_id}" + ) + allocation_records = ( + self.get_allocation_records_by_agreement_id( + tfrs_cursor, tfrs_exclusion_agreement_id + ) + ) + + if not allocation_records: + logger.warning( + f"No allocation records found in TFRS for agreement ID: {tfrs_exclusion_agreement_id}" + ) + else: + # Insert allocation agreement records + for record_data in allocation_records: + self.insert_allocation_agreement_version_row( + lcfs_cursor, + new_lcfs_report_id, + record_data, + "CREATE", + ) + + processed_count += 1 + + # Commit all changes + lcfs_conn.commit() + logger.info( + f"Successfully committed all orphaned allocation agreement migrations" + ) + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + return orphaned_count, processed_count, skipped_count + + +def main(): + setup_logging() + logger.info("Starting Orphaned Allocation Agreement Migration") + + migrator = OrphanedAllocationAgreementMigrator() + + try: + orphaned, processed, skipped = migrator.migrate() + logger.info( + f"Migration completed. Found {orphaned} orphaned reports. Processed: {processed}, Skipped: {skipped}" + ) + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/migrate_other_uses.py b/etl/python_migration/migrations/migrate_other_uses.py new file mode 100644 index 000000000..656eaa507 --- /dev/null +++ b/etl/python_migration/migrations/migrate_other_uses.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +""" +Other Uses Migration Script + +Migrates Schedule C (other uses) data from TFRS to LCFS database. +This script replicates the functionality of other_uses.groovy +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import argparse +import logging +import sys +import uuid +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import ( + setup_logging, + safe_decimal, + safe_int, + safe_str, + build_legacy_mapping, +) + +logger = logging.getLogger(__name__) + + +class OtherUsesMigrator: + def __init__(self, dry_run: bool = False): + self.record_uuid_map: Dict[int, str] = {} + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + self.dry_run = dry_run + + # Statistics for dry run + self.stats = { + 'tfrs_reports_found': 0, + 'schedule_c_records_found': 0, + 'creates': 0, + 'updates': 0, + 'skipped_no_lcfs_match': 0, + 'fuel_type_mappings': {}, + 'fuel_category_mappings': {}, + 'expected_use_mappings': {} + } + + def map_fuel_category_id(self, fuel_class_id: int) -> Optional[int]: + """Maps TFRS fuel_class_id to LCFS fuel_category_id""" + mapping = { + 1: 2, # Diesel + 2: 1, # Gasoline + } + result = mapping.get(fuel_class_id) + # Track mapping usage for stats + if result: + key = f"{fuel_class_id}->{result}" + self.stats['fuel_category_mappings'][key] = self.stats['fuel_category_mappings'].get(key, 0) + 1 + return result + + def map_fuel_type_id(self, tfrs_type_id: int) -> int: + """Maps TFRS fuel type ID to LCFS fuel type ID""" + mapping = { + 1: 1, # Biodiesel + 2: 2, # CNG + 3: 3, # Electricity + 4: 4, # Ethanol + 5: 5, # HDRD + 6: 6, # Hydrogen + 7: 7, # LNG + 8: 13, # Propane + 9: 5, # Renewable diesel -> HDRD + 10: 14, # Renewable gasoline + 11: 17, # Natural gas-based gasoline -> Fossil-derived gasoline + 19: 16, # Petroleum-based diesel -> Fossil-derived diesel + 20: 17, # Petroleum-based gasoline -> Fossil-derived gasoline + 21: 15, # Renewable naphtha + } + result = mapping.get(tfrs_type_id, 19) # Default to 'Other' if no match found + # Track mapping usage for stats + key = f"{tfrs_type_id}->{result}" + self.stats['fuel_type_mappings'][key] = self.stats['fuel_type_mappings'].get(key, 0) + 1 + return result + + def map_expected_use_id(self, tfrs_expected_use_id: int) -> int: + """Maps TFRS expected use ID to LCFS expected use ID""" + mapping = { + 2: 1, # Heating Oil + 1: 2, # Other + } + result = mapping.get(tfrs_expected_use_id, 2) # Default to 'Other' (id: 2) + # Track mapping usage for stats + key = f"{tfrs_expected_use_id}->{result}" + self.stats['expected_use_mappings'][key] = self.stats['expected_use_mappings'].get(key, 0) + 1 + return result + + def is_record_changed(self, old_row: Optional[Dict], new_row: Dict) -> bool: + """Checks if any relevant fields in a schedule_c record differ between old and new""" + if not old_row or not new_row: + return True + + return ( + old_row.get("quantity") != new_row.get("quantity") + or old_row.get("fuel_type_id") != new_row.get("fuel_type_id") + or old_row.get("fuel_class_id") != new_row.get("fuel_class_id") + or old_row.get("expected_use_id") != new_row.get("expected_use_id") + or old_row.get("rationale") != new_row.get("rationale") + ) + + def get_current_version(self, lcfs_cursor, group_uuid: str) -> int: + """Get current highest version for a group UUID""" + query = """ + SELECT version + FROM other_uses + WHERE group_uuid = %s + ORDER BY version DESC + LIMIT 1 + """ + lcfs_cursor.execute(query, (group_uuid,)) + result = lcfs_cursor.fetchone() + return result[0] if result else -1 + + def insert_version_row( + self, lcfs_cursor, lcfs_cr_id: int, row_data: Dict, action: str + ) -> bool: + """Inserts a new row in other_uses with action=CREATE/UPDATE""" + try: + record_id = row_data["schedule_c_record_id"] + + # Get/create stable group UUID for version chain + group_uuid = self.record_uuid_map.get(record_id) + if not group_uuid: + group_uuid = str(uuid.uuid4()) + self.record_uuid_map[record_id] = group_uuid + + # Get current highest version + current_ver = self.get_current_version(lcfs_cursor, group_uuid) + next_ver = 0 if current_ver < 0 else current_ver + 1 + + # Map TFRS fields to LCFS fields + expected_use_id = self.map_expected_use_id( + row_data.get("expected_use_id", 1) + ) + fuel_cat_id = self.map_fuel_category_id(row_data.get("fuel_class_id")) + fuel_type_id = self.map_fuel_type_id(row_data.get("fuel_type_id", 1)) + quantity = safe_decimal(row_data.get("quantity", 0)) + rationale = safe_str(row_data.get("rationale", "")) + units = safe_str(row_data.get("unit_of_measure", "")) + ci_of_fuel = safe_decimal(row_data.get("ci_of_fuel", 0)) + + # Insert the record + insert_sql = """ + INSERT INTO other_uses ( + compliance_report_id, + fuel_type_id, + fuel_category_id, + provision_of_the_act_id, + ci_of_fuel, + quantity_supplied, + units, + expected_use_id, + rationale, + group_uuid, + version, + action_type, + create_user, + update_user + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::actiontypeenum, %s, %s) + """ + + params = [ + lcfs_cr_id, + fuel_type_id, + fuel_cat_id, + 7, # provision_of_the_act_id + float(ci_of_fuel), + float(quantity), + units, + expected_use_id, + rationale, + group_uuid, + next_ver, + action, + "ETL", + "ETL", + ] + + if self.dry_run: + logger.info( + f"[DRY RUN] Would insert other_uses row: recordId={record_id}, action={action}, groupUuid={group_uuid}, version={next_ver}" + ) + else: + lcfs_cursor.execute(insert_sql, params) + logger.info( + f"Inserted other_uses row: recordId={record_id}, action={action}, groupUuid={group_uuid}, version={next_ver}" + ) + return True + + except Exception as e: + logger.error(f"Failed to insert other_uses record: {e}") + return False + + def get_lcfs_reports_with_legacy_ids(self, lcfs_cursor) -> List[int]: + """Get all LCFS compliance reports with legacy IDs""" + query = """ + SELECT compliance_report_id, legacy_id + FROM compliance_report + WHERE legacy_id IS NOT NULL + """ + lcfs_cursor.execute(query) + return [row[1] for row in lcfs_cursor.fetchall()] # Return legacy_ids + + def get_root_report_id(self, tfrs_cursor, tfrs_id: int) -> Optional[int]: + """Find root report ID for supplemental chain""" + query = """ + SELECT root_report_id + FROM compliance_report + WHERE id = %s + """ + tfrs_cursor.execute(query, (tfrs_id,)) + result = tfrs_cursor.fetchone() + return result[0] if result else None + + def get_report_chain(self, tfrs_cursor, root_id: int) -> List[int]: + """Get full chain of reports""" + query = """ + SELECT + c.id AS tfrs_report_id, + c.traversal + FROM compliance_report c + WHERE c.root_report_id = %s + ORDER BY c.traversal, c.id + """ + tfrs_cursor.execute(query, (root_id,)) + return [row[0] for row in tfrs_cursor.fetchall()] + + def get_schedule_c_records( + self, tfrs_cursor, chain_tfrs_id: int + ) -> Dict[int, Dict]: + """Get current Schedule C records for a report""" + query = """ + SELECT + scr.id AS schedule_c_record_id, + scr.quantity, + scr.fuel_type_id, + scr.fuel_class_id, + scr.expected_use_id, + scr.rationale, + cr.id AS compliance_report_id, + uom.name AS unit_of_measure, + dci.density AS default_ci_of_fuel + FROM compliance_report_schedule_c_record scr + JOIN compliance_report_schedule_c sc ON sc.id = scr.schedule_id + JOIN compliance_report cr ON cr.schedule_c_id = sc.id + LEFT JOIN approved_fuel_type aft ON aft.id = scr.fuel_type_id + LEFT JOIN default_carbon_intensity dci ON dci.category_id = aft.default_carbon_intensity_category_id + LEFT JOIN unit_of_measure uom ON uom.id = aft.unit_of_measure_id + WHERE cr.id = %s + """ + + tfrs_cursor.execute(query, (chain_tfrs_id,)) + records = {} + + for row in tfrs_cursor.fetchall(): + records[row[0]] = { # schedule_c_record_id as key + "schedule_c_record_id": row[0], + "quantity": row[1], + "fuel_type_id": row[2], + "fuel_class_id": row[3], + "expected_use_id": row[4], + "rationale": row[5], + "unit_of_measure": row[7], + "ci_of_fuel": row[8], + } + + return records + + def get_lcfs_compliance_report_id( + self, lcfs_cursor, chain_tfrs_id: int + ) -> Optional[int]: + """Find corresponding LCFS compliance report""" + query = """ + SELECT compliance_report_id + FROM compliance_report + WHERE legacy_id = %s + """ + lcfs_cursor.execute(query, (chain_tfrs_id,)) + result = lcfs_cursor.fetchone() + return result[0] if result else None + + def migrate(self) -> Tuple[int, int]: + """Main migration logic""" + total_processed = 0 + total_skipped = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Get all LCFS compliance reports with legacy IDs + logger.info( + "Retrieving LCFS compliance_report with legacy_id != null" + ) + tfrs_ids = self.get_lcfs_reports_with_legacy_ids(lcfs_cursor) + self.stats['tfrs_reports_found'] = len(tfrs_ids) + logger.info(f"Found {len(tfrs_ids)} reports to process") + + # Process each TFRS compliance report + for tfrs_id in tfrs_ids: + logger.info(f"Processing TFRS compliance_report.id = {tfrs_id}") + + # Find root report ID for supplemental chain + root_id = self.get_root_report_id(tfrs_cursor, tfrs_id) + if not root_id: + logger.warning( + f"No root_report_id found for TFRS #{tfrs_id}; skipping." + ) + total_skipped += 1 + continue + + # Get full chain of reports + chain_ids = self.get_report_chain(tfrs_cursor, root_id) + if not chain_ids: + logger.warning(f"Chain empty for root={root_id}? skipping.") + total_skipped += 1 + continue + + # Track previous records for change detection + previous_records = {} + + # Process each report in the chain + for idx, chain_tfrs_id in enumerate(chain_ids): + logger.info(f"TFRS #{chain_tfrs_id} (chain idx={idx})") + + # Get current Schedule C records + current_records = self.get_schedule_c_records( + tfrs_cursor, chain_tfrs_id + ) + self.stats['schedule_c_records_found'] += len(current_records) + + # Find corresponding LCFS compliance report + lcfs_cr_id = self.get_lcfs_compliance_report_id( + lcfs_cursor, chain_tfrs_id + ) + if not lcfs_cr_id: + logger.warning( + f"TFRS #{chain_tfrs_id} not found in LCFS; skipping diff." + ) + self.stats['skipped_no_lcfs_match'] += len(current_records) + previous_records = current_records + continue + + # Compare and insert records + for rec_id, new_data in current_records.items(): + if rec_id not in previous_records: + self.insert_version_row( + lcfs_cursor, lcfs_cr_id, new_data, "CREATE" + ) + self.stats['creates'] += 1 + total_processed += 1 + elif self.is_record_changed( + previous_records.get(rec_id), new_data + ): + self.insert_version_row( + lcfs_cursor, lcfs_cr_id, new_data, "UPDATE" + ) + self.stats['updates'] += 1 + total_processed += 1 + + previous_records = current_records + + # Commit all changes + if not self.dry_run: + lcfs_conn.commit() + logger.info(f"Successfully committed {total_processed} records") + else: + logger.info(f"[DRY RUN] Would commit {total_processed} records") + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Migration failed: {e}") + raise + + return total_processed, total_skipped + + def print_statistics(self): + """Print comprehensive migration statistics""" + logger.info("=" * 60) + logger.info("OTHER USES MIGRATION STATISTICS") + logger.info("=" * 60) + + logger.info(f"📊 Source Data:") + logger.info(f" • TFRS Compliance Reports Found: {self.stats['tfrs_reports_found']}") + logger.info(f" • Schedule C Records Found: {self.stats['schedule_c_records_found']}") + + logger.info(f"🔄 Actions to Perform:") + logger.info(f" • CREATE operations: {self.stats['creates']}") + logger.info(f" • UPDATE operations: {self.stats['updates']}") + logger.info(f" • Records skipped (no LCFS match): {self.stats['skipped_no_lcfs_match']}") + + total_actions = self.stats['creates'] + self.stats['updates'] + logger.info(f" • Total records to process: {total_actions}") + + if self.stats['fuel_type_mappings']: + logger.info(f"🔗 Fuel Type Mappings:") + for mapping, count in sorted(self.stats['fuel_type_mappings'].items()): + logger.info(f" • {mapping}: {count} records") + + if self.stats['fuel_category_mappings']: + logger.info(f"🔗 Fuel Category Mappings:") + for mapping, count in sorted(self.stats['fuel_category_mappings'].items()): + logger.info(f" • {mapping}: {count} records") + + if self.stats['expected_use_mappings']: + logger.info(f"🔗 Expected Use Mappings:") + for mapping, count in sorted(self.stats['expected_use_mappings'].items()): + logger.info(f" • {mapping}: {count} records") + + logger.info("=" * 60) + + +def main(): + parser = argparse.ArgumentParser(description="Migrate Other Uses (Schedule C) data from TFRS to LCFS") + parser.add_argument( + "--dry-run", + action="store_true", + help="Run migration in dry-run mode (no database changes, statistics only)" + ) + args = parser.parse_args() + + setup_logging() + mode = "DRY RUN" if args.dry_run else "PRODUCTION" + logger.info(f"Starting Other Uses (Schedule C) Migration - {mode} MODE") + + migrator = OtherUsesMigrator(dry_run=args.dry_run) + + try: + processed, skipped = migrator.migrate() + + # Print statistics + migrator.print_statistics() + + if args.dry_run: + logger.info(f"[DRY RUN] Migration analysis completed. Would process: {processed}, Would skip: {skipped}") + else: + logger.info(f"Migration completed successfully. Processed: {processed}, Skipped: {skipped}") + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/migrations/run_all_migrations.py b/etl/python_migration/migrations/run_all_migrations.py new file mode 100644 index 000000000..2bc59ac11 --- /dev/null +++ b/etl/python_migration/migrations/run_all_migrations.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +""" +Run All Migrations Script + +Executes all TFRS to LCFS migration scripts in the correct order. +This script provides a centralized way to run the complete migration process. +""" + +import logging +import sys +import time +import os +from typing import List, Tuple + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.utils import setup_logging +from core.database import tfrs_db, lcfs_db + +# Import all migration modules +from .migrate_data_cleanup import DataCleanupMigrator +from .migrate_compliance_summaries import ComplianceSummaryMigrator +from .migrate_compliance_summary_updates import ComplianceSummaryUpdater +from .migrate_compliance_report_history import ComplianceReportHistoryMigrator +from .migrate_allocation_agreements import AllocationAgreementMigrator +from .migrate_other_uses import OtherUsesMigrator +from .migrate_notional_transfers import NotionalTransferMigrator +from .migrate_fuel_supply import FuelSupplyMigrator +from .migrate_orphaned_allocation_agreements import OrphanedAllocationAgreementMigrator + +logger = logging.getLogger(__name__) + + +class MigrationRunner: + def __init__(self): + self.results = [] + + def test_connections(self) -> bool: + """Test database connections before starting migrations""" + logger.info("Testing database connections...") + + tfrs_ok = tfrs_db.test_connection() + lcfs_ok = lcfs_db.test_connection() + + if tfrs_ok: + logger.info("✅ TFRS database connection successful") + else: + logger.error("❌ TFRS database connection failed") + + if lcfs_ok: + logger.info("✅ LCFS database connection successful") + else: + logger.error("❌ LCFS database connection failed") + + return tfrs_ok and lcfs_ok + + def run_migration(self, migrator_class, name: str) -> Tuple[bool, str, int, int]: + """Run a single migration and return results""" + logger.info(f"\n{'='*60}") + logger.info(f"Starting {name}") + logger.info(f"{'='*60}") + + start_time = time.time() + + try: + migrator = migrator_class() + + # Run the migration - handle different return signatures + if hasattr(migrator, "migrate"): + result = migrator.migrate() + + # Handle different return types + if isinstance(result, tuple) and len(result) == 2: + processed, skipped_or_failed = result + total = processed + skipped_or_failed + elif isinstance(result, tuple) and len(result) == 3: + # For orphaned_allocation_agreement which returns (orphaned, processed, skipped) + orphaned, processed, skipped = result + total = processed + skipped + else: + processed = result if isinstance(result, int) else 0 + total = processed + + elif hasattr(migrator, "update_summaries"): + # For compliance_summary_update + processed, skipped = migrator.update_summaries() + total = processed + skipped + else: + raise Exception( + f"Migrator {migrator_class.__name__} has no migrate method" + ) + + end_time = time.time() + duration = end_time - start_time + + logger.info(f"✅ {name} completed successfully") + logger.info(f" 📊 Processed: {processed}") + logger.info(f" ⏱️ Duration: {duration:.2f} seconds") + + return True, f"Success - Processed: {processed}", processed, total + + except Exception as e: + end_time = time.time() + duration = end_time - start_time + + logger.error(f"❌ {name} failed after {duration:.2f} seconds") + logger.error(f" Error: {e}") + + return False, f"Failed: {e}", 0, 0 + + def run_all_migrations(self) -> bool: + """Run all migrations in the correct order""" + + # Define migration order and configurations + migrations = [ + (DataCleanupMigrator, "Data Cleanup Migration (Pre-migration)"), + (ComplianceSummaryMigrator, "Compliance Summary Migration"), + (ComplianceSummaryUpdater, "Compliance Summary Update"), + (ComplianceReportHistoryMigrator, "Compliance Report History Migration"), + (AllocationAgreementMigrator, "Allocation Agreement Migration"), + (OtherUsesMigrator, "Other Uses (Schedule C) Migration"), + (NotionalTransferMigrator, "Notional Transfer (Schedule A) Migration"), + (FuelSupplyMigrator, "Fuel Supply (Schedule B) Migration"), + ( + OrphanedAllocationAgreementMigrator, + "Orphaned Allocation Agreement Migration", + ), + ] + + logger.info("🚀 Starting complete TFRS to LCFS migration process") + logger.info(f"📋 Total migrations to run: {len(migrations)}") + + overall_start_time = time.time() + total_processed = 0 + total_records = 0 + failed_migrations = [] + + # Run each migration + for migrator_class, name in migrations: + success, message, processed, total = self.run_migration( + migrator_class, name + ) + + self.results.append( + { + "name": name, + "success": success, + "message": message, + "processed": processed, + "total": total, + } + ) + + if success: + total_processed += processed + total_records += total + else: + failed_migrations.append(name) + # Continue with other migrations even if one fails + logger.warning( + f"⚠️ Continuing with remaining migrations despite {name} failure" + ) + + overall_end_time = time.time() + overall_duration = overall_end_time - overall_start_time + + # Print summary + self.print_summary( + overall_duration, total_processed, total_records, failed_migrations + ) + + # Return detailed results for orchestrator + results_dict = {} + for result in self.results: + results_dict[result["name"]] = { + "success": result["success"], + "message": result["message"], + "records_processed": result["processed"], + "total_records": result["total"], + } + + return results_dict + + def print_summary( + self, + duration: float, + total_processed: int, + total_records: int, + failed_migrations: List[str], + ): + """Print migration summary""" + logger.info(f"\n{'='*80}") + logger.info("📋 MIGRATION SUMMARY") + logger.info(f"{'='*80}") + + logger.info( + f"⏱️ Total Duration: {duration:.2f} seconds ({duration/60:.1f} minutes)" + ) + logger.info(f"📊 Total Records Processed: {total_processed:,}") + logger.info(f"📈 Total Records Encountered: {total_records:,}") + + # Print individual results + logger.info(f"\n📝 Individual Migration Results:") + for result in self.results: + status = "✅" if result["success"] else "❌" + logger.info(f" {status} {result['name']}: {result['message']}") + + # Print failures if any + if failed_migrations: + logger.error(f"\n❌ Failed Migrations ({len(failed_migrations)}):") + for migration in failed_migrations: + logger.error(f" • {migration}") + else: + logger.info(f"\n🎉 All migrations completed successfully!") + + logger.info(f"{'='*80}") + + +def main(): + """Main entry point""" + setup_logging() + + runner = MigrationRunner() + + # Test connections first + if not runner.test_connections(): + logger.error("❌ Database connection tests failed. Aborting migrations.") + sys.exit(1) + + # Run all migrations + results = runner.run_all_migrations() + + # Check if all migrations succeeded + all_success = all(result.get("success", False) for result in results.values()) + + if all_success: + logger.info("🎉 All migrations completed successfully!") + sys.exit(0) + else: + logger.error("❌ Some migrations failed. Check logs for details.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/quick-start.sh b/etl/python_migration/quick-start.sh new file mode 100755 index 000000000..22caf7944 --- /dev/null +++ b/etl/python_migration/quick-start.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# Quick start script for TFRS to LCFS migration with automatic Docker setup + +set -e + +echo "🚀 TFRS to LCFS Migration - Quick Start" +echo "======================================" +echo + +# Check if we're in the right directory +if [ ! -f "docker-compose.yml" ]; then + echo "❌ Error: Please run this script from the python_migration directory" + exit 1 +fi + +# Check if Docker is running +if ! docker info >/dev/null 2>&1; then + echo "❌ Error: Docker is not running. Please start Docker first." + exit 1 +fi + +# Parse command line arguments +ENV="dev" +SKIP_VALIDATION="" +ONLY_SETUP="" + +while [[ $# -gt 0 ]]; do + case $1 in + --env) + ENV="$2" + shift 2 + ;; + --skip-validation) + SKIP_VALIDATION="--skip-validation" + shift + ;; + --setup-only) + ONLY_SETUP="true" + shift + ;; + -h|--help) + echo "Usage: $0 [--env dev|test|prod] [--skip-validation] [--setup-only]" + echo + echo "Options:" + echo " --env ENV Environment to use (dev, test, prod) [default: dev]" + echo " --skip-validation Skip validation after migration" + echo " --setup-only Only setup Docker environment, don't run migration" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "❌ Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +echo "Environment: $ENV" +if [ -n "$SKIP_VALIDATION" ]; then + echo "Validation: Skipped" +else + echo "Validation: Enabled" +fi + +if [ -n "$ONLY_SETUP" ]; then + echo "Mode: Setup only" +else + echo "Mode: Complete migration" +fi + +echo + +# Security check - only import operations allowed +echo "🔒 SECURITY: This script only performs IMPORT operations from OpenShift" +echo "🔒 SECURITY: NO EXPORT to production databases is allowed" +echo + +# Check OpenShift login if not using dev +if [ "$ENV" != "dev" ]; then + echo "🔐 Checking OpenShift login..." + if ! oc whoami >/dev/null 2>&1; then + echo "❌ Error: Not logged into OpenShift. Please run 'oc login' first." + exit 1 + fi + echo "✅ OpenShift login verified" + echo +fi + +if [ -n "$ONLY_SETUP" ]; then + echo "🐳 Setting up Docker environment automatically..." + python setup/migration_orchestrator.py auto-setup --env "$ENV" + + if [ $? -eq 0 ]; then + echo + echo "✅ Docker environment setup completed!" + echo "You can now run migrations manually with:" + echo " python setup/migration_orchestrator.py migrate" + echo " python setup/migration_orchestrator.py validate" + else + echo "❌ Docker environment setup failed" + exit 1 + fi +else + echo "🚀 Running complete migration with automatic Docker setup..." + python setup/migration_orchestrator.py auto-complete --env "$ENV" $SKIP_VALIDATION + + if [ $? -eq 0 ]; then + echo + echo "🎉 Complete migration process successful!" + echo + echo "📊 What happened:" + echo " 1. Started TFRS Docker container automatically" + echo " 2. Started LCFS environment (if available)" + echo " 3. Imported $ENV data from OpenShift" + echo " 4. Ran all migration scripts" + if [ -z "$SKIP_VALIDATION" ]; then + echo " 5. Ran comprehensive validation" + fi + echo + echo "🔍 Check the logs above for detailed results" + echo "📁 Validation reports saved in setup/ directory" + else + echo "❌ Migration process failed" + exit 1 + fi +fi + +echo +echo "🐳 Running containers:" +docker ps --format "table {{.ID}}\\t{{.Names}}\\t{{.Status}}" | grep -E "(tfrs|lcfs|postgres)" + +echo +echo "✨ Migration complete! You can now access:" +echo " - TFRS data in the tfrs-migration container" +echo " - LCFS application at http://localhost:3000 (if running)" +echo " - LCFS database in the LCFS environment" \ No newline at end of file diff --git a/etl/python_migration/requirements.txt b/etl/python_migration/requirements.txt new file mode 100644 index 000000000..b41a8ae34 --- /dev/null +++ b/etl/python_migration/requirements.txt @@ -0,0 +1,2 @@ +psycopg2-binary==2.9.7 +python-dotenv==1.0.0 \ No newline at end of file diff --git a/etl/python_migration/setup-paths.sh b/etl/python_migration/setup-paths.sh new file mode 100755 index 000000000..0188bb8c5 --- /dev/null +++ b/etl/python_migration/setup-paths.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Script to help users set up correct paths for their environment + +echo "🔧 TFRS to LCFS Migration Path Setup" +echo "=====================================" +echo + +# Check current directory +if [ ! -f "docker-compose.yml" ]; then + echo "❌ Error: Please run this script from the python_migration directory" + echo "Current directory: $(pwd)" + exit 1 +fi + +echo "✅ Running from correct directory: $(pwd)" +echo + +# Look for LCFS docker-compose.yml in common locations +echo "🔍 Looking for LCFS docker-compose.yml..." + +LCFS_PATHS=( + "../../../docker-compose.yml" + "../../docker-compose.yml" + "../lcfs/docker-compose.yml" + "../../../../lcfs/docker-compose.yml" +) + +FOUND_LCFS="" + +for path in "${LCFS_PATHS[@]}"; do + if [ -f "$path" ]; then + echo "✅ Found LCFS docker-compose.yml at: $path" + FOUND_LCFS="$path" + break + else + echo "❌ Not found at: $path" + fi +done + +echo + +if [ -n "$FOUND_LCFS" ]; then + echo "🎉 Setup complete!" + echo "LCFS docker-compose.yml found at: $FOUND_LCFS" + echo + echo "You can now use:" + echo " make setup-prod # Setup with production data" + echo " make setup-dev # Setup with dev data" + echo " make quick-start # Complete migration with dev data" +else + echo "⚠️ LCFS docker-compose.yml not found in expected locations" + echo + echo "Please either:" + echo "1. Move this migration project to be relative to the main LCFS project" + echo "2. Update the paths in Makefile and setup/docker_manager.py" + echo "3. Start LCFS environment manually before running migrations" + echo + echo "Expected structure:" + echo " lcfs/" + echo " ├── docker-compose.yml (main LCFS)" + echo " └── etl/" + echo " └── python_migration/" + echo " ├── docker-compose.yml (TFRS only)" + echo " └── Makefile" +fi + +echo +echo "📋 Current Docker containers:" +docker ps --format "table {{.Names}}\\t{{.Status}}" 2>/dev/null || echo "Docker not running" + +echo +echo "🔧 Environment setup:" +echo " Python: $(python --version 2>&1)" +echo " Docker: $(docker --version 2>/dev/null || echo 'Not found')" +echo " OpenShift CLI: $(oc version --client --short 2>/dev/null || echo 'Not found')" +echo " OpenShift User: $(oc whoami 2>/dev/null || echo 'Not logged in')" \ No newline at end of file diff --git a/etl/python_migration/setup/__init__.py b/etl/python_migration/setup/__init__.py new file mode 100644 index 000000000..2e5567155 --- /dev/null +++ b/etl/python_migration/setup/__init__.py @@ -0,0 +1 @@ +# Setup and orchestration scripts \ No newline at end of file diff --git a/etl/python_migration/setup/database_manager.py b/etl/python_migration/setup/database_manager.py new file mode 100644 index 000000000..54b33fda4 --- /dev/null +++ b/etl/python_migration/setup/database_manager.py @@ -0,0 +1,770 @@ +""" +Database setup and verification utilities for TFRS to LCFS migration. +Python equivalent of data-transfer.sh with additional validation capabilities. +""" + +import os +import sys +import logging +import subprocess +import tempfile +import shutil +from typing import Dict, List, Optional, Tuple +from dataclasses import dataclass +from enum import Enum +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from core.database import get_source_connection, get_destination_connection + + +class Environment(Enum): + DEV = "dev" + TEST = "test" + PROD = "prod" + + +class Application(Enum): + TFRS = "tfrs" + LCFS = "lcfs" + + +class Direction(Enum): + IMPORT = "import" + EXPORT = "export" + + +@dataclass +class DatabaseConfig: + project_name: str + app_label: str + db_name: str + remote_db_user: str + local_db_user: str + + +class DatabaseSetup: + """Handle database setup and data transfer operations.""" + + def __init__(self): + self.logger = logging.getLogger(__name__) + self.setup_logging() + + def setup_logging(self): + """Set up logging configuration.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + def get_database_config(self, app: Application, env: Environment) -> DatabaseConfig: + """Get database configuration for the specified application and environment.""" + if app == Application.TFRS: + project_name = f"0ab226-{env.value}" + app_label = f"tfrs-crunchy-{env.value}-tfrs" + return DatabaseConfig( + project_name=project_name, + app_label=app_label, + db_name="tfrs", + remote_db_user="postgres", + local_db_user="tfrs", + ) + elif app == Application.LCFS: + project_name = f"d2bd59-{env.value}" + app_label = f"lcfs-crunchy-{env.value}-lcfs" + return DatabaseConfig( + project_name=project_name, + app_label=app_label, + db_name="lcfs", + remote_db_user="postgres", + local_db_user="lcfs", + ) + else: + raise ValueError(f"Invalid application: {app}") + + def check_openshift_login(self) -> bool: + """Check if user is logged into OpenShift.""" + try: + result = subprocess.run( + ["oc", "whoami"], capture_output=True, text=True, check=True + ) + self.logger.info(f"OpenShift user: {result.stdout.strip()}") + return True + except (subprocess.CalledProcessError, FileNotFoundError): + self.logger.error("Not logged into OpenShift or 'oc' command not found") + return False + + def check_docker_container(self, container_name: str) -> bool: + """Check if Docker container exists and is running.""" + try: + result = subprocess.run( + ["docker", "inspect", container_name, "--format={{.State.Running}}"], + capture_output=True, + text=True, + check=True, + ) + is_running = result.stdout.strip() == "true" + if is_running: + self.logger.info(f"Docker container {container_name} is running") + else: + self.logger.error(f"Docker container {container_name} is not running") + return is_running + except (subprocess.CalledProcessError, FileNotFoundError): + self.logger.error( + f"Docker container {container_name} not found or Docker not available" + ) + return False + + def set_oc_project(self, db_config: DatabaseConfig) -> Optional[str]: + # Set the OpenShift project + subprocess.run( + ["oc", "project", db_config.project_name], + capture_output=True, + text=True, + check=True, + ) + + def get_leader_pod(self, db_config: DatabaseConfig) -> Optional[str]: + """Find the leader pod in the PostgreSQL cluster.""" + try: + # Set the OpenShift project + self.set_oc_project(db_config) + + # Get all pods with the app label + result = subprocess.run( + ["oc", "get", "pods", "-o", "name"], + capture_output=True, + text=True, + check=True, + ) + + pods = [ + pod.strip() + for pod in result.stdout.split("\n") + if db_config.app_label in pod + ] + + # Find the leader pod + for pod in pods: + try: + cmd = [ + "oc", + "exec", + pod, + "--", + "bash", + "-c", + f'psql -U {db_config.remote_db_user} -tAc "SELECT pg_is_in_recovery()"', + ] + result = subprocess.run( + cmd, capture_output=True, text=True, check=True + ) + + if result.stdout.strip() == "f": + self.logger.info(f"Leader pod identified: {pod}") + return pod + except subprocess.CalledProcessError: + continue + + self.logger.error("No leader pod found") + return None + + except subprocess.CalledProcessError as e: + self.logger.error(f"Error finding leader pod: {e}") + return None + + def transfer_data( + self, + app: Application, + env: Environment, + direction: Direction, + container_name: str, + table_name: Optional[str] = None, + ) -> bool: + """Transfer data between OpenShift and local container.""" + + # Validation - NO EXPORTS TO PRODUCTION ALLOWED + if app == Application.TFRS and direction == Direction.EXPORT: + self.logger.error("Export operation is not supported for TFRS application") + return False + + if direction == Direction.EXPORT and env == Environment.PROD: + self.logger.error( + "SECURITY: Export to production environment is strictly prohibited" + ) + return False + + if not self.check_openshift_login(): + return False + + if not self.check_docker_container(container_name): + return False + + db_config = self.get_database_config(app, env) + + # pod_name = self.get_leader_pod(db_config) + # manually set pod name because getting leader pod is challenging + if app == Application.TFRS: + pod_name = "tfrs-crunchy-prod-tfrs-n4pf-0" + elif app == Application.LCFS: + pod_name = "lcfs-crunchy-prod-lcfs-2znf-0" + + if not pod_name: + return False + + # Set the OpenShift project + self.set_oc_project(db_config) + + # Set up file naming + table_option = f"-t {table_name}" if table_name else "" + file_suffix = ( + f"{db_config.db_name}_{table_name}" if table_name else db_config.db_name + ) + dump_file = f"{file_suffix}.tar" + + try: + if direction == Direction.IMPORT: + return self._import_data( + pod_name, db_config, container_name, table_option, dump_file + ) + # else: + # return self._export_data( + # pod_name, db_config, container_name, table_option, dump_file + # ) + except Exception as e: + self.logger.error(f"Data transfer failed: {e}") + return False + + def _get_dump_dir(self) -> str: + """Get the directory for storing database dumps.""" + # Create a dumps directory relative to the script location + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + dump_dir = os.path.join(script_dir, "dumps") + if not os.path.exists(dump_dir): + os.makedirs(dump_dir) + return dump_dir + + def _import_data( + self, + pod_name: str, + db_config: DatabaseConfig, + container_name: str, + table_option: str, + dump_file: str, + ) -> bool: + """Import data from OpenShift to local container.""" + dump_dir = self._get_dump_dir() + local_dump_path = os.path.join(dump_dir, dump_file) + + # Check if .tar file already exists + if os.path.exists(local_dump_path): + self.logger.info(f"Dump file {dump_file} already exists, skipping download from OpenShift") + else: + self.logger.info("Selecting openshift namespace") + + self.logger.info("Starting pg_dump on OpenShift pod") + cmd = [ + "oc", + "exec", + pod_name, + "--", + "bash", + "-c", + f"pg_dump -U {db_config.remote_db_user} {table_option} -F t --no-privileges --no-owner -c -d {db_config.db_name} > /tmp/{dump_file}", + ] + subprocess.run(cmd, check=True) + + self.logger.info("Downloading dump file from OpenShift pod") + subprocess.run( + ["oc", "cp", f"{pod_name}:/tmp/{dump_file}", local_dump_path], + check=True, + ) + + # Cleanup OpenShift pod dump file + self.logger.info("Cleaning up dump file from OpenShift pod") + subprocess.run( + ["oc", "exec", pod_name, "--", "rm", f"/tmp/{dump_file}"], check=False + ) + + try: + self.logger.info(f"Copying dump file to local container {container_name}") + subprocess.run( + ["docker", "cp", local_dump_path, f"{container_name}:/tmp/{dump_file}"], + check=True, + ) + + self.logger.info("Restoring local database") + cmd = [ + "docker", + "exec", + container_name, + "bash", + "-c", + f"pg_restore -U {db_config.local_db_user} --dbname={db_config.db_name} --no-owner --clean --if-exists --verbose /tmp/{dump_file}", + ] + subprocess.run(cmd, check=False) # Allow some errors in restore + + # Cleanup only Docker container dump file + self.logger.info("Cleaning up dump file from Docker container") + subprocess.run( + ["docker", "exec", container_name, "rm", f"/tmp/{dump_file}"], + check=False, + ) + + return True + + except subprocess.CalledProcessError as e: + self.logger.error(f"Import failed: {e}") + return False + + # def _export_data( + # self, + # pod_name: str, + # db_config: DatabaseConfig, + # container_name: str, + # table_option: str, + # dump_file: str, + # ) -> bool: + # """Export data from local container to OpenShift.""" + # temp_dir = None + # try: + # temp_dir = tempfile.mkdtemp() + # local_dump_path = os.path.join(temp_dir, dump_file) + + # self.logger.info("Starting pg_dump on local container") + # cmd = [ + # "docker", + # "exec", + # container_name, + # "bash", + # "-c", + # f"pg_dump -U {db_config.local_db_user} {table_option} -F t --no-privileges --no-owner -c -d {db_config.db_name} > /tmp/{dump_file}", + # ] + # subprocess.run(cmd, check=True) + + # self.logger.info("Copying dump file from local container") + # subprocess.run( + # ["docker", "cp", f"{container_name}:/tmp/{dump_file}", local_dump_path], + # check=True, + # ) + + # # Create transfer directory structure + # transfer_dir = os.path.join(temp_dir, "tmp_transfer") + # os.makedirs(transfer_dir) + # shutil.move(local_dump_path, os.path.join(transfer_dir, dump_file)) + + # self.logger.info("Uploading dump file to OpenShift pod") + # subprocess.run( + # ["oc", "rsync", transfer_dir, f"{pod_name}:/tmp/"], check=True + # ) + + # self.logger.info("Restoring database on OpenShift pod") + # cmd = [ + # "oc", + # "exec", + # pod_name, + # "--", + # "bash", + # "-c", + # f"pg_restore -U {db_config.remote_db_user} --dbname={db_config.db_name} --no-owner --clean --if-exists --verbose /tmp/tmp_transfer/{dump_file}", + # ] + # subprocess.run(cmd, check=False) # Allow some errors in restore + + # # Cleanup + # self.logger.info("Cleaning up temporary files") + # subprocess.run( + # ["oc", "exec", pod_name, "--", "rm", "-rf", "/tmp/tmp_transfer"], + # check=False, + # ) + # subprocess.run( + # ["docker", "exec", container_name, "rm", f"/tmp/{dump_file}"], + # check=False, + # ) + + # return True + + # except subprocess.CalledProcessError as e: + # self.logger.error(f"Export failed: {e}") + # return False + # finally: + # if temp_dir and os.path.exists(temp_dir): + # shutil.rmtree(temp_dir) + + def verify_database_population(self, app: Application) -> Dict[str, int]: + """Verify that databases are properly populated with data.""" + self.logger.info(f"Verifying {app.value} database population") + + if app == Application.TFRS: + return self._verify_tfrs_population() + else: + return self._verify_lcfs_population() + + def _verify_tfrs_population(self) -> Dict[str, int]: + """Verify TFRS database has required tables and data.""" + required_tables = { + "compliance_report": "SELECT COUNT(*) FROM compliance_report", + "compliance_report_schedule_b_record": "SELECT COUNT(*) FROM compliance_report_schedule_b_record", + "compliance_report_schedule_a_record": "SELECT COUNT(*) FROM compliance_report_schedule_a_record", + "compliance_report_schedule_c_record": "SELECT COUNT(*) FROM compliance_report_schedule_c_record", + "compliance_report_exclusion_agreement_record": "SELECT COUNT(*) FROM compliance_report_exclusion_agreement_record", + } + + results = {} + try: + with get_source_connection() as conn: + with conn.cursor() as cursor: + for table, query in required_tables.items(): + try: + cursor.execute(query) + count = cursor.fetchone()[0] + results[table] = count + self.logger.info(f"TFRS {table}: {count} records") + except Exception as e: + self.logger.warning(f"Could not query {table}: {e}") + results[table] = -1 + except Exception as e: + self.logger.error(f"Could not connect to TFRS database: {e}") + + return results + + def _verify_lcfs_population(self) -> Dict[str, int]: + """Verify LCFS database has required tables and structure.""" + required_tables = { + "compliance_report": "SELECT COUNT(*) FROM compliance_report", + "fuel_supply": "SELECT COUNT(*) FROM fuel_supply", + "notional_transfer": "SELECT COUNT(*) FROM notional_transfer", + "other_uses": "SELECT COUNT(*) FROM other_uses", + "allocation_agreement": "SELECT COUNT(*) FROM allocation_agreement", + } + + results = {} + try: + with get_destination_connection() as conn: + with conn.cursor() as cursor: + for table, query in required_tables.items(): + try: + cursor.execute(query) + count = cursor.fetchone()[0] + results[table] = count + self.logger.info(f"LCFS {table}: {count} records") + except Exception as e: + self.logger.warning(f"Could not query {table}: {e}") + results[table] = -1 + except Exception as e: + self.logger.error(f"Could not connect to LCFS database: {e}") + + return results + + def setup_test_environment( + self, + tfrs_container: str, + lcfs_container: str, + env: Environment = Environment.DEV, + ) -> bool: + """Automatically set up both databases for testing.""" + self.logger.info("Setting up test environment...") + + # Import TFRS data + self.logger.info("Setting up TFRS database...") + tfrs_success = self.transfer_data( + Application.TFRS, env, Direction.IMPORT, tfrs_container + ) + + if not tfrs_success: + self.logger.error("Failed to set up TFRS database") + return False + + # Import LCFS data + self.logger.info("Setting up LCFS database...") + lcfs_success = self.transfer_data( + Application.LCFS, env, Direction.IMPORT, lcfs_container + ) + + if not lcfs_success: + self.logger.error("Failed to set up LCFS database") + return False + + # Verify both databases + tfrs_data = self.verify_database_population(Application.TFRS) + lcfs_data = self.verify_database_population(Application.LCFS) + + # Check if we have sufficient data for migration testing + min_required_records = 10 # Minimum records needed for meaningful testing + + tfrs_ready = ( + tfrs_data.get("compliance_report", 0) >= min_required_records + and tfrs_data.get("compliance_report_schedule_b_record", 0) + >= min_required_records + ) + + lcfs_ready = ( + lcfs_data.get("compliance_report", 0) >= 0 # LCFS can be empty initially + ) + + if tfrs_ready and lcfs_ready: + self.logger.info("✅ Test environment setup completed successfully") + self.logger.info("Databases are ready for migration testing") + return True + else: + self.logger.error("❌ Test environment setup incomplete") + self.logger.error("Insufficient data for migration testing") + return False + + def setup_prod_environment(self, tfrs_container: str, lcfs_container: str) -> bool: + """Set up both databases using production data for testing.""" + self.logger.info("Setting up production environment...") + self.logger.warning("⚠️ This will import PRODUCTION data to local containers") + + # Import TFRS production data + self.logger.info("Setting up TFRS database with production data...") + tfrs_success = self.transfer_data( + Application.TFRS, Environment.PROD, Direction.IMPORT, tfrs_container + ) + + if not tfrs_success: + self.logger.error("Failed to set up TFRS database with production data") + return False + + # Import LCFS production data + self.logger.info("Setting up LCFS database with production data...") + lcfs_success = self.transfer_data( + Application.LCFS, Environment.PROD, Direction.IMPORT, lcfs_container + ) + + if not lcfs_success: + self.logger.error("Failed to set up LCFS database with production data") + return False + + # Verify both databases + tfrs_data = self.verify_database_population(Application.TFRS) + lcfs_data = self.verify_database_population(Application.LCFS) + + # For production data, we expect substantial amounts of data + min_prod_records = 100 # Minimum records expected in production + + tfrs_ready = ( + tfrs_data.get("compliance_report", 0) >= min_prod_records + and tfrs_data.get("compliance_report_schedule_b_record", 0) + >= min_prod_records + ) + + lcfs_ready = ( + lcfs_data.get("compliance_report", 0) >= 0 # LCFS can have existing data + ) + + if tfrs_ready and lcfs_ready: + self.logger.info("✅ Production environment setup completed successfully") + self.logger.info("Databases are loaded with production data") + self.logger.info( + f"TFRS compliance reports: {tfrs_data.get('compliance_report', 0)}" + ) + self.logger.info( + f"TFRS fuel supply records: {tfrs_data.get('compliance_report_schedule_b_record', 0)}" + ) + self.logger.info( + f"LCFS compliance reports: {lcfs_data.get('compliance_report', 0)}" + ) + return True + else: + self.logger.error("❌ Production environment setup incomplete") + self.logger.error("Insufficient production data loaded") + if not tfrs_ready: + self.logger.error( + f"TFRS data insufficient - need at least {min_prod_records} records" + ) + return False + + def check_migration_readiness(self) -> Tuple[bool, List[str]]: + """Check if databases are ready for migration.""" + issues = [] + + # Verify TFRS source data + tfrs_data = self.verify_database_population(Application.TFRS) + if tfrs_data.get("compliance_report", 0) == 0: + issues.append("No compliance reports found in TFRS database") + + # Verify LCFS destination is accessible + lcfs_data = self.verify_database_population(Application.LCFS) + if not lcfs_data: # Empty dict means connection failed + issues.append("Cannot connect to LCFS database") + + # Check for required TFRS tables + required_tfrs_tables = [ + "compliance_report_schedule_b_record", + "compliance_report_schedule_a_record", + "compliance_report_schedule_c_record", + "compliance_report_exclusion_agreement_record", + ] + + for table in required_tfrs_tables: + if tfrs_data.get(table, -1) == -1: + issues.append(f"Cannot access TFRS table: {table}") + + is_ready = len(issues) == 0 + + if is_ready: + self.logger.info("✅ Databases are ready for migration") + else: + self.logger.error("❌ Migration readiness check failed") + for issue in issues: + self.logger.error(f" - {issue}") + + return is_ready, issues + + def reset_database(self, app: Application, container_name: str, env: Environment = Environment.DEV) -> bool: + """Reset database from existing .tar file.""" + db_config = self.get_database_config(app, env) + dump_file = f"{db_config.db_name}.tar" + dump_dir = self._get_dump_dir() + local_dump_path = os.path.join(dump_dir, dump_file) + + # Check if dump file exists + if not os.path.exists(local_dump_path): + self.logger.error(f"Dump file {dump_file} not found in {dump_dir}") + self.logger.error(f"Please run 'make setup-{env.value}' first to download the dump file") + return False + + # Check if Docker container is running + if not self.check_docker_container(container_name): + return False + + self.logger.info(f"Resetting {app.value} database from {dump_file}") + + try: + # Copy dump file to container + self.logger.info(f"Copying dump file to local container {container_name}") + subprocess.run( + ["docker", "cp", local_dump_path, f"{container_name}:/tmp/{dump_file}"], + check=True, + ) + + # Restore database + self.logger.info("Restoring database") + cmd = [ + "docker", + "exec", + container_name, + "bash", + "-c", + f"pg_restore -U {db_config.local_db_user} --dbname={db_config.db_name} --no-owner --clean --if-exists --verbose /tmp/{dump_file}", + ] + subprocess.run(cmd, check=False) # Allow some errors in restore + + # Cleanup container dump file + self.logger.info("Cleaning up dump file from Docker container") + subprocess.run( + ["docker", "exec", container_name, "rm", f"/tmp/{dump_file}"], + check=False, + ) + + self.logger.info(f"✅ {app.value} database reset successfully") + return True + + except subprocess.CalledProcessError as e: + self.logger.error(f"Reset failed: {e}") + return False + + +def main(): + """Main CLI interface.""" + if len(sys.argv) < 2: + print("Usage:") + print( + " python database_manager.py setup [env]" + ) + print( + " python database_manager.py setup-prod " + ) + print(" python database_manager.py verify-tfrs") + print(" python database_manager.py verify-lcfs") + print(" python database_manager.py check-readiness") + print(" python database_manager.py reset [env]") + print( + " python database_manager.py transfer [table]" + ) + sys.exit(1) + + setup = DatabaseSetup() + command = sys.argv[1] + + if command == "setup": + if len(sys.argv) < 4: + print( + "Usage: python database_manager.py setup [env]" + ) + sys.exit(1) + + tfrs_container = sys.argv[2] + lcfs_container = sys.argv[3] + env = Environment(sys.argv[4]) if len(sys.argv) > 4 else Environment.DEV + + success = setup.setup_test_environment(tfrs_container, lcfs_container, env) + sys.exit(0 if success else 1) + + elif command == "setup-prod": + if len(sys.argv) < 4: + print( + "Usage: python database_manager.py setup-prod " + ) + sys.exit(1) + + tfrs_container = sys.argv[2] + lcfs_container = sys.argv[3] + + # Confirm production data import + print("⚠️ WARNING: This will import PRODUCTION data to your local containers!") + print("This may take significant time and will overwrite existing data.") + confirmation = input("Are you sure you want to proceed? (yes/no): ") + + if confirmation.lower() != "yes": + print("Production setup cancelled.") + sys.exit(0) + + success = setup.setup_prod_environment(tfrs_container, lcfs_container) + sys.exit(0 if success else 1) + + elif command == "verify-tfrs": + setup.verify_database_population(Application.TFRS) + + elif command == "verify-lcfs": + setup.verify_database_population(Application.LCFS) + + elif command == "check-readiness": + is_ready, issues = setup.check_migration_readiness() + sys.exit(0 if is_ready else 1) + + elif command == "reset": + if len(sys.argv) < 4: + print("Usage: python database_manager.py reset [env]") + sys.exit(1) + + app = Application(sys.argv[2]) + container = sys.argv[3] + env = Environment(sys.argv[4]) if len(sys.argv) > 4 else Environment.DEV + + success = setup.reset_database(app, container, env) + sys.exit(0 if success else 1) + + elif command == "transfer": + if len(sys.argv) < 6: + print( + "Usage: python database_setup.py transfer [table]" + ) + sys.exit(1) + + app = Application(sys.argv[2]) + env = Environment(sys.argv[3]) + direction = Direction(sys.argv[4]) + container = sys.argv[5] + table = sys.argv[6] if len(sys.argv) > 6 else None + + success = setup.transfer_data(app, env, direction, container, table) + sys.exit(0 if success else 1) + + else: + print(f"Unknown command: {command}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/setup/docker_manager.py b/etl/python_migration/setup/docker_manager.py new file mode 100644 index 000000000..7f17cc6d3 --- /dev/null +++ b/etl/python_migration/setup/docker_manager.py @@ -0,0 +1,473 @@ +""" +Docker management utilities for TFRS to LCFS migration. +Handles automatic container startup and container ID retrieval. +""" + +import os +import sys +import logging +import subprocess +import time +from typing import Dict, List, Optional, Tuple +from pathlib import Path + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from database_manager import DatabaseSetup, Environment, Application + + +class DockerManager: + """Manages Docker containers for migration testing.""" + + def __init__(self): + self.logger = logging.getLogger(__name__) + self.setup_logging() + self.migration_compose_path = ( + Path(__file__).parent.parent / "docker-compose.yml" + ) + self.lcfs_compose_path = ( + Path(__file__).parent.parent / "../../docker-compose.yml" + ) + + def setup_logging(self): + """Set up logging configuration.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + def check_docker_available(self) -> bool: + """Check if Docker is available and running.""" + try: + result = subprocess.run( + ["docker", "--version"], capture_output=True, text=True, check=True + ) + self.logger.info(f"Docker available: {result.stdout.strip()}") + + # Check if Docker daemon is running + subprocess.run(["docker", "ps"], capture_output=True, text=True, check=True) + return True + + except (subprocess.CalledProcessError, FileNotFoundError): + self.logger.error("Docker is not available or not running") + return False + + def check_docker_compose_available(self) -> bool: + """Check if Docker Compose is available.""" + try: + result = subprocess.run( + ["docker", "compose", "version"], + capture_output=True, + text=True, + check=True, + ) + self.logger.info(f"Docker Compose available: {result.stdout.strip()}") + return True + except (subprocess.CalledProcessError, FileNotFoundError): + # Try legacy docker-compose command + try: + result = subprocess.run( + ["docker-compose", "--version"], + capture_output=True, + text=True, + check=True, + ) + self.logger.info( + f"Docker Compose (legacy) available: {result.stdout.strip()}" + ) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + self.logger.error("Docker Compose is not available") + return False + + def get_compose_command(self) -> List[str]: + """Get the appropriate docker compose command.""" + try: + subprocess.run( + ["docker", "compose", "version"], + capture_output=True, + text=True, + check=True, + ) + return ["docker", "compose"] + except (subprocess.CalledProcessError, FileNotFoundError): + return ["docker-compose"] + + def start_tfrs_container(self) -> Optional[str]: + """Start TFRS container using docker-compose and return container ID.""" + if not self.migration_compose_path.exists(): + self.logger.error( + f"Docker compose file not found: {self.migration_compose_path}" + ) + return None + + try: + compose_cmd = self.get_compose_command() + + self.logger.info("Starting TFRS container...") + + # Start the TFRS service + cmd = compose_cmd + [ + "-f", + str(self.migration_compose_path), + "up", + "-d", + "tfrs", + ] + subprocess.run(cmd, check=True, cwd=self.migration_compose_path.parent) + + # Wait for container to be healthy + self.logger.info("Waiting for TFRS container to be ready...") + if self.wait_for_container_healthy("tfrs-migration", timeout=60): + container_id = self.get_container_id("tfrs-migration") + if container_id: + self.logger.info(f"✅ TFRS container ready: {container_id}") + return container_id + else: + self.logger.error("Could not get TFRS container ID") + return None + else: + self.logger.error("TFRS container failed to become healthy") + return None + + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to start TFRS container: {e}") + return None + + def start_lcfs_environment(self) -> Optional[str]: + """Start LCFS environment and return LCFS database container ID.""" + if not self.lcfs_compose_path.exists(): + self.logger.error( + f"LCFS docker compose file not found: {self.lcfs_compose_path}" + ) + return None + + try: + compose_cmd = self.get_compose_command() + + self.logger.info("Starting LCFS environment...") + + # Start the LCFS environment + cmd = compose_cmd + ["-f", str(self.lcfs_compose_path), "up", "-d"] + subprocess.run(cmd, check=True, cwd=self.lcfs_compose_path.parent) + + # Wait for database to be ready + self.logger.info("Waiting for LCFS database to be ready...") + + # Try common LCFS database container names + possible_names = [ + "lcfs-crunchy-postgres-primary", + "lcfs-postgres", + "postgres", + "lcfs-db", + "db", + ] + + for container_name in possible_names: + if self.container_exists(container_name): + if self.wait_for_container_healthy(container_name, timeout=120): + container_id = self.get_container_id(container_name) + if container_id: + self.logger.info( + f"✅ LCFS database ready: {container_id} ({container_name})" + ) + return container_id + + # If no specific container found, list all containers and let user choose + self.logger.warning( + "Could not automatically identify LCFS database container" + ) + self.list_running_containers() + return None + + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to start LCFS environment: {e}") + return None + + def get_container_id(self, container_name: str) -> Optional[str]: + """Get container ID by container name.""" + try: + result = subprocess.run( + ["docker", "ps", "-q", "-f", f"name={container_name}"], + capture_output=True, + text=True, + check=True, + ) + container_id = result.stdout.strip() + return container_id if container_id else None + except subprocess.CalledProcessError: + return None + + def container_exists(self, container_name: str) -> bool: + """Check if a container exists and is running.""" + try: + result = subprocess.run( + [ + "docker", + "ps", + "-f", + f"name={container_name}", + "--format", + "{{.Names}}", + ], + capture_output=True, + text=True, + check=True, + ) + return container_name in result.stdout + except subprocess.CalledProcessError: + return False + + def wait_for_container_healthy( + self, container_name: str, timeout: int = 60 + ) -> bool: + """Wait for container to become healthy or ready.""" + start_time = time.time() + + while time.time() - start_time < timeout: + try: + # Check if container is running + result = subprocess.run( + [ + "docker", + "ps", + "-f", + f"name={container_name}", + "--format", + "{{.Status}}", + ], + capture_output=True, + text=True, + check=True, + ) + + if not result.stdout.strip(): + self.logger.warning(f"Container {container_name} not found") + time.sleep(2) + continue + + # Check health status + health_result = subprocess.run( + [ + "docker", + "inspect", + container_name, + "--format", + "{{.State.Health.Status}}", + ], + capture_output=True, + text=True, + check=False, + ) + + if health_result.returncode == 0: + health_status = health_result.stdout.strip() + if health_status == "healthy": + return True + elif health_status == "unhealthy": + self.logger.warning(f"Container {container_name} is unhealthy") + return False + else: + self.logger.debug( + f"Container {container_name} health status: {health_status}" + ) + else: + # No health check defined, check if container is running + status_result = subprocess.run( + [ + "docker", + "inspect", + container_name, + "--format", + "{{.State.Status}}", + ], + capture_output=True, + text=True, + check=True, + ) + + if status_result.stdout.strip() == "running": + # Additional check for PostgreSQL containers + if self.check_postgres_ready(container_name): + return True + + time.sleep(2) + + except subprocess.CalledProcessError as e: + self.logger.debug(f"Error checking container {container_name}: {e}") + time.sleep(2) + + return False + + def check_postgres_ready(self, container_name: str) -> bool: + """Check if PostgreSQL is ready to accept connections.""" + try: + # Try to connect to PostgreSQL + result = subprocess.run( + ["docker", "exec", container_name, "pg_isready"], + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 + except subprocess.CalledProcessError: + return False + + def list_running_containers(self): + """List all running containers for user reference.""" + try: + result = subprocess.run( + [ + "docker", + "ps", + "--format", + "table {{.ID}}\\t{{.Names}}\\t{{.Status}}", + ], + capture_output=True, + text=True, + check=True, + ) + self.logger.info("Running containers:") + self.logger.info(result.stdout) + except subprocess.CalledProcessError: + self.logger.error("Could not list running containers") + + def stop_containers(self): + """Stop migration-related containers.""" + try: + compose_cmd = self.get_compose_command() + + # Stop TFRS container + if self.migration_compose_path.exists(): + self.logger.info("Stopping TFRS container...") + cmd = compose_cmd + ["-f", str(self.migration_compose_path), "down"] + subprocess.run(cmd, check=True, cwd=self.migration_compose_path.parent) + + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to stop containers: {e}") + + def setup_migration_environment( + self, env: Environment = Environment.DEV, start_lcfs: bool = True + ) -> Tuple[Optional[str], Optional[str]]: + """ + Set up complete migration environment with automatic container management. + + Returns: + Tuple of (tfrs_container_id, lcfs_container_id) + """ + if ( + not self.check_docker_available() + or not self.check_docker_compose_available() + ): + return None, None + + self.logger.info("🚀 Setting up migration environment...") + + # Start TFRS container + tfrs_container_id = self.start_tfrs_container() + if not tfrs_container_id: + self.logger.error("Failed to start TFRS container") + return None, None + + # Start LCFS environment if requested + lcfs_container_id = None + if start_lcfs: + lcfs_container_id = self.start_lcfs_environment() + if not lcfs_container_id: + self.logger.warning("Could not automatically identify LCFS container") + self.logger.info("Please identify the LCFS database container manually") + + # Now set up databases with data + db_setup = DatabaseSetup() + + # Setup TFRS database + self.logger.info("Setting up TFRS database with data...") + from database_manager import Direction + + tfrs_success = db_setup.transfer_data( + Application.TFRS, env, Direction.IMPORT, tfrs_container_id + ) + + if not tfrs_success: + self.logger.error("Failed to setup TFRS database") + return tfrs_container_id, lcfs_container_id + + # Setup LCFS database if container available + if lcfs_container_id: + self.logger.info("Setting up LCFS database with data...") + lcfs_success = db_setup.transfer_data( + Application.LCFS, env, Direction.IMPORT, lcfs_container_id + ) + + if not lcfs_success: + self.logger.warning("Failed to setup LCFS database") + + self.logger.info("✅ Migration environment setup complete") + self.logger.info(f"TFRS Container ID: {tfrs_container_id}") + if lcfs_container_id: + self.logger.info(f"LCFS Container ID: {lcfs_container_id}") + + return tfrs_container_id, lcfs_container_id + + +def main(): + """Main CLI interface.""" + if len(sys.argv) < 2: + print("Usage:") + print(" python docker_manager.py setup [env] [--no-lcfs]") + print(" python docker_manager.py start-tfrs") + print(" python docker_manager.py start-lcfs") + print(" python docker_manager.py stop") + print(" python docker_manager.py list") + sys.exit(1) + + manager = DockerManager() + command = sys.argv[1] + + if command == "setup": + env = ( + Environment(sys.argv[2]) + if len(sys.argv) > 2 and sys.argv[2] in ["dev", "test", "prod"] + else Environment.DEV + ) + start_lcfs = "--no-lcfs" not in sys.argv + + tfrs_id, lcfs_id = manager.setup_migration_environment(env, start_lcfs) + + if tfrs_id: + print(f"TFRS_CONTAINER_ID={tfrs_id}") + if lcfs_id: + print(f"LCFS_CONTAINER_ID={lcfs_id}") + + sys.exit(0 if tfrs_id else 1) + + elif command == "start-tfrs": + container_id = manager.start_tfrs_container() + if container_id: + print(f"TFRS_CONTAINER_ID={container_id}") + sys.exit(0) + else: + sys.exit(1) + + elif command == "start-lcfs": + container_id = manager.start_lcfs_environment() + if container_id: + print(f"LCFS_CONTAINER_ID={container_id}") + sys.exit(0) + else: + sys.exit(1) + + elif command == "stop": + manager.stop_containers() + + elif command == "list": + manager.list_running_containers() + + else: + print(f"Unknown command: {command}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/setup/migration_orchestrator.py b/etl/python_migration/setup/migration_orchestrator.py new file mode 100644 index 000000000..e5564a001 --- /dev/null +++ b/etl/python_migration/setup/migration_orchestrator.py @@ -0,0 +1,510 @@ +""" +Complete migration orchestrator that sets up databases, runs migrations, and validates results. +""" + +import sys +import os +import logging +import argparse +from datetime import datetime +from typing import List, Dict, Any, Optional, Tuple + +# Add current directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from setup.database_manager import DatabaseSetup, Environment, Application +from setup.docker_manager import DockerManager +from migrations.run_all_migrations import MigrationRunner +from setup.validation_runner import run_all_validations, save_results_to_file + + +class MigrationOrchestrator: + """Orchestrates the complete migration process from setup to validation.""" + + def __init__(self): + self.setup_logging() + self.logger = logging.getLogger(__name__) + self.db_setup = DatabaseSetup() + self.docker_manager = DockerManager() + + def setup_logging(self): + """Set up logging configuration.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + def run_complete_migration( + self, + tfrs_container: str = None, + lcfs_container: str = None, + env: Environment = Environment.DEV, + skip_setup: bool = False, + skip_validation: bool = False, + ) -> bool: + """ + Run the complete migration process: + 1. Set up databases (optional) + 2. Verify migration readiness + 3. Run migrations + 4. Run validations + """ + start_time = datetime.now() + + self.logger.info("=" * 80) + self.logger.info("TFRS TO LCFS COMPLETE MIGRATION PROCESS") + self.logger.info(f"Started: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + self.logger.info("=" * 80) + + try: + # Step 1: Database Setup (optional) + if not skip_setup and tfrs_container and lcfs_container: + self.logger.info("\n🔄 STEP 1: Setting up test databases") + if not self.db_setup.setup_test_environment( + tfrs_container, lcfs_container, env + ): + self.logger.error("❌ Database setup failed") + return False + self.logger.info("✅ Database setup completed") + else: + self.logger.info("\n⏭️ STEP 1: Skipping database setup") + + # Step 2: Migration Readiness Check + self.logger.info("\n🔍 STEP 2: Checking migration readiness") + is_ready, issues = self.db_setup.check_migration_readiness() + + if not is_ready: + self.logger.error("❌ Migration readiness check failed:") + for issue in issues: + self.logger.error(f" - {issue}") + return False + + self.logger.info("✅ Migration readiness check passed") + + # Step 3: Run Migrations + self.logger.info("\n🚀 STEP 3: Running data migrations") + migration_success = self._run_migrations() + + if not migration_success: + self.logger.error("❌ Migration process failed") + return False + + self.logger.info("✅ Migration process completed") + + # Step 4: Run Validations (optional) + if not skip_validation: + self.logger.info("\n🔎 STEP 4: Running validation scripts") + validation_success = self._run_validations() + + if not validation_success: + self.logger.error("❌ Validation process failed") + return False + + self.logger.info("✅ Validation process completed") + else: + self.logger.info("\n⏭️ STEP 4: Skipping validation") + + # Success Summary + end_time = datetime.now() + duration = end_time - start_time + + self.logger.info("\n" + "=" * 80) + self.logger.info("🎉 COMPLETE MIGRATION PROCESS SUCCESSFUL!") + self.logger.info(f"Duration: {duration}") + self.logger.info(f"Completed: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + self.logger.info("=" * 80) + + return True + + except Exception as e: + self.logger.error(f"❌ Migration process failed with error: {e}") + return False + + def _run_migrations(self) -> bool: + """Run all migration scripts.""" + try: + runner = MigrationRunner() + + # Import and run migrations + migration_results = runner.run_all_migrations() + + # Check for any failures + failed_migrations = [ + name + for name, result in migration_results.items() + if not result.get("success", False) + ] + + if failed_migrations: + self.logger.error(f"Failed migrations: {failed_migrations}") + return False + + # Log summary + total_migrations = len(migration_results) + self.logger.info( + f"All {total_migrations} migrations completed successfully" + ) + + # Log record counts + for migration_name, result in migration_results.items(): + if "records_processed" in result: + self.logger.info( + f" - {migration_name}: {result['records_processed']} records" + ) + + return True + + except Exception as e: + self.logger.error(f"Migration execution failed: {e}") + return False + + def _run_validations(self) -> bool: + """Run all validation scripts.""" + try: + # Run validations + validation_results = run_all_validations() + + # Save detailed results + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"complete_migration_validation_{timestamp}.json" + save_results_to_file(validation_results, filename) + + # Check for any failures + failed_validations = [ + name + for name, result in validation_results.items() + if result.get("status") == "failed" + ] + + if failed_validations: + self.logger.error(f"Failed validations: {failed_validations}") + return False + + # Log summary + total_validations = len(validation_results) + self.logger.info( + f"All {total_validations} validations completed successfully" + ) + + # Log key metrics + for validation_name, result in validation_results.items(): + if result.get("status") == "success" and "results" in result: + results = result["results"] + if "record_counts" in results: + counts = results["record_counts"] + if isinstance(counts, dict) and "source_count" in counts: + self.logger.info( + f" - {validation_name}: {counts['source_count']} → {counts['dest_count']} records" + ) + + return True + + except Exception as e: + self.logger.error(f"Validation execution failed: {e}") + return False + + def run_database_setup_only( + self, + tfrs_container: str, + lcfs_container: str, + env: Environment = Environment.DEV, + ) -> bool: + """Run only the database setup portion.""" + self.logger.info("🔄 Running database setup only") + return self.db_setup.setup_test_environment(tfrs_container, lcfs_container, env) + + def run_prod_database_setup_only( + self, tfrs_container: str, lcfs_container: str + ) -> bool: + """Run only the database setup portion using production data.""" + self.logger.info("🔄 Running production database setup only") + self.logger.warning("⚠️ This will import PRODUCTION data to local containers") + return self.db_setup.setup_prod_environment(tfrs_container, lcfs_container) + + def run_migration_only(self) -> bool: + """Run only the migration portion (assumes databases are already set up).""" + self.logger.info("🚀 Running migration only") + + # Check readiness first + is_ready, issues = self.db_setup.check_migration_readiness() + if not is_ready: + self.logger.error("Migration readiness check failed:") + for issue in issues: + self.logger.error(f" - {issue}") + return False + + return self._run_migrations() + + def run_validation_only(self) -> bool: + """Run only the validation portion (assumes migration is complete).""" + self.logger.info("🔎 Running validation only") + return self._run_validations() + + def run_auto_environment_setup( + self, env: Environment = Environment.DEV, start_lcfs: bool = True + ) -> Tuple[Optional[str], Optional[str]]: + """ + Automatically set up migration environment with Docker containers. + + Returns: + Tuple of (tfrs_container_id, lcfs_container_id) + """ + self.logger.info("🐳 Setting up automatic Docker environment") + + tfrs_id, lcfs_id = self.docker_manager.setup_migration_environment( + env, start_lcfs + ) + + if tfrs_id: + self.logger.info(f"✅ TFRS container ready: {tfrs_id}") + else: + self.logger.error("❌ Failed to set up TFRS container") + + if start_lcfs: + if lcfs_id: + self.logger.info(f"✅ LCFS container ready: {lcfs_id}") + else: + self.logger.warning("⚠️ LCFS container not automatically identified") + + return tfrs_id, lcfs_id + + def run_complete_migration_auto( + self, env: Environment = Environment.DEV, skip_validation: bool = False + ) -> bool: + """ + Run complete migration with automatic Docker environment setup. + """ + start_time = datetime.now() + + self.logger.info("=" * 80) + self.logger.info("TFRS TO LCFS COMPLETE MIGRATION PROCESS (AUTO DOCKER)") + self.logger.info(f"Started: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + self.logger.info("=" * 80) + + try: + # Step 1: Auto Docker Environment Setup + self.logger.info("\n🐳 STEP 1: Setting up Docker environment automatically") + tfrs_id, lcfs_id = self.run_auto_environment_setup(env, start_lcfs=True) + + if not tfrs_id: + self.logger.error("❌ Auto environment setup failed") + return False + + self.logger.info("✅ Auto environment setup completed") + + # Step 2: Migration Readiness Check + self.logger.info("\n🔍 STEP 2: Checking migration readiness") + is_ready, issues = self.db_setup.check_migration_readiness() + + if not is_ready: + self.logger.error("❌ Migration readiness check failed:") + for issue in issues: + self.logger.error(f" - {issue}") + return False + + self.logger.info("✅ Migration readiness check passed") + + # Step 3: Run Migrations + self.logger.info("\n🚀 STEP 3: Running data migrations") + migration_success = self._run_migrations() + + if not migration_success: + self.logger.error("❌ Migration process failed") + return False + + self.logger.info("✅ Migration process completed") + + # Step 4: Run Validations (optional) + if not skip_validation: + self.logger.info("\n🔎 STEP 4: Running validation scripts") + validation_success = self._run_validations() + + if not validation_success: + self.logger.error("❌ Validation process failed") + return False + + self.logger.info("✅ Validation process completed") + else: + self.logger.info("\n⏭️ STEP 4: Skipping validation") + + # Success Summary + end_time = datetime.now() + duration = end_time - start_time + + self.logger.info("\n" + "=" * 80) + self.logger.info("🎉 COMPLETE AUTO MIGRATION PROCESS SUCCESSFUL!") + self.logger.info(f"TFRS Container: {tfrs_id}") + if lcfs_id: + self.logger.info(f"LCFS Container: {lcfs_id}") + self.logger.info(f"Duration: {duration}") + self.logger.info(f"Completed: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + self.logger.info("=" * 80) + + return True + + except Exception as e: + self.logger.error(f"❌ Auto migration process failed with error: {e}") + return False + + +def main(): + """Main CLI interface.""" + parser = argparse.ArgumentParser( + description="TFRS to LCFS Complete Migration Orchestrator" + ) + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Complete migration command + complete_parser = subparsers.add_parser( + "complete", help="Run complete migration process" + ) + complete_parser.add_argument("--tfrs-container", help="TFRS Docker container name") + complete_parser.add_argument("--lcfs-container", help="LCFS Docker container name") + complete_parser.add_argument( + "--env", + choices=["dev", "test", "prod"], + default="dev", + help="Environment to use for data transfer", + ) + complete_parser.add_argument( + "--skip-setup", + action="store_true", + help="Skip database setup (use existing data)", + ) + complete_parser.add_argument( + "--skip-validation", action="store_true", help="Skip validation after migration" + ) + + # Setup only command + setup_parser = subparsers.add_parser("setup", help="Setup databases only") + setup_parser.add_argument("tfrs_container", help="TFRS Docker container name") + setup_parser.add_argument("lcfs_container", help="LCFS Docker container name") + setup_parser.add_argument( + "--env", + choices=["dev", "test", "prod"], + default="dev", + help="Environment to use for data transfer", + ) + + # Production setup command + setup_prod_parser = subparsers.add_parser( + "setup-prod", help="Setup databases with production data" + ) + setup_prod_parser.add_argument("tfrs_container", help="TFRS Docker container name") + setup_prod_parser.add_argument("lcfs_container", help="LCFS Docker container name") + + # Auto setup command (Docker containers managed automatically) + auto_setup_parser = subparsers.add_parser( + "auto-setup", help="Automatically setup Docker environment" + ) + auto_setup_parser.add_argument( + "--env", + choices=["dev", "test", "prod"], + default="dev", + help="Environment to use for data transfer", + ) + auto_setup_parser.add_argument( + "--no-lcfs", action="store_true", help="Skip LCFS environment setup" + ) + + # Complete auto migration command + auto_complete_parser = subparsers.add_parser( + "auto-complete", help="Complete migration with auto Docker setup" + ) + auto_complete_parser.add_argument( + "--env", + choices=["dev", "test", "prod"], + default="dev", + help="Environment to use for data transfer", + ) + auto_complete_parser.add_argument( + "--skip-validation", action="store_true", help="Skip validation after migration" + ) + + # Migration only command + subparsers.add_parser("migrate", help="Run migration only (assumes setup complete)") + + # Validation only command + subparsers.add_parser( + "validate", help="Run validation only (assumes migration complete)" + ) + + # Readiness check command + subparsers.add_parser("check", help="Check migration readiness") + + args = parser.parse_args() + + if not args.command: + parser.print_help() + sys.exit(1) + + orchestrator = MigrationOrchestrator() + + if args.command == "complete": + success = orchestrator.run_complete_migration( + tfrs_container=args.tfrs_container, + lcfs_container=args.lcfs_container, + env=Environment(args.env), + skip_setup=args.skip_setup, + skip_validation=args.skip_validation, + ) + + elif args.command == "setup": + success = orchestrator.run_database_setup_only( + args.tfrs_container, args.lcfs_container, Environment(args.env) + ) + + elif args.command == "setup-prod": + # Confirm production data import + print("⚠️ WARNING: This will import PRODUCTION data to your local containers!") + print("This may take significant time and will overwrite existing data.") + confirmation = input("Are you sure you want to proceed? (yes/no): ") + + if confirmation.lower() != "yes": + print("Production setup cancelled.") + sys.exit(0) + + success = orchestrator.run_prod_database_setup_only( + args.tfrs_container, args.lcfs_container + ) + + elif args.command == "auto-setup": + tfrs_id, lcfs_id = orchestrator.run_auto_environment_setup( + Environment(args.env), start_lcfs=not args.no_lcfs + ) + + success = tfrs_id is not None + if success: + print(f"✅ Auto setup completed") + print(f"TFRS Container ID: {tfrs_id}") + if lcfs_id: + print(f"LCFS Container ID: {lcfs_id}") + + elif args.command == "auto-complete": + success = orchestrator.run_complete_migration_auto( + Environment(args.env), skip_validation=args.skip_validation + ) + + elif args.command == "migrate": + success = orchestrator.run_migration_only() + + elif args.command == "validate": + success = orchestrator.run_validation_only() + + elif args.command == "check": + success, issues = orchestrator.db_setup.check_migration_readiness() + if not success: + print("Migration readiness issues:") + for issue in issues: + print(f" - {issue}") + + else: + parser.print_help() + sys.exit(1) + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/setup/validation_runner.py b/etl/python_migration/setup/validation_runner.py new file mode 100644 index 000000000..02924b52f --- /dev/null +++ b/etl/python_migration/setup/validation_runner.py @@ -0,0 +1,113 @@ +""" +Run all validation scripts for TFRS to LCFS migration. +""" +import sys +import os +from typing import Dict, Any +import json +from datetime import datetime + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from validation.validate_allocation_agreements import AllocationAgreementValidator +from validation.validate_fuel_supply import FuelSupplyValidator +from validation.validate_notional_transfers import NotionalTransferValidator +from validation.validate_other_uses import OtherUsesValidator + + +def run_all_validations() -> Dict[str, Any]: + """Run all validation scripts and return consolidated results.""" + print("=" * 60) + print("TFRS TO LCFS MIGRATION VALIDATION REPORT") + print(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 60) + + all_results = {} + + # List of validators to run + validators = [ + ("allocation_agreement", AllocationAgreementValidator()), + ("fuel_supply", FuelSupplyValidator()), + ("notional_transfer", NotionalTransferValidator()), + ("other_uses", OtherUsesValidator()) + ] + + for validator_name, validator in validators: + print(f"\n{'='*20} {validator_name.upper().replace('_', ' ')} {'='*20}") + + try: + results = validator.run_validation() + all_results[validator_name] = { + 'status': 'success', + 'results': results + } + print(f"✓ {validator_name} validation completed successfully") + + except Exception as e: + print(f"✗ {validator_name} validation failed: {str(e)}") + all_results[validator_name] = { + 'status': 'failed', + 'error': str(e) + } + + # Generate summary report + print("\n" + "=" * 60) + print("VALIDATION SUMMARY") + print("=" * 60) + + for validator_name, result in all_results.items(): + status = result['status'] + status_symbol = "✓" if status == 'success' else "✗" + print(f"{status_symbol} {validator_name.replace('_', ' ').title()}: {status.upper()}") + + if status == 'success' and 'results' in result: + # Show key metrics + results = result['results'] + if 'record_counts' in results: + counts = results['record_counts'] + if isinstance(counts, dict) and 'source_count' in counts: + print(f" Source: {counts['source_count']}, Dest: {counts['dest_count']}, " + f"Diff: {counts['difference']}") + elif isinstance(counts, dict) and 'standard' in counts: + # Fuel supply has nested structure + std = counts['standard'] + print(f" Source: {std['source_count']}, Dest: {std['dest_count']}, " + f"Diff: {std['difference']}") + + return all_results + + +def save_results_to_file(results: Dict[str, Any], filename: str = None): + """Save validation results to a JSON file.""" + if filename is None: + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + filename = f"validation_results_{timestamp}.json" + + filepath = os.path.join(os.path.dirname(__file__), filename) + + with open(filepath, 'w') as f: + json.dump(results, f, indent=2, default=str) + + print(f"\nValidation results saved to: {filepath}") + + +if __name__ == "__main__": + try: + results = run_all_validations() + save_results_to_file(results) + + # Check if any validations failed + failed_validations = [name for name, result in results.items() + if result['status'] == 'failed'] + + if failed_validations: + print(f"\n⚠️ {len(failed_validations)} validation(s) failed: {', '.join(failed_validations)}") + sys.exit(1) + else: + print(f"\n🎉 All {len(results)} validations completed successfully!") + sys.exit(0) + + except Exception as e: + print(f"\n❌ Critical error running validations: {e}") + sys.exit(1) \ No newline at end of file diff --git a/etl/python_migration/validation/README.md b/etl/python_migration/validation/README.md new file mode 100644 index 000000000..2df68b024 --- /dev/null +++ b/etl/python_migration/validation/README.md @@ -0,0 +1,128 @@ +# LCFS Migration Validation Scripts + +This folder contains Python validation scripts that verify the integrity and correctness of data migrated from TFRS to LCFS systems. + +## Overview + +The validation scripts are Python equivalents of the original Groovy validation scripts, designed to: + +1. **Compare record counts** between source (TFRS) and destination (LCFS) systems +2. **Validate sample records** to ensure data integrity and correct mapping +3. **Check for data anomalies** such as NULL values in critical fields +4. **Verify version chain integrity** for records with group_uuid versioning +5. **Validate action type distributions** (CREATE vs UPDATE) +6. **Ensure no new-period records were impacted** by the ETL process +7. **Check for duplicate records** and other data quality issues + +## Available Validators + +### 1. validate_allocation_agreements.py +- Validates migration of allocation agreement records +- Checks transaction type mapping +- Verifies partner information and quantities + +### 2. validate_fuel_supply.py +- Validates Schedule B (fuel supply) record migration +- Includes special validation for GHGenius records +- Checks calculation consistency (energy, compliance units) +- Verifies carbon intensity mappings + +### 3. validate_notional_transfers.py +- Validates Schedule A (notional transfer) record migration +- Checks transfer type mapping (Received vs Transferred) +- Verifies trading partner information + +### 4. validate_other_uses.py +- Validates Schedule C (other uses) record migration +- Checks expected use type mappings +- Handles cases where expected_use table may not exist in source + +## Usage + +### Run Individual Validators + +```bash +# From the validation directory +python validate_allocation_agreements.py +python validate_fuel_supply.py +python validate_notional_transfers.py +python validate_other_uses.py +``` + +### Run All Validations + +```bash +# From the setup directory +python ../setup/validation_runner.py +``` + +This will: +- Execute all validators in sequence +- Generate a comprehensive report +- Save results to a timestamped JSON file +- Exit with appropriate status codes + +## Configuration + +The validators use the same configuration as the migration scripts: +- Database connections are managed through `../config.py` +- Environment variables should be set in `.env` file in the parent directory + +## Output + +Validators produce: +1. **Console output** with detailed validation results +2. **JSON report files** with structured results for further analysis +3. **Status indicators** (✓ for success, ✗ for failures) + +## Key Validation Patterns + +### Record Count Comparison +```python +results['record_counts'] = self.compare_record_counts( + source_query="SELECT COUNT(*) FROM source_table", + dest_query="SELECT COUNT(*) FROM dest_table WHERE create_user = 'ETL'" +) +``` + +### Sample Record Validation +- Validates 10 sample records by default +- Matches records based on key fields (legacy_id, quantities, etc.) +- Reports match success rate + +### Version Chain Validation +- Identifies records with multiple versions (group_uuid) +- Verifies versions are sequential +- Reports version chain integrity + +### Data Anomaly Detection +- Checks for NULL values in critical fields +- Reports counts of problematic records +- Helps identify mapping issues + +## GHGenius Processing Note + +The fuel supply validator includes special handling for GHGenius records, which require carbon intensity values to be calculated from Schedule D data. See `ghgenius_processing.md` for detailed information about this process. + +## Error Handling + +- Validators gracefully handle missing tables or connection issues +- Detailed error messages help diagnose problems +- Failed validations don't prevent other validators from running + +## Integration + +These validation scripts are designed to be run: +1. **After migration scripts** to verify results +2. **As part of CI/CD pipelines** for automated validation +3. **During testing** to catch regressions +4. **Before go-live** as final verification + +## Extending Validators + +To add new validation checks: + +1. Inherit from `BaseValidator` +2. Implement required abstract methods +3. Use provided utility methods for common patterns +4. Add to `run_all_validations.py` for integration \ No newline at end of file diff --git a/etl/python_migration/validation/__init__.py b/etl/python_migration/validation/__init__.py new file mode 100644 index 000000000..ff0aee822 --- /dev/null +++ b/etl/python_migration/validation/__init__.py @@ -0,0 +1 @@ +# Validation package for TFRS to LCFS migration scripts \ No newline at end of file diff --git a/etl/python_migration/validation/validate_2019_fixes.py b/etl/python_migration/validation/validate_2019_fixes.py new file mode 100644 index 000000000..ab86520f9 --- /dev/null +++ b/etl/python_migration/validation/validate_2019_fixes.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +""" +Validation Script for 2019-2023 Migration Fixes + +Tests the fixes applied to resolve: +1. Missing standalone exclusion reports (Eco-energy LLC, 102078290 Saskatchewan, Idemitsu Apollo) +2. Missing allocation sections for combo compliance/exclusion reports +3. Incorrect Line 22 balance calculations +4. Credit issuance display issues (City of Surrey) + +This script validates that all reported issues have been resolved across historical years 2019-2023. +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +import sys +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import setup_logging + +logger = logging.getLogger(__name__) + + +class MigrationFixesValidator: + def __init__(self): + self.test_results = { + "orphaned_reports": {"passed": 0, "failed": 0, "details": []}, + "allocation_sections": {"passed": 0, "failed": 0, "details": []}, + "line_22_balances": {"passed": 0, "failed": 0, "details": []}, + "credit_issuance": {"passed": 0, "failed": 0, "details": []}, + } + + # Test configuration - no specific organization names for privacy + self.test_years = ["2019", "2020", "2021", "2022", "2023"] + + def test_orphaned_reports_fixed(self, lcfs_cursor) -> bool: + """Test that standalone exclusion reports without main compliance reports are properly migrated""" + logger.info("Testing orphaned exclusion reports...") + + # Count standalone exclusion reports (reports with allocation agreements but no main compliance data) + query = """ + SELECT COUNT(DISTINCT cr.compliance_report_id) as orphaned_count + FROM compliance_report cr + JOIN compliance_period cp ON cr.compliance_period_id = cp.compliance_period_id + JOIN allocation_agreement aa ON cr.compliance_report_id = aa.compliance_report_id + WHERE cp.description IN ('2019', '2020', '2021', '2022', '2023') + AND cr.legacy_id IS NOT NULL + AND NOT EXISTS ( + -- Check if there's a main compliance report for the same org/period + SELECT 1 FROM compliance_report cr2 + JOIN compliance_report_summary crs ON cr2.compliance_report_id = crs.compliance_report_id + WHERE cr2.organization_id = cr.organization_id + AND cr2.compliance_period_id = cr.compliance_period_id + AND cr2.compliance_report_id != cr.compliance_report_id + AND (crs.line_1_fossil_derived_base_fuel_gasoline > 0 + OR crs.line_1_fossil_derived_base_fuel_diesel > 0) + ) + """ + + lcfs_cursor.execute(query) + result = lcfs_cursor.fetchone() + orphaned_count = result[0] if result else 0 + + if orphaned_count > 0: + self.test_results["orphaned_reports"]["passed"] += orphaned_count + self.test_results["orphaned_reports"]["details"].append( + f"✓ Found {orphaned_count} standalone exclusion reports with allocation data" + ) + logger.info(f"✓ Found {orphaned_count} properly migrated orphaned exclusion reports") + else: + self.test_results["orphaned_reports"]["failed"] += 1 + self.test_results["orphaned_reports"]["details"].append( + "? No standalone exclusion reports found - this may be expected" + ) + logger.info("? No standalone exclusion reports found - this may be expected") + + return True # This test is informational, not a failure condition + + def test_allocation_sections_present(self, lcfs_cursor) -> bool: + """Test that reports with both compliance and allocation data are properly migrated""" + logger.info("Testing allocation sections for combo reports...") + + # Find reports that have both main compliance data AND allocation agreements + query = """ + SELECT + cp.description as period, + COUNT(DISTINCT cr.compliance_report_id) as reports_with_allocations, + SUM(allocation_counts.allocation_count) as total_allocations + FROM compliance_report cr + JOIN compliance_period cp ON cr.compliance_period_id = cp.compliance_period_id + JOIN compliance_report_summary crs ON cr.compliance_report_id = crs.compliance_report_id + JOIN ( + SELECT + compliance_report_id, + COUNT(*) as allocation_count + FROM allocation_agreement + GROUP BY compliance_report_id + ) allocation_counts ON cr.compliance_report_id = allocation_counts.compliance_report_id + WHERE cp.description IN ('2019', '2020', '2021', '2022', '2023') + AND cr.legacy_id IS NOT NULL + AND (crs.line_1_fossil_derived_base_fuel_gasoline > 0 + OR crs.line_1_fossil_derived_base_fuel_diesel > 0) + GROUP BY cp.description + ORDER BY cp.description + """ + + lcfs_cursor.execute(query) + results = lcfs_cursor.fetchall() + + total_combo_reports = 0 + total_allocations = 0 + + for row in results: + period, report_count, allocation_count = row + total_combo_reports += report_count + total_allocations += allocation_count or 0 + + self.test_results["allocation_sections"]["details"].append( + f"✓ {period}: {report_count} combo reports with {allocation_count or 0} total allocations" + ) + logger.info(f"✓ {period}: {report_count} combo reports with {allocation_count or 0} allocations") + + if total_combo_reports > 0: + self.test_results["allocation_sections"]["passed"] = total_combo_reports + self.test_results["allocation_sections"]["details"].append( + f"✓ Total: {total_combo_reports} combo reports with {total_allocations} allocation agreements" + ) + logger.info(f"✓ Found {total_combo_reports} combo reports with allocation sections") + else: + self.test_results["allocation_sections"]["failed"] += 1 + self.test_results["allocation_sections"]["details"].append( + "✗ No combo compliance/allocation reports found" + ) + logger.warning("✗ No combo compliance/allocation reports found") + + return total_combo_reports > 0 + + def test_line_22_calculations(self, lcfs_cursor) -> bool: + """Test that Line 22 shows correct balance values""" + logger.info("Testing Line 22 balance calculations...") + + query = """ + SELECT + cr.compliance_report_id, + cp.description as period, + crs.line_22_compliance_units_issued, + crs.line_18_units_to_be_banked, + crs.line_17_non_banked_units_used + FROM compliance_report cr + JOIN compliance_period cp ON cr.compliance_period_id = cp.compliance_period_id + LEFT JOIN compliance_report_summary crs ON cr.compliance_report_id = crs.compliance_report_id + WHERE cp.description IN ('2019', '2020', '2021', '2022', '2023') + AND cr.legacy_id IS NOT NULL + AND crs.line_22_compliance_units_issued IS NOT NULL + ORDER BY cp.description, cr.compliance_report_id + """ + + lcfs_cursor.execute(query) + results = lcfs_cursor.fetchall() + + issues_found = 0 + valid_calculations = 0 + + for row in results: + cr_id, period, line_22, line_18, line_17 = row + + # Line 22 should not be the same as Line 18 (credits issued) + # It should represent available balance at period end + if line_22 == line_18 and line_18 != 0: + issues_found += 1 + self.test_results["line_22_balances"]["details"].append( + f"✗ Report {cr_id} ({period}): Line 22 ({line_22}) equals Line 18 ({line_18}) - likely showing issued credits instead of balance" + ) + else: + valid_calculations += 1 + + if issues_found == 0: + self.test_results["line_22_balances"]["passed"] = len(results) + self.test_results["line_22_balances"]["details"].append( + f"✓ All {len(results)} Line 22 calculations appear correct" + ) + logger.info(f"✓ All {len(results)} Line 22 calculations validated") + else: + self.test_results["line_22_balances"]["failed"] = issues_found + self.test_results["line_22_balances"]["passed"] = valid_calculations + logger.warning(f"✗ Found {issues_found} Line 22 calculation issues") + + return issues_found == 0 + + def test_credit_issuance_display(self, lcfs_cursor) -> bool: + """Test that credit issuance values are properly displayed (not showing as 0 when credits exist)""" + logger.info("Testing credit issuance display...") + + query = """ + SELECT + cp.description as period, + COUNT(CASE WHEN crs.line_18_units_to_be_banked > 0 THEN 1 END) as reports_with_line_18, + COUNT(CASE WHEN crs.line_14_low_carbon_fuel_surplus > 0 THEN 1 END) as reports_with_line_14, + SUM(crs.line_18_units_to_be_banked) as total_line_18, + SUM(crs.line_14_low_carbon_fuel_surplus) as total_line_14, + COUNT(*) as total_reports + FROM compliance_report cr + JOIN compliance_period cp ON cr.compliance_period_id = cp.compliance_period_id + LEFT JOIN compliance_report_summary crs ON cr.compliance_report_id = crs.compliance_report_id + WHERE cp.description IN ('2019', '2020', '2021', '2022', '2023') + AND cr.legacy_id IS NOT NULL + AND crs.compliance_report_id IS NOT NULL + GROUP BY cp.description + ORDER BY cp.description + """ + + lcfs_cursor.execute(query) + results = lcfs_cursor.fetchall() + + total_reports_with_credits = 0 + + for row in results: + period, reports_18, reports_14, total_18, total_14, total_reports = row + + reports_with_credits = reports_18 + reports_14 + total_reports_with_credits += reports_with_credits + + self.test_results["credit_issuance"]["details"].append( + f"✓ {period}: {reports_with_credits}/{total_reports} reports show credit issuance (Line 18: {reports_18}, Line 14: {reports_14})" + ) + logger.info(f"✓ {period}: {reports_with_credits} reports with credit issuance") + + if total_reports_with_credits > 0: + self.test_results["credit_issuance"]["passed"] = total_reports_with_credits + self.test_results["credit_issuance"]["details"].append( + f"✓ Total: {total_reports_with_credits} reports show proper credit issuance values" + ) + logger.info(f"✓ Found {total_reports_with_credits} reports with proper credit issuance display") + else: + self.test_results["credit_issuance"]["failed"] += 1 + self.test_results["credit_issuance"]["details"].append( + "✗ No reports show credit issuance values > 0" + ) + logger.warning("✗ All credit issuance values showing as 0") + + return total_reports_with_credits > 0 + + def validate_all_fixes(self) -> bool: + """Run all validation tests""" + logger.info("Starting validation of migration fixes for 2019-2023...") + + all_tests_passed = True + + try: + with get_destination_connection() as lcfs_conn: + lcfs_cursor = lcfs_conn.cursor() + + # Run all validation tests + tests = [ + ("Orphaned Reports", self.test_orphaned_reports_fixed), + ("Allocation Sections", self.test_allocation_sections_present), + ("Line 22 Calculations", self.test_line_22_calculations), + ("Credit Issuance Display", self.test_credit_issuance_display), + ] + + for test_name, test_func in tests: + logger.info(f"\n--- Running {test_name} Test ---") + try: + result = test_func(lcfs_cursor) + if not result: + all_tests_passed = False + logger.error(f"❌ {test_name} test FAILED") + else: + logger.info(f"✅ {test_name} test PASSED") + except Exception as e: + logger.error(f"❌ {test_name} test ERROR: {e}") + all_tests_passed = False + + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Validation failed with error: {e}") + all_tests_passed = False + + return all_tests_passed + + def print_summary(self): + """Print detailed validation summary""" + logger.info("\n" + "="*80) + logger.info("MIGRATION FIXES VALIDATION SUMMARY") + logger.info("="*80) + + for test_name, results in self.test_results.items(): + passed = results["passed"] + failed = results["failed"] + total = passed + failed + + if total > 0: + success_rate = (passed / total) * 100 + logger.info(f"\n📊 {test_name.replace('_', ' ').title()}:") + logger.info(f" ✅ Passed: {passed}") + logger.info(f" ❌ Failed: {failed}") + logger.info(f" 📈 Success Rate: {success_rate:.1f}%") + + # Print details + if results["details"]: + logger.info(" 📋 Details:") + for detail in results["details"]: + logger.info(f" {detail}") + else: + logger.info(f"\n📊 {test_name.replace('_', ' ').title()}: No tests run") + + # Overall summary + total_passed = sum(r["passed"] for r in self.test_results.values()) + total_failed = sum(r["failed"] for r in self.test_results.values()) + total_tests = total_passed + total_failed + + if total_tests > 0: + overall_success = (total_passed / total_tests) * 100 + logger.info(f"\n🎯 OVERALL RESULTS:") + logger.info(f" Total Tests: {total_tests}") + logger.info(f" Passed: {total_passed}") + logger.info(f" Failed: {total_failed}") + logger.info(f" Success Rate: {overall_success:.1f}%") + + if total_failed == 0: + logger.info(" 🎉 ALL TESTS PASSED! Migration fixes are working correctly.") + else: + logger.warning(f" ⚠️ {total_failed} tests failed. Review the details above.") + + logger.info("="*80) + + +def main(): + setup_logging() + logger.info("Starting Migration Fixes Validation for 2019-2023") + + validator = MigrationFixesValidator() + + try: + success = validator.validate_all_fixes() + validator.print_summary() + + if success: + logger.info("🎉 All migration fixes validated successfully!") + else: + logger.error("❌ Some migration fixes need attention.") + sys.exit(1) + + except Exception as e: + logger.error(f"Validation failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/etl/python_migration/validation/validate_allocation_agreements.py b/etl/python_migration/validation/validate_allocation_agreements.py new file mode 100644 index 000000000..01ef36143 --- /dev/null +++ b/etl/python_migration/validation/validate_allocation_agreements.py @@ -0,0 +1,193 @@ +""" +Allocation Agreement validation script for TFRS to LCFS migration. +""" + +from typing import Dict, Any, List +from .validation_base import BaseValidator +from core.database import get_source_connection, get_destination_connection + + +class AllocationAgreementValidator(BaseValidator): + """Validator for allocation agreement migration.""" + + def get_validation_name(self) -> str: + return "Allocation Agreement" + + def validate(self) -> Dict[str, Any]: + """Run allocation agreement validation.""" + results = {} + + # 1. Compare record counts + results["record_counts"] = self.compare_record_counts( + source_query="SELECT COUNT(*) FROM compliance_report_exclusion_agreement_record", + dest_query="SELECT COUNT(*) FROM allocation_agreement WHERE user_type = 'SUPPLIER'", + ) + + # 2. Sample validation + results["sample_validation"] = self.validate_sample_records() + + # 3. Transaction type mapping validation + results["transaction_type_distribution"] = self.check_transaction_type_mapping() + + # 4. NULL value checks + results["null_checks"] = self.check_null_values( + table_name="allocation_agreement", + fields=[ + "fuel_type_id", + "allocation_transaction_type_id", + "transaction_partner", + "quantity", + "quantity_not_sold", + ], + where_clause="WHERE user_type = 'SUPPLIER'", + ) + + # 5. Version chain validation + results["version_chains"] = self.validate_version_chains( + table_name="allocation_agreement", + where_clause="WHERE user_type = 'SUPPLIER'", + ) + + # 6. Action type distribution + results["action_type_distribution"] = self.check_action_type_distribution() + + # 7. New period impact check + results["new_period_impact"] = self.check_new_period_allocation_impact() + + return results + + def validate_sample_records(self) -> Dict[str, int]: + """Validate a sample of records for data integrity.""" + sample_size = 10 + + source_query = """ + SELECT + crear.id AS agreement_record_id, + cr.id AS cr_legacy_id, + CASE WHEN tt.the_type = 'Purchased' THEN 'Allocated from' ELSE 'Allocated to' END AS responsibility, + aft.name AS fuel_type, + crear.transaction_partner, + crear.postal_address, + crear.quantity, + crear.quantity_not_sold + FROM compliance_report cr + JOIN compliance_report_exclusion_agreement crea ON cr.exclusion_agreement_id = crea.id + JOIN compliance_report_exclusion_agreement_record crear ON crear.exclusion_agreement_id = crea.id + JOIN transaction_type tt ON crear.transaction_type_id = tt.id + JOIN approved_fuel_type aft ON crear.fuel_type_id = aft.id + WHERE cr.exclusion_agreement_id IS NOT NULL + ORDER BY cr.id + LIMIT %s + """ + + match_count = 0 + total_count = 0 + + with get_source_connection() as source_conn: + with source_conn.cursor() as source_cursor: + source_cursor.execute(source_query, (sample_size,)) + source_records = source_cursor.fetchall() + + with get_destination_connection() as dest_conn: + with dest_conn.cursor() as dest_cursor: + for record in source_records: + total_count += 1 + _, legacy_id, _, fuel_type, partner, _, quantity, _ = record + + dest_query = """ + SELECT aa.*, cr.legacy_id, ft.fuel_type, att.type AS allocation_type + FROM allocation_agreement aa + JOIN compliance_report cr ON cr.compliance_report_id = aa.compliance_report_id + JOIN fuel_type ft ON ft.fuel_type_id = aa.fuel_type_id + JOIN allocation_transaction_type att ON att.allocation_transaction_type_id = aa.allocation_transaction_type_id + WHERE cr.legacy_id = %s + AND aa.transaction_partner = %s + AND ABS(aa.quantity - %s) < 0.01 + AND ft.fuel_type = %s + LIMIT 1 + """ + + dest_cursor.execute( + dest_query, (legacy_id, partner, quantity, fuel_type) + ) + if dest_cursor.fetchone(): + match_count += 1 + self.logger.info( + f"✓ Record for compliance report {legacy_id}, partner {partner} matches" + ) + else: + self.logger.info( + f"✗ No match found for compliance report {legacy_id}, partner {partner}" + ) + + return {"matches": match_count, "total": total_count} + + def check_transaction_type_mapping(self) -> List[Dict[str, Any]]: + """Check allocation transaction type mapping integrity.""" + query = """ + SELECT att.type, COUNT(*) AS count + FROM allocation_agreement aa + JOIN allocation_transaction_type att ON att.allocation_transaction_type_id = aa.allocation_transaction_type_id + WHERE aa.user_type = 'SUPPLIER' + GROUP BY att.type + ORDER BY count DESC + """ + + distribution = [] + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + self.logger.info("\nAllocation transaction type distribution:") + for type_name, count in results: + self.logger.info(f"{type_name}: {count} records") + distribution.append({"type": type_name, "count": count}) + + return distribution + + def check_action_type_distribution(self) -> List[Dict[str, Any]]: + """Check action type distribution.""" + query = """ + SELECT action_type, COUNT(*) as count + FROM allocation_agreement + WHERE user_type = 'SUPPLIER' + GROUP BY action_type + """ + + distribution = [] + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + self.logger.info("\nAction type distribution:") + for action_type, count in results: + self.logger.info(f"{action_type}: {count} records") + distribution.append({"action_type": action_type, "count": count}) + + return distribution + + def check_new_period_allocation_impact(self) -> int: + """Check if new-period allocation agreement records were impacted.""" + query = """ + SELECT COUNT(*) as count + FROM allocation_agreement aa + JOIN compliance_report cr ON cr.compliance_report_id = aa.compliance_report_id + WHERE aa.user_type != 'SUPPLIER' + AND EXISTS ( + SELECT 1 FROM allocation_agreement aa2 + WHERE aa2.group_uuid = aa.group_uuid + AND aa2.user_type = 'SUPPLIER' + ) + """ + + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + return cursor.fetchone()[0] + + +if __name__ == "__main__": + validator = AllocationAgreementValidator() + results = validator.run_validation() diff --git a/etl/python_migration/validation/validate_compliance_summaries.py b/etl/python_migration/validation/validate_compliance_summaries.py new file mode 100644 index 000000000..791d127fe --- /dev/null +++ b/etl/python_migration/validation/validate_compliance_summaries.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +""" +Compliance Summary Validation Script + +Validates the migration of compliance summaries from TFRS to LCFS by comparing +the source snapshot data with the migrated LCFS summary records. +""" + +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import json +import logging +from decimal import Decimal +from typing import Dict, List, Optional, Tuple + +from core.database import get_source_connection, get_destination_connection +from core.utils import setup_logging, safe_decimal, build_legacy_mapping + +logger = logging.getLogger(__name__) + + +class ComplianceSummaryValidator: + def __init__(self): + self.legacy_to_lcfs_mapping: Dict[int, int] = {} + self.validation_errors: List[Dict] = [] + self.warnings: List[Dict] = [] + + def load_mappings(self, lcfs_cursor): + logger.info("Loading legacy ID to LCFS compliance_report_id mappings") + self.legacy_to_lcfs_mapping = build_legacy_mapping(lcfs_cursor) + logger.info(f"Loaded {len(self.legacy_to_lcfs_mapping)} legacy mappings") + + def fetch_tfrs_snapshot_data(self, tfrs_cursor) -> List[Dict]: + """Fetch snapshot data from TFRS""" + query = """ + SELECT compliance_report_id, snapshot + FROM public.compliance_report_snapshot + WHERE snapshot IS NOT NULL + """ + + tfrs_cursor.execute(query) + records = [] + + for row in tfrs_cursor.fetchall(): + try: + snapshot_data = json.loads(row[1]) + records.append( + {"compliance_report_id": row[0], "snapshot": snapshot_data} + ) + except json.JSONDecodeError as e: + logger.warning( + f"Failed to parse JSON for compliance_report_id {row[0]}: {e}" + ) + continue + + logger.info(f"Fetched {len(records)} TFRS snapshot records") + return records + + def fetch_lcfs_summary_data(self, lcfs_cursor) -> Dict[int, Dict]: + """Fetch summary data from LCFS""" + query = """ + SELECT + compliance_report_id, + line_1_fossil_derived_base_fuel_gasoline, + line_2_eligible_renewable_fuel_supplied_gasoline, + line_3_total_tracked_fuel_supplied_gasoline, + line_4_eligible_renewable_fuel_required_gasoline, + line_5_net_notionally_transferred_gasoline, + line_6_renewable_fuel_retained_gasoline, + line_7_previously_retained_gasoline, + line_8_obligation_deferred_gasoline, + line_9_obligation_added_gasoline, + line_10_net_renewable_fuel_supplied_gasoline, + line_11_non_compliance_penalty_gasoline, + line_1_fossil_derived_base_fuel_diesel, + line_2_eligible_renewable_fuel_supplied_diesel, + line_3_total_tracked_fuel_supplied_diesel, + line_4_eligible_renewable_fuel_required_diesel, + line_5_net_notionally_transferred_diesel, + line_6_renewable_fuel_retained_diesel, + line_7_previously_retained_diesel, + line_8_obligation_deferred_diesel, + line_9_obligation_added_diesel, + line_10_net_renewable_fuel_supplied_diesel, + line_11_non_compliance_penalty_diesel, + line_15_banked_units_used, + line_22_compliance_units_issued, + line_21_non_compliance_penalty_payable, + total_non_compliance_penalty_payable + FROM compliance_report_summary + """ + + lcfs_cursor.execute(query) + records = {} + + for row in lcfs_cursor.fetchall(): + compliance_report_id = row[0] + records[compliance_report_id] = { + "line_1_fossil_derived_base_fuel_gasoline": row[1], + "line_2_eligible_renewable_fuel_supplied_gasoline": row[2], + "line_3_total_tracked_fuel_supplied_gasoline": row[3], + "line_4_eligible_renewable_fuel_required_gasoline": row[4], + "line_5_net_notionally_transferred_gasoline": row[5], + "line_6_renewable_fuel_retained_gasoline": row[6], + "line_7_previously_retained_gasoline": row[7], + "line_8_obligation_deferred_gasoline": row[8], + "line_9_obligation_added_gasoline": row[9], + "line_10_net_renewable_fuel_supplied_gasoline": row[10], + "line_11_non_compliance_penalty_gasoline": row[11], + "line_1_fossil_derived_base_fuel_diesel": row[12], + "line_2_eligible_renewable_fuel_supplied_diesel": row[13], + "line_3_total_tracked_fuel_supplied_diesel": row[14], + "line_4_eligible_renewable_fuel_required_diesel": row[15], + "line_5_net_notionally_transferred_diesel": row[16], + "line_6_renewable_fuel_retained_diesel": row[17], + "line_7_previously_retained_diesel": row[18], + "line_8_obligation_deferred_diesel": row[19], + "line_9_obligation_added_diesel": row[20], + "line_10_net_renewable_fuel_supplied_diesel": row[21], + "line_11_non_compliance_penalty_diesel": row[22], + "line_15_banked_units_used": row[23], + "line_22_compliance_units_issued": row[24], + "line_21_non_compliance_penalty_payable": row[25], + "total_non_compliance_penalty_payable": row[26], + } + + logger.info(f"Fetched {len(records)} LCFS summary records") + return records + + def parse_tfrs_snapshot(self, snapshot: Dict) -> Dict: + """Parse TFRS snapshot data into expected format""" + summary_lines = snapshot.get("summary", {}).get("lines", {}) + + # Extract gasoline class mappings (lines 1-11) + gasoline_values = {} + for i in range(1, 12): + gasoline_values[f"line_{i}_gasoline"] = safe_decimal( + summary_lines.get(str(i), 0) + ) + + # Extract diesel class mappings (lines 12-22) + diesel_values = {} + for i in range(1, 12): + diesel_values[f"line_{i}_diesel"] = safe_decimal( + summary_lines.get(str(i + 11), 0) + ) + + # Extract special fields + compliance_units_issued = safe_decimal(summary_lines.get("25", 0)) + banked_used = safe_decimal(summary_lines.get("26", 0)) + line28_penalty = safe_decimal(summary_lines.get("28", 0)) + total_payable = safe_decimal( + snapshot.get("summary", {}).get("total_payable", 0) + ) + + return { + **gasoline_values, + **diesel_values, + "compliance_units_issued": compliance_units_issued, + "banked_used": banked_used, + "line28_penalty": line28_penalty, + "total_payable": total_payable, + } + + def validate_field_mapping( + self, + tfrs_data: Dict, + lcfs_data: Dict, + legacy_report_id: int, + lcfs_report_id: int, + ) -> bool: + """Validate that TFRS fields are correctly mapped to LCFS fields""" + validation_passed = True + tolerance = Decimal("0.01") # Allow small floating point differences + + # Define field mappings for validation + field_mappings = [ + # Gasoline class mappings + ("line_1_gasoline", "line_1_fossil_derived_base_fuel_gasoline"), + ("line_2_gasoline", "line_2_eligible_renewable_fuel_supplied_gasoline"), + ("line_3_gasoline", "line_3_total_tracked_fuel_supplied_gasoline"), + ("line_4_gasoline", "line_4_eligible_renewable_fuel_required_gasoline"), + ("line_5_gasoline", "line_5_net_notionally_transferred_gasoline"), + ("line_6_gasoline", "line_6_renewable_fuel_retained_gasoline"), + ("line_7_gasoline", "line_7_previously_retained_gasoline"), + ("line_8_gasoline", "line_8_obligation_deferred_gasoline"), + ("line_9_gasoline", "line_9_obligation_added_gasoline"), + ("line_10_gasoline", "line_10_net_renewable_fuel_supplied_gasoline"), + ("line_11_gasoline", "line_11_non_compliance_penalty_gasoline"), + # Diesel class mappings + ("line_1_diesel", "line_1_fossil_derived_base_fuel_diesel"), + ("line_2_diesel", "line_2_eligible_renewable_fuel_supplied_diesel"), + ("line_3_diesel", "line_3_total_tracked_fuel_supplied_diesel"), + ("line_4_diesel", "line_4_eligible_renewable_fuel_required_diesel"), + ("line_5_diesel", "line_5_net_notionally_transferred_diesel"), + ("line_6_diesel", "line_6_renewable_fuel_retained_diesel"), + ("line_7_diesel", "line_7_previously_retained_diesel"), + ("line_8_diesel", "line_8_obligation_deferred_diesel"), + ("line_9_diesel", "line_9_obligation_added_diesel"), + ("line_10_diesel", "line_10_net_renewable_fuel_supplied_diesel"), + ("line_11_diesel", "line_11_non_compliance_penalty_diesel"), + # Special field mappings + ("compliance_units_issued", "line_22_compliance_units_issued"), + ("banked_used", "line_15_banked_units_used"), + ("line28_penalty", "line_21_non_compliance_penalty_payable"), + ("total_payable", "total_non_compliance_penalty_payable"), + ] + + # Validate decimal/float fields + for tfrs_field, lcfs_field in field_mappings: + tfrs_value = Decimal(str(tfrs_data.get(tfrs_field, 0))) + lcfs_value = Decimal(str(lcfs_data.get(lcfs_field, 0))) + + if abs(tfrs_value - lcfs_value) > tolerance: + self.validation_errors.append( + { + "type": "field_mismatch", + "legacy_report_id": legacy_report_id, + "lcfs_report_id": lcfs_report_id, + "field": lcfs_field, + "tfrs_value": float(tfrs_value), + "lcfs_value": float(lcfs_value), + "difference": float(abs(tfrs_value - lcfs_value)), + } + ) + validation_passed = False + + # No integer fields to validate since credits_offset fields are not in LCFS + + return validation_passed + + def validate_migration(self) -> Tuple[int, int, int]: + """Run full validation of compliance summary migration""" + total_validated = 0 + total_passed = 0 + total_failed = 0 + + try: + with get_source_connection() as tfrs_conn: + with get_destination_connection() as lcfs_conn: + tfrs_cursor = tfrs_conn.cursor() + lcfs_cursor = lcfs_conn.cursor() + + # Load mappings + self.load_mappings(lcfs_cursor) + + # Fetch data from both systems + tfrs_snapshots = self.fetch_tfrs_snapshot_data(tfrs_cursor) + lcfs_summaries = self.fetch_lcfs_summary_data(lcfs_cursor) + + logger.info("Starting validation process") + + for tfrs_record in tfrs_snapshots: + legacy_report_id = tfrs_record["compliance_report_id"] + lcfs_report_id = self.legacy_to_lcfs_mapping.get( + legacy_report_id + ) + + if lcfs_report_id is None: + self.warnings.append( + { + "type": "missing_mapping", + "legacy_report_id": legacy_report_id, + "message": "No LCFS mapping found for TFRS report", + } + ) + continue + + if lcfs_report_id not in lcfs_summaries: + self.validation_errors.append( + { + "type": "missing_lcfs_summary", + "legacy_report_id": legacy_report_id, + "lcfs_report_id": lcfs_report_id, + "message": "LCFS summary record not found", + } + ) + total_failed += 1 + continue + + # Parse TFRS snapshot data + tfrs_parsed = self.parse_tfrs_snapshot(tfrs_record["snapshot"]) + lcfs_data = lcfs_summaries[lcfs_report_id] + + # Validate field mappings + if self.validate_field_mapping( + tfrs_parsed, lcfs_data, legacy_report_id, lcfs_report_id + ): + total_passed += 1 + else: + total_failed += 1 + + total_validated += 1 + + tfrs_cursor.close() + lcfs_cursor.close() + + except Exception as e: + logger.error(f"Validation failed: {e}") + raise + + return total_validated, total_passed, total_failed + + def generate_report(self) -> str: + """Generate validation report""" + report = [] + report.append("=" * 60) + report.append("COMPLIANCE SUMMARY VALIDATION REPORT") + report.append("=" * 60) + + if self.validation_errors: + report.append(f"\nCRITICAL ERRORS ({len(self.validation_errors)}):") + report.append("-" * 40) + for error in self.validation_errors: + report.append( + f"• {error['type']}: Legacy ID {error['legacy_report_id']}" + ) + if "field" in error: + report.append(f" Field: {error['field']}") + if "tfrs_value" in error: + report.append( + f" TFRS: {error['tfrs_value']}, LCFS: {error['lcfs_value']}" + ) + if "message" in error: + report.append(f" {error['message']}") + report.append("") + + if self.warnings: + report.append(f"\nWARNINGS ({len(self.warnings)}):") + report.append("-" * 40) + for warning in self.warnings: + report.append( + f"• {warning['type']}: Legacy ID {warning['legacy_report_id']}" + ) + report.append(f" {warning['message']}") + report.append("") + + return "\n".join(report) + + +def main(): + setup_logging() + logger.info("Starting Compliance Summary Validation") + + validator = ComplianceSummaryValidator() + + try: + validated, passed, failed = validator.validate_migration() + + logger.info( + f"Validation completed: {validated} records validated, {passed} passed, {failed} failed" + ) + + # Generate and display report + report = validator.generate_report() + print(report) + + # Save report to file + with open("compliance_summary_validation_report.txt", "w") as f: + f.write(report) + + logger.info( + "Validation report saved to compliance_summary_validation_report.txt" + ) + + # Exit with error code if validation failed + if failed > 0: + sys.exit(1) + + except Exception as e: + logger.error(f"Validation failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/etl/python_migration/validation/validate_fuel_supply.py b/etl/python_migration/validation/validate_fuel_supply.py new file mode 100644 index 000000000..78c13f184 --- /dev/null +++ b/etl/python_migration/validation/validate_fuel_supply.py @@ -0,0 +1,315 @@ +""" +Fuel Supply validation script for TFRS to LCFS migration. +""" + +from typing import Dict, Any, List +from .validation_base import BaseValidator +from core.database import get_source_connection, get_destination_connection + + +class FuelSupplyValidator(BaseValidator): + """Validator for fuel supply migration.""" + + def get_validation_name(self) -> str: + return "Fuel Supply" + + def validate(self) -> Dict[str, Any]: + """Run fuel supply validation.""" + results = {} + + # 1. Validate source database structure + self.validate_source_structure() + + # 2. Compare record counts (including GHGenius) + results["record_counts"] = self.compare_record_counts_with_ghgenius() + + # 3. Sample validation + results["sample_validation"] = self.validate_sample_records() + + # 4. NULL value checks + results["null_checks"] = self.check_null_values( + table_name="fuel_supply", + fields=[ + "fuel_category_id", + "fuel_type_id", + "provision_of_the_act_id", + "quantity", + ], + where_clause="WHERE create_user = 'ETL'", + ) + + # 5. Calculation consistency check + results["calculation_validation"] = self.validate_calculations() + + # 6. GHGenius record validation + results["ghgenius_validation"] = self.validate_ghgenius_records() + + # 7. Duplicate record check + results["duplicate_check"] = self.check_duplicate_records() + + # 8. New period impact check + results["new_period_impact"] = self.check_new_period_impact( + table_name="fuel_supply fs JOIN compliance_report cr ON cr.compliance_report_id = fs.compliance_report_id", + user_filter="", + ) + + return results + + def validate_source_structure(self): + """Validate that required source tables exist.""" + query = """ + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'compliance_report_schedule_b_record' + ) AS table_exists + """ + + with get_source_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + table_exists = cursor.fetchone()[0] + + if not table_exists: + raise Exception( + "Table 'compliance_report_schedule_b_record' does not exist in source database" + ) + + def compare_record_counts_with_ghgenius(self) -> Dict[str, Any]: + """Compare record counts including GHGenius specific records.""" + # Standard record count comparison + standard_counts = self.compare_record_counts( + source_query=""" + SELECT COUNT(*) + FROM compliance_report_schedule_b_record crsbr + JOIN compliance_report cr ON cr.schedule_b_id = crsbr.schedule_id + """, + dest_query="SELECT COUNT(*) FROM fuel_supply WHERE create_user = 'ETL'", + ) + + # GHGenius record count comparison + ghgenius_source_query = """ + SELECT COUNT(*) + FROM compliance_report_schedule_b_record crsbr + JOIN carbon_intensity_fuel_determination cifd + ON cifd.fuel_id = crsbr.fuel_type_id + AND cifd.provision_act_id = crsbr.provision_of_the_act_id + JOIN determination_type dt ON dt.id = cifd.determination_type_id + WHERE dt.the_type = 'GHGenius' + """ + + ghgenius_dest_query = """ + SELECT COUNT(*) + FROM fuel_supply fs + JOIN provision_of_the_act pota ON pota.provision_of_the_act_id = fs.provision_of_the_act_id + WHERE fs.create_user = 'ETL' + AND pota.name = 'GHGenius modelled - Section 6 (5) (d) (ii) (A)' + """ + + ghgenius_counts = self.compare_record_counts( + source_query=ghgenius_source_query, dest_query=ghgenius_dest_query + ) + + self.logger.info(f"\nGHGenius specific records:") + self.logger.info(f"GHGenius source count: {ghgenius_counts['source_count']}") + self.logger.info(f"GHGenius destination count: {ghgenius_counts['dest_count']}") + self.logger.info(f"GHGenius difference: {ghgenius_counts['difference']}") + + return {"standard": standard_counts, "ghgenius": ghgenius_counts} + + def validate_sample_records(self) -> Dict[str, int]: + """Validate a sample of records for data integrity.""" + sample_size = 10 + + source_query = """ + WITH schedule_b AS ( + SELECT crsbr.id as fuel_supply_id, + cr.id as cr_legacy_id, + crsbr.quantity, + uom.name as unit_of_measure, + fc.fuel_class as fuel_category, + fc1.fuel_code as fuel_code_prefix, + aft.name as fuel_type, + CONCAT(TRIM(pa.description), ' - ', TRIM(pa.provision)) as provision_act + FROM compliance_report_schedule_b_record crsbr + INNER JOIN fuel_class fc ON fc.id = crsbr.fuel_class_id + INNER JOIN approved_fuel_type aft ON aft.id = crsbr.fuel_type_id + INNER JOIN provision_act pa ON pa.id = crsbr.provision_of_the_act_id + INNER JOIN compliance_report cr ON cr.schedule_b_id = crsbr.schedule_id + LEFT JOIN unit_of_measure uom ON uom.id = aft.unit_of_measure_id + LEFT JOIN fuel_code fc1 ON fc1.id = crsbr.fuel_code_id + ORDER BY cr.id + LIMIT %s + ) + SELECT * FROM schedule_b + """ + + match_count = 0 + total_count = 0 + + with get_source_connection() as source_conn: + with source_conn.cursor() as source_cursor: + source_cursor.execute(source_query, (sample_size,)) + source_records = source_cursor.fetchall() + + with get_destination_connection() as dest_conn: + with dest_conn.cursor() as dest_cursor: + for record in source_records: + total_count += 1 + _, legacy_id, quantity, _, _, _, fuel_type, _ = record + + dest_query = """ + SELECT fs.*, cr.legacy_id + FROM fuel_supply fs + JOIN compliance_report cr ON cr.compliance_report_id = fs.compliance_report_id + JOIN fuel_type ft ON ft.fuel_type_id = fs.fuel_type_id + WHERE cr.legacy_id = %s + AND ft.fuel_type = %s + AND ABS(fs.quantity - %s) < 0.01 + AND fs.create_user = 'ETL' + LIMIT 1 + """ + + dest_cursor.execute( + dest_query, (legacy_id, fuel_type, quantity) + ) + if dest_cursor.fetchone(): + match_count += 1 + self.logger.info( + f"✓ Record for compliance report {legacy_id}, fuel type {fuel_type} matches" + ) + else: + self.logger.info( + f"✗ No match found for compliance report {legacy_id}, fuel type {fuel_type}" + ) + + return {"matches": match_count, "total": total_count} + + def validate_calculations(self) -> Dict[str, Any]: + """Validate calculation consistency.""" + query = """ + SELECT + compliance_report_id, + quantity, + energy_density, + energy, + ci_of_fuel, + target_ci, + eer, + compliance_units, + ABS(energy - (quantity * energy_density)) > 0.01 as energy_calc_error, + ABS(compliance_units - ((((target_ci * eer) - ci_of_fuel) * (energy_density * quantity)) / 1000000)) > 0.1 as compliance_unit_calc_error + FROM fuel_supply + WHERE create_user = 'ETL' + LIMIT 20 + """ + + energy_errors = 0 + cu_errors = 0 + total_checked = 0 + + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + for result in results: + total_checked += 1 + energy_error = result[8] # energy_calc_error + cu_error = result[9] # compliance_unit_calc_error + + if energy_error: + energy_errors += 1 + if cu_error: + cu_errors += 1 + + self.logger.info(f"\nCalculation validation (from 20 sample records):") + self.logger.info(f"Energy calculation errors: {energy_errors}/{total_checked}") + self.logger.info( + f"Compliance units calculation errors: {cu_errors}/{total_checked}" + ) + + return { + "energy_errors": energy_errors, + "cu_errors": cu_errors, + "total_checked": total_checked, + } + + def validate_ghgenius_records(self) -> Dict[str, Any]: + """Validate GHGenius records specifically.""" + query = """ + SELECT fs.*, pota.name AS provision_name + FROM fuel_supply fs + JOIN provision_of_the_act pota ON pota.provision_of_the_act_id = fs.provision_of_the_act_id + WHERE pota.name = 'GHGenius modelled - Section 6 (5) (d) (ii) (A)' + AND fs.create_user = 'ETL' + LIMIT 10 + """ + + ghgenius_count = 0 + ghgenius_with_ci_count = 0 + + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + self.logger.info(f"\nGHGenius record validation:") + for result in results: + ghgenius_count += 1 + report_id = result[1] # compliance_report_id + ci_of_fuel = result[10] # ci_of_fuel (approximate position) + + if ci_of_fuel is not None and ci_of_fuel != 0: + ghgenius_with_ci_count += 1 + self.logger.info( + f"✓ GHGenius record for compliance report {report_id} has CI: {ci_of_fuel}" + ) + else: + self.logger.info( + f"✗ GHGenius record for compliance report {report_id} missing CI value" + ) + + self.logger.info( + f"Found {ghgenius_with_ci_count}/{ghgenius_count} GHGenius records with correct CI values" + ) + + return {"total_ghgenius": ghgenius_count, "with_ci": ghgenius_with_ci_count} + + def check_duplicate_records(self) -> int: + """Check for duplicate records.""" + query = """ + SELECT compliance_report_id, fuel_type_id, quantity, COUNT(*) as count + FROM fuel_supply + WHERE create_user = 'ETL' + GROUP BY compliance_report_id, fuel_type_id, quantity + HAVING COUNT(*) > 1 + LIMIT 10 + """ + + duplicate_count = 0 + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + for result in results: + duplicate_count += 1 + cr_id, fuel_type_id, quantity, count = result + self.logger.info( + f"Duplicate found: CR:{cr_id}, fuel_type:{fuel_type_id}, " + f"quantity:{quantity}, count:{count}" + ) + + status = ( + f"{duplicate_count} duplicates found" + if duplicate_count > 0 + else "No duplicates found" + ) + self.logger.info(f"\nDuplicate records check: {status}") + + return duplicate_count + + +if __name__ == "__main__": + validator = FuelSupplyValidator() + results = validator.run_validation() diff --git a/etl/python_migration/validation/validate_notional_transfers.py b/etl/python_migration/validation/validate_notional_transfers.py new file mode 100644 index 000000000..054063439 --- /dev/null +++ b/etl/python_migration/validation/validate_notional_transfers.py @@ -0,0 +1,212 @@ +""" +Notional Transfer validation script for TFRS to LCFS migration. +""" + +from typing import Dict, Any, List +from .validation_base import BaseValidator +from core.database import get_source_connection, get_destination_connection + + +class NotionalTransferValidator(BaseValidator): + """Validator for notional transfer migration.""" + + def get_validation_name(self) -> str: + return "Notional Transfer" + + def validate(self) -> Dict[str, Any]: + """Run notional transfer validation.""" + results = {} + + # 1. Compare record counts + results["record_counts"] = self.compare_record_counts( + source_query="SELECT COUNT(*) FROM compliance_report_schedule_a_record", + dest_query="SELECT COUNT(*) FROM notional_transfer WHERE user_type::text = 'SUPPLIER'", + ) + + # 2. Sample validation + results["sample_validation"] = self.validate_sample_records() + + # 3. NULL value checks + results["null_checks"] = self.check_null_values( + table_name="notional_transfer", + fields=[ + "fuel_category_id", + "legal_name", + "received_or_transferred", + "quantity", + ], + where_clause="WHERE user_type::text = 'SUPPLIER'", + ) + + # 4. Transfer type mapping validation + results["transfer_type_distribution"] = self.check_transfer_type_mapping() + + # 5. Version chain validation + results["version_chains"] = self.validate_version_chains( + table_name="notional_transfer", + where_clause="WHERE user_type::text = 'SUPPLIER'", + ) + + # 6. Duplicate record check + results["duplicate_check"] = self.check_duplicate_records() + + # 7. New period impact check + results["new_period_impact"] = self.check_new_period_notional_impact() + + return results + + def validate_sample_records(self) -> Dict[str, int]: + """Validate a sample of records for data integrity.""" + sample_size = 10 + + source_query = """ + SELECT + sar.id AS schedule_a_record_id, + cr.id AS cr_legacy_id, + sar.quantity, + sar.trading_partner, + sar.postal_address, + fc.fuel_class AS fuel_category, + CASE + WHEN sar.transfer_type_id = 1 THEN 'Received' + ELSE 'Transferred' + END AS transfer_type + FROM compliance_report_schedule_a_record sar + JOIN compliance_report_schedule_a sa ON sa.id = sar.schedule_id + JOIN compliance_report cr ON cr.schedule_a_id = sa.id + JOIN fuel_class fc ON fc.id = sar.fuel_class_id + ORDER BY cr.id + LIMIT %s + """ + + match_count = 0 + total_count = 0 + + with get_source_connection() as source_conn: + with source_conn.cursor() as source_cursor: + source_cursor.execute(source_query, (sample_size,)) + source_records = source_cursor.fetchall() + + with get_destination_connection() as dest_conn: + with dest_conn.cursor() as dest_cursor: + for record in source_records: + total_count += 1 + ( + _, + legacy_id, + quantity, + trading_partner, + _, + _, + transfer_type, + ) = record + + dest_query = """ + SELECT nt.*, cr.legacy_id + FROM notional_transfer nt + JOIN compliance_report cr ON cr.compliance_report_id = nt.compliance_report_id + WHERE cr.legacy_id = %s + AND nt.legal_name = %s + AND ABS(nt.quantity - %s) < 0.01 + AND nt.received_or_transferred::text = %s + LIMIT 1 + """ + + dest_cursor.execute( + dest_query, + (legacy_id, trading_partner, quantity, transfer_type), + ) + if dest_cursor.fetchone(): + match_count += 1 + self.logger.info( + f"✓ Record for compliance report {legacy_id}, partner {trading_partner} matches" + ) + else: + self.logger.info( + f"✗ No match found for compliance report {legacy_id}, partner {trading_partner}" + ) + + return {"matches": match_count, "total": total_count} + + def check_transfer_type_mapping(self) -> List[Dict[str, Any]]: + """Verify transfer type mapping.""" + query = """ + SELECT received_or_transferred::text, COUNT(*) as count + FROM notional_transfer + WHERE user_type::text = 'SUPPLIER' + GROUP BY received_or_transferred + """ + + distribution = [] + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + self.logger.info("\nTransfer type distribution:") + for transfer_type, count in results: + self.logger.info(f"{transfer_type}: {count} records") + distribution.append( + {"transfer_type": transfer_type, "count": count} + ) + + return distribution + + def check_duplicate_records(self) -> int: + """Check for duplicate records within same compliance report.""" + query = """ + SELECT compliance_report_id, legal_name, quantity, received_or_transferred::text, + COUNT(*) as count + FROM notional_transfer + WHERE version = 0 AND user_type::text = 'SUPPLIER' + GROUP BY compliance_report_id, legal_name, quantity, received_or_transferred + HAVING COUNT(*) > 1 + LIMIT 10 + """ + + duplicate_count = 0 + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + for result in results: + duplicate_count += 1 + cr_id, partner, quantity, transfer_type, count = result + self.logger.info( + f"Duplicate found: CR:{cr_id}, partner:{partner}, " + f"type:{transfer_type}, count:{count}" + ) + + status = ( + f"{duplicate_count} duplicates found" + if duplicate_count > 0 + else "No duplicates found" + ) + self.logger.info(f"\nDuplicate records check: {status}") + + return duplicate_count + + def check_new_period_notional_impact(self) -> int: + """Check if new-period notional transfer records were impacted.""" + query = """ + SELECT COUNT(*) as count + FROM notional_transfer nt + JOIN compliance_report cr ON cr.compliance_report_id = nt.compliance_report_id + WHERE nt.user_type::text != 'SUPPLIER' + AND EXISTS ( + SELECT 1 FROM notional_transfer nt2 + WHERE nt2.group_uuid = nt.group_uuid + AND nt2.user_type::text = 'SUPPLIER' + ) + """ + + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + return cursor.fetchone()[0] + + +if __name__ == "__main__": + validator = NotionalTransferValidator() + results = validator.run_validation() diff --git a/etl/python_migration/validation/validate_other_uses.py b/etl/python_migration/validation/validate_other_uses.py new file mode 100644 index 000000000..172921dbb --- /dev/null +++ b/etl/python_migration/validation/validate_other_uses.py @@ -0,0 +1,236 @@ +""" +Other Uses validation script for TFRS to LCFS migration. +""" + +from typing import Dict, Any, List +from .validation_base import BaseValidator +from core.database import get_source_connection, get_destination_connection + + +class OtherUsesValidator(BaseValidator): + """Validator for other uses migration.""" + + def get_validation_name(self) -> str: + return "Other Uses (Schedule C)" + + def validate(self) -> Dict[str, Any]: + """Run other uses validation.""" + results = {} + + # 1. Validate source database structure + self.validate_source_structure() + + # 2. Compare record counts + results["record_counts"] = self.compare_record_counts( + source_query="SELECT COUNT(*) FROM compliance_report_schedule_c_record", + dest_query="SELECT COUNT(*) FROM other_uses WHERE create_user::text = 'ETL'", + ) + + # Skip further validation if no destination records + if results["record_counts"]["dest_count"] == 0: + self.logger.info( + "\nNo other_uses records found in destination - skipping validation checks" + ) + return results + + # 3. Sample validation + results["sample_validation"] = self.validate_sample_records() + + # 4. Expected use mapping validation + results["expected_use_distribution"] = self.check_expected_use_mapping() + + # 5. NULL value checks + results["null_checks"] = self.check_null_values( + table_name="other_uses", + fields=[ + "fuel_category_id", + "fuel_type_id", + "expected_use_id", + "quantity_supplied", + ], + where_clause="WHERE create_user::text = 'ETL'", + ) + + # 6. Version chain validation + results["version_chains"] = self.validate_version_chains( + table_name="other_uses", where_clause="WHERE create_user::text = 'ETL'" + ) + + # 7. Action type distribution + results["action_type_distribution"] = self.check_action_type_distribution() + + # 8. New period impact check + results["new_period_impact"] = self.check_new_period_impact( + table_name="other_uses ou JOIN compliance_report cr ON cr.compliance_report_id = ou.compliance_report_id", + user_filter="", + ) + + return results + + def validate_source_structure(self): + """Validate that required source tables exist.""" + # Check main table + query = """ + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'compliance_report_schedule_c_record' + ) AS table_exists + """ + + with get_source_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + table_exists = cursor.fetchone()[0] + + if not table_exists: + raise Exception( + "Table 'compliance_report_schedule_c_record' does not exist in source database" + ) + + # Check expected_use table existence + expected_use_query = """ + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'expected_use' + ) AS table_exists + """ + + with get_source_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(expected_use_query) + self.expected_use_table_exists = cursor.fetchone()[0] + + def validate_sample_records(self) -> Dict[str, int]: + """Validate a sample of records for data integrity.""" + sample_size = 10 + + # Build source query based on whether expected_use table exists + base_query = """ + SELECT + scr.id AS schedule_c_record_id, + cr.id AS cr_legacy_id, + scr.quantity, + aft.name AS fuel_type, + fc.fuel_class AS fuel_category + """ + + if self.expected_use_table_exists: + base_query += """, + eu.description AS expected_use, + scr.rationale""" + else: + base_query += """, + 'Other' AS expected_use, + scr.rationale""" + + base_query += """ + FROM compliance_report_schedule_c_record scr + JOIN compliance_report_schedule_c sc ON sc.id = scr.schedule_id + JOIN compliance_report cr ON cr.schedule_c_id = sc.id + JOIN approved_fuel_type aft ON aft.id = scr.fuel_type_id + JOIN fuel_class fc ON fc.id = scr.fuel_class_id + """ + + if self.expected_use_table_exists: + base_query += "JOIN expected_use eu ON eu.id = scr.expected_use_id" + + source_query = ( + base_query + + """ + ORDER BY cr.id + LIMIT %s + """ + ) + + match_count = 0 + total_count = 0 + + with get_source_connection() as source_conn: + with source_conn.cursor() as source_cursor: + source_cursor.execute(source_query, (sample_size,)) + source_records = source_cursor.fetchall() + + with get_destination_connection() as dest_conn: + with dest_conn.cursor() as dest_cursor: + for record in source_records: + total_count += 1 + _, legacy_id, quantity, fuel_type, _, _, _ = record + + dest_query = """ + SELECT ou.*, cr.legacy_id, ft.fuel_type, et.name AS expected_use_type + FROM other_uses ou + JOIN compliance_report cr ON cr.compliance_report_id = ou.compliance_report_id + JOIN fuel_type ft ON ft.fuel_type_id = ou.fuel_type_id + JOIN expected_use_type et ON et.expected_use_type_id = ou.expected_use_id + WHERE cr.legacy_id = %s + AND ft.fuel_type = %s + AND ABS(ou.quantity_supplied - %s) < 0.01 + AND ou.create_user::text = 'ETL' + LIMIT 1 + """ + + dest_cursor.execute( + dest_query, (legacy_id, fuel_type, quantity) + ) + if dest_cursor.fetchone(): + match_count += 1 + self.logger.info( + f"✓ Record for compliance report {legacy_id}, fuel type {fuel_type} matches" + ) + else: + self.logger.info( + f"✗ No match found for compliance report {legacy_id}, fuel type {fuel_type}" + ) + + return {"matches": match_count, "total": total_count} + + def check_expected_use_mapping(self) -> List[Dict[str, Any]]: + """Check expected use type mapping distribution.""" + query = """ + SELECT et.name, COUNT(*) AS count + FROM other_uses ou + JOIN expected_use_type et ON et.expected_use_type_id = ou.expected_use_id + WHERE ou.create_user::text = 'ETL' + GROUP BY et.name + ORDER BY count DESC + """ + + distribution = [] + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + self.logger.info("\nExpected use mapping distribution:") + for use_type, count in results: + self.logger.info(f"{use_type}: {count} records") + distribution.append({"expected_use": use_type, "count": count}) + + return distribution + + def check_action_type_distribution(self) -> List[Dict[str, Any]]: + """Check action type distribution.""" + query = """ + SELECT action_type::text, COUNT(*) as count + FROM other_uses + WHERE create_user::text = 'ETL' + GROUP BY action_type + """ + + distribution = [] + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + self.logger.info("\nAction type distribution:") + for action_type, count in results: + self.logger.info(f"{action_type}: {count} records") + distribution.append({"action_type": action_type, "count": count}) + + return distribution + + +if __name__ == "__main__": + validator = OtherUsesValidator() + results = validator.run_validation() diff --git a/etl/python_migration/validation/validation_base.py b/etl/python_migration/validation/validation_base.py new file mode 100644 index 000000000..2da987891 --- /dev/null +++ b/etl/python_migration/validation/validation_base.py @@ -0,0 +1,222 @@ +""" +Base validator class for TFRS to LCFS migration validation. +""" + +import logging +from abc import ABC, abstractmethod +from typing import Dict, Any, List, Tuple +import sys +import os + +# Add parent directory to path to import config and database modules +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.database import get_source_connection, get_destination_connection + + +class BaseValidator(ABC): + """Base class for migration validation scripts.""" + + def __init__(self): + self.logger = logging.getLogger(self.__class__.__name__) + self.validation_results = {} + + def setup_logging(self): + """Set up logging configuration.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + @abstractmethod + def get_validation_name(self) -> str: + """Return the name of this validation.""" + pass + + @abstractmethod + def validate(self) -> Dict[str, Any]: + """Run the validation and return results.""" + pass + + def compare_record_counts( + self, + source_query: str, + dest_query: str, + source_params: List = None, + dest_params: List = None, + ) -> Dict[str, int]: + """Compare record counts between source and destination.""" + with get_source_connection() as source_conn: + with source_conn.cursor() as cursor: + cursor.execute(source_query, source_params or []) + source_count = cursor.fetchone()[0] + + with get_destination_connection() as dest_conn: + with dest_conn.cursor() as cursor: + cursor.execute(dest_query, dest_params or []) + dest_count = cursor.fetchone()[0] + + return { + "source_count": source_count, + "dest_count": dest_count, + "difference": dest_count - source_count, + } + + def check_null_values( + self, table_name: str, fields: List[str], where_clause: str = "" + ) -> Dict[str, int]: + """Check for NULL values in key fields.""" + null_checks = [] + for field in fields: + null_checks.append( + f"SUM(CASE WHEN {field} IS NULL THEN 1 ELSE 0 END) as null_{field.replace('.', '_')}" + ) + + query = f""" + SELECT {', '.join(null_checks)} + FROM {table_name} + {where_clause} + """ + + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + result = cursor.fetchone() + + null_counts = {} + for i, field in enumerate(fields): + field_key = field.replace(".", "_") + null_counts[f"null_{field_key}"] = result[i] + + return null_counts + + def validate_version_chains( + self, table_name: str, where_clause: str = "" + ) -> List[Dict[str, Any]]: + """Validate version chain integrity.""" + query = f""" + SELECT group_uuid, COUNT(*) as version_count, + MIN(version) as min_version, MAX(version) as max_version + FROM {table_name} + {where_clause} + GROUP BY group_uuid + HAVING COUNT(*) > 1 + ORDER BY version_count DESC + LIMIT 10 + """ + + version_chains = [] + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + for row in results: + group_uuid, version_count, min_version, max_version = row + + # Check if versions are sequential + version_query = f""" + SELECT version FROM {table_name} + WHERE group_uuid = %s + ORDER BY version + """ + cursor.execute(version_query, (group_uuid,)) + versions = [r[0] for r in cursor.fetchall()] + + is_sequential = len(versions) == (versions[-1] - versions[0] + 1) + + version_chains.append( + { + "group_uuid": group_uuid, + "version_count": version_count, + "min_version": min_version, + "max_version": max_version, + "versions": versions, + "is_sequential": is_sequential, + } + ) + + return version_chains + + def check_new_period_impact(self, table_name: str, user_filter: str) -> int: + """Check if any new-period records were impacted by ETL.""" + query = f""" + SELECT COUNT(*) as count + FROM {table_name} + WHERE create_user != 'ETL' + AND update_user = 'ETL' + {user_filter} + """ + + with get_destination_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + return cursor.fetchone()[0] + + def log_validation_results(self, results: Dict[str, Any]): + """Log validation results in a formatted way.""" + validation_name = self.get_validation_name() + self.logger.info(f"**** BEGIN {validation_name.upper()} VALIDATION ****") + + # Record counts + if "record_counts" in results: + counts = results["record_counts"] + self.logger.info(f"Source record count: {counts['source_count']}") + self.logger.info(f"Destination record count: {counts['dest_count']}") + self.logger.info(f"Difference: {counts['difference']}") + + # Sample validation + if "sample_validation" in results: + sample = results["sample_validation"] + self.logger.info( + f"Found {sample['matches']}/{sample['total']} matching records" + ) + + # NULL value checks + if "null_checks" in results: + self.logger.info("\nData anomalies check:") + for field, count in results["null_checks"].items(): + self.logger.info(f"Records with {field}: {count}") + + # Version chains + if "version_chains" in results: + chains = results["version_chains"] + self.logger.info(f"\nVersion chain validation:") + if chains: + for chain in chains: + self.logger.info( + f"Group {chain['group_uuid']}: {chain['version_count']} versions " + f"({chain['min_version']} to {chain['max_version']})" + ) + seq_status = ( + "sequential" if chain["is_sequential"] else "non-sequential" + ) + self.logger.info( + f" Versions are {seq_status}: {', '.join(map(str, chain['versions']))}" + ) + else: + self.logger.info("No version chains found") + + # New period impact + if "new_period_impact" in results: + impact = results["new_period_impact"] + self.logger.info(f"\nNew period records impacted: {impact}") + if impact > 0: + self.logger.error( + f"WARNING: {impact} records from the latest reporting period were modified by ETL process" + ) + else: + self.logger.info("✓ No latest reporting period records were modified") + + self.logger.info(f"**** END {validation_name.upper()} VALIDATION ****") + + def run_validation(self) -> Dict[str, Any]: + """Run the complete validation process.""" + try: + self.setup_logging() + results = self.validate() + self.log_validation_results(results) + return results + except Exception as e: + self.logger.error(f"Error in {self.get_validation_name()} validation: {e}") + raise diff --git a/frontend/src/assets/locales/en/legacy.json b/frontend/src/assets/locales/en/legacy.json index b3a5276a6..7fdb13f9b 100644 --- a/frontend/src/assets/locales/en/legacy.json +++ b/frontend/src/assets/locales/en/legacy.json @@ -2,7 +2,8 @@ "activityLists": { "scheduleA": "Schedule A - Notional transfers of renewable fuel", "scheduleB": "Schedule B - Part 3 fuel supply", - "scheduleC": "Schedule C - Fuels used for other purposes" + "scheduleC": "Schedule C - Fuels used for other purposes", + "exclusionAgreement": "Exclusion Agreements - Part 3 fuels either purchased or sold under an exclusion agreement" }, "columnLabels": { "fuelClass": "Fuel Class", diff --git a/frontend/src/components/BCDataGrid/BCGridBase.jsx b/frontend/src/components/BCDataGrid/BCGridBase.jsx index 8eca56ab6..78dc9e769 100644 --- a/frontend/src/components/BCDataGrid/BCGridBase.jsx +++ b/frontend/src/components/BCDataGrid/BCGridBase.jsx @@ -24,6 +24,13 @@ export const BCGridBase = forwardRef( { autoSizeStrategy, autoHeight, + enableCellTextSelection, + getRowId, + overlayNoRowsTemplate, + queryData, + dataKey, + paginationOptions, + onPaginationChange, onRowClicked, ...props }, @@ -150,21 +157,19 @@ export const BCGridBase = forwardRef( loadingMessage: 'One moment please...' }} animateRows - overlayNoRowsTemplate="No rows found" - autoSizeStrategy={{ - type: 'fitGridWidth', - defaultMinWidth: 100, - ...autoSizeStrategy - }} + autoSizeStrategy={{ type: 'fitCellContents', ...autoSizeStrategy }} suppressDragLeaveHidesColumns suppressMovableColumns suppressColumnMoveAnimation={false} suppressCsvExport={false} suppressColumnVirtualisation={true} enableBrowserTooltips={true} + enableCellTextSelection={enableCellTextSelection} + getRowId={getRowId} suppressPaginationPanel suppressScrollOnNewData onRowDataUpdated={determineHeight} + overlayNoRowsTemplate={overlayNoRowsTemplate || "No rows found"} getRowStyle={(params) => { const defaultStyle = typeof getRowStyle === 'function' ? getRowStyle(params) : {} diff --git a/frontend/src/components/Login.jsx b/frontend/src/components/Login.jsx index 55a6ca3bc..1c79a507c 100644 --- a/frontend/src/components/Login.jsx +++ b/frontend/src/components/Login.jsx @@ -50,8 +50,8 @@ const season = const seasonImages = { spring: { count: 250, - radius: [1, 4], - wind: [2, 1], + radius: [5, 5], + wind: [0, 0], image: bgSpringImage }, summer: { diff --git a/frontend/src/constants/common.js b/frontend/src/constants/common.js index 6ae1b8237..725bd7b1a 100644 --- a/frontend/src/constants/common.js +++ b/frontend/src/constants/common.js @@ -77,6 +77,36 @@ export const FILTER_KEYS = { export const MAX_FILE_SIZE_BYTES = 52428800 // 50MB +export const LEGISLATION_TRANSITION_YEAR = 2024 + +export const NEW_REGULATION_YEAR = 2025 + +export const CURRENT_COMPLIANCE_YEAR = ( + LEGISLATION_TRANSITION_YEAR + 1 +).toString() + +export const DEFAULT_CI_FUEL_CODE = + 'Default carbon intensity - section 19 (b) (ii)' + +export const isLegacyCompliancePeriod = (compliancePeriod) => { + // If it's already a number, use it directly + if (typeof compliancePeriod === 'number') { + return compliancePeriod < LEGISLATION_TRANSITION_YEAR + } + + // Try to parse it as a number + const parsedPeriod = Number(compliancePeriod) + + // If parsing failed or resulted in NaN, return false + if (isNaN(parsedPeriod)) { + return false + } + + return parsedPeriod < LEGISLATION_TRANSITION_YEAR +} + +export const FUEL_CATEGORIES = ['Diesel', 'Gasoline', 'Jet fuel'] + // File upload constants for compliance reports export const COMPLIANCE_REPORT_FILE_TYPES = { MIME_TYPES: [ @@ -107,13 +137,3 @@ export const SCHEDULE_IMPORT_FILE_TYPES = { return this.MIME_TYPES.join(',') } } - -export const FUEL_CATEGORIES = ['Diesel', 'Gasoline', 'Jet fuel'] -export const LEGISLATION_TRANSITION_YEAR = 2024 -export const NEW_REGULATION_YEAR = 2025 - -export const CURRENT_COMPLIANCE_YEAR = ( - LEGISLATION_TRANSITION_YEAR + 1 -).toString() -export const DEFAULT_CI_FUEL_CODE = - 'Default carbon intensity - section 19 (b) (ii)' \ No newline at end of file diff --git a/frontend/src/constants/config.js b/frontend/src/constants/config.js index ba85acc65..0d0d928f6 100644 --- a/frontend/src/constants/config.js +++ b/frontend/src/constants/config.js @@ -1,3 +1,6 @@ +import { useUserStore } from '@/stores/useUserStore' +import { roles } from '@/constants/roles' + export function getApiBaseUrl() { // Split the hostname const hostnameParts = window.location.hostname.split('.') @@ -29,7 +32,14 @@ export function getApiBaseUrl() { } export const isFeatureEnabled = (featureFlag) => { - return CONFIG.feature_flags[featureFlag] + // Get user roles from the store + const userRoles = useUserStore.getState().user?.roles || [] + + // Check if user is a beta tester + const isBetaTester = userRoles.some((role) => role.name === roles.beta_tester) + + // Feature is enabled if the flag is set OR the user is a beta tester + return CONFIG.feature_flags[featureFlag] || isBetaTester } export const FEATURE_FLAGS = { diff --git a/frontend/src/constants/roles.js b/frontend/src/constants/roles.js index 657ada2b9..b600458d0 100644 --- a/frontend/src/constants/roles.js +++ b/frontend/src/constants/roles.js @@ -9,7 +9,8 @@ export const roles = { transfers: 'Transfer', compliance_reporting: 'Compliance Reporting', signing_authority: 'Signing Authority', - read_only: 'Read Only' + read_only: 'Read Only', + beta_tester: 'Beta Tester' } export const govRoles = [ diff --git a/frontend/src/views/ComplianceReports/ComplianceReportViewSelector.jsx b/frontend/src/views/ComplianceReports/ComplianceReportViewSelector.jsx index d063f5eab..ba24276fc 100644 --- a/frontend/src/views/ComplianceReports/ComplianceReportViewSelector.jsx +++ b/frontend/src/views/ComplianceReports/ComplianceReportViewSelector.jsx @@ -7,6 +7,7 @@ import { useLocation, useParams } from 'react-router-dom' import { EditViewComplianceReport } from '@/views/ComplianceReports/EditViewComplianceReport.jsx' import { useEffect } from 'react' import useComplianceReportStore from '@/stores/useComplianceReportStore' +import { FEATURE_FLAGS, isFeatureEnabled } from '@/constants/config.js' export const ComplianceReportViewSelector = () => { const { complianceReportId } = useParams() @@ -45,7 +46,17 @@ export const ComplianceReportViewSelector = () => { return } - return reportData?.report?.legacyId ? ( + // Determine which view to show: + // - 2024+ reports: always show full view + // - Pre-2024 reports: use feature flag to control view + const reportYear = + reportData?.report?.compliancePeriod?.description && + parseInt(reportData.report.compliancePeriod.description) + + const showLegacyView = + reportYear < 2024 && !isFeatureEnabled(FEATURE_FLAGS.LEGACY_REPORT_DETAILS) + + return showLegacyView ? ( { const [isScrollingUp, setIsScrollingUp] = useState(false) const [lastScrollTop, setLastScrollTop] = useState(0) + const [isSigningAuthorityDeclared, setIsSigningAuthorityDeclared] = + useState(false) const { compliancePeriod, complianceReportId } = useParams() const scrollToTopOrBottom = () => { @@ -91,6 +91,11 @@ export const ViewLegacyComplianceReport = ({ reportData, error, isError }) => { return } + const dummyButtonClusterConfig = {} + const dummyMethods = { + handleSubmit: () => () => {} + } + if (isError) { return ( <> @@ -129,25 +134,15 @@ export const ViewLegacyComplianceReport = ({ reportData, error, isError }) => { - - {isFeatureEnabled(FEATURE_FLAGS.LEGACY_REPORT_DETAILS) && ( - <> - - - - )} {t('report:questions')} diff --git a/frontend/src/views/ComplianceReports/__tests__/ComplianceReportViewSelector.test.jsx b/frontend/src/views/ComplianceReports/__tests__/ComplianceReportViewSelector.test.jsx index d681cd787..44c9ae6bf 100644 --- a/frontend/src/views/ComplianceReports/__tests__/ComplianceReportViewSelector.test.jsx +++ b/frontend/src/views/ComplianceReports/__tests__/ComplianceReportViewSelector.test.jsx @@ -1,8 +1,11 @@ import React from 'react' -import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen } from '@testing-library/react' -import { ComplianceReportViewSelector } from '../ComplianceReportViewSelector' -import { wrapper } from '@/tests/utils/wrapper.jsx' +import { render, screen, waitFor } from '@testing-library/react' +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { ComplianceReportViewSelector } from '../ComplianceReportViewSelector.jsx' +import * as useComplianceReportsHook from '@/hooks/useComplianceReports' +import * as useCurrentUserHook from '@/hooks/useCurrentUser' +import { wrapper } from '@/tests/utils/wrapper' +import { isFeatureEnabled } from '@/constants/config.js' // Create mock functions at the top level const mockUseParams = vi.fn() @@ -17,6 +20,17 @@ vi.mock('react-router-dom', () => ({ useLocation: () => mockUseLocation() })) +vi.mock('@/constants/config.js', () => ({ + FEATURE_FLAGS: { + LEGACY_REPORT_DETAILS: 'fullLegacyReports' + }, + isFeatureEnabled: vi.fn() +})) + +vi.mock('@/components/Loading', () => ({ + default: () =>
Loading...
+})) + vi.mock('@tanstack/react-query', () => ({ useQueryClient: () => mockUseQueryClient() })) @@ -70,6 +84,9 @@ describe('ComplianceReportViewSelector', () => { id: 'report-123', currentStatus: { status: 'Draft' + }, + compliancePeriod: { + description: '2024' } } }, @@ -81,6 +98,9 @@ describe('ComplianceReportViewSelector', () => { beforeEach(() => { vi.clearAllMocks() + + // Default to feature flag disabled + isFeatureEnabled.mockReturnValue(false) // Default mock implementations mockUseParams.mockReturnValue({ complianceReportId: 'test-report-id' }) @@ -90,6 +110,24 @@ describe('ComplianceReportViewSelector', () => { mockUseGetComplianceReport.mockReturnValue(defaultReportData) }) + const setupMocks = ({ + currentUser = defaultCurrentUser, + reportData = defaultReportData, + complianceReportId = 'test-report-id', + locationState = null, + isError = false, + error = null + } = {}) => { + mockUseCurrentUser.mockReturnValue(currentUser) + mockUseGetComplianceReport.mockReturnValue({ + ...reportData, + isError, + error + }) + mockUseParams.mockReturnValue({ complianceReportId }) + mockUseLocation.mockReturnValue({ state: locationState }) + } + describe('Component rendering', () => { it('renders the component function correctly', () => { render(, { wrapper }) @@ -185,17 +223,20 @@ describe('ComplianceReportViewSelector', () => { }) describe('Report type rendering', () => { - it('renders ViewLegacyComplianceReport when legacyId exists', () => { - const reportWithLegacyId = { + it('renders ViewLegacyComplianceReport for pre-2024 reports when feature flag is disabled', () => { + // Feature flag disabled (default) + isFeatureEnabled.mockReturnValue(false) + + const pre2024Report = { ...defaultReportData, data: { report: { ...defaultReportData.data.report, - legacyId: 'legacy-123' + compliancePeriod: { description: '2023' } } } } - mockUseGetComplianceReport.mockReturnValue(reportWithLegacyId) + mockUseGetComplianceReport.mockReturnValue(pre2024Report) render(, { wrapper }) @@ -203,45 +244,94 @@ describe('ComplianceReportViewSelector', () => { expect(screen.queryByTestId('edit-report')).not.toBeInTheDocument() }) - it('renders EditViewComplianceReport when legacyId does not exist', () => { - const reportWithoutLegacyId = { + it('renders EditViewComplianceReport for 2024+ reports', () => { + const report2024Plus = { ...defaultReportData, data: { report: { ...defaultReportData.data.report, - legacyId: null + compliancePeriod: { description: '2024' } } } } - mockUseGetComplianceReport.mockReturnValue(reportWithoutLegacyId) + mockUseGetComplianceReport.mockReturnValue(report2024Plus) render(, { wrapper }) expect(screen.getByTestId('edit-report')).toBeInTheDocument() expect(screen.queryByTestId('legacy-report')).not.toBeInTheDocument() }) + }) - it('passes correct props to ViewLegacyComplianceReport', () => { - const reportWithLegacyId = { - data: { - report: { - id: 'report-123', - legacyId: 'legacy-123', - currentStatus: { status: 'Draft' } + describe('Feature flag behavior', () => { + it('renders EditViewComplianceReport when feature flag is enabled for pre-2024 reports', async () => { + // Mock feature flag as enabled + isFeatureEnabled.mockReturnValue(true) + + setupMocks({ + reportData: { + data: { + report: { + compliancePeriod: { description: '2015' }, + currentStatus: { status: 'DRAFT' } + } } - }, - isLoading: false, + } + }) + + render(, { wrapper }) + + await waitFor(() => { + expect(screen.getByTestId('edit-report')).toBeInTheDocument() + expect(screen.queryByTestId('legacy-report')).not.toBeInTheDocument() + }) + }) + + it('renders ViewLegacyComplianceReport when feature flag is disabled for pre-2024 reports', async () => { + // Mock feature flag as disabled + isFeatureEnabled.mockReturnValue(false) + + setupMocks({ + reportData: { + data: { + report: { + compliancePeriod: { description: '2023' }, + currentStatus: { status: 'DRAFT' } + } + } + } + }) + + render(, { wrapper }) + + await waitFor(() => { + expect(screen.getByTestId('legacy-report')).toBeInTheDocument() + expect(screen.queryByTestId('edit-report')).not.toBeInTheDocument() + }) + }) + }) + + describe('Props passing', () => { + it('passes error and isError props to the rendered component', async () => { + const testError = { message: 'Test error' } + setupMocks({ isError: true, - error: 'Test error', - refetch: mockRefetch - } - mockUseGetComplianceReport.mockReturnValue(reportWithLegacyId) + error: testError, + reportData: { + data: { + report: { + compliancePeriod: { description: '2024' }, + currentStatus: { status: 'DRAFT' } + } + } + } + }) render(, { wrapper }) - const legacyReportElement = screen.getByTestId('legacy-report') - expect(legacyReportElement.textContent).toContain('"error":"Test error"') - expect(legacyReportElement.textContent).toContain('"isError":true') + const editReportElement = screen.getByTestId('edit-report') + expect(editReportElement.textContent).toContain('"error":{"message":"Test error"}') + expect(editReportElement.textContent).toContain('"isError":true') }) it('passes correct props to EditViewComplianceReport', () => { @@ -406,4 +496,52 @@ describe('ComplianceReportViewSelector', () => { }) }) -}) \ No newline at end of file + + describe('Cache invalidation tests', () => { + it('does not invalidate cache when location state is null', async () => { + setupMocks({ + reportData: { + data: { + report: { + compliancePeriod: { description: '2024' }, + currentStatus: { status: 'DRAFT' } + } + } + }, + locationState: null // No location state + }) + + render(, { wrapper }) + + await waitFor(() => { + expect(screen.getByTestId('edit-report')).toBeInTheDocument() + }) + + // Should not invalidate cache since there's no location state + expect(mockQueryClient.invalidateQueries).not.toHaveBeenCalled() + expect(mockRefetch).not.toHaveBeenCalled() + }) + }) + + describe('Hook integration tests', () => { + it('calls useGetComplianceReport with correct parameters', async () => { + const currentUser = { data: { organization: { organizationId: '456' } }, isLoading: false } + const complianceReportId = '789' + + setupMocks({ + currentUser, + complianceReportId + }) + + render(, { wrapper }) + + expect(mockUseGetComplianceReport).toHaveBeenCalledWith( + '456', // organizationId + '789', // complianceReportId + { + enabled: true // !isCurrentUserLoading + } + ) + }) + }) +}) diff --git a/frontend/src/views/ComplianceReports/__tests__/ViewLegacyComplianceReport.test.jsx b/frontend/src/views/ComplianceReports/__tests__/ViewLegacyComplianceReport.test.jsx index 7b209d5da..f66798d5c 100644 --- a/frontend/src/views/ComplianceReports/__tests__/ViewLegacyComplianceReport.test.jsx +++ b/frontend/src/views/ComplianceReports/__tests__/ViewLegacyComplianceReport.test.jsx @@ -165,7 +165,8 @@ describe('ViewLegacyComplianceReport', () => { // Set default mock returns first mockUseCurrentUser.mockReturnValue({ data: { isGovernmentUser: false }, - isLoading: false + isLoading: false, + hasRoles: vi.fn(() => false) }) mockUseOrganization.mockReturnValue({ data: { id: 1, name: 'Test Org' }, diff --git a/frontend/src/views/ComplianceReports/components/LegacyAssessmentCard.jsx b/frontend/src/views/ComplianceReports/components/LegacyAssessmentCard.jsx index c29a8f717..e2b3aa0f2 100644 --- a/frontend/src/views/ComplianceReports/components/LegacyAssessmentCard.jsx +++ b/frontend/src/views/ComplianceReports/components/LegacyAssessmentCard.jsx @@ -90,20 +90,22 @@ export const LegacyAssessmentCard = ({ > {t('report:supplementalWarning')} - - } - sx={{ mt: 2 }} - onClick={viewLegacyReport} - > - {t('report:viewLegacyBtn')} - - + {legacyReportId && ( + + } + sx={{ mt: 2 }} + onClick={viewLegacyReport} + > + {t('report:viewLegacyBtn')} + + + )} } diff --git a/frontend/src/views/ComplianceReports/components/NewComplianceReportButton.jsx b/frontend/src/views/ComplianceReports/components/NewComplianceReportButton.jsx index 0b7b50e93..0deb4e819 100644 --- a/frontend/src/views/ComplianceReports/components/NewComplianceReportButton.jsx +++ b/frontend/src/views/ComplianceReports/components/NewComplianceReportButton.jsx @@ -47,7 +47,7 @@ export const NewComplianceReportButton = forwardRef((props, ref) => { const periodsArray = periods?.data || periods || [] return periodsArray.filter((item) => { const effectiveYear = new Date(item.effectiveDate).getFullYear() - return effectiveYear <= currentYear && effectiveYear >= 2024 + return effectiveYear >= 2018 && effectiveYear <= currentYear }) } diff --git a/frontend/src/views/ComplianceReports/components/_schema.jsx b/frontend/src/views/ComplianceReports/components/_schema.jsx index 317e7fa5b..be320ba12 100644 --- a/frontend/src/views/ComplianceReports/components/_schema.jsx +++ b/frontend/src/views/ComplianceReports/components/_schema.jsx @@ -359,7 +359,8 @@ export const renewableFuelColumns = ( jetFuelEditableCells = [] } - return [ + // Define all potential columns + const allColumns = [ { id: 'line', label: t('report:summaryLabels.line'), @@ -415,6 +416,16 @@ export const renewableFuelColumns = ( } } ] + + // Filter out the jetFuel column if the compliance period year is less than 2024 + const filteredColumns = allColumns.filter((col) => { + if (col.id === 'jetFuel' && parseInt(compliancePeriodYear) < 2024) { + return false // Exclude jet fuel column for years before 2024 + } + return true // Include all other columns + }) + + return filteredColumns } export const lowCarbonColumns = (t) => [ diff --git a/frontend/src/views/ComplianceReports/legacy-deprecated/ExclusionAgreementSummary.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/ExclusionAgreementSummary.jsx new file mode 100644 index 000000000..458f502e0 --- /dev/null +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/ExclusionAgreementSummary.jsx @@ -0,0 +1,108 @@ +import BCAlert from '@/components/BCAlert' +import BCBox from '@/components/BCBox' +import Grid2 from '@mui/material/Grid2' +import { formatNumberWithCommas as valueFormatter } from '@/utils/formatters' +import { useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useLocation, useParams } from 'react-router-dom' +import { COMPLIANCE_REPORT_STATUSES } from '@/constants/statuses.js' +import { LinkRenderer } from '@/utils/grid/cellRenderers.jsx' +import { BCGridViewer } from '@/components/BCDataGrid/BCGridViewer.jsx' +import { defaultInitialPagination } from '@/constants/schedules.js' +import { useGetAllocationAgreements } from '@/hooks/useAllocationAgreement.js' +import { exclusionSummaryColDefs } from './_schema.jsx' + +export const ExclusionAgreementSummary = ({ data, status }) => { + const gridRef = useRef() + const [alertMessage, setAlertMessage] = useState('') + const [alertSeverity, setAlertSeverity] = useState('info') + const { complianceReportId: reportIdString } = useParams() + + const [paginationOptions, setPaginationOptions] = useState( + defaultInitialPagination + ) + + const { t } = useTranslation(['common', 'exclusionAgreement', 'legacy']) + const location = useLocation() + + const complianceReportId = parseInt(reportIdString) + + const queryData = useGetAllocationAgreements( + complianceReportId, + paginationOptions, + { + cacheTime: 0, + staleTime: 0, + enabled: !isNaN(complianceReportId) + } + ) + + useEffect(() => { + if (location.state?.message) { + setAlertMessage(location.state.message) + setAlertSeverity(location.state.severity || 'info') + } + }, [location.state]) + + const defaultColDef = useMemo( + () => ({ + floatingFilter: false, + filter: false, + cellRenderer: + status === COMPLIANCE_REPORT_STATUSES.DRAFT ? LinkRenderer : undefined, + cellRendererParams: { + url: () => 'exclusion-agreements' + } + }), + [status] + ) + + const columns = useMemo(() => exclusionSummaryColDefs(t), [t]) + + const getRowId = (params) => { + return params.data.allocationAgreementId.toString() + } + + return ( + +
+ {alertMessage && ( + + {alertMessage} + + )} +
+ + + setPaginationOptions((prev) => ({ + ...prev, + ...newPagination + })) + } + autoSizeStrategy={{ + type: 'fitCellContents', + defaultMinWidth: 50, + defaultMaxWidth: 600 + }} + enableCellTextSelection + /> + + + ) +} + +ExclusionAgreementSummary.displayName = 'ExclusionAgreementSummary' diff --git a/frontend/src/views/ComplianceReports/legacy/LegacyReportDetails.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/LegacyReportDetails.jsx similarity index 85% rename from frontend/src/views/ComplianceReports/legacy/LegacyReportDetails.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/LegacyReportDetails.jsx index f7e7e9a76..bf8c5f025 100644 --- a/frontend/src/views/ComplianceReports/legacy/LegacyReportDetails.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/LegacyReportDetails.jsx @@ -1,4 +1,4 @@ -import { useCallback, useMemo, useState } from 'react' +import { useMemo, useState } from 'react' import { useNavigate, useParams } from 'react-router-dom' import { useTranslation } from 'react-i18next' import { @@ -9,19 +9,21 @@ import { Link } from '@mui/material' import BCTypography from '@/components/BCTypography' +import { ScheduleASummary } from './ScheduleASummary' +import { ScheduleBSummary } from './ScheduleBSummary' +import { ScheduleCSummary } from './ScheduleCSummary' +import { ExclusionAgreementSummary } from './ExclusionAgreementSummary' import { ROUTES, buildPath } from '@/routes/routes' import { useGetAllNotionalTransfers } from '@/hooks/useNotionalTransfer' -import { ScheduleASummary } from '@/views/ComplianceReports/legacy/ScheduleASummary.jsx' import { useGetAllOtherUses } from '@/hooks/useOtherUses.js' -import { ScheduleCSummary } from '@/views/ComplianceReports/legacy/ScheduleCSummary.jsx' import { useGetFuelSupplies } from '@/hooks/useFuelSupply.js' -import { ScheduleBSummary } from '@/views/ComplianceReports/legacy/ScheduleBSummary.jsx' +import { useGetAllAllocationAgreements } from '@/hooks/useAllocationAgreement' import { isArrayEmpty } from '@/utils/array.js' import { ExpandMore } from '@mui/icons-material' const LegacyReportDetails = ({ currentStatus = 'Draft' }) => { - const { t } = useTranslation(['legacy']) + const { t } = useTranslation(['legacy', 'report']) const { compliancePeriod, complianceReportId } = useParams() const navigate = useNavigate() @@ -71,6 +73,21 @@ const LegacyReportDetails = ({ currentStatus = 'Draft' }) => { data.otherUses.length > 0 && ( ) + }, + { + name: t('legacy:activityLists.exclusionAgreement'), + action: () => + navigate( + buildPath(ROUTES.REPORTS.ADD.ALLOCATION_AGREEMENTS, { + compliancePeriod, + complianceReportId + }) + ), + useFetch: useGetAllAllocationAgreements, + component: (data) => + data?.allocationAgreements?.length > 0 && ( + + ) } ], [t, complianceReportId, navigate, compliancePeriod, currentStatus] diff --git a/frontend/src/views/ComplianceReports/legacy/LegacyReportSummary.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/LegacyReportSummary.jsx similarity index 100% rename from frontend/src/views/ComplianceReports/legacy/LegacyReportSummary.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/LegacyReportSummary.jsx diff --git a/frontend/src/views/ComplianceReports/legacy/ScheduleASummary.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleASummary.jsx similarity index 98% rename from frontend/src/views/ComplianceReports/legacy/ScheduleASummary.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleASummary.jsx index afcf36fcd..97a7be238 100644 --- a/frontend/src/views/ComplianceReports/legacy/ScheduleASummary.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleASummary.jsx @@ -7,7 +7,7 @@ import { useTranslation } from 'react-i18next' import { useLocation, useParams } from 'react-router-dom' import { COMPLIANCE_REPORT_STATUSES } from '@/constants/statuses.js' import { LinkRenderer } from '@/utils/grid/cellRenderers.jsx' -import { scheduleASummaryColDefs } from '@/views/ComplianceReports/legacy/_schema.jsx' +import { scheduleASummaryColDefs } from '@/views/ComplianceReports/legacy-deprecated/_schema.jsx' import { BCGridViewer } from '@/components/BCDataGrid/BCGridViewer.jsx' import { defaultInitialPagination } from '@/constants/schedules.js' diff --git a/frontend/src/views/ComplianceReports/legacy/ScheduleBSummary.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleBSummary.jsx similarity index 98% rename from frontend/src/views/ComplianceReports/legacy/ScheduleBSummary.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleBSummary.jsx index c2148a1ed..5355fc3cb 100644 --- a/frontend/src/views/ComplianceReports/legacy/ScheduleBSummary.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleBSummary.jsx @@ -9,7 +9,7 @@ import { useLocation, useParams } from 'react-router-dom' import { v4 as uuid } from 'uuid' import { COMPLIANCE_REPORT_STATUSES } from '@/constants/statuses.js' import { LinkRenderer } from '@/utils/grid/cellRenderers.jsx' -import { scheduleBSummaryColDefs } from '@/views/ComplianceReports/legacy/_schema.jsx' +import { scheduleBSummaryColDefs } from '@/views/ComplianceReports/legacy-deprecated/_schema.jsx' export const ScheduleBSummary = ({ data, status }) => { const [alertMessage, setAlertMessage] = useState('') diff --git a/frontend/src/views/ComplianceReports/legacy/ScheduleCSummary.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleCSummary.jsx similarity index 98% rename from frontend/src/views/ComplianceReports/legacy/ScheduleCSummary.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleCSummary.jsx index 771d896f8..c38217bfe 100644 --- a/frontend/src/views/ComplianceReports/legacy/ScheduleCSummary.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/ScheduleCSummary.jsx @@ -6,7 +6,7 @@ import { useLocation, useParams } from 'react-router-dom' import { COMPLIANCE_REPORT_STATUSES } from '@/constants/statuses.js' import { LinkRenderer } from '@/utils/grid/cellRenderers.jsx' import Grid2 from '@mui/material/Grid2' -import { scheduleCSummaryColDefs } from '@/views/ComplianceReports/legacy/_schema.jsx' +import { scheduleCSummaryColDefs } from '@/views/ComplianceReports/legacy-deprecated/_schema.jsx' import { BCGridViewer } from '@/components/BCDataGrid/BCGridViewer.jsx' import { defaultInitialPagination } from '@/constants/schedules.js' import { useGetOtherUses } from '@/hooks/useOtherUses.js' diff --git a/frontend/src/views/ComplianceReports/legacy/__test__/LegacyReportSummary.test.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/__test__/LegacyReportSummary.test.jsx similarity index 97% rename from frontend/src/views/ComplianceReports/legacy/__test__/LegacyReportSummary.test.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/__test__/LegacyReportSummary.test.jsx index 6457c4642..3efe4a9dd 100644 --- a/frontend/src/views/ComplianceReports/legacy/__test__/LegacyReportSummary.test.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/__test__/LegacyReportSummary.test.jsx @@ -1,4 +1,4 @@ -// src/views/ComplianceReports/legacy/__test__/LegacyReportSummary.test.jsx +// src/views/ComplianceReports/legacy-deprecated/__test__/LegacyReportSummary.test.jsx import { describe, it, expect, beforeEach, vi } from 'vitest' import { render, screen, waitFor } from '@testing-library/react' import LegacyReportSummary from '../LegacyReportSummary' @@ -29,7 +29,7 @@ vi.mock('../_schema', () => ({ nonComplianceColumns: vi.fn(() => []) })) -describe('LegacyReportSummary', () => { +describe.skip('LegacyReportSummary', () => { const reportID = '123' const currentStatus = 'active' const compliancePeriodYear = '2025' diff --git a/frontend/src/views/ComplianceReports/legacy/__test__/ScheduleASummary.test.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleASummary.test.jsx similarity index 98% rename from frontend/src/views/ComplianceReports/legacy/__test__/ScheduleASummary.test.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleASummary.test.jsx index 728e88d8b..121054476 100644 --- a/frontend/src/views/ComplianceReports/legacy/__test__/ScheduleASummary.test.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleASummary.test.jsx @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { cleanup, render, screen } from '@testing-library/react' import { useLocation, useNavigate } from 'react-router-dom' -import { ScheduleASummary } from '@/views/ComplianceReports/legacy/ScheduleASummary' +import { ScheduleASummary } from '@/views/ComplianceReports/legacy-deprecated/ScheduleASummary' import { wrapper } from '@/tests/utils/wrapper.jsx' vi.mock('react-router-dom', async () => { diff --git a/frontend/src/views/ComplianceReports/legacy/__test__/ScheduleBSummary.test.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleBSummary.test.jsx similarity index 100% rename from frontend/src/views/ComplianceReports/legacy/__test__/ScheduleBSummary.test.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleBSummary.test.jsx diff --git a/frontend/src/views/ComplianceReports/legacy/__test__/ScheduleCSummary.test.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleCSummary.test.jsx similarity index 100% rename from frontend/src/views/ComplianceReports/legacy/__test__/ScheduleCSummary.test.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/__test__/ScheduleCSummary.test.jsx diff --git a/frontend/src/views/ComplianceReports/legacy/_schema.jsx b/frontend/src/views/ComplianceReports/legacy-deprecated/_schema.jsx similarity index 71% rename from frontend/src/views/ComplianceReports/legacy/_schema.jsx rename to frontend/src/views/ComplianceReports/legacy-deprecated/_schema.jsx index 2d4b37e9d..6144293ad 100644 --- a/frontend/src/views/ComplianceReports/legacy/_schema.jsx +++ b/frontend/src/views/ComplianceReports/legacy-deprecated/_schema.jsx @@ -42,22 +42,22 @@ export const scheduleBSummaryColDefs = (t) => [ { headerName: t('fuelSupply:fuelSupplyColLabels.fuelType'), field: 'fuelType', - valueGetter: (params) => params.data.fuelType?.fuelType + valueGetter: (params) => params.data.fuelType }, { headerName: t('legacy:columnLabels.fuelClass'), field: 'fuelCategory', - valueGetter: (params) => params.data.fuelCategory?.category + valueGetter: (params) => params.data.fuelCategory }, { headerName: t('fuelSupply:fuelSupplyColLabels.determiningCarbonIntensity'), field: 'determiningCarbonIntensity', - valueGetter: (params) => params.data.provisionOfTheAct?.name + valueGetter: (params) => params.data.provisionOfTheAct }, { headerName: t('fuelSupply:fuelSupplyColLabels.fuelCode'), field: 'fuelCode', - valueGetter: (params) => params.data.fuelCode?.fuelCode + valueGetter: (params) => params.data.fuelCode }, { headerName: t('fuelSupply:fuelSupplyColLabels.quantity'), @@ -124,6 +124,72 @@ export const scheduleCSummaryColDefs = (t) => [ } ] +export const exclusionSummaryColDefs = (t) => [ + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.allocationTransactionType' + ), + field: 'allocationTransactionType' + }, + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.transactionPartner' + ), + field: 'transactionPartner' + }, + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.postalAddress' + ), + field: 'postalAddress' + }, + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.transactionPartnerEmail' + ), + field: 'transactionPartnerEmail' + }, + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.transactionPartnerPhone' + ), + field: 'transactionPartnerPhone' + }, + { + headerName: t('allocationAgreement:allocationAgreementColLabels.fuelType'), + field: 'fuelType' + }, + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.fuelCategory' + ), + field: 'fuelCategory' + }, + { + headerName: t( + 'allocationAgreement:allocationAgreementColLabels.carbonIntensity' + ), + field: 'provisionOfTheAct' + }, + { + headerName: t('allocationAgreement:allocationAgreementColLabels.fuelCode'), + field: 'fuelCode' + }, + { + headerName: t('allocationAgreement:allocationAgreementColLabels.ciOfFuel'), + field: 'ciOfFuel' + }, + { + headerName: t('allocationAgreement:allocationAgreementColLabels.quantity'), + field: 'quantity', + valueFormatter + }, + { + headerName: t('allocationAgreement:allocationAgreementColLabels.units'), + field: 'units' + } +] + export const renewableFuelColumns = (t, data) => { return [ { diff --git a/frontend/src/views/FuelSupplies/AddEditFuelSupplies.jsx b/frontend/src/views/FuelSupplies/AddEditFuelSupplies.jsx index 7c4e27b5e..631f5a90e 100644 --- a/frontend/src/views/FuelSupplies/AddEditFuelSupplies.jsx +++ b/frontend/src/views/FuelSupplies/AddEditFuelSupplies.jsx @@ -1,7 +1,14 @@ import BCBox from '@/components/BCBox' import { BCGridEditor } from '@/components/BCDataGrid/BCGridEditor' import BCTypography from '@/components/BCTypography' -import { NEW_REGULATION_YEAR, REPORT_SCHEDULES } from '@/constants/common' +import { + DEFAULT_CI_FUEL, + NEW_REGULATION_YEAR, + REPORT_SCHEDULES, + isLegacyCompliancePeriod +} from '@/constants/common' +import { useGetComplianceReport } from '@/hooks/useComplianceReports' +import { useCurrentUser } from '@/hooks/useCurrentUser' import { buildPath, ROUTES } from '@/routes/routes' import { useFuelSupplyOptions, @@ -24,7 +31,10 @@ import Grid2 from '@mui/material/Grid2' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useLocation, useNavigate, useParams } from 'react-router-dom' -import { defaultColDef, fuelSupplyColDefs } from './_schema' +import { v4 as uuid } from 'uuid' +import * as schema from './_schema' +import * as legacySchema from './_legacySchema' +// import { defaultColDef, fuelSupplyColDefs } from './_schema' import { REPORT_SCHEDULES_VIEW } from '@/constants/statuses' import { useComplianceReportWithCache } from '@/hooks/useComplianceReports' import Loading from '@/components/Loading' @@ -160,6 +170,23 @@ export const AddEditFuelSupplies = () => { }, 100) }, []) + const getSchema = (compliancePeriod) => { + return isLegacyCompliancePeriod(compliancePeriod) ? legacySchema : schema + } + + useEffect(() => { + const currentSchema = getSchema(compliancePeriod) + const updatedColumnDefs = currentSchema.fuelSupplyColDefs( + optionsData, + errors, + warnings, + compliancePeriod, + isSupplemental, + isEarlyIssuance + ) + setColumnDefs(updatedColumnDefs) + }, [isSupplemental, isEarlyIssuance, errors, optionsData, warnings]) + const onFirstDataRendered = useCallback((params) => { params.api?.autoSizeAllColumns?.() }, []) @@ -275,37 +302,71 @@ export const AddEditFuelSupplies = () => { {t('fuelSupply:fuelSupplyNote')} + + + ) : ( - - {t('fuelSupply:fuelSupplyGuide')} - + <> + + {t('fuelSupply:fuelSupplyGuide')} + + + + + )} - - - ) } diff --git a/frontend/src/views/FuelSupplies/__tests__/AddEditFuelSupplies.test.jsx b/frontend/src/views/FuelSupplies/__tests__/AddEditFuelSupplies.test.jsx index 1304378f1..1018ef33a 100644 --- a/frontend/src/views/FuelSupplies/__tests__/AddEditFuelSupplies.test.jsx +++ b/frontend/src/views/FuelSupplies/__tests__/AddEditFuelSupplies.test.jsx @@ -25,7 +25,8 @@ vi.mock('@/utils/array.js', () => ({ })) vi.mock('@/utils/formatters', () => ({ - cleanEmptyStringValues: vi.fn((data) => ({ ...data, cleaned: true })) + cleanEmptyStringValues: vi.fn((data) => ({ ...data, cleaned: true })), + formatNumberWithCommas: vi.fn((value) => value?.toString()) })) vi.mock('@/utils/schedules.js', () => ({ @@ -79,6 +80,39 @@ vi.mock('@/constants/common', () => ({ })) vi.mock('@/constants/statuses', () => ({ + TRANSFER_STATUSES: { + NEW: 'New', + DRAFT: 'Draft', + DELETED: 'Deleted', + SENT: 'Sent', + SUBMITTED: 'Submitted', + RECOMMENDED: 'Recommended', + RECORDED: 'Recorded', + REFUSED: 'Refused', + DECLINED: 'Declined', + RESCINDED: 'Rescinded' + }, + COMPLIANCE_REPORT_STATUSES: { + DRAFT: 'Draft', + DELETED: 'Deleted', + SUBMITTED: 'Submitted', + ANALYST_ADJUSTMENT: 'Analyst adjustment', + RECOMMENDED_BY_ANALYST: 'Recommended by analyst', + RECOMMENDED_BY_MANAGER: 'Recommended by manager', + SUPPLEMENTAL_REQUESTED: 'Supplemental requested', + ASSESSED: 'Assessed', + REJECTED: 'Rejected', + RETURN_TO_ANALYST: 'Return to analyst', + RETURN_TO_MANAGER: 'Return to manager', + RETURN_TO_SUPPLIER: 'Return to supplier' + }, + TRANSACTION_STATUSES: { + NEW: 'New', + DRAFT: 'Draft', + RECOMMENDED: 'Recommended', + APPROVED: 'Approved', + DELETED: 'Deleted' + }, REPORT_SCHEDULES_VIEW: { EDIT: 'EDIT', VIEW: 'VIEW' diff --git a/frontend/src/views/FuelSupplies/_legacySchema.jsx b/frontend/src/views/FuelSupplies/_legacySchema.jsx new file mode 100644 index 000000000..7617c5e83 --- /dev/null +++ b/frontend/src/views/FuelSupplies/_legacySchema.jsx @@ -0,0 +1,692 @@ +import { actions, validation } from '@/components/BCDataGrid/columns' +import { + AsyncSuggestionEditor, + AutocompleteCellEditor, + NumberEditor, + RequiredHeader +} from '@/components/BCDataGrid/components' +import { apiRoutes } from '@/constants/routes' +import i18n from '@/i18n' +import colors from '@/themes/base/colors' +import { + decimalFormatter, + formatNumberWithCommas, + numberFormatter +} from '@/utils/formatters' +import { + fuelTypeOtherConditionalStyle, + isFuelTypeOther +} from '@/utils/fuelTypeOther' +import { changelogCellStyle } from '@/utils/grid/changelogCellStyle' +import { + StandardCellStyle, + StandardCellWarningAndErrors +} from '@/utils/grid/errorRenderers' +import { suppressKeyboardEvent } from '@/utils/grid/eventHandlers' +import { SelectRenderer } from '@/utils/grid/cellRenderers.jsx' +import { ACTION_STATUS_MAP } from '@/constants/schemaConstants' + +export const PROVISION_APPROVED_FUEL_CODE = + 'Approved fuel code - Section 6 (5) (c)' +export const PROVISION_GHGENIUS = + 'GHGenius modelled - Section 6 (5) (d) (ii) (A)' + +export const fuelSupplyColDefs = ( + optionsData, + errors, + warnings, + isSupplemental +) => [ + validation, + actions((params) => { + return { + enableDuplicate: false, + enableDelete: !params.data.isNewSupplementalEntry, + enableUndo: isSupplemental && params.data.isNewSupplementalEntry, + enableStatus: + isSupplemental && + params.data.isNewSupplementalEntry && + ACTION_STATUS_MAP[params.data.actionType] + } + }), + { + field: 'id', + cellEditor: 'agTextCellEditor', + cellDataType: 'text', + hide: true + }, + { + field: 'complianceReportId', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.complianceReportId'), + cellEditor: 'agTextCellEditor', + cellDataType: 'text', + hide: true + }, + { + field: 'fuelSupplyId', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelSupplyId'), + cellEditor: 'agTextCellEditor', + cellDataType: 'text', + hide: true + }, + { + field: 'fuelCodeId', + cellEditor: 'agTextCellEditor', + cellDataType: 'text', + hide: true + }, + { + field: 'endUseId', + cellEditor: 'agTextCellEditor', + cellDataType: 'text', + hide: true + }, + { + field: 'complianceUnits', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.complianceUnits'), + minWidth: 100, + valueFormatter: formatNumberWithCommas, + editable: false, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental) + }, + { + field: 'fuelType', + headerComponent: RequiredHeader, + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelType'), + cellEditor: AutocompleteCellEditor, + cellRenderer: SelectRenderer, + cellEditorParams: { + options: optionsData?.fuelTypes?.map((obj) => obj.fuelType).sort(), + multiple: false, + disableCloseOnSelect: false, + freeSolo: false, + openOnFocus: true + }, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental), + suppressKeyboardEvent, + minWidth: 260, + editable: true, + valueGetter: (params) => params.data.fuelType, + valueSetter: (params) => { + if (params.newValue) { + const fuelType = optionsData?.fuelTypes?.find( + (obj) => obj.fuelType === params.newValue + ) + params.data.fuelType = params.newValue + params.data.fuelTypeId = fuelType?.fuelTypeId + params.data.fuelTypeOther = null + params.data.fuelCategory = null + params.data.fuelCategoryId = null + params.data.endUseId = null + params.data.endUseType = null + params.data.eer = null + params.data.provisionOfTheAct = null + params.data.provisionOfTheActId = null + params.data.fuelCode = null + params.data.fuelCodeId = null + params.data.units = fuelType?.unit + params.data.unrecognized = fuelType?.unrecognized + params.data.energyDensity = fuelType?.energyDensity + } + return true + }, + tooltipValueGetter: () => 'Select the fuel type from the list' + }, + { + field: 'fuelTypeOther', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelTypeOther'), + cellEditor: AsyncSuggestionEditor, + cellEditorParams: (params) => ({ + queryKey: 'fuel-type-others', + queryFn: async ({ queryKey, client }) => { + const path = apiRoutes.getFuelTypeOthers + const response = await client.get(path) + params.node.data.apiDataCache = response.data + return response.data + }, + title: 'transactionPartner', + api: params.api, + minWords: 1 + }), + cellStyle: (params) => { + if (isSupplemental && params.data.isNewSupplementalEntry) { + if (params.data.actionType === 'UPDATE') { + return { backgroundColor: colors.alerts.warning.background } + } + } else { + return StandardCellStyle( + params, + errors, + warnings, + fuelTypeOtherConditionalStyle + ) + } + }, + valueSetter: (params) => { + const { newValue: selectedFuelTypeOther, data } = params + data.fuelTypeOther = selectedFuelTypeOther + return true + }, + editable: (params) => isFuelTypeOther(params), + minWidth: 250 + }, + { + field: 'fuelCategory', + headerComponent: RequiredHeader, + headerName: i18n.t('legacy:columnLabels.fuelClass'), + cellEditor: AutocompleteCellEditor, + cellRenderer: SelectRenderer, + cellEditorParams: (params) => ({ + options: optionsData?.fuelTypes + ?.find((obj) => params.data.fuelType === obj.fuelType) + ?.fuelCategories.map((item) => item.fuelCategory) + .sort(), + multiple: false, + disableCloseOnSelect: false, + freeSolo: false, + openOnFocus: true + }), + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental), + valueSetter: (params) => { + if (params.newValue) { + params.data.fuelCategory = params.newValue + params.data.fuelCategoryId = optionsData?.fuelTypes + ?.find((obj) => params.data.fuelType === obj.fuelType) + ?.fuelCategories?.find( + (obj) => params.newValue === obj.fuelCategory + )?.fuelCategoryId + params.data.endUseId = null + params.data.endUseType = null + params.data.eer = null + params.data.provisionOfTheAct = null + params.data.provisionOfTheActId = null + params.data.fuelCode = null + } + return true + }, + suppressKeyboardEvent, + minWidth: 135, + valueGetter: (params) => params.data.fuelCategory, + editable: (params) => + optionsData?.fuelTypes + ?.find((obj) => params.data.fuelType === obj.fuelType) + ?.fuelCategories.map((item) => item.fuelCategory).length > 1, + tooltipValueGetter: () => 'Select the fuel class from the list' + }, + { + field: 'provisionOfTheAct', + headerComponent: RequiredHeader, + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.provisionOfTheActId'), + cellEditor: AutocompleteCellEditor, + cellRenderer: SelectRenderer, + valueGetter: (params) => params.data.provisionOfTheAct, + cellEditorParams: (params) => ({ + options: optionsData?.fuelTypes + ?.find((obj) => params.data.fuelType === obj.fuelType) + ?.provisions.map((item) => item.name) + .sort(), + multiple: false, + disableCloseOnSelect: false, + freeSolo: false, + openOnFocus: true + }), + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental), + suppressKeyboardEvent, + minWidth: 370, + valueSetter: (params) => { + if (params.newValue) { + params.data.provisionOfTheAct = params.newValue + params.data.provisionOfTheActId = optionsData?.fuelTypes + ?.find((obj) => params.data.fuelType === obj.fuelType) + ?.provisions.find( + (item) => item.name === params.newValue + )?.provisionOfTheActId + + if (params.newValue === PROVISION_GHGENIUS) { + params.data.fuelCode = null + params.data.fuelCodeId = null + } else if (params.newValue === PROVISION_APPROVED_FUEL_CODE) { + params.data.fuelCode = null + params.data.fuelCodeId = null + } + params.data.eer = null + params.data.energyDensity = null + } + return true + }, + editable: true, + tooltipValueGetter: () => + 'Act Relied Upon to Determine Carbon Intensity: Identify the appropriate provision of the Act relied upon to determine the carbon intensity of each fuel.' + }, + { + field: 'fuelCode', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelCode'), + cellEditor: AutocompleteCellEditor, + cellRenderer: SelectRenderer, + cellEditorParams: (params) => { + const fuelType = optionsData?.fuelTypes?.find( + (obj) => params.data.fuelType === obj.fuelType + ) + const uniqueFuelCodes = new Map() + fuelType?.fuelCodes?.forEach((code) => { + if (!uniqueFuelCodes.has(code.fuelCode)) { + uniqueFuelCodes.set(code.fuelCode, code) + } + }) + return { + options: + Array.from(uniqueFuelCodes.values()).map((item) => item.fuelCode) || + [], + multiple: false, + disableCloseOnSelect: false, + freeSolo: false, + openOnFocus: true + } + }, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental), + suppressKeyboardEvent, + minWidth: 135, + editable: (params) => { + const fuelType = optionsData?.fuelTypes?.find( + (obj) => params.data.fuelType === obj.fuelType + ) + return ( + params.data.provisionOfTheAct === PROVISION_APPROVED_FUEL_CODE && + fuelType?.fuelCodes?.length > 0 + ) + }, + valueGetter: (params) => { + const fuelType = optionsData?.fuelTypes?.find( + (obj) => params.data.fuelType === obj.fuelType + ) + if (!fuelType) return params.data.fuelCode + + const isFuelCodeScenario = + params.data.provisionOfTheAct === PROVISION_APPROVED_FUEL_CODE + const uniqueFuelCodes = new Map() + fuelType?.fuelCodes?.forEach((code) => { + if (!uniqueFuelCodes.has(code.fuelCode)) { + uniqueFuelCodes.set(code.fuelCode, code) + } + }) + const fuelCodes = Array.from(uniqueFuelCodes.values()).map( + (item) => item.fuelCode + ) + + if (isFuelCodeScenario && !params.data.fuelCode) { + if (fuelCodes.length === 1) { + const singleFuelCode = Array.from(uniqueFuelCodes.values())[0] + params.data.fuelCode = singleFuelCode.fuelCode + params.data.fuelCodeId = singleFuelCode.fuelCodeId + } + } + + return params.data.fuelCode + }, + valueSetter: (params) => { + if (params.newValue) { + params.data.fuelCode = params.newValue + const fuelType = optionsData?.fuelTypes?.find( + (obj) => params.data.fuelType === obj.fuelType + ) + if (params.data.provisionOfTheAct === PROVISION_APPROVED_FUEL_CODE) { + const matchingFuelCode = fuelType?.fuelCodes?.find( + (fuelCode) => params.data.fuelCode === fuelCode.fuelCode + ) + if (matchingFuelCode) { + params.data.fuelCodeId = matchingFuelCode.fuelCodeId + } + } + } else { + params.data.fuelCode = undefined + params.data.fuelCodeId = undefined + } + return true + } + }, + { + field: 'quantity', + headerComponent: RequiredHeader, + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.quantity'), + valueFormatter: formatNumberWithCommas, + cellEditor: NumberEditor, + cellEditorParams: { + precision: 0, + min: 0, + showStepperButtons: false + }, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental) + }, + { + field: 'units', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.units'), + minWidth: 60, + cellEditor: AutocompleteCellEditor, + cellEditorParams: (params) => ({ + options: ['L', 'kg', 'kWh', 'm³'], + multiple: false, + disableCloseOnSelect: false, + freeSolo: false, + openOnFocus: true + }), + cellRenderer: SelectRenderer, + suppressKeyboardEvent, + editable: (params) => isFuelTypeOther(params), + cellStyle: (params) => { + if (isSupplemental && params.data.isNewSupplementalEntry) { + if (params.data.actionType === 'UPDATE') { + return { backgroundColor: colors.alerts.warning.background } + } + } else { + return StandardCellStyle( + params, + errors, + warnings, + fuelTypeOtherConditionalStyle + ) + } + } + }, + { + field: 'targetCi', + headerName: i18n.t('legacy:columnLabels.ciLimit'), + editable: false, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental) + }, + { + field: 'ciOfFuel', + headerName: i18n.t('legacy:columnLabels.fuelCi'), + cellEditor: NumberEditor, + cellEditorParams: { + precision: 2, + min: 0, + showStepperButtons: false + }, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental), + editable: (params) => params.data.provisionOfTheAct === PROVISION_GHGENIUS, + valueGetter: (params) => params.data.ciOfFuel, + valueSetter: (params) => { + if (params.newValue !== undefined) { + params.data.ciOfFuel = params.newValue + return true + } + return false + } + }, + { + field: 'energyDensity', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.energyDensity'), + cellEditor: 'agNumberCellEditor', + cellStyle: (params) => { + if (isSupplemental && params.data.isNewSupplementalEntry) { + if (params.data.actionType === 'UPDATE') { + return { backgroundColor: colors.alerts.warning.background } + } + } else { + return StandardCellStyle( + params, + errors, + warnings, + fuelTypeOtherConditionalStyle + ) + } + }, + cellEditorParams: { + precision: 2, + min: 0, + showStepperButtons: false + }, + valueGetter: (params) => { + if (isFuelTypeOther(params)) { + return params.data?.energyDensity + ? params.data?.energyDensity + ' MJ/' + params.data?.units + : 0 + } else { + return params.data?.energyDensity + ? params.data?.energyDensity + ' MJ/' + params.data?.units + : '' + } + }, + editable: (params) => isFuelTypeOther(params) + }, + { + field: 'eer', + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.eer'), + editable: false, + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental) + }, + { + field: 'energy', + cellStyle: (params) => + StandardCellWarningAndErrors(params, errors, warnings, isSupplemental), + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.energy'), + valueFormatter: formatNumberWithCommas, + minWidth: 100, + editable: false + } +] + +export const fuelSupplySummaryColDef = [ + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.complianceUnits'), + field: 'complianceUnits', + valueFormatter: formatNumberWithCommas + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelType'), + field: 'fuelType', + valueGetter: (params) => params.data.fuelType + }, + { + headerName: i18n.t('legacy:columnLabels.fuelClass'), + field: 'fuelCategory', + valueGetter: (params) => params.data.fuelCategory + }, + { + headerName: i18n.t( + 'fuelSupply:fuelSupplyColLabels.determiningCarbonIntensity' + ), + field: 'determiningCarbonIntensity', + valueGetter: (params) => params.data.provisionOfTheAct + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelCode'), + field: 'fuelCode', + valueGetter: (params) => params.data.fuelCode + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.quantity'), + field: 'quantity', + valueFormatter: formatNumberWithCommas + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.units'), + field: 'units' + }, + { + headerName: i18n.t('legacy:columnLabels.ciLimit'), + field: 'targetCi' + }, + { + headerName: i18n.t('legacy:columnLabels.fuelCi'), + field: 'ciOfFuel' + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.energyDensity'), + field: 'energyDensity' + }, + { headerName: i18n.t('fuelSupply:fuelSupplyColLabels.eer'), field: 'eer' }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.energy'), + field: 'energy', + valueFormatter: formatNumberWithCommas + } +] + +export const defaultColDef = { + editable: true, + resizable: true, + filter: false, + floatingFilter: false, + sortable: false, + singleClickEdit: true +} + +export const changelogCommonColDefs = [ + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.complianceUnits'), + field: 'complianceUnits', + valueFormatter: formatNumberWithCommas, + cellStyle: (params) => changelogCellStyle(params, 'complianceUnits') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelType'), + field: 'fuelType', + valueGetter: (params) => params.data.fuelType, + cellStyle: (params) => changelogCellStyle(params, 'fuelTypeId') + }, + { + headerName: i18n.t('legacy:columnLabels.fuelClass'), + field: 'fuelCategory', + valueGetter: (params) => params.data.fuelCategory, + cellStyle: (params) => changelogCellStyle(params, 'fuelCategoryId') + }, + { + headerName: i18n.t( + 'fuelSupply:fuelSupplyColLabels.determiningCarbonIntensity' + ), + field: 'determiningCarbonIntensity', + valueGetter: (params) => params.data.provisionOfTheAct, + cellStyle: (params) => changelogCellStyle(params, 'provisionOfTheActId') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.fuelCode'), + field: 'fuelCode', + valueGetter: (params) => params.data.fuelCode, + cellStyle: (params) => changelogCellStyle(params, 'fuelCodeId') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.quantity'), + field: 'quantity', + valueFormatter: formatNumberWithCommas, + cellStyle: (params) => changelogCellStyle(params, 'quantity') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.units'), + field: 'units', + cellStyle: (params) => changelogCellStyle(params, 'units') + }, + { + headerName: i18n.t('legacy:columnLabels.ciLimit'), + field: 'targetCi', + cellStyle: (params) => changelogCellStyle(params, 'targetCi') + }, + { + headerName: i18n.t('legacy:columnLabels.fuelCi'), + field: 'ciOfFuel', + cellStyle: (params) => changelogCellStyle(params, 'ciOfFuel') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.energyDensity'), + field: 'energyDensity', + cellStyle: (params) => changelogCellStyle(params, 'energyDensity') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.eer'), + field: 'eer', + cellStyle: (params) => changelogCellStyle(params, 'eer') + }, + { + headerName: i18n.t('fuelSupply:fuelSupplyColLabels.energy'), + field: 'energy', + valueFormatter: formatNumberWithCommas, + cellStyle: (params) => changelogCellStyle(params, 'energy') + } +] + +export const changelogColDefs = [ + { + field: 'groupUuid', + hide: true, + sort: 'desc', + sortIndex: 1 + }, + { + field: 'version', + hide: true, + sort: 'desc', + sortIndex: 2 + }, + { + field: 'actionType', + headerName: 'Action', + valueGetter: (params) => { + if (!params?.data) return '' + + if (params.data.actionType === 'UPDATE') { + if (params.data.updated) { + return 'Edited old' + } else { + return 'Edited new' + } + } + if (params.data.actionType === 'DELETE') { + return 'Deleted' + } + if (params.data.actionType === 'CREATE') { + return 'Added' + } + return params.data.actionType + }, + cellStyle: (params) => { + if (!params?.data) return {} + + if (params.data.actionType === 'UPDATE') { + return { backgroundColor: colors.alerts.warning.background } + } + return {} + } + }, + ...changelogCommonColDefs +] + +export const changelogDefaultColDefs = { + floatingFilter: false, + filter: false +} + +export const changelogCommonGridOptions = { + overlayNoRowsTemplate: i18n.t('fuelSupply:noFuelSuppliesFound'), + autoSizeStrategy: { + type: 'fitCellContents', + defaultMinWidth: 50, + defaultMaxWidth: 600 + }, + enableCellTextSelection: true, + ensureDomOrder: true +} + +export const changelogGridOptions = { + ...changelogCommonGridOptions, + getRowStyle: (params) => { + if (params.data.actionType === 'DELETE') { + return { + backgroundColor: colors.alerts.error.background + } + } + if (params.data.actionType === 'CREATE') { + return { + backgroundColor: colors.alerts.success.background + } + } + } +} diff --git a/frontend/src/views/Organizations/AddEditOrg/__tests__/AddEditOrg.test.jsx b/frontend/src/views/Organizations/AddEditOrg/__tests__/AddEditOrg.test.jsx index 6c37ea7dc..3e796368f 100644 --- a/frontend/src/views/Organizations/AddEditOrg/__tests__/AddEditOrg.test.jsx +++ b/frontend/src/views/Organizations/AddEditOrg/__tests__/AddEditOrg.test.jsx @@ -1,10 +1,25 @@ -import { render, screen } from '@testing-library/react' +import React from 'react' +import { + render, + screen, + fireEvent, + waitFor, + within +} from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { AddEditOrgForm } from '../AddEditOrgForm' +import { useForm, FormProvider } from 'react-hook-form' import { vi, describe, it, expect, beforeEach } from 'vitest' import { AddEditOrg } from '../AddEditOrg' import { useTranslation } from 'react-i18next' import { useParams, useNavigate } from 'react-router-dom' import { useOrganization, useOrganizationTypes } from '@/hooks/useOrganization' import { useApiService } from '@/services/useApiService' +import { ROUTES } from '@/routes/routes' +import { wrapper } from '@/tests/utils/wrapper' +import { yupResolver } from '@hookform/resolvers/yup' +import { schemaValidation } from '@/views/Organizations/AddEditOrg/_schema.js' +import { useMutation } from '@tanstack/react-query' // Mock react-i18next vi.mock('react-i18next', () => ({ @@ -17,6 +32,49 @@ vi.mock('react-router-dom', () => ({ useNavigate: vi.fn() })) +// Mock AddressAutocomplete to prevent network requests +vi.mock('@/components/BCForm/AddressAutocomplete', () => ({ + AddressAutocomplete: React.forwardRef( + ( + { + name, + placeholder = 'Start typing address...', + value, + onChange, + onBlur, + error + }, + ref + ) => ( + onChange && onChange(e.target.value)} + onBlur={onBlur} + data-testid={`address-autocomplete-${name}`} + aria-label={name?.includes('street') ? 'org:streetAddrLabel' : name} + /> + ) + ) +})) + +// Mock the useMutation hook to properly handle onSuccess callback +const mockMutate = vi.fn() +vi.mock('@tanstack/react-query', async () => { + const actual = await vi.importActual('@tanstack/react-query') + return { + ...actual, + useMutation: vi.fn(() => ({ + mutate: mockMutate, + isPending: false, + isError: false + })) + } +}) + // Mock hooks vi.mock('@/hooks/useOrganization') vi.mock('@/services/useApiService') @@ -78,6 +136,85 @@ describe('AddEditOrg', () => { post: vi.fn(), put: vi.fn() }) + + mockNavigate = vi.fn() + useNavigate.mockReturnValue(mockNavigate) + useParams.mockReturnValue({ orgID: undefined }) + + useOrganization.mockReturnValue({ + data: null, + isFetched: true + }) + + apiSpy = { + post: vi.fn().mockResolvedValue({}), + put: vi.fn().mockResolvedValue({}) + } + useApiService.mockReturnValue(apiSpy) + }) + + it('renders correctly with provided organization data and maps all address fields correctly', () => { + useOrganization.mockReturnValue({ + data: mockedOrg, + isFetched: true + }) + useParams.mockReturnValue({ orgID: '123' }) + + render( + + + , + { wrapper } + ) + + expect(screen.getByLabelText(/org:legalNameLabel/i)).toHaveValue('Test Org') + expect(screen.getByLabelText(/org:operatingNameLabel/i)).toHaveValue( + 'Test Operating Org' + ) + expect(screen.getByLabelText(/org:emailAddrLabel/i)).toHaveValue( + 'test@example.com' + ) + expect(screen.getByLabelText(/org:phoneNbrLabel/i)).toHaveValue( + '123-456-7890' + ) + expect(screen.getAllByLabelText(/org:streetAddrLabel/i)[0]).toHaveValue( + '123 Test St' + ) + expect(screen.getAllByLabelText(/org:cityLabel/i)[0]).toHaveValue( + 'Test City' + ) + // Also check attorney address if there are multiple address fields + const streetAddressFields = screen.getAllByLabelText(/org:streetAddrLabel/i) + if (streetAddressFields.length > 1) { + expect(streetAddressFields[1]).toHaveValue('456 Attorney Rd') + } + }) + + it('renders required errors in the form correctly', async () => { + render( + + + , + { wrapper } + ) + + fireEvent.click(screen.getByTestId('saveOrganization')) + + await waitFor(async () => { + const errorMessages = await screen.findAllByText(/required/i) + expect(errorMessages.length).toBeGreaterThan(0) + + expect( + screen.getByText(/Legal Name of Organization is required./i) + ).toBeInTheDocument() + expect( + screen.getByText(/Operating Name of Organization is required./i) + ).toBeInTheDocument() + expect( + screen.getByText(/Email Address is required./i) + ).toBeInTheDocument() + expect(screen.getByText(/Phone Number is required./i)).toBeInTheDocument() + }) }) it('renders in edit mode when orgID is present', () => { diff --git a/frontend/src/views/Users/AddEditUser/__tests__/AddEditUser.test.jsx b/frontend/src/views/Users/AddEditUser/__tests__/AddEditUser.test.jsx index 62cddc84d..4ecedb44f 100644 --- a/frontend/src/views/Users/AddEditUser/__tests__/AddEditUser.test.jsx +++ b/frontend/src/views/Users/AddEditUser/__tests__/AddEditUser.test.jsx @@ -4,6 +4,8 @@ import userEvent from '@testing-library/user-event' import { describe, it, expect, beforeEach, vi } from 'vitest' import { AddEditUser } from '../AddEditUser' import { wrapper } from '@/tests/utils/wrapper' +import { HttpResponse } from 'msw' +import { httpOverwrite } from '@/tests/utils/handlers' import * as currentUserHooks from '@/hooks/useCurrentUser' import * as userHooks from '@/hooks/useUser' import * as organizationUserHooks from '@/hooks/useOrganization'