From 512c08bff3d12f01efdbd0ece57fd234bce61ee4 Mon Sep 17 00:00:00 2001 From: Debby Date: Fri, 5 Jun 2026 17:49:42 +0700 Subject: [PATCH] add sustainability ind version --- dags/etl_food_security.py | 38 ++++--------------- .../bigquery_aggraget_fact_selected_layer.py | 2 + scripts/bigquery_aggregate_layer.py | 3 ++ scripts/bigquery_analytical_layer.py | 11 ++++-- 4 files changed, 20 insertions(+), 34 deletions(-) diff --git a/dags/etl_food_security.py b/dags/etl_food_security.py index 778eda5..ca6eaf9 100644 --- a/dags/etl_food_security.py +++ b/dags/etl_food_security.py @@ -51,10 +51,9 @@ Scripts folder harus berisi: - bigquery_helpers.py - bigquery_datasource.py """ - from airflow import DAG from airflow.operators.python import PythonOperator -from datetime import datetime +from datetime import datetime, timedelta from scripts.bigquery_raw_layer import ( run_verify_connection, @@ -63,37 +62,25 @@ from scripts.bigquery_raw_layer import ( run_load_unicef, run_staging_integration, ) -from scripts.bigquery_cleaned_layer import ( - run_cleaned_integration, -) -from scripts.bigquery_dimensional_model import ( - run_dimensional_model, -) -from scripts.bigquery_analytical_layer import ( - run_analytical_layer, -) -from scripts.bigquery_aggregate_layer import ( - run_aggregation, -) -from scripts.bigquery_aggraget_fact_selected_layer import ( - run_indicator_norm_aggregation, -) +from scripts.bigquery_cleaned_layer import run_cleaned_integration +from scripts.bigquery_dimensional_model import run_dimensional_model +from scripts.bigquery_analytical_layer import run_analytical_layer +from scripts.bigquery_aggregate_layer import run_aggregation +from scripts.bigquery_aggraget_fact_selected_layer import run_indicator_norm_aggregation # DEFAULT ARGS - default_args = { 'owner': 'Debby Seftia', 'email': ['d1041221004@student.untan.ac.id'], } # DAG DEFINITION - with DAG( dag_id = "etl_food_security_bigquery", description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold)", default_args = default_args, - start_date = datetime(2026, 3, 1), - schedule_interval = "0 0 1 */3 *", + start_date = datetime.now(), + schedule_interval = timedelta(days=60), catchup = False, tags = ["food-security", "bigquery", "kimball"], ) as dag: @@ -102,47 +89,38 @@ with DAG( task_id = "verify_bigquery_connection", python_callable = run_verify_connection ) - task_fao = PythonOperator( task_id = "load_fao_to_bronze", python_callable = run_load_fao ) - task_worldbank = PythonOperator( task_id = "load_worldbank_to_bronze", python_callable = run_load_worldbank ) - task_unicef = PythonOperator( task_id = "load_unicef_to_bronze", python_callable = run_load_unicef ) - task_staging = PythonOperator( task_id = "staging_integration_to_silver", python_callable = run_staging_integration ) - task_cleaned = PythonOperator( task_id = "cleaned_integration_to_silver", python_callable = run_cleaned_integration ) - task_dimensional = PythonOperator( task_id = "dimensional_model_to_gold", python_callable = run_dimensional_model ) - task_analytical = PythonOperator( task_id = "analytical_layer_to_gold", python_callable = run_analytical_layer ) - task_aggregation = PythonOperator( task_id = "aggregation_to_gold", python_callable = run_aggregation ) - task_indicator_norm = PythonOperator( task_id = "indicator_norm_aggregation_to_gold", python_callable = run_indicator_norm_aggregation diff --git a/scripts/bigquery_aggraget_fact_selected_layer.py b/scripts/bigquery_aggraget_fact_selected_layer.py index bd32fab..450e8b6 100644 --- a/scripts/bigquery_aggraget_fact_selected_layer.py +++ b/scripts/bigquery_aggraget_fact_selected_layer.py @@ -102,10 +102,12 @@ PILLAR_NAME_ID_MAP: dict = { "Access" : "Akses", "Utilization" : "Pemanfaatan", "Stability" : "Stabilitas", + "Sustainability": "Keberlanjutan", "availability" : "Ketersediaan", "access" : "Akses", "utilization" : "Pemanfaatan", "stability" : "Stabilitas", + "sustainability": "Keberlanjutan", } # Mapping nama indikator (Inggris -> Indonesia) diff --git a/scripts/bigquery_aggregate_layer.py b/scripts/bigquery_aggregate_layer.py index 96b26a0..61a666a 100644 --- a/scripts/bigquery_aggregate_layer.py +++ b/scripts/bigquery_aggregate_layer.py @@ -98,15 +98,18 @@ PILLAR_TRANSLATION_ID: dict = { "Access" : "Keterjangkauan", "Utilization" : "Pemanfaatan", "Stability" : "Stabilitas", + "Sustainability" : "Keberlanjutan", # Variasi penulisan yang mungkin muncul "availability" : "Ketersediaan", "access" : "Keterjangkauan", "utilization" : "Pemanfaatan", "stability" : "Stabilitas", + "sustainability" : "Keberlanjutan", "Food Availability" : "Ketersediaan Pangan", "Food Access" : "Keterjangkauan Pangan", "Food Utilization" : "Pemanfaatan Pangan", "Food Stability" : "Stabilitas Pangan", + "Food Sustainability": "Keberlanjutan Pangan", } INDICATOR_TRANSLATION_ID: dict = { diff --git a/scripts/bigquery_analytical_layer.py b/scripts/bigquery_analytical_layer.py index 117e19a..c68df06 100644 --- a/scripts/bigquery_analytical_layer.py +++ b/scripts/bigquery_analytical_layer.py @@ -46,15 +46,18 @@ PILLAR_TRANSLATION_ID: dict = { "Access" : "Keterjangkauan", "Utilization" : "Pemanfaatan", "Stability" : "Stabilitas", + "Sustainability": "Keberlanjutan", # Variasi penulisan yang mungkin muncul "availability" : "Ketersediaan", "access" : "Keterjangkauan", "utilization" : "Pemanfaatan", "stability" : "Stabilitas", - "Food Availability" : "Ketersediaan Pangan", - "Food Access" : "Keterjangkauan Pangan", - "Food Utilization" : "Pemanfaatan Pangan", - "Food Stability" : "Stabilitas Pangan", + "sustainability": "Keberlanjutan", + "Food Availability" : "Ketersediaan Pangan", + "Food Access" : "Keterjangkauan Pangan", + "Food Utilization" : "Pemanfaatan Pangan", + "Food Stability" : "Stabilitas Pangan", + "Food Sustainability": "Keberlanjutan Pangan", } INDICATOR_TRANSLATION_ID: dict = {