From 74be63226a55138eb2fb6f07ce1a60421470560f Mon Sep 17 00:00:00 2001 From: Debby Date: Wed, 15 Apr 2026 13:56:55 +0700 Subject: [PATCH] dag etl --- dags/etl_food_security.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/dags/etl_food_security.py b/dags/etl_food_security.py index 2f99c23..778eda5 100644 --- a/dags/etl_food_security.py +++ b/dags/etl_food_security.py @@ -2,9 +2,7 @@ AIRFLOW DAG — ETL Food Security BigQuery Kimball Data Warehouse Architecture -Schedule : Setiap 3 bulan sekali (tanggal 1, pukul 00:00) - Cron: "0 0 1 */3 *" - -> 1 Jan, 1 Apr, 1 Jul, 1 Okt +Schedule : Setiap 3 hari sekali (timedelta(days=3)) Catchup : False Kimball ETL Flow: @@ -74,8 +72,7 @@ from scripts.bigquery_dimensional_model import ( from scripts.bigquery_analytical_layer import ( run_analytical_layer, ) -# FIXED: nama modul disesuaikan dengan nama file yang benar -from scripts.bigquery_analysis_aggregation import ( +from scripts.bigquery_aggregate_layer import ( run_aggregation, ) from scripts.bigquery_aggraget_fact_selected_layer import ( @@ -85,28 +82,20 @@ from scripts.bigquery_aggraget_fact_selected_layer import ( # DEFAULT ARGS default_args = { - 'owner': 'data-engineering', + 'owner': 'Debby Seftia', 'email': ['d1041221004@student.untan.ac.id'], } # DAG DEFINITION -# -# schedule_interval = "0 0 1 */3 *" -# ┌───── menit : 0 -# │ ┌─── jam : 0 (tengah malam) -# │ │ ┌─ hari : 1 (tanggal 1 setiap bulan yang cocok) -# │ │ │ ┌ bulan : */3 (setiap 3 bulan -> Jan, Apr, Jul, Okt) -# │ │ │ │ ┌ hari minggu : * (semua) -# 0 0 1 */3 * with DAG( dag_id = "etl_food_security_bigquery", - description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold) | Schedule: setiap 3 bulan", + description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold)", default_args = default_args, - start_date = datetime(2026, 1, 1), - schedule_interval = "0 0 1 */3 *", # Setiap 3 bulan sekali + start_date = datetime(2026, 3, 1), + schedule_interval = "0 0 1 */3 *", catchup = False, - tags = ["food-security", "bigquery", "kimball", "quarterly"], + tags = ["food-security", "bigquery", "kimball"], ) as dag: task_verify = PythonOperator(