schdule /3 bulan dan tambah metadata

This commit is contained in:
Debby
2026-04-15 13:37:40 +07:00
parent fa2cf75634
commit 76b451b2c1
2 changed files with 659 additions and 572 deletions

View File

@@ -2,7 +2,9 @@
AIRFLOW DAG — ETL Food Security BigQuery AIRFLOW DAG — ETL Food Security BigQuery
Kimball Data Warehouse Architecture Kimball Data Warehouse Architecture
Schedule : Setiap 3 hari sekali (timedelta(days=3)) Schedule : Setiap 3 bulan sekali (tanggal 1, pukul 00:00)
Cron: "0 0 1 */3 *"
-> 1 Jan, 1 Apr, 1 Jul, 1 Okt
Catchup : False Catchup : False
Kimball ETL Flow: Kimball ETL Flow:
@@ -72,7 +74,8 @@ from scripts.bigquery_dimensional_model import (
from scripts.bigquery_analytical_layer import ( from scripts.bigquery_analytical_layer import (
run_analytical_layer, run_analytical_layer,
) )
from scripts.bigquery_aggregate_layer import ( # FIXED: nama modul disesuaikan dengan nama file yang benar
from scripts.bigquery_analysis_aggregation import (
run_aggregation, run_aggregation,
) )
from scripts.bigquery_aggraget_fact_selected_layer import ( from scripts.bigquery_aggraget_fact_selected_layer import (
@@ -87,15 +90,23 @@ default_args = {
} }
# DAG DEFINITION # DAG DEFINITION
#
# schedule_interval = "0 0 1 */3 *"
# ┌───── menit : 0
# │ ┌─── jam : 0 (tengah malam)
# │ │ ┌─ hari : 1 (tanggal 1 setiap bulan yang cocok)
# │ │ │ ┌ bulan : */3 (setiap 3 bulan -> Jan, Apr, Jul, Okt)
# │ │ │ │ ┌ hari minggu : * (semua)
# 0 0 1 */3 *
with DAG( with DAG(
dag_id = "etl_food_security_bigquery", dag_id = "etl_food_security_bigquery",
description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold)", description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold) | Schedule: setiap 3 bulan",
default_args = default_args, default_args = default_args,
start_date = datetime(2026, 3, 1), start_date = datetime(2026, 1, 1),
schedule_interval = "0 0 */3 * *", schedule_interval = "0 0 1 */3 *", # Setiap 3 bulan sekali
catchup = False, catchup = False,
tags = ["food-security", "bigquery", "kimball"], tags = ["food-security", "bigquery", "kimball", "quarterly"],
) as dag: ) as dag:
task_verify = PythonOperator( task_verify = PythonOperator(

File diff suppressed because it is too large Load Diff