schdule /3 bulan dan tambah metadata
This commit is contained in:
@@ -2,7 +2,9 @@
|
||||
AIRFLOW DAG — ETL Food Security BigQuery
|
||||
Kimball Data Warehouse Architecture
|
||||
|
||||
Schedule : Setiap 3 hari sekali (timedelta(days=3))
|
||||
Schedule : Setiap 3 bulan sekali (tanggal 1, pukul 00:00)
|
||||
Cron: "0 0 1 */3 *"
|
||||
-> 1 Jan, 1 Apr, 1 Jul, 1 Okt
|
||||
Catchup : False
|
||||
|
||||
Kimball ETL Flow:
|
||||
@@ -72,7 +74,8 @@ from scripts.bigquery_dimensional_model import (
|
||||
from scripts.bigquery_analytical_layer import (
|
||||
run_analytical_layer,
|
||||
)
|
||||
from scripts.bigquery_aggregate_layer import (
|
||||
# FIXED: nama modul disesuaikan dengan nama file yang benar
|
||||
from scripts.bigquery_analysis_aggregation import (
|
||||
run_aggregation,
|
||||
)
|
||||
from scripts.bigquery_aggraget_fact_selected_layer import (
|
||||
@@ -87,15 +90,23 @@ default_args = {
|
||||
}
|
||||
|
||||
# DAG DEFINITION
|
||||
#
|
||||
# schedule_interval = "0 0 1 */3 *"
|
||||
# ┌───── menit : 0
|
||||
# │ ┌─── jam : 0 (tengah malam)
|
||||
# │ │ ┌─ hari : 1 (tanggal 1 setiap bulan yang cocok)
|
||||
# │ │ │ ┌ bulan : */3 (setiap 3 bulan -> Jan, Apr, Jul, Okt)
|
||||
# │ │ │ │ ┌ hari minggu : * (semua)
|
||||
# 0 0 1 */3 *
|
||||
|
||||
with DAG(
|
||||
dag_id = "etl_food_security_bigquery",
|
||||
description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold)",
|
||||
description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold) | Schedule: setiap 3 bulan",
|
||||
default_args = default_args,
|
||||
start_date = datetime(2026, 3, 1),
|
||||
schedule_interval = "0 0 */3 * *",
|
||||
start_date = datetime(2026, 1, 1),
|
||||
schedule_interval = "0 0 1 */3 *", # Setiap 3 bulan sekali
|
||||
catchup = False,
|
||||
tags = ["food-security", "bigquery", "kimball"],
|
||||
tags = ["food-security", "bigquery", "kimball", "quarterly"],
|
||||
) as dag:
|
||||
|
||||
task_verify = PythonOperator(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user