schdule /3 bulan dan tambah metadata
This commit is contained in:
@@ -2,7 +2,9 @@
|
|||||||
AIRFLOW DAG — ETL Food Security BigQuery
|
AIRFLOW DAG — ETL Food Security BigQuery
|
||||||
Kimball Data Warehouse Architecture
|
Kimball Data Warehouse Architecture
|
||||||
|
|
||||||
Schedule : Setiap 3 hari sekali (timedelta(days=3))
|
Schedule : Setiap 3 bulan sekali (tanggal 1, pukul 00:00)
|
||||||
|
Cron: "0 0 1 */3 *"
|
||||||
|
-> 1 Jan, 1 Apr, 1 Jul, 1 Okt
|
||||||
Catchup : False
|
Catchup : False
|
||||||
|
|
||||||
Kimball ETL Flow:
|
Kimball ETL Flow:
|
||||||
@@ -72,7 +74,8 @@ from scripts.bigquery_dimensional_model import (
|
|||||||
from scripts.bigquery_analytical_layer import (
|
from scripts.bigquery_analytical_layer import (
|
||||||
run_analytical_layer,
|
run_analytical_layer,
|
||||||
)
|
)
|
||||||
from scripts.bigquery_aggregate_layer import (
|
# FIXED: nama modul disesuaikan dengan nama file yang benar
|
||||||
|
from scripts.bigquery_analysis_aggregation import (
|
||||||
run_aggregation,
|
run_aggregation,
|
||||||
)
|
)
|
||||||
from scripts.bigquery_aggraget_fact_selected_layer import (
|
from scripts.bigquery_aggraget_fact_selected_layer import (
|
||||||
@@ -87,15 +90,23 @@ default_args = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# DAG DEFINITION
|
# DAG DEFINITION
|
||||||
|
#
|
||||||
|
# schedule_interval = "0 0 1 */3 *"
|
||||||
|
# ┌───── menit : 0
|
||||||
|
# │ ┌─── jam : 0 (tengah malam)
|
||||||
|
# │ │ ┌─ hari : 1 (tanggal 1 setiap bulan yang cocok)
|
||||||
|
# │ │ │ ┌ bulan : */3 (setiap 3 bulan -> Jan, Apr, Jul, Okt)
|
||||||
|
# │ │ │ │ ┌ hari minggu : * (semua)
|
||||||
|
# 0 0 1 */3 *
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
dag_id = "etl_food_security_bigquery",
|
dag_id = "etl_food_security_bigquery",
|
||||||
description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold)",
|
description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold) | Schedule: setiap 3 bulan",
|
||||||
default_args = default_args,
|
default_args = default_args,
|
||||||
start_date = datetime(2026, 3, 1),
|
start_date = datetime(2026, 1, 1),
|
||||||
schedule_interval = "0 0 */3 * *",
|
schedule_interval = "0 0 1 */3 *", # Setiap 3 bulan sekali
|
||||||
catchup = False,
|
catchup = False,
|
||||||
tags = ["food-security", "bigquery", "kimball"],
|
tags = ["food-security", "bigquery", "kimball", "quarterly"],
|
||||||
) as dag:
|
) as dag:
|
||||||
|
|
||||||
task_verify = PythonOperator(
|
task_verify = PythonOperator(
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user