izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dag etl

Browse Source
This commit is contained in:
Debby
2026-04-15 13:56:55 +07:00
parent 76b451b2c1
commit 74be63226a
1 changed files with 7 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
dags/etl_food_security.py
View File

@@ -2,9 +2,7 @@
AIRFLOW DAG — ETL Food Security BigQuery AIRFLOW DAG — ETL Food Security BigQuery
Kimball Data Warehouse Architecture Kimball Data Warehouse Architecture
Schedule : Setiap 3 bulan sekali (tanggal 1, pukul 00:00) Schedule : Setiap 3 hari sekali (timedelta(days=3))
Cron: "0 0 1 */3 *"
-> 1 Jan, 1 Apr, 1 Jul, 1 Okt
Catchup : False Catchup : False
Kimball ETL Flow: Kimball ETL Flow:
@@ -74,8 +72,7 @@ from scripts.bigquery_dimensional_model import (
from scripts.bigquery_analytical_layer import ( from scripts.bigquery_analytical_layer import (
run_analytical_layer, run_analytical_layer,
) )
# FIXED: nama modul disesuaikan dengan nama file yang benar from scripts.bigquery_aggregate_layer import (
from scripts.bigquery_analysis_aggregation import (
run_aggregation, run_aggregation,
) )
from scripts.bigquery_aggraget_fact_selected_layer import ( from scripts.bigquery_aggraget_fact_selected_layer import (
@@ -85,28 +82,20 @@ from scripts.bigquery_aggraget_fact_selected_layer import (
# DEFAULT ARGS # DEFAULT ARGS
default_args = { default_args = {
'owner': 'data-engineering', 'owner': 'Debby Seftia',
'email': ['d1041221004@student.untan.ac.id'], 'email': ['d1041221004@student.untan.ac.id'],
} }
# DAG DEFINITION # DAG DEFINITION
#
# schedule_interval = "0 0 1 */3 *"
# ┌───── menit : 0
# │ ┌─── jam : 0 (tengah malam)
# │ │ ┌─ hari : 1 (tanggal 1 setiap bulan yang cocok)
# │ │ │ ┌ bulan : */3 (setiap 3 bulan -> Jan, Apr, Jul, Okt)
# │ │ │ │ ┌ hari minggu : * (semua)
# 0 0 1 */3 *
with DAG( with DAG(
dag_id = "etl_food_security_bigquery", dag_id = "etl_food_security_bigquery",
description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold) | Schedule: setiap 3 bulan", description = "Kimball ETL: FAO, World Bank, UNICEF → BigQuery (Bronze → Silver → Gold)",
default_args = default_args, default_args = default_args,
start_date = datetime(2026, 1, 1), start_date = datetime(2026, 3, 1),
schedule_interval = "0 0 1 */3 *", # Setiap 3 bulan sekali schedule_interval = "0 0 1 */3 *",
catchup = False, catchup = False,
tags = ["food-security", "bigquery", "kimball", "quarterly"], tags = ["food-security", "bigquery", "kimball"],
) as dag: ) as dag:
task_verify = PythonOperator( task_verify = PythonOperator(