create analytical and agg

This commit is contained in:
Debby
2026-03-14 23:54:24 +07:00
parent 453fb3ef52
commit 2f29e42e3f
3 changed files with 1353 additions and 2 deletions

View File

@@ -1,6 +1,6 @@
from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import datetime
from datetime import datetime, timedelta
# Import fungsi dari folder scripts
from scripts.bigquery_raw_layer import (
@@ -19,11 +19,21 @@ from scripts.bigquery_dimesional_model import (
run_dimensional_model,
)
from scripts.bigquery_analytical_layer import (
run_analytical_layer,
)
from scripts.bigquery_aggregate_layer import (
run_aggregation,
)
with DAG(
dag_id = "etl_food_security_bigquery",
description = "Kimball ETL: FAO, World Bank, UNICEF to BigQuery (Bronze to Silver to Gold)",
start_date = datetime(2026, 3, 1),
schedule_interval = "@daily",
schedule_interval = timedelta(days=3),
catchup = False,
tags = ["food-security", "bigquery", "kimball"]
) as dag:
@@ -63,5 +73,15 @@ with DAG(
python_callable = run_dimensional_model
)
task_analytical = PythonOperator(
task_id = "analytical_layer_to_gold",
python_callable = run_analytical_layer
)
task_aggregation = PythonOperator(
task_id = "aggregation_to_gold",
python_callable = run_aggregation
)
task_verify >> task_fao >> task_worldbank >> task_unicef >> task_staging >> task_cleaned >> task_dimensional
task_verify >> task_fao >> task_worldbank >> task_unicef >> task_staging >> task_cleaned >> task_dimensional >> task_analytical >> task_aggregation