create analytical and agg
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
from airflow import DAG
|
||||
from airflow.operators.python import PythonOperator
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Import fungsi dari folder scripts
|
||||
from scripts.bigquery_raw_layer import (
|
||||
@@ -19,11 +19,21 @@ from scripts.bigquery_dimesional_model import (
|
||||
run_dimensional_model,
|
||||
)
|
||||
|
||||
from scripts.bigquery_analytical_layer import (
|
||||
run_analytical_layer,
|
||||
)
|
||||
|
||||
from scripts.bigquery_aggregate_layer import (
|
||||
run_aggregation,
|
||||
)
|
||||
|
||||
|
||||
with DAG(
|
||||
dag_id = "etl_food_security_bigquery",
|
||||
description = "Kimball ETL: FAO, World Bank, UNICEF to BigQuery (Bronze to Silver to Gold)",
|
||||
start_date = datetime(2026, 3, 1),
|
||||
schedule_interval = "@daily",
|
||||
schedule_interval = timedelta(days=3),
|
||||
catchup = False,
|
||||
tags = ["food-security", "bigquery", "kimball"]
|
||||
) as dag:
|
||||
@@ -63,5 +73,15 @@ with DAG(
|
||||
python_callable = run_dimensional_model
|
||||
)
|
||||
|
||||
task_analytical = PythonOperator(
|
||||
task_id = "analytical_layer_to_gold",
|
||||
python_callable = run_analytical_layer
|
||||
)
|
||||
|
||||
task_aggregation = PythonOperator(
|
||||
task_id = "aggregation_to_gold",
|
||||
python_callable = run_aggregation
|
||||
)
|
||||
|
||||
|
||||
task_verify >> task_fao >> task_worldbank >> task_unicef >> task_staging >> task_cleaned >> task_dimensional
|
||||
task_verify >> task_fao >> task_worldbank >> task_unicef >> task_staging >> task_cleaned >> task_dimensional >> task_analytical >> task_aggregation
|
||||
Reference in New Issue
Block a user