izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename pillar

Browse Source
This commit is contained in:
Debby
2026-06-27 13:23:02 +07:00
parent 17cc7833ad
commit ebc189f2a0
4 changed files with 807 additions and 1456 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bigquery_aggraget_fact_selected_layer.py
+200 -425
View File
File diff suppressed because it is too large Load Diff
scripts/bigquery_aggregate_layer.py
+272 -864
View File
File diff suppressed because it is too large Load Diff
scripts/bigquery_analytical_layer.py
+46 -21
View File
@@ -11,6 +11,8 @@ Filtering Order:
6. Save analytical table (dengan nama/label lengkap untuk Looker Studio)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
CHANGED: pillar_name sekarang pakai prefix 'Food ' (Food Availability, Food Access, dst.)
'Sustainability' -> 'Food Other', nama Indonesia: Ketersediaan Pangan, Akses Pangan, dst.
"""
import pandas as pd
@@ -38,26 +40,36 @@ from google.cloud import bigquery
# =============================================================================
# TRANSLATION DICTIONARIES
# CHANGED: "Sustainability" / "sustainability" -> "Other" / "Lainnya"
# =============================================================================
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability": "Keberlanjutan",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability": "Keberlanjutan",
# Mapping nama pilar (Inggris dengan prefix Food) -> Bahasa Indonesia
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Access" : "Akses Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Food Sustainability": "Keberlanjutan Pangan",
"Food Other" : "Indikator Tambahan",
# Variasi tanpa prefix Food (dari data lama)
"Availability" : "Ketersediaan Pangan",
"Access" : "Akses Pangan",
"Utilization" : "Pemanfaatan Pangan",
"Stability" : "Stabilitas Pangan",
"Other" : "Indikator Tambahan",
# Legacy Sustainability -> Food Other -> Indikator Tambahan
"Sustainability" : "Indikator Tambahan",
"sustainability" : "Indikator Tambahan",
# lowercase
"food availability" : "Ketersediaan Pangan",
"food access" : "Akses Pangan",
"food utilization" : "Pemanfaatan Pangan",
"food stability" : "Stabilitas Pangan",
"food other" : "Indikator Tambahan",
"availability" : "Ketersediaan Pangan",
"access" : "Akses Pangan",
"utilization" : "Pemanfaatan Pangan",
"stability" : "Stabilitas Pangan",
"other" : "Indikator Tambahan",
}
@@ -194,7 +206,11 @@ def translate_indicator(name: str) -> str:
def translate_pillar(name: str) -> str:
"""Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli."""
"""
Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli.
CHANGED: pillar_name menggunakan prefix 'Food ' (Food Availability, Food Access, dll.)
'Sustainability' -> 'Food Other' (EN) / 'Indikator Tambahan' (ID).
"""
if not name:
return name
return PILLAR_TRANSLATION_ID.get(name, name)
@@ -284,6 +300,18 @@ class AnalyticalLayerLoader:
self.df_clean = self.client.query(query).result().to_dataframe(create_bqstorage_client=False)
self.logger.info(f" Loaded: {len(self.df_clean):,} rows")
# Rename pillar_name: add 'Food ' prefix, remove Sustainability
PILLAR_RENAME_MAP = {
'Availability' : 'Food Availability',
'Access' : 'Food Access',
'Utilization' : 'Food Utilization',
'Stability' : 'Food Stability',
'Other' : 'Food Other',
'Sustainability': 'Food Other',
'sustainability': 'Food Other',
}
self.df_clean['pillar_name'] = self.df_clean['pillar_name'].replace(PILLAR_RENAME_MAP)
if 'is_year_range' in self.df_clean.columns:
yr = self.df_clean['is_year_range'].value_counts()
self.logger.info(f" Breakdown:")
@@ -614,11 +642,7 @@ class AnalyticalLayerLoader:
'value',
]].copy()
# ------------------------------------------------------------------
# TAMBAHAN: kolom terjemahan Bahasa Indonesia
# indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
# pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name
# ------------------------------------------------------------------
# Terjemahan Bahasa Indonesia
analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator)
analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar)
@@ -701,7 +725,8 @@ class AnalyticalLayerLoader:
'fixed_countries': len(self.selected_country_ids),
'no_gaps' : True,
'layer' : 'gold',
'columns' : 'id + name + name_id (Looker Studio ready)'
'columns' : 'id + name + name_id (Looker Studio ready)',
'pillar_change' : 'Sustainability -> Food Other; all pillars use Food prefix',
}),
'validation_metrics' : json.dumps({
'fixed_countries' : len(self.selected_country_ids),
scripts/bigquery_dimensional_model.py
+199 -56
View File
@@ -10,6 +10,13 @@ Kimball ETL Flow yang dijalankan file ini:
Classes:
DimensionalModelLoader — Build Star Schema & load ke Gold layer
Pilar resmi FAO yang digunakan (5 pilar dengan prefix "Food "):
- Food Availability (Ketersediaan Pangan)
- Food Access (Akses Pangan)
- Food Utilization (Pemanfaatan Pangan)
- Food Stability (Stabilitas Pangan)
- Food Other (Indikator Tambahan) — additional useful statistics
Usage:
python bigquery_dimensional_model.py
"""
@@ -37,6 +44,53 @@ if hasattr(sys.stdout, 'reconfigure'):
sys.stdout.reconfigure(encoding='utf-8')
# =============================================================================
# PILLAR CONSTANTS
# Satu-satunya sumber kebenaran untuk nama pilar di seluruh pipeline.
# Tidak ada lagi "Sustainability" — digantikan "Food Other".
# =============================================================================
# Mapping dari nilai lama/raw -> nama pilar resmi (dengan prefix "Food ")
PILLAR_RENAME_MAP: dict = {
# Nilai lama tanpa prefix
'Availability' : 'Food Availability',
'Access' : 'Food Access',
'Utilization' : 'Food Utilization',
'Stability' : 'Food Stability',
'Other' : 'Food Other',
# Nilai yang sudah benar (idempotent)
'Food Availability': 'Food Availability',
'Food Access' : 'Food Access',
'Food Utilization' : 'Food Utilization',
'Food Stability' : 'Food Stability',
'Food Other' : 'Food Other',
# lowercase
'food availability': 'Food Availability',
'food access' : 'Food Access',
'food utilization' : 'Food Utilization',
'food stability' : 'Food Stability',
'food other' : 'Food Other',
}
# Kode resmi per pilar
PILLAR_CODE_MAP: dict = {
'Food Availability': 'AVL',
'Food Access' : 'ACC',
'Food Utilization' : 'UTL',
'Food Stability' : 'STB',
'Food Other' : 'OTH',
}
# Nama 5 pilar resmi (urutan tampilan)
OFFICIAL_PILLARS: list = [
'Food Availability',
'Food Access',
'Food Utilization',
'Food Stability',
'Food Other',
]
# =============================================================================
# DIMENSIONAL MODEL LOADER
# =============================================================================
@@ -62,11 +116,28 @@ class DimensionalModelLoader:
def __init__(self, client: bigquery.Client, df_clean: pd.DataFrame):
self.client = client
self.df_clean = df_clean
self.df_clean = df_clean.copy()
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.propagate = False
self.target_layer = 'gold'
# Normalisasi pillar column sekarang, satu kali, di awal
if 'pillar' in self.df_clean.columns:
self.df_clean['pillar'] = (
self.df_clean['pillar']
.astype(str)
.str.strip()
.map(lambda x: PILLAR_RENAME_MAP.get(x, 'Food Other'))
)
unknown = set(self.df_clean['pillar'].unique()) - set(OFFICIAL_PILLARS)
if unknown:
self.logger.warning(
f" [WARN] Pillar values tidak dikenali (di-set ke 'Food Other'): {unknown}"
)
self.df_clean['pillar'] = self.df_clean['pillar'].replace(
{u: 'Food Other' for u in unknown}
)
self.load_metadata = {
'dim_country' : {'start_time': None, 'end_time': None, 'rows_loaded': 0, 'status': 'pending'},
'dim_indicator' : {'start_time': None, 'end_time': None, 'rows_loaded': 0, 'status': 'pending'},
@@ -117,7 +188,7 @@ class DimensionalModelLoader:
"""
try:
self.client.query(query).result()
self.logger.info(f" [OK] FK: {table_name}.{fk_column} → {ref_table}.{ref_column}")
self.logger.info(f" [OK] FK: {table_name}.{fk_column} -> {ref_table}.{ref_column}")
except Exception as e:
if "already exists" in str(e).lower():
self.logger.info(f" [INFO] FK already exists: {constraint_name}")
@@ -140,12 +211,16 @@ class DimensionalModelLoader:
'rows_transformed' : meta['rows_loaded'],
'rows_loaded' : meta['rows_loaded'],
'completeness_pct' : 100.0 if meta['status'] == 'success' else 0.0,
'config_snapshot' : json.dumps({'load_mode': 'full_refresh', 'layer': 'gold'}),
'config_snapshot' : json.dumps({
'load_mode' : 'full_refresh',
'layer' : 'gold',
'pillar_names': OFFICIAL_PILLARS,
}),
'validation_metrics' : json.dumps({'status': meta['status'], 'rows': meta['rows_loaded']})
}
try:
save_etl_metadata(self.client, metadata)
self.logger.info(f" Metadata → [AUDIT] etl_metadata")
self.logger.info(f" Metadata -> [AUDIT] etl_metadata")
except Exception as e:
self.logger.warning(f" [WARN] Could not save metadata for {table_name}: {e}")
@@ -156,7 +231,7 @@ class DimensionalModelLoader:
def load_dim_time(self):
table_name = 'dim_time'
self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_time → [DW/Gold] fs_asean_gold...")
self.logger.info("Loading dim_time -> [DW/Gold] fs_asean_gold...")
try:
if 'year_range' in self.df_clean.columns:
@@ -229,7 +304,7 @@ class DimensionalModelLoader:
)
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_time: {rows_loaded} rows\n")
self.logger.info(f" [OK] dim_time: {rows_loaded} rows\n")
return rows_loaded
except Exception as e:
@@ -240,7 +315,7 @@ class DimensionalModelLoader:
def load_dim_country(self):
table_name = 'dim_country'
self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_country → [DW/Gold] fs_asean_gold...")
self.logger.info("Loading dim_country -> [DW/Gold] fs_asean_gold...")
try:
dim_country = self.df_clean[['country']].drop_duplicates().copy()
@@ -293,7 +368,7 @@ class DimensionalModelLoader:
)
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_country: {rows_loaded} rows\n")
self.logger.info(f" [OK] dim_country: {rows_loaded} rows\n")
return rows_loaded
except Exception as e:
@@ -304,7 +379,7 @@ class DimensionalModelLoader:
def load_dim_indicator(self):
table_name = 'dim_indicator'
self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_indicator → [DW/Gold] fs_asean_gold...")
self.logger.info("Loading dim_indicator -> [DW/Gold] fs_asean_gold...")
try:
has_direction = 'direction' in self.df_clean.columns
@@ -335,13 +410,18 @@ class DimensionalModelLoader:
cat_map.columns = ['indicator_name', 'indicator_category']
dim_indicator = dim_indicator.merge(cat_map, on='indicator_name', how='left')
else:
def categorize_indicator(name):
# Kategorisasi otomatis — tidak ada lagi kata "Sustainability"
def categorize_indicator(name: str) -> str:
n = str(name).lower()
if any(w in n for w in ['undernourishment', 'malnutrition', 'stunting',
'wasting', 'anemia', 'food security', 'food insecure', 'hunger']):
if any(w in n for w in [
'undernourishment', 'malnutrition', 'stunting',
'wasting', 'anemia', 'food security', 'food insecure', 'hunger'
]):
return 'Health & Nutrition'
elif any(w in n for w in ['production', 'yield', 'cereal', 'crop',
'import dependency', 'share of dietary']):
elif any(w in n for w in [
'production', 'yield', 'cereal', 'crop',
'import dependency', 'share of dietary'
]):
return 'Agricultural Production'
elif any(w in n for w in ['import', 'export', 'trade']):
return 'Trade'
@@ -350,8 +430,12 @@ class DimensionalModelLoader:
elif any(w in n for w in ['water', 'sanitation', 'infrastructure', 'rail']):
return 'Infrastructure'
else:
return 'Sustainability'
dim_indicator['indicator_category'] = dim_indicator['indicator_name'].apply(categorize_indicator)
# Fallback: "Additional Statistics" menggantikan "Sustainability"
return 'Additional Statistics'
dim_indicator['indicator_category'] = (
dim_indicator['indicator_name'].apply(categorize_indicator)
)
dim_indicator = dim_indicator.drop_duplicates(subset=['indicator_name'], keep='first')
dim_indicator_final = dim_indicator[
@@ -384,7 +468,7 @@ class DimensionalModelLoader:
)
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_indicator: {rows_loaded} rows\n")
self.logger.info(f" [OK] dim_indicator: {rows_loaded} rows\n")
return rows_loaded
except Exception as e:
@@ -395,7 +479,7 @@ class DimensionalModelLoader:
def load_dim_source(self):
table_name = 'dim_source'
self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_source → [DW/Gold] fs_asean_gold...")
self.logger.info("Loading dim_source -> [DW/Gold] fs_asean_gold...")
try:
source_details = {
@@ -455,7 +539,7 @@ class DimensionalModelLoader:
)
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_source: {rows_loaded} rows\n")
self.logger.info(f" [OK] dim_source: {rows_loaded} rows\n")
return rows_loaded
except Exception as e:
@@ -464,26 +548,45 @@ class DimensionalModelLoader:
raise
def load_dim_pillar(self):
"""
Load dim_pillar dengan 5 pilar resmi FAO (prefix 'Food ').
'Sustainability' tidak ada — digantikan 'Food Other' (Indikator Tambahan).
Pilar dibuat dari OFFICIAL_PILLARS (bukan dari data) agar selalu lengkap.
"""
table_name = 'dim_pillar'
self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_pillar → [DW/Gold] fs_asean_gold...")
self.logger.info("Loading dim_pillar -> [DW/Gold] fs_asean_gold...")
try:
pillar_codes = {
'Availability': 'AVL', 'Access' : 'ACC',
'Utilization' : 'UTL', 'Stability': 'STB', 'Sustainability': 'STN',
}
pillars_data = [
{'pillar_name': p, 'pillar_code': pillar_codes.get(p, 'STN')}
for p in self.df_clean['pillar'].unique()
]
# Ambil pilar yang benar-benar ada di data (sudah dinormalisasi di __init__)
pillars_in_data = set(self.df_clean['pillar'].unique()) if 'pillar' in self.df_clean.columns else set()
dim_pillar_final = pd.DataFrame(pillars_data).sort_values('pillar_name')[
['pillar_name', 'pillar_code']
].copy()
# Bangun dari OFFICIAL_PILLARS — urutan tampilan konsisten
pillars_data = []
for pillar_name in OFFICIAL_PILLARS:
code = PILLAR_CODE_MAP.get(pillar_name, 'OTH')
pillars_data.append({
'pillar_name': pillar_name,
'pillar_code': code,
})
if pillar_name not in pillars_in_data:
self.logger.warning(
f" [INFO] Pilar '{pillar_name}' tidak ada di data — "
f"tetap disertakan di dim_pillar untuk kelengkapan."
)
dim_pillar_final = pd.DataFrame(pillars_data)[['pillar_name', 'pillar_code']].copy()
dim_pillar_final = dim_pillar_final.reset_index(drop=True)
dim_pillar_final.insert(0, 'pillar_id', range(1, len(dim_pillar_final) + 1))
self.logger.info(" Pillar list yang akan di-load:")
for _, row in dim_pillar_final.iterrows():
in_data = "[ada di data]" if row['pillar_name'] in pillars_in_data else "[tidak ada di data]"
self.logger.info(
f" {int(row['pillar_id'])}. {row['pillar_name']:<20} "
f"({row['pillar_code']}) {in_data}"
)
schema = [
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
@@ -501,7 +604,7 @@ class DimensionalModelLoader:
)
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_pillar: {rows_loaded} rows\n")
self.logger.info(f" [OK] dim_pillar: {rows_loaded} rows\n")
return rows_loaded
except Exception as e:
@@ -516,7 +619,7 @@ class DimensionalModelLoader:
def load_fact_food_security(self):
table_name = 'fact_food_security'
self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading fact_food_security → [DW/Gold] fs_asean_gold...")
self.logger.info("Loading fact_food_security -> [DW/Gold] fs_asean_gold...")
try:
# Load dims dari Gold untuk FK resolution
@@ -563,7 +666,8 @@ class DimensionalModelLoader:
# Resolve FKs
fact_table = fact_table.merge(
dim_country[['country_id', 'country_name']].rename(columns={'country_name': 'country'}),
dim_country[['country_id', 'country_name']].rename(
columns={'country_name': 'country'}),
on='country', how='left'
)
fact_table = fact_table.merge(
@@ -576,14 +680,28 @@ class DimensionalModelLoader:
on=['start_year', 'end_year'], how='left'
)
fact_table = fact_table.merge(
dim_source[['source_id', 'source_name']].rename(columns={'source_name': 'source'}),
dim_source[['source_id', 'source_name']].rename(
columns={'source_name': 'source'}),
on='source', how='left'
)
# pillar kolom sudah dinormalisasi ke nama resmi di __init__
fact_table = fact_table.merge(
dim_pillar[['pillar_id', 'pillar_name']].rename(columns={'pillar_name': 'pillar'}),
dim_pillar[['pillar_id', 'pillar_name']].rename(
columns={'pillar_name': 'pillar'}),
on='pillar', how='left'
)
# Log FK resolution stats
n_total = len(fact_table)
n_no_pillar = fact_table['pillar_id'].isna().sum()
if n_no_pillar > 0:
self.logger.warning(
f" [WARN] {n_no_pillar}/{n_total} rows tidak dapat di-resolve pillar_id"
)
unmatched = fact_table[fact_table['pillar_id'].isna()]['pillar'].value_counts()
for val, cnt in unmatched.items():
self.logger.warning(f" pillar='{val}': {cnt} rows")
# Filter hanya row dengan FK lengkap
fact_table = fact_table[
fact_table['country_id'].notna() &
@@ -634,7 +752,7 @@ class DimensionalModelLoader:
)
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name)
self.logger.info(f" ✓ fact_food_security: {rows_loaded:,} rows\n")
self.logger.info(f" [OK] fact_food_security: {rows_loaded:,} rows\n")
return rows_loaded
except Exception as e:
@@ -712,12 +830,28 @@ class DimensionalModelLoader:
self.logger.info(f" Unique Sources : {int(stats['unique_sources']):>10,}")
self.logger.info(f" Unique Pillars : {int(stats['unique_pillars']):>10,}")
# Validasi pillar — pastikan tidak ada "Sustainability" di BigQuery
query_pillar = f"""
SELECT pillar_name, pillar_code
FROM `{get_table_id('dim_pillar', layer='gold')}`
ORDER BY pillar_id
"""
df_pillar = self.client.query(query_pillar).result().to_dataframe(
create_bqstorage_client=False
)
self.logger.info(f"\n Pillar Dimension:")
for _, row in df_pillar.iterrows():
self.logger.info(f" [{row['pillar_code']}] {row['pillar_name']}")
# Cek arah indikator
query_dir = f"""
SELECT direction, COUNT(*) AS count
FROM `{get_table_id('dim_indicator', layer='gold')}`
GROUP BY direction ORDER BY direction
"""
df_dir = self.client.query(query_dir).result().to_dataframe(create_bqstorage_client=False)
df_dir = self.client.query(query_dir).result().to_dataframe(
create_bqstorage_client=False
)
if len(df_dir) > 0:
self.logger.info(f"\n Direction Distribution:")
for _, row in df_dir.iterrows():
@@ -738,11 +872,14 @@ class DimensionalModelLoader:
self.pipeline_metadata['rows_fetched'] = len(self.df_clean)
self.logger.info("\n" + "=" * 60)
self.logger.info("DIMENSIONAL MODEL LOAD — DW (Gold) → fs_asean_gold")
self.logger.info("DIMENSIONAL MODEL LOAD — DW (Gold) -> fs_asean_gold")
self.logger.info("=" * 60)
self.logger.info(" Pilar resmi (5 pilar, prefix 'Food '):")
for p in OFFICIAL_PILLARS:
self.logger.info(f" - {p} [{PILLAR_CODE_MAP[p]}]")
# Dimensions
self.logger.info("\nLOADING DIMENSION TABLES → fs_asean_gold")
self.logger.info("\nLOADING DIMENSION TABLES -> fs_asean_gold")
self.load_dim_country()
self.load_dim_indicator()
self.load_dim_time()
@@ -750,7 +887,7 @@ class DimensionalModelLoader:
self.load_dim_pillar()
# Fact
self.logger.info("\nLOADING FACT TABLE → fs_asean_gold")
self.logger.info("\nLOADING FACT TABLE -> fs_asean_gold")
self.load_fact_food_security()
# Validate
@@ -768,8 +905,12 @@ class DimensionalModelLoader:
'rows_loaded' : total_loaded,
'execution_timestamp': self.pipeline_metadata['start_time'],
'completeness_pct' : 100.0,
'config_snapshot' : json.dumps({'load_mode': 'full_refresh', 'layer': 'gold'}),
'validation_metrics': json.dumps({t: m['status'] for t, m in self.load_metadata.items()}),
'config_snapshot' : json.dumps({
'load_mode' : 'full_refresh',
'layer' : 'gold',
'pillar_names': OFFICIAL_PILLARS,
}),
'validation_metrics' : json.dumps({t: m['status'] for t, m in self.load_metadata.items()}),
'table_name' : 'dimensional_model_pipeline',
})
try:
@@ -785,20 +926,19 @@ class DimensionalModelLoader:
self.logger.info(f" Duration : {duration:.2f}s")
self.logger.info(f" Tables :")
for tbl, meta in self.load_metadata.items():
icon = "✓" if meta['status'] == 'success' else "✗"
icon = "[OK]" if meta['status'] == 'success' else "[FAIL]"
self.logger.info(f" {icon} {tbl:25s}: {meta['rows_loaded']:>10,} rows")
self.logger.info(f"\n Metadata → [AUDIT] etl_metadata")
self.logger.info(f"\n Metadata -> [AUDIT] etl_metadata")
self.logger.info("=" * 60)
# =============================================================================
# AIRFLOW TASK FUNCTIONS ← sama polanya dengan raw & cleaned layer
# AIRFLOW TASK FUNCTIONS
# =============================================================================
def run_dimensional_model():
"""
Airflow task: Load dimensional model dari cleaned_integrated.
Dipanggil oleh DAG setelah task cleaned_integration_to_silver selesai.
"""
from scripts.bigquery_config import get_bigquery_client
@@ -817,9 +957,10 @@ if __name__ == "__main__":
print("=" * 60)
print("BIGQUERY DIMENSIONAL MODEL LOAD")
print("Kimball DW Architecture")
print(" Input : STAGING (Silver) → cleaned_integrated (fs_asean_silver)")
print(" Output : DW (Gold) → dim_*, fact_* (fs_asean_gold)")
print(" Audit : AUDIT → etl_logs, etl_metadata (fs_asean_audit)")
print(" Input : STAGING (Silver) -> cleaned_integrated (fs_asean_silver)")
print(" Output : DW (Gold) -> dim_*, fact_* (fs_asean_gold)")
print(" Audit : AUDIT -> etl_logs, etl_metadata (fs_asean_audit)")
print(f" Pillars: {', '.join(OFFICIAL_PILLARS)}")
print("=" * 60)
logger = setup_logging()
@@ -827,24 +968,26 @@ if __name__ == "__main__":
print("\nLoading cleaned_integrated (fs_asean_silver)...")
df_clean = read_from_bigquery(client, 'cleaned_integrated', layer='silver')
print(f" ✓ Loaded : {len(df_clean):,} rows")
print(f" [OK] Loaded : {len(df_clean):,} rows")
print(f" Columns : {len(df_clean.columns)}")
print(f" Sources : {df_clean['source'].nunique()}")
print(f" Indicators : {df_clean['indicator_standardized'].nunique()}")
print(f" Countries : {df_clean['country'].nunique()}")
print(f" Year range : {int(df_clean['year'].min())}–{int(df_clean['year'].max())}")
print(f" Year range : {int(df_clean['year'].min())}-{int(df_clean['year'].max())}")
if 'pillar' in df_clean.columns:
print(f" Pillars raw : {sorted(df_clean['pillar'].unique())}")
if 'direction' in df_clean.columns:
print(f" Direction : {df_clean['direction'].value_counts().to_dict()}")
else:
print(f" [WARN] direction column not found — run bigquery_cleaned_layer.py first")
print("\n[1/1] Dimensional Model Load → DW (Gold)...")
print("\n[1/1] Dimensional Model Load -> DW (Gold)...")
loader = DimensionalModelLoader(client, df_clean)
loader.run()
print("\n" + "=" * 60)
print("✓ DIMENSIONAL MODEL ETL COMPLETED")
print(" 🥇 DW (Gold) : dim_country, dim_indicator, dim_time,")
print("[OK] DIMENSIONAL MODEL ETL COMPLETED")
print(" DW (Gold) : dim_country, dim_indicator, dim_time,")
print(" dim_source, dim_pillar, fact_food_security")
print(" 📋 AUDIT : etl_logs, etl_metadata")
print(" AUDIT : etl_logs, etl_metadata")
print("=" * 60)