izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename pillar

Browse Source
This commit is contained in:
Debby
2026-06-27 13:23:02 +07:00
parent 17cc7833ad
commit ebc189f2a0
4 changed files with 807 additions and 1456 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bigquery_aggraget_fact_selected_layer.py
+232 -457
View File
File diff suppressed because it is too large Load Diff
scripts/bigquery_aggregate_layer.py
+301 -893
View File
File diff suppressed because it is too large Load Diff
scripts/bigquery_analytical_layer.py
+46 -21
View File
@@ -11,6 +11,8 @@ Filtering Order:
6. Save analytical table (dengan nama/label lengkap untuk Looker Studio) 6. Save analytical table (dengan nama/label lengkap untuk Looker Studio)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
CHANGED: pillar_name sekarang pakai prefix 'Food ' (Food Availability, Food Access, dst.)
'Sustainability' -> 'Food Other', nama Indonesia: Ketersediaan Pangan, Akses Pangan, dst.
""" """
import pandas as pd import pandas as pd
@@ -38,26 +40,36 @@ from google.cloud import bigquery
# ============================================================================= # =============================================================================
# TRANSLATION DICTIONARIES # TRANSLATION DICTIONARIES
# CHANGED: "Sustainability" / "sustainability" -> "Other" / "Lainnya"
# ============================================================================= # =============================================================================
PILLAR_TRANSLATION_ID: dict = { PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security # Mapping nama pilar (Inggris dengan prefix Food) -> Bahasa Indonesia
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability": "Keberlanjutan",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability": "Keberlanjutan",
"Food Availability" : "Ketersediaan Pangan", "Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan", "Food Access" : "Akses Pangan",
"Food Utilization" : "Pemanfaatan Pangan", "Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan", "Food Stability" : "Stabilitas Pangan",
"Food Sustainability": "Keberlanjutan Pangan", "Food Other" : "Indikator Tambahan",
# Variasi tanpa prefix Food (dari data lama)
"Availability" : "Ketersediaan Pangan",
"Access" : "Akses Pangan",
"Utilization" : "Pemanfaatan Pangan",
"Stability" : "Stabilitas Pangan",
"Other" : "Indikator Tambahan",
# Legacy Sustainability -> Food Other -> Indikator Tambahan
"Sustainability" : "Indikator Tambahan",
"sustainability" : "Indikator Tambahan",
# lowercase
"food availability" : "Ketersediaan Pangan",
"food access" : "Akses Pangan",
"food utilization" : "Pemanfaatan Pangan",
"food stability" : "Stabilitas Pangan",
"food other" : "Indikator Tambahan",
"availability" : "Ketersediaan Pangan",
"access" : "Akses Pangan",
"utilization" : "Pemanfaatan Pangan",
"stability" : "Stabilitas Pangan",
"other" : "Indikator Tambahan",
} }
@@ -194,7 +206,11 @@ def translate_indicator(name: str) -> str:
def translate_pillar(name: str) -> str: def translate_pillar(name: str) -> str:
"""Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli.""" """
Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli.
CHANGED: pillar_name menggunakan prefix 'Food ' (Food Availability, Food Access, dll.)
'Sustainability' -> 'Food Other' (EN) / 'Indikator Tambahan' (ID).
"""
if not name: if not name:
return name return name
return PILLAR_TRANSLATION_ID.get(name, name) return PILLAR_TRANSLATION_ID.get(name, name)
@@ -284,6 +300,18 @@ class AnalyticalLayerLoader:
self.df_clean = self.client.query(query).result().to_dataframe(create_bqstorage_client=False) self.df_clean = self.client.query(query).result().to_dataframe(create_bqstorage_client=False)
self.logger.info(f" Loaded: {len(self.df_clean):,} rows") self.logger.info(f" Loaded: {len(self.df_clean):,} rows")
# Rename pillar_name: add 'Food ' prefix, remove Sustainability
PILLAR_RENAME_MAP = {
'Availability' : 'Food Availability',
'Access' : 'Food Access',
'Utilization' : 'Food Utilization',
'Stability' : 'Food Stability',
'Other' : 'Food Other',
'Sustainability': 'Food Other',
'sustainability': 'Food Other',
}
self.df_clean['pillar_name'] = self.df_clean['pillar_name'].replace(PILLAR_RENAME_MAP)
if 'is_year_range' in self.df_clean.columns: if 'is_year_range' in self.df_clean.columns:
yr = self.df_clean['is_year_range'].value_counts() yr = self.df_clean['is_year_range'].value_counts()
self.logger.info(f" Breakdown:") self.logger.info(f" Breakdown:")
@@ -614,11 +642,7 @@ class AnalyticalLayerLoader:
'value', 'value',
]].copy() ]].copy()
# ------------------------------------------------------------------ # Terjemahan Bahasa Indonesia
# TAMBAHAN: kolom terjemahan Bahasa Indonesia
# indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
# pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name
# ------------------------------------------------------------------
analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator) analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator)
analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar) analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar)
@@ -701,7 +725,8 @@ class AnalyticalLayerLoader:
'fixed_countries': len(self.selected_country_ids), 'fixed_countries': len(self.selected_country_ids),
'no_gaps' : True, 'no_gaps' : True,
'layer' : 'gold', 'layer' : 'gold',
'columns' : 'id + name + name_id (Looker Studio ready)' 'columns' : 'id + name + name_id (Looker Studio ready)',
'pillar_change' : 'Sustainability -> Food Other; all pillars use Food prefix',
}), }),
'validation_metrics' : json.dumps({ 'validation_metrics' : json.dumps({
'fixed_countries' : len(self.selected_country_ids), 'fixed_countries' : len(self.selected_country_ids),
scripts/bigquery_dimensional_model.py
+228 -85
View File
@@ -10,6 +10,13 @@ Kimball ETL Flow yang dijalankan file ini:
Classes: Classes:
DimensionalModelLoader — Build Star Schema & load ke Gold layer DimensionalModelLoader — Build Star Schema & load ke Gold layer
Pilar resmi FAO yang digunakan (5 pilar dengan prefix "Food "):
- Food Availability (Ketersediaan Pangan)
- Food Access (Akses Pangan)
- Food Utilization (Pemanfaatan Pangan)
- Food Stability (Stabilitas Pangan)
- Food Other (Indikator Tambahan) — additional useful statistics
Usage: Usage:
python bigquery_dimensional_model.py python bigquery_dimensional_model.py
""" """
@@ -37,6 +44,53 @@ if hasattr(sys.stdout, 'reconfigure'):
sys.stdout.reconfigure(encoding='utf-8') sys.stdout.reconfigure(encoding='utf-8')
# =============================================================================
# PILLAR CONSTANTS
# Satu-satunya sumber kebenaran untuk nama pilar di seluruh pipeline.
# Tidak ada lagi "Sustainability" — digantikan "Food Other".
# =============================================================================
# Mapping dari nilai lama/raw -> nama pilar resmi (dengan prefix "Food ")
PILLAR_RENAME_MAP: dict = {
# Nilai lama tanpa prefix
'Availability' : 'Food Availability',
'Access' : 'Food Access',
'Utilization' : 'Food Utilization',
'Stability' : 'Food Stability',
'Other' : 'Food Other',
# Nilai yang sudah benar (idempotent)
'Food Availability': 'Food Availability',
'Food Access' : 'Food Access',
'Food Utilization' : 'Food Utilization',
'Food Stability' : 'Food Stability',
'Food Other' : 'Food Other',
# lowercase
'food availability': 'Food Availability',
'food access' : 'Food Access',
'food utilization' : 'Food Utilization',
'food stability' : 'Food Stability',
'food other' : 'Food Other',
}
# Kode resmi per pilar
PILLAR_CODE_MAP: dict = {
'Food Availability': 'AVL',
'Food Access' : 'ACC',
'Food Utilization' : 'UTL',
'Food Stability' : 'STB',
'Food Other' : 'OTH',
}
# Nama 5 pilar resmi (urutan tampilan)
OFFICIAL_PILLARS: list = [
'Food Availability',
'Food Access',
'Food Utilization',
'Food Stability',
'Food Other',
]
# ============================================================================= # =============================================================================
# DIMENSIONAL MODEL LOADER # DIMENSIONAL MODEL LOADER
# ============================================================================= # =============================================================================
@@ -62,11 +116,28 @@ class DimensionalModelLoader:
def __init__(self, client: bigquery.Client, df_clean: pd.DataFrame): def __init__(self, client: bigquery.Client, df_clean: pd.DataFrame):
self.client = client self.client = client
self.df_clean = df_clean self.df_clean = df_clean.copy()
self.logger = logging.getLogger(self.__class__.__name__) self.logger = logging.getLogger(self.__class__.__name__)
self.logger.propagate = False self.logger.propagate = False
self.target_layer = 'gold' self.target_layer = 'gold'
# Normalisasi pillar column sekarang, satu kali, di awal
if 'pillar' in self.df_clean.columns:
self.df_clean['pillar'] = (
self.df_clean['pillar']
.astype(str)
.str.strip()
.map(lambda x: PILLAR_RENAME_MAP.get(x, 'Food Other'))
)
unknown = set(self.df_clean['pillar'].unique()) - set(OFFICIAL_PILLARS)
if unknown:
self.logger.warning(
f" [WARN] Pillar values tidak dikenali (di-set ke 'Food Other'): {unknown}"
)
self.df_clean['pillar'] = self.df_clean['pillar'].replace(
{u: 'Food Other' for u in unknown}
)
self.load_metadata = { self.load_metadata = {
'dim_country' : {'start_time': None, 'end_time': None, 'rows_loaded': 0, 'status': 'pending'}, 'dim_country' : {'start_time': None, 'end_time': None, 'rows_loaded': 0, 'status': 'pending'},
'dim_indicator' : {'start_time': None, 'end_time': None, 'rows_loaded': 0, 'status': 'pending'}, 'dim_indicator' : {'start_time': None, 'end_time': None, 'rows_loaded': 0, 'status': 'pending'},
@@ -117,7 +188,7 @@ class DimensionalModelLoader:
""" """
try: try:
self.client.query(query).result() self.client.query(query).result()
self.logger.info(f" [OK] FK: {table_name}.{fk_column} → {ref_table}.{ref_column}") self.logger.info(f" [OK] FK: {table_name}.{fk_column} -> {ref_table}.{ref_column}")
except Exception as e: except Exception as e:
if "already exists" in str(e).lower(): if "already exists" in str(e).lower():
self.logger.info(f" [INFO] FK already exists: {constraint_name}") self.logger.info(f" [INFO] FK already exists: {constraint_name}")
@@ -140,12 +211,16 @@ class DimensionalModelLoader:
'rows_transformed' : meta['rows_loaded'], 'rows_transformed' : meta['rows_loaded'],
'rows_loaded' : meta['rows_loaded'], 'rows_loaded' : meta['rows_loaded'],
'completeness_pct' : 100.0 if meta['status'] == 'success' else 0.0, 'completeness_pct' : 100.0 if meta['status'] == 'success' else 0.0,
'config_snapshot' : json.dumps({'load_mode': 'full_refresh', 'layer': 'gold'}), 'config_snapshot' : json.dumps({
'load_mode' : 'full_refresh',
'layer' : 'gold',
'pillar_names': OFFICIAL_PILLARS,
}),
'validation_metrics' : json.dumps({'status': meta['status'], 'rows': meta['rows_loaded']}) 'validation_metrics' : json.dumps({'status': meta['status'], 'rows': meta['rows_loaded']})
} }
try: try:
save_etl_metadata(self.client, metadata) save_etl_metadata(self.client, metadata)
self.logger.info(f" Metadata → [AUDIT] etl_metadata") self.logger.info(f" Metadata -> [AUDIT] etl_metadata")
except Exception as e: except Exception as e:
self.logger.warning(f" [WARN] Could not save metadata for {table_name}: {e}") self.logger.warning(f" [WARN] Could not save metadata for {table_name}: {e}")
@@ -156,13 +231,13 @@ class DimensionalModelLoader:
def load_dim_time(self): def load_dim_time(self):
table_name = 'dim_time' table_name = 'dim_time'
self.load_metadata[table_name]['start_time'] = datetime.now() self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_time → [DW/Gold] fs_asean_gold...") self.logger.info("Loading dim_time -> [DW/Gold] fs_asean_gold...")
try: try:
if 'year_range' in self.df_clean.columns: if 'year_range' in self.df_clean.columns:
dim_time = self.df_clean[['year', 'year_range']].drop_duplicates().copy() dim_time = self.df_clean[['year', 'year_range']].drop_duplicates().copy()
else: else:
dim_time = self.df_clean[['year']].drop_duplicates().copy() dim_time = self.df_clean[['year']].drop_duplicates().copy()
dim_time['year_range'] = None dim_time['year_range'] = None
dim_time['year'] = dim_time['year'].astype(int) dim_time['year'] = dim_time['year'].astype(int)
@@ -194,10 +269,10 @@ class DimensionalModelLoader:
pass pass
return pd.Series({'year': year, 'start_year': start_year, 'end_year': end_year}) return pd.Series({'year': year, 'start_year': start_year, 'end_year': end_year})
parsed = dim_time.apply(parse_year_range, axis=1) parsed = dim_time.apply(parse_year_range, axis=1)
dim_time['year'] = parsed['year'].astype(int) dim_time['year'] = parsed['year'].astype(int)
dim_time['start_year'] = parsed['start_year'].astype(int) dim_time['start_year'] = parsed['start_year'].astype(int)
dim_time['end_year'] = parsed['end_year'].astype(int) dim_time['end_year'] = parsed['end_year'].astype(int)
dim_time['is_year_range'] = (dim_time['start_year'] != dim_time['end_year']) dim_time['is_year_range'] = (dim_time['start_year'] != dim_time['end_year'])
dim_time['decade'] = (dim_time['year'] // 10) * 10 dim_time['decade'] = (dim_time['year'] // 10) * 10
dim_time['is_range'] = (dim_time['start_year'] != dim_time['end_year']).astype(int) dim_time['is_range'] = (dim_time['start_year'] != dim_time['end_year']).astype(int)
@@ -229,7 +304,7 @@ class DimensionalModelLoader:
) )
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded) log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name) self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_time: {rows_loaded} rows\n") self.logger.info(f" [OK] dim_time: {rows_loaded} rows\n")
return rows_loaded return rows_loaded
except Exception as e: except Exception as e:
@@ -240,11 +315,11 @@ class DimensionalModelLoader:
def load_dim_country(self): def load_dim_country(self):
table_name = 'dim_country' table_name = 'dim_country'
self.load_metadata[table_name]['start_time'] = datetime.now() self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_country → [DW/Gold] fs_asean_gold...") self.logger.info("Loading dim_country -> [DW/Gold] fs_asean_gold...")
try: try:
dim_country = self.df_clean[['country']].drop_duplicates().copy() dim_country = self.df_clean[['country']].drop_duplicates().copy()
dim_country.columns = ['country_name'] dim_country.columns = ['country_name']
region_mapping = { region_mapping = {
'Brunei Darussalam': ('Southeast Asia', 'ASEAN'), 'Brunei Darussalam': ('Southeast Asia', 'ASEAN'),
@@ -293,7 +368,7 @@ class DimensionalModelLoader:
) )
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded) log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name) self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_country: {rows_loaded} rows\n") self.logger.info(f" [OK] dim_country: {rows_loaded} rows\n")
return rows_loaded return rows_loaded
except Exception as e: except Exception as e:
@@ -304,27 +379,27 @@ class DimensionalModelLoader:
def load_dim_indicator(self): def load_dim_indicator(self):
table_name = 'dim_indicator' table_name = 'dim_indicator'
self.load_metadata[table_name]['start_time'] = datetime.now() self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_indicator → [DW/Gold] fs_asean_gold...") self.logger.info("Loading dim_indicator -> [DW/Gold] fs_asean_gold...")
try: try:
has_direction = 'direction' in self.df_clean.columns has_direction = 'direction' in self.df_clean.columns
has_unit = 'unit' in self.df_clean.columns has_unit = 'unit' in self.df_clean.columns
has_category = 'indicator_category' in self.df_clean.columns has_category = 'indicator_category' in self.df_clean.columns
dim_indicator = self.df_clean[['indicator_standardized']].drop_duplicates().copy() dim_indicator = self.df_clean[['indicator_standardized']].drop_duplicates().copy()
dim_indicator.columns = ['indicator_name'] dim_indicator.columns = ['indicator_name']
if has_unit: if has_unit:
unit_map = self.df_clean[['indicator_standardized', 'unit']].drop_duplicates() unit_map = self.df_clean[['indicator_standardized', 'unit']].drop_duplicates()
unit_map.columns = ['indicator_name', 'unit'] unit_map.columns = ['indicator_name', 'unit']
dim_indicator = dim_indicator.merge(unit_map, on='indicator_name', how='left') dim_indicator = dim_indicator.merge(unit_map, on='indicator_name', how='left')
else: else:
dim_indicator['unit'] = None dim_indicator['unit'] = None
if has_direction: if has_direction:
dir_map = self.df_clean[['indicator_standardized', 'direction']].drop_duplicates() dir_map = self.df_clean[['indicator_standardized', 'direction']].drop_duplicates()
dir_map.columns = ['indicator_name', 'direction'] dir_map.columns = ['indicator_name', 'direction']
dim_indicator = dim_indicator.merge(dir_map, on='indicator_name', how='left') dim_indicator = dim_indicator.merge(dir_map, on='indicator_name', how='left')
self.logger.info(" [OK] direction column from cleaned_integrated") self.logger.info(" [OK] direction column from cleaned_integrated")
else: else:
dim_indicator['direction'] = 'higher_better' dim_indicator['direction'] = 'higher_better'
@@ -332,16 +407,21 @@ class DimensionalModelLoader:
if has_category: if has_category:
cat_map = self.df_clean[['indicator_standardized', 'indicator_category']].drop_duplicates() cat_map = self.df_clean[['indicator_standardized', 'indicator_category']].drop_duplicates()
cat_map.columns = ['indicator_name', 'indicator_category'] cat_map.columns = ['indicator_name', 'indicator_category']
dim_indicator = dim_indicator.merge(cat_map, on='indicator_name', how='left') dim_indicator = dim_indicator.merge(cat_map, on='indicator_name', how='left')
else: else:
def categorize_indicator(name): # Kategorisasi otomatis — tidak ada lagi kata "Sustainability"
def categorize_indicator(name: str) -> str:
n = str(name).lower() n = str(name).lower()
if any(w in n for w in ['undernourishment', 'malnutrition', 'stunting', if any(w in n for w in [
'wasting', 'anemia', 'food security', 'food insecure', 'hunger']): 'undernourishment', 'malnutrition', 'stunting',
'wasting', 'anemia', 'food security', 'food insecure', 'hunger'
]):
return 'Health & Nutrition' return 'Health & Nutrition'
elif any(w in n for w in ['production', 'yield', 'cereal', 'crop', elif any(w in n for w in [
'import dependency', 'share of dietary']): 'production', 'yield', 'cereal', 'crop',
'import dependency', 'share of dietary'
]):
return 'Agricultural Production' return 'Agricultural Production'
elif any(w in n for w in ['import', 'export', 'trade']): elif any(w in n for w in ['import', 'export', 'trade']):
return 'Trade' return 'Trade'
@@ -350,10 +430,14 @@ class DimensionalModelLoader:
elif any(w in n for w in ['water', 'sanitation', 'infrastructure', 'rail']): elif any(w in n for w in ['water', 'sanitation', 'infrastructure', 'rail']):
return 'Infrastructure' return 'Infrastructure'
else: else:
return 'Sustainability' # Fallback: "Additional Statistics" menggantikan "Sustainability"
dim_indicator['indicator_category'] = dim_indicator['indicator_name'].apply(categorize_indicator) return 'Additional Statistics'
dim_indicator = dim_indicator.drop_duplicates(subset=['indicator_name'], keep='first') dim_indicator['indicator_category'] = (
dim_indicator['indicator_name'].apply(categorize_indicator)
)
dim_indicator = dim_indicator.drop_duplicates(subset=['indicator_name'], keep='first')
dim_indicator_final = dim_indicator[ dim_indicator_final = dim_indicator[
['indicator_name', 'indicator_category', 'unit', 'direction'] ['indicator_name', 'indicator_category', 'unit', 'direction']
].copy() ].copy()
@@ -384,7 +468,7 @@ class DimensionalModelLoader:
) )
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded) log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name) self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_indicator: {rows_loaded} rows\n") self.logger.info(f" [OK] dim_indicator: {rows_loaded} rows\n")
return rows_loaded return rows_loaded
except Exception as e: except Exception as e:
@@ -395,7 +479,7 @@ class DimensionalModelLoader:
def load_dim_source(self): def load_dim_source(self):
table_name = 'dim_source' table_name = 'dim_source'
self.load_metadata[table_name]['start_time'] = datetime.now() self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_source → [DW/Gold] fs_asean_gold...") self.logger.info("Loading dim_source -> [DW/Gold] fs_asean_gold...")
try: try:
source_details = { source_details = {
@@ -455,7 +539,7 @@ class DimensionalModelLoader:
) )
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded) log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name) self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_source: {rows_loaded} rows\n") self.logger.info(f" [OK] dim_source: {rows_loaded} rows\n")
return rows_loaded return rows_loaded
except Exception as e: except Exception as e:
@@ -464,26 +548,45 @@ class DimensionalModelLoader:
raise raise
def load_dim_pillar(self): def load_dim_pillar(self):
"""
Load dim_pillar dengan 5 pilar resmi FAO (prefix 'Food ').
'Sustainability' tidak ada — digantikan 'Food Other' (Indikator Tambahan).
Pilar dibuat dari OFFICIAL_PILLARS (bukan dari data) agar selalu lengkap.
"""
table_name = 'dim_pillar' table_name = 'dim_pillar'
self.load_metadata[table_name]['start_time'] = datetime.now() self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading dim_pillar → [DW/Gold] fs_asean_gold...") self.logger.info("Loading dim_pillar -> [DW/Gold] fs_asean_gold...")
try: try:
pillar_codes = { # Ambil pilar yang benar-benar ada di data (sudah dinormalisasi di __init__)
'Availability': 'AVL', 'Access' : 'ACC', pillars_in_data = set(self.df_clean['pillar'].unique()) if 'pillar' in self.df_clean.columns else set()
'Utilization' : 'UTL', 'Stability': 'STB', 'Sustainability': 'STN',
}
pillars_data = [
{'pillar_name': p, 'pillar_code': pillar_codes.get(p, 'STN')}
for p in self.df_clean['pillar'].unique()
]
dim_pillar_final = pd.DataFrame(pillars_data).sort_values('pillar_name')[ # Bangun dari OFFICIAL_PILLARS — urutan tampilan konsisten
['pillar_name', 'pillar_code'] pillars_data = []
].copy() for pillar_name in OFFICIAL_PILLARS:
code = PILLAR_CODE_MAP.get(pillar_name, 'OTH')
pillars_data.append({
'pillar_name': pillar_name,
'pillar_code': code,
})
if pillar_name not in pillars_in_data:
self.logger.warning(
f" [INFO] Pilar '{pillar_name}' tidak ada di data — "
f"tetap disertakan di dim_pillar untuk kelengkapan."
)
dim_pillar_final = pd.DataFrame(pillars_data)[['pillar_name', 'pillar_code']].copy()
dim_pillar_final = dim_pillar_final.reset_index(drop=True) dim_pillar_final = dim_pillar_final.reset_index(drop=True)
dim_pillar_final.insert(0, 'pillar_id', range(1, len(dim_pillar_final) + 1)) dim_pillar_final.insert(0, 'pillar_id', range(1, len(dim_pillar_final) + 1))
self.logger.info(" Pillar list yang akan di-load:")
for _, row in dim_pillar_final.iterrows():
in_data = "[ada di data]" if row['pillar_name'] in pillars_in_data else "[tidak ada di data]"
self.logger.info(
f" {int(row['pillar_id'])}. {row['pillar_name']:<20} "
f"({row['pillar_code']}) {in_data}"
)
schema = [ schema = [
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
@@ -501,7 +604,7 @@ class DimensionalModelLoader:
) )
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded) log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name) self._save_table_metadata(table_name)
self.logger.info(f" ✓ dim_pillar: {rows_loaded} rows\n") self.logger.info(f" [OK] dim_pillar: {rows_loaded} rows\n")
return rows_loaded return rows_loaded
except Exception as e: except Exception as e:
@@ -516,7 +619,7 @@ class DimensionalModelLoader:
def load_fact_food_security(self): def load_fact_food_security(self):
table_name = 'fact_food_security' table_name = 'fact_food_security'
self.load_metadata[table_name]['start_time'] = datetime.now() self.load_metadata[table_name]['start_time'] = datetime.now()
self.logger.info("Loading fact_food_security → [DW/Gold] fs_asean_gold...") self.logger.info("Loading fact_food_security -> [DW/Gold] fs_asean_gold...")
try: try:
# Load dims dari Gold untuk FK resolution # Load dims dari Gold untuk FK resolution
@@ -563,7 +666,8 @@ class DimensionalModelLoader:
# Resolve FKs # Resolve FKs
fact_table = fact_table.merge( fact_table = fact_table.merge(
dim_country[['country_id', 'country_name']].rename(columns={'country_name': 'country'}), dim_country[['country_id', 'country_name']].rename(
columns={'country_name': 'country'}),
on='country', how='left' on='country', how='left'
) )
fact_table = fact_table.merge( fact_table = fact_table.merge(
@@ -576,14 +680,28 @@ class DimensionalModelLoader:
on=['start_year', 'end_year'], how='left' on=['start_year', 'end_year'], how='left'
) )
fact_table = fact_table.merge( fact_table = fact_table.merge(
dim_source[['source_id', 'source_name']].rename(columns={'source_name': 'source'}), dim_source[['source_id', 'source_name']].rename(
columns={'source_name': 'source'}),
on='source', how='left' on='source', how='left'
) )
# pillar kolom sudah dinormalisasi ke nama resmi di __init__
fact_table = fact_table.merge( fact_table = fact_table.merge(
dim_pillar[['pillar_id', 'pillar_name']].rename(columns={'pillar_name': 'pillar'}), dim_pillar[['pillar_id', 'pillar_name']].rename(
columns={'pillar_name': 'pillar'}),
on='pillar', how='left' on='pillar', how='left'
) )
# Log FK resolution stats
n_total = len(fact_table)
n_no_pillar = fact_table['pillar_id'].isna().sum()
if n_no_pillar > 0:
self.logger.warning(
f" [WARN] {n_no_pillar}/{n_total} rows tidak dapat di-resolve pillar_id"
)
unmatched = fact_table[fact_table['pillar_id'].isna()]['pillar'].value_counts()
for val, cnt in unmatched.items():
self.logger.warning(f" pillar='{val}': {cnt} rows")
# Filter hanya row dengan FK lengkap # Filter hanya row dengan FK lengkap
fact_table = fact_table[ fact_table = fact_table[
fact_table['country_id'].notna() & fact_table['country_id'].notna() &
@@ -634,7 +752,7 @@ class DimensionalModelLoader:
) )
log_update(self.client, 'DW', table_name, 'full_load', rows_loaded) log_update(self.client, 'DW', table_name, 'full_load', rows_loaded)
self._save_table_metadata(table_name) self._save_table_metadata(table_name)
self.logger.info(f" ✓ fact_food_security: {rows_loaded:,} rows\n") self.logger.info(f" [OK] fact_food_security: {rows_loaded:,} rows\n")
return rows_loaded return rows_loaded
except Exception as e: except Exception as e:
@@ -712,12 +830,28 @@ class DimensionalModelLoader:
self.logger.info(f" Unique Sources : {int(stats['unique_sources']):>10,}") self.logger.info(f" Unique Sources : {int(stats['unique_sources']):>10,}")
self.logger.info(f" Unique Pillars : {int(stats['unique_pillars']):>10,}") self.logger.info(f" Unique Pillars : {int(stats['unique_pillars']):>10,}")
# Validasi pillar — pastikan tidak ada "Sustainability" di BigQuery
query_pillar = f"""
SELECT pillar_name, pillar_code
FROM `{get_table_id('dim_pillar', layer='gold')}`
ORDER BY pillar_id
"""
df_pillar = self.client.query(query_pillar).result().to_dataframe(
create_bqstorage_client=False
)
self.logger.info(f"\n Pillar Dimension:")
for _, row in df_pillar.iterrows():
self.logger.info(f" [{row['pillar_code']}] {row['pillar_name']}")
# Cek arah indikator
query_dir = f""" query_dir = f"""
SELECT direction, COUNT(*) AS count SELECT direction, COUNT(*) AS count
FROM `{get_table_id('dim_indicator', layer='gold')}` FROM `{get_table_id('dim_indicator', layer='gold')}`
GROUP BY direction ORDER BY direction GROUP BY direction ORDER BY direction
""" """
df_dir = self.client.query(query_dir).result().to_dataframe(create_bqstorage_client=False) df_dir = self.client.query(query_dir).result().to_dataframe(
create_bqstorage_client=False
)
if len(df_dir) > 0: if len(df_dir) > 0:
self.logger.info(f"\n Direction Distribution:") self.logger.info(f"\n Direction Distribution:")
for _, row in df_dir.iterrows(): for _, row in df_dir.iterrows():
@@ -738,11 +872,14 @@ class DimensionalModelLoader:
self.pipeline_metadata['rows_fetched'] = len(self.df_clean) self.pipeline_metadata['rows_fetched'] = len(self.df_clean)
self.logger.info("\n" + "=" * 60) self.logger.info("\n" + "=" * 60)
self.logger.info("DIMENSIONAL MODEL LOAD — DW (Gold) → fs_asean_gold") self.logger.info("DIMENSIONAL MODEL LOAD — DW (Gold) -> fs_asean_gold")
self.logger.info("=" * 60) self.logger.info("=" * 60)
self.logger.info(" Pilar resmi (5 pilar, prefix 'Food '):")
for p in OFFICIAL_PILLARS:
self.logger.info(f" - {p} [{PILLAR_CODE_MAP[p]}]")
# Dimensions # Dimensions
self.logger.info("\nLOADING DIMENSION TABLES → fs_asean_gold") self.logger.info("\nLOADING DIMENSION TABLES -> fs_asean_gold")
self.load_dim_country() self.load_dim_country()
self.load_dim_indicator() self.load_dim_indicator()
self.load_dim_time() self.load_dim_time()
@@ -750,7 +887,7 @@ class DimensionalModelLoader:
self.load_dim_pillar() self.load_dim_pillar()
# Fact # Fact
self.logger.info("\nLOADING FACT TABLE → fs_asean_gold") self.logger.info("\nLOADING FACT TABLE -> fs_asean_gold")
self.load_fact_food_security() self.load_fact_food_security()
# Validate # Validate
@@ -762,15 +899,19 @@ class DimensionalModelLoader:
total_loaded = sum(m['rows_loaded'] for m in self.load_metadata.values()) total_loaded = sum(m['rows_loaded'] for m in self.load_metadata.values())
self.pipeline_metadata.update({ self.pipeline_metadata.update({
'end_time' : pipeline_end, 'end_time' : pipeline_end,
'duration_seconds' : duration, 'duration_seconds' : duration,
'rows_transformed' : total_loaded, 'rows_transformed' : total_loaded,
'rows_loaded' : total_loaded, 'rows_loaded' : total_loaded,
'execution_timestamp': self.pipeline_metadata['start_time'], 'execution_timestamp': self.pipeline_metadata['start_time'],
'completeness_pct' : 100.0, 'completeness_pct' : 100.0,
'config_snapshot' : json.dumps({'load_mode': 'full_refresh', 'layer': 'gold'}), 'config_snapshot' : json.dumps({
'validation_metrics': json.dumps({t: m['status'] for t, m in self.load_metadata.items()}), 'load_mode' : 'full_refresh',
'table_name' : 'dimensional_model_pipeline', 'layer' : 'gold',
'pillar_names': OFFICIAL_PILLARS,
}),
'validation_metrics' : json.dumps({t: m['status'] for t, m in self.load_metadata.items()}),
'table_name' : 'dimensional_model_pipeline',
}) })
try: try:
save_etl_metadata(self.client, self.pipeline_metadata) save_etl_metadata(self.client, self.pipeline_metadata)
@@ -785,20 +926,19 @@ class DimensionalModelLoader:
self.logger.info(f" Duration : {duration:.2f}s") self.logger.info(f" Duration : {duration:.2f}s")
self.logger.info(f" Tables :") self.logger.info(f" Tables :")
for tbl, meta in self.load_metadata.items(): for tbl, meta in self.load_metadata.items():
icon = "✓" if meta['status'] == 'success' else "✗" icon = "[OK]" if meta['status'] == 'success' else "[FAIL]"
self.logger.info(f" {icon} {tbl:25s}: {meta['rows_loaded']:>10,} rows") self.logger.info(f" {icon} {tbl:25s}: {meta['rows_loaded']:>10,} rows")
self.logger.info(f"\n Metadata → [AUDIT] etl_metadata") self.logger.info(f"\n Metadata -> [AUDIT] etl_metadata")
self.logger.info("=" * 60) self.logger.info("=" * 60)
# ============================================================================= # =============================================================================
# AIRFLOW TASK FUNCTIONS ← sama polanya dengan raw & cleaned layer # AIRFLOW TASK FUNCTIONS
# ============================================================================= # =============================================================================
def run_dimensional_model(): def run_dimensional_model():
""" """
Airflow task: Load dimensional model dari cleaned_integrated. Airflow task: Load dimensional model dari cleaned_integrated.
Dipanggil oleh DAG setelah task cleaned_integration_to_silver selesai. Dipanggil oleh DAG setelah task cleaned_integration_to_silver selesai.
""" """
from scripts.bigquery_config import get_bigquery_client from scripts.bigquery_config import get_bigquery_client
@@ -817,9 +957,10 @@ if __name__ == "__main__":
print("=" * 60) print("=" * 60)
print("BIGQUERY DIMENSIONAL MODEL LOAD") print("BIGQUERY DIMENSIONAL MODEL LOAD")
print("Kimball DW Architecture") print("Kimball DW Architecture")
print(" Input : STAGING (Silver) → cleaned_integrated (fs_asean_silver)") print(" Input : STAGING (Silver) -> cleaned_integrated (fs_asean_silver)")
print(" Output : DW (Gold) → dim_*, fact_* (fs_asean_gold)") print(" Output : DW (Gold) -> dim_*, fact_* (fs_asean_gold)")
print(" Audit : AUDIT → etl_logs, etl_metadata (fs_asean_audit)") print(" Audit : AUDIT -> etl_logs, etl_metadata (fs_asean_audit)")
print(f" Pillars: {', '.join(OFFICIAL_PILLARS)}")
print("=" * 60) print("=" * 60)
logger = setup_logging() logger = setup_logging()
@@ -827,24 +968,26 @@ if __name__ == "__main__":
print("\nLoading cleaned_integrated (fs_asean_silver)...") print("\nLoading cleaned_integrated (fs_asean_silver)...")
df_clean = read_from_bigquery(client, 'cleaned_integrated', layer='silver') df_clean = read_from_bigquery(client, 'cleaned_integrated', layer='silver')
print(f" ✓ Loaded : {len(df_clean):,} rows") print(f" [OK] Loaded : {len(df_clean):,} rows")
print(f" Columns : {len(df_clean.columns)}") print(f" Columns : {len(df_clean.columns)}")
print(f" Sources : {df_clean['source'].nunique()}") print(f" Sources : {df_clean['source'].nunique()}")
print(f" Indicators : {df_clean['indicator_standardized'].nunique()}") print(f" Indicators : {df_clean['indicator_standardized'].nunique()}")
print(f" Countries : {df_clean['country'].nunique()}") print(f" Countries : {df_clean['country'].nunique()}")
print(f" Year range : {int(df_clean['year'].min())}–{int(df_clean['year'].max())}") print(f" Year range : {int(df_clean['year'].min())}-{int(df_clean['year'].max())}")
if 'pillar' in df_clean.columns:
print(f" Pillars raw : {sorted(df_clean['pillar'].unique())}")
if 'direction' in df_clean.columns: if 'direction' in df_clean.columns:
print(f" Direction : {df_clean['direction'].value_counts().to_dict()}") print(f" Direction : {df_clean['direction'].value_counts().to_dict()}")
else: else:
print(f" [WARN] direction column not found — run bigquery_cleaned_layer.py first") print(f" [WARN] direction column not found — run bigquery_cleaned_layer.py first")
print("\n[1/1] Dimensional Model Load → DW (Gold)...") print("\n[1/1] Dimensional Model Load -> DW (Gold)...")
loader = DimensionalModelLoader(client, df_clean) loader = DimensionalModelLoader(client, df_clean)
loader.run() loader.run()
print("\n" + "=" * 60) print("\n" + "=" * 60)
print("✓ DIMENSIONAL MODEL ETL COMPLETED") print("[OK] DIMENSIONAL MODEL ETL COMPLETED")
print(" 🥇 DW (Gold) : dim_country, dim_indicator, dim_time,") print(" DW (Gold) : dim_country, dim_indicator, dim_time,")
print(" dim_source, dim_pillar, fact_food_security") print(" dim_source, dim_pillar, fact_food_security")
print(" 📋 AUDIT : etl_logs, etl_metadata") print(" AUDIT : etl_logs, etl_metadata")
print("=" * 60) print("=" * 60)