izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
cfb0df3a151400064a6ff5260f36c6188e5492ef
airflow-coolify/scripts/bigquery_aggregate_layer.py
Debby cfb0df3a15 indonesian version column
2026-05-19 10:09:48 +07:00

1824 lines
87 KiB
Python
Raw Blame History

"""
BIGQUERY ANALYSIS LAYER - FOOD SECURITY AGGREGATION
Semua agregasi pakai norm_value dari _get_norm_value_df()
UPDATED: Simpan 6 tabel ke fs_asean_gold (layer='gold'):
- agg_pillar_composite
- agg_pillar_by_country
- agg_framework_by_country
- agg_framework_asean (+ kolom performance_status: 'Good'/'Bad', threshold=60)
- agg_narrative_overview (bilingual: narrative_en, narrative_id)
- agg_narrative_pillar (bilingual: narrative_en, narrative_id)
Narrative style:
- Plain text, tanpa markdown bold (**)
- Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata
- Bilingual: narrative_en (Inggris) + narrative_id (Indonesia)
- Granularity: per tahun (Overview & Pillar)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
- agg_pillar_composite : + pillar_name_id
- agg_pillar_by_country : + pillar_name_id
- agg_framework_by_country : (framework tidak diterjemahkan, sudah singkat)
- agg_narrative_pillar : + pillar_name_id
"""
import pandas as pd
import numpy as np
from datetime import datetime
import logging
import json
import sys as _sys
from scripts.bigquery_config import get_bigquery_client
from scripts.bigquery_helpers import (
log_update,
load_to_bigquery,
read_from_bigquery,
setup_logging,
save_etl_metadata,
)
from google.cloud import bigquery
# =============================================================================
# KONSTANTA GLOBAL
# =============================================================================
DIRECTION_INVERT_KEYWORDS = frozenset({
"negative", "lower_better", "lower_is_better", "inverse", "neg",
})
DIRECTION_POSITIVE_KEYWORDS = frozenset({
"positive", "higher_better", "higher_is_better",
})
NORMALIZE_FRAMEWORKS_JOINTLY = False
PERFORMANCE_THRESHOLD = 60.0
SDG_ONLY_KEYWORDS: frozenset = frozenset([
"prevalence of undernourishment (percent) (3-year average)",
"number of people undernourished (million) (3-year average)",
"prevalence of severe food insecurity in the total population (percent) (3-year average)",
"prevalence of severe food insecurity in the male adult population (percent) (3-year average)",
"prevalence of severe food insecurity in the female adult population (percent) (3-year average)",
"prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)",
"prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)",
"prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)",
"number of severely food insecure people (million) (3-year average)",
"number of severely food insecure male adults (million) (3-year average)",
"number of severely food insecure female adults (million) (3-year average)",
"number of moderately or severely food insecure people (million) (3-year average)",
"number of moderately or severely food insecure male adults (million) (3-year average)",
"number of moderately or severely food insecure female adults (million) (3-year average)",
"percentage of children under 5 years of age who are stunted (modelled estimates) (percent)",
"number of children under 5 years of age who are stunted (modeled estimates) (million)",
"percentage of children under 5 years affected by wasting (percent)",
"number of children under 5 years affected by wasting (million)",
"percentage of children under 5 years of age who are overweight (modelled estimates) (percent)",
"number of children under 5 years of age who are overweight (modeled estimates) (million)",
"prevalence of anemia among women of reproductive age (15-49 years) (percent)",
"number of women of reproductive age (15-49 years) affected by anemia (million)",
])
_SDG_ONLY_LOWER: frozenset = frozenset(k.lower() for k in SDG_ONLY_KEYWORDS)
_FIES_DETECTION_LOWER: frozenset = frozenset([
"prevalence of severe food insecurity in the total population (percent) (3-year average)",
"prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)",
"number of severely food insecure people (million) (3-year average)",
"number of moderately or severely food insecure people (million) (3-year average)",
])
# =============================================================================
# TRANSLATION DICTIONARIES
# =============================================================================
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
}
INDICATOR_TRANSLATION_ID: dict = {
# -------------------------------------------------------------------------
# AVAILABILITY
# -------------------------------------------------------------------------
"Average dietary energy supply adequacy (percent) (3-year average)":
"Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)",
"Average value of food production (constant 2014-2016 thousand US$) (3-year average)":
"Nilai rata-rata produksi pangan (ribu US$ konstan 2014-2016) (rata-rata 3 tahun)",
"Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
"Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)",
"Average protein supply (g/cap/day) (3-year average)":
"Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)",
"Average supply of protein of animal origin (g/cap/day) (3-year average)":
"Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)",
"Cereal import dependency ratio (percent) (3-year average)":
"Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
"Percent of arable land equipped for irrigation (percent) (3-year average)":
"Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)",
"Crop production index (2014-2016 = 100)":
"Indeks produksi tanaman pangan (2014-2016 = 100)",
"Livestock production index (2014-2016 = 100)":
"Indeks produksi peternakan (2014-2016 = 100)",
"Value of food imports over total merchandise exports (percent) (3-year average)":
"Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)",
"Food production variability (constant 2014-2016 thousand US$ per capita)":
"Variabilitas produksi pangan (ribu US$ konstan 2014-2016 per kapita)",
"Food supply variability (kcal/cap/day)":
"Variabilitas pasokan pangan (kkal/kapita/hari)",
# -------------------------------------------------------------------------
# ACCESS
# -------------------------------------------------------------------------
"Gross domestic product per capita, PPP (constant 2017 international $)":
"Produk domestik bruto per kapita, PPP (internasional konstan 2017 US$)",
"Domestic food price level index (2015 = 1.00)":
"Indeks tingkat harga pangan domestik (2015 = 1,00)",
"Domestic food price volatility index":
"Indeks volatilitas harga pangan domestik",
"Prevalence of undernourishment (percent) (3-year average)":
"Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)",
"Number of people undernourished (million) (3-year average)":
"Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)",
"Depth of the food deficit (kcal/capita/day) (3-year average)":
"Kedalaman defisit pangan (kkal/kapita/hari) (rata-rata 3 tahun)",
"Percentage of population using at least basic drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum dasar (persen)",
"Percentage of population using safely managed drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)",
"Percentage of population using at least basic sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)",
"Percentage of population using safely managed sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)",
"Access to electricity (percent of rural population)":
"Akses listrik (persen penduduk pedesaan)",
"Proportion of population with access to electricity (percent)":
"Proporsi penduduk dengan akses listrik (persen)",
"Road infrastructure index":
"Indeks infrastruktur jalan",
"Rail lines density (total route-km per 100 square km of land area)":
"Kepadatan jalur kereta api (total rute-km per 100 km2 lahan)",
"Gross national income per capita (Atlas method, current US$)":
"Pendapatan nasional bruto per kapita (metode Atlas, US$ terkini)",
"Food Insecurity Experience Scale (FIES)":
"Skala Pengalaman Ketidakamanan Pangan (FIES)",
# -------------------------------------------------------------------------
# UTILIZATION
# -------------------------------------------------------------------------
"Prevalence of severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Number of severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)",
"Number of children under 5 years of age who are stunted (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)",
"Percentage of children under 5 years affected by wasting (percent)":
"Persentase anak di bawah 5 tahun yang mengalami wasting (persen)",
"Number of children under 5 years affected by wasting (million)":
"Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)",
"Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)",
"Number of children under 5 years of age who are overweight (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)",
"Prevalence of anemia among women of reproductive age (15-49 years) (percent)":
"Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)",
"Number of women of reproductive age (15-49 years) affected by anemia (million)":
"Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)",
"Prevalence of obesity in the adult population (18 years and older) (percent)":
"Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)",
"Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)":
"Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)",
"Minimum dietary diversity for women (MDD-W) (percent)":
"Keragaman pola makan minimum untuk perempuan (MDD-W) (persen)",
# -------------------------------------------------------------------------
# STABILITY
# -------------------------------------------------------------------------
"Cereal import dependency ratio (percent)":
"Rasio ketergantungan impor sereal (persen)",
"Political stability and absence of violence/terrorism (index)":
"Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)",
"Domestic food price volatility":
"Volatilitas harga pangan domestik",
"Per capita food supply variability (kcal/cap/day)":
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
"Percentage of arable land equipped for irrigation (percent)":
"Persentase lahan pertanian yang dilengkapi irigasi (persen)",
"GDP per capita growth (annual %)":
"Pertumbuhan PDB per kapita (% tahunan)",
"GDP growth (annual %)":
"Pertumbuhan PDB (% tahunan)",
}
def translate_indicator(name: str) -> str:
"""Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return INDICATOR_TRANSLATION_ID.get(name, name)
def translate_pillar(name: str) -> str:
"""Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return PILLAR_TRANSLATION_ID.get(name, name)
# =============================================================================
# WINDOWS CP1252 SAFE LOGGING
# =============================================================================
class _SafeStreamHandler(logging.StreamHandler):
def emit(self, record):
try:
super().emit(record)
except UnicodeEncodeError:
try:
msg = self.format(record)
self.stream.write(
msg.encode("utf-8", errors="replace").decode("ascii", errors="replace")
+ self.terminator
)
self.flush()
except Exception:
self.handleError(record)
# =============================================================================
# HELPERS
# =============================================================================
def _should_invert(direction: str, logger=None, context: str = "") -> bool:
d = str(direction).lower().strip()
if d in DIRECTION_INVERT_KEYWORDS:
return True
if d in DIRECTION_POSITIVE_KEYWORDS:
return False
if logger:
logger.warning(
f" [DIRECTION WARNING] Unknown direction '{direction}' "
f"{'(' + context + ')' if context else ''}. Defaulting to positive (no invert)."
)
return False
def global_minmax(series: pd.Series, lo: float = 1.0, hi: float = 100.0) -> pd.Series:
values = series.dropna().values
if len(values) == 0:
return pd.Series(np.nan, index=series.index)
v_min, v_max = values.min(), values.max()
if v_min == v_max:
return pd.Series((lo + hi) / 2.0, index=series.index)
result = np.full(len(series), np.nan)
not_nan = series.notna()
raw = series[not_nan].values
result[not_nan.values] = lo + (raw - v_min) / (v_max - v_min) * (hi - lo)
return pd.Series(result, index=series.index)
def add_yoy(df: pd.DataFrame, group_cols: list, score_col: str) -> pd.DataFrame:
df = df.sort_values(group_cols + ["year"]).reset_index(drop=True)
if group_cols:
df["year_over_year_change"] = df.groupby(group_cols)[score_col].diff()
else:
df["year_over_year_change"] = df[score_col].diff()
return df
def safe_int(series: pd.Series, fill: int = 0, col_name: str = "", logger=None) -> pd.Series:
n_nan = series.isna().sum()
if n_nan > 0 and logger:
logger.warning(
f" [NaN WARNING] Kolom '{col_name}' punya {n_nan} NaN -> di-fill dengan {fill}"
)
return series.fillna(fill).astype(int)
def check_and_dedup(df: pd.DataFrame, key_cols: list, context: str = "", logger=None) -> pd.DataFrame:
dupes = df.duplicated(subset=key_cols, keep=False)
if dupes.any():
n_dupes = dupes.sum()
if logger:
logger.warning(
f" [DEDUP WARNING] {context}: {n_dupes} duplikat rows pada {key_cols}. "
f"Di-aggregate dengan mean."
)
numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
agg_dict = {
c: ("mean" if c in numeric_cols else "first")
for c in df.columns if c not in key_cols
}
df = df.groupby(key_cols, as_index=False).agg(agg_dict)
return df
def _performance_status(score) -> str:
if score is None or (isinstance(score, float) and np.isnan(score)):
return "N/A"
return "Good" if score >= PERFORMANCE_THRESHOLD else "Bad"
def _fmt_score(score) -> str:
if score is None or (isinstance(score, float) and np.isnan(score)):
return "N/A"
return f"{score:.2f}"
def _fmt_delta(delta) -> str:
if delta is None or (isinstance(delta, float) and np.isnan(delta)):
return "N/A"
sign = "+" if delta >= 0 else ""
return f"{sign}{delta:.2f}"
# =============================================================================
# NARRATIVE CONDITION DETECTORS (shared)
# =============================================================================
def _detect_series_trend(scores: list) -> str:
if len(scores) < 3:
return "insufficient"
x = np.arange(len(scores))
slope = np.polyfit(x, scores, 1)[0]
cv = np.std(scores) / (np.mean(scores) + 1e-9)
if cv > 0.20:
return "fluctuating"
mid = len(scores) // 2
slope1 = np.polyfit(np.arange(mid), scores[:mid], 1)[0] if mid > 1 else slope
slope2 = np.polyfit(np.arange(len(scores) - mid), scores[mid:], 1)[0] if (len(scores) - mid) > 1 else slope
if slope > 0:
slowing = slope2 < slope1
return "improving_slowing" if slowing else "improving_consistent"
else:
return "deteriorating"
def _detect_country_gap(scores_by_country_year: pd.DataFrame, score_col: str) -> str:
std_by_year = (
scores_by_country_year.groupby("year")[score_col]
.std().dropna()
)
if len(std_by_year) < 3:
return "unknown"
years = sorted(std_by_year.index)
stds = [std_by_year[y] for y in years]
slope = np.polyfit(np.arange(len(stds)), stds, 1)[0]
mean_s = np.mean(stds)
if abs(slope) < 0.02 * mean_s:
return "stable"
return "widening" if slope > 0 else "narrowing"
def _find_anomaly_year(values_by_year: dict) -> tuple:
years = sorted(values_by_year.keys())
deltas = {}
for i in range(1, len(years)):
y0, y1 = years[i-1], years[i]
v0, v1 = values_by_year.get(y0), values_by_year.get(y1)
if v0 is not None and v1 is not None and not (pd.isna(v0) or pd.isna(v1)):
deltas[y1] = v1 - v0
if not deltas:
return None, None
threshold = 1.5 * np.std(list(deltas.values()))
min_y = min(deltas, key=deltas.get)
max_y = max(deltas, key=deltas.get)
if abs(deltas[min_y]) > threshold and deltas[min_y] < 0:
return min_y, "drop"
if abs(deltas[max_y]) > threshold and deltas[max_y] > 0:
return max_y, "rise"
return None, None
# =============================================================================
# NARRATIVE BUILDER — OVERVIEW (per tahun)
# =============================================================================
def _build_overview_narrative(
year: int,
score: float,
performance_status: str,
yoy_val,
n_mdg: int,
n_sdg: int,
ranking_list: list,
most_improved_country,
most_improved_delta,
most_declined_country,
most_declined_delta,
historical_scores: dict,
country_scores_all: pd.DataFrame,
) -> tuple:
sentences_en = []
sentences_id = []
perf_word_en = "good" if performance_status == "Good" else "below target"
perf_word_id = "baik" if performance_status == "Good" else "di bawah target"
s1_en = (
f"In {year}, ASEAN food security scored {_fmt_score(score)} out of 100 "
f"({perf_word_en}), covering {n_mdg + n_sdg} indicators "
f"({n_mdg} MDGs and {n_sdg} SDGs)."
)
s1_id = (
f"Pada tahun {year}, skor ketahanan pangan ASEAN mencapai {_fmt_score(score)} dari 100 "
f"({perf_word_id}), mencakup {n_mdg + n_sdg} indikator "
f"({n_mdg} MDGs dan {n_sdg} SDGs)."
)
sentences_en.append(s1_en)
sentences_id.append(s1_id)
if yoy_val is not None and not pd.isna(yoy_val):
if abs(yoy_val) < 0.5:
s2_en = f"The score was relatively stable compared to the previous year."
s2_id = f"Skor relatif stabil dibandingkan tahun sebelumnya."
elif yoy_val > 0:
s2_en = f"This represents an improvement of {abs(yoy_val):.2f} points from the previous year."
s2_id = f"Ini merupakan peningkatan sebesar {abs(yoy_val):.2f} poin dari tahun sebelumnya."
else:
s2_en = f"This represents a decline of {abs(yoy_val):.2f} points from the previous year."
s2_id = f"Ini merupakan penurunan sebesar {abs(yoy_val):.2f} poin dari tahun sebelumnya."
sentences_en.append(s2_en)
sentences_id.append(s2_id)
hist_years = sorted(historical_scores.keys())
hist_scores = [historical_scores[y] for y in hist_years if not pd.isna(historical_scores.get(y, np.nan))]
if len(hist_scores) >= 3:
trend = _detect_series_trend(hist_scores)
if trend == "improving_consistent":
s3_en = f"The overall trajectory since {hist_years[0]} has been consistently upward."
s3_id = f"Trajektori keseluruhan sejak {hist_years[0]} menunjukkan tren yang konsisten meningkat."
elif trend == "improving_slowing":
s3_en = f"While the long-term trend since {hist_years[0]} is positive, the pace of improvement has slowed in recent years."
s3_id = f"Meskipun tren jangka panjang sejak {hist_years[0]} positif, laju perbaikan melambat dalam beberapa tahun terakhir."
elif trend == "deteriorating":
s3_en = f"The overall trend since {hist_years[0]} shows a declining trajectory that warrants attention."
s3_id = f"Tren keseluruhan sejak {hist_years[0]} menunjukkan trajektori yang menurun dan perlu perhatian."
elif trend == "fluctuating":
s3_en = f"Progress since {hist_years[0]} has been uneven, with scores fluctuating across years."
s3_id = f"Kemajuan sejak {hist_years[0]} tidak merata, dengan skor yang berfluktuasi antar tahun."
else:
s3_en = ""
s3_id = ""
if s3_en:
sentences_en.append(s3_en)
sentences_id.append(s3_id)
if not country_scores_all.empty:
gap_trend = _detect_country_gap(
country_scores_all[country_scores_all["year"] <= year],
"framework_score_1_100"
)
if gap_trend == "widening":
s4_en = "The performance gap among ASEAN member states has widened over time, indicating unequal progress."
s4_id = "Kesenjangan performa antar negara anggota ASEAN semakin melebar, mengindikasikan kemajuan yang tidak merata."
elif gap_trend == "narrowing":
s4_en = "The performance gap among ASEAN member states has narrowed, reflecting more balanced regional development."
s4_id = "Kesenjangan performa antar negara anggota ASEAN semakin menyempit, mencerminkan pembangunan regional yang lebih merata."
elif gap_trend == "stable":
s4_en = "The performance gap among ASEAN member states has remained relatively stable."
s4_id = "Kesenjangan performa antar negara anggota ASEAN relatif stabil."
else:
s4_en = ""
s4_id = ""
if s4_en:
sentences_en.append(s4_en)
sentences_id.append(s4_id)
if ranking_list and len(ranking_list) >= 2:
top = ranking_list[0]
bottom = ranking_list[-1]
s5_en = (
f"In {year}, {top['country_name']} led the region with a score of "
f"{_fmt_score(top['score'])}, while {bottom['country_name']} ranked last "
f"at {_fmt_score(bottom['score'])}."
)
s5_id = (
f"Pada tahun {year}, {top['country_name']} memimpin kawasan dengan skor "
f"{_fmt_score(top['score'])}, sementara {bottom['country_name']} berada di "
f"posisi terbawah dengan skor {_fmt_score(bottom['score'])}."
)
sentences_en.append(s5_en)
sentences_id.append(s5_id)
if most_improved_country and most_declined_country:
if most_improved_country != most_declined_country:
s6_en = (
f"{most_improved_country} showed the biggest improvement "
f"({_fmt_delta(most_improved_delta)} pts), "
f"while {most_declined_country} experienced the largest decline "
f"({_fmt_delta(most_declined_delta)} pts)."
)
s6_id = (
f"{most_improved_country} mencatat peningkatan terbesar "
f"({_fmt_delta(most_improved_delta)} poin), "
f"sementara {most_declined_country} mengalami penurunan terbesar "
f"({_fmt_delta(most_declined_delta)} poin)."
)
sentences_en.append(s6_en)
sentences_id.append(s6_id)
narrative_en = " ".join(s for s in sentences_en if s)
narrative_id = " ".join(s for s in sentences_id if s)
return narrative_en, narrative_id
# =============================================================================
# NARRATIVE BUILDER — PILLAR (per tahun per pilar)
# =============================================================================
def _build_pillar_narrative(
year: int,
pillar_name: str,
pillar_score: float,
rank_in_year: int,
n_pillars: int,
yoy_val,
top_country: str,
top_country_score,
bot_country: str,
bot_country_score,
pillar_scores_history: dict,
all_pillar_scores_year: pd.DataFrame,
country_pillar_all: pd.DataFrame,
) -> tuple:
sentences_en = []
sentences_id = []
rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th")
perf_word_en = "good" if pillar_score >= PERFORMANCE_THRESHOLD else "below target"
perf_word_id = "baik" if pillar_score >= PERFORMANCE_THRESHOLD else "di bawah target"
s1_en = (
f"In {year}, the {pillar_name} pillar ranked {rank_in_year}{rank_suffix} out of "
f"{n_pillars} pillars with a score of {_fmt_score(pillar_score)} ({perf_word_en})."
)
s1_id = (
f"Pada tahun {year}, pilar {pillar_name} menempati peringkat {rank_in_year} dari "
f"{n_pillars} pilar dengan skor {_fmt_score(pillar_score)} ({perf_word_id})."
)
sentences_en.append(s1_en)
sentences_id.append(s1_id)
if yoy_val is not None and not pd.isna(yoy_val):
if abs(yoy_val) < 0.5:
s2_en = "Performance was relatively stable compared to the previous year."
s2_id = "Performa relatif stabil dibandingkan tahun sebelumnya."
elif yoy_val > 0:
s2_en = f"This is an improvement of {abs(yoy_val):.2f} points from the previous year."
s2_id = f"Ini merupakan peningkatan {abs(yoy_val):.2f} poin dari tahun sebelumnya."
else:
s2_en = f"This marks a decline of {abs(yoy_val):.2f} points from the previous year."
s2_id = f"Ini menandai penurunan {abs(yoy_val):.2f} poin dari tahun sebelumnya."
sentences_en.append(s2_en)
sentences_id.append(s2_id)
hist_years = sorted(pillar_scores_history.keys())
hist_scores = [
pillar_scores_history[y]
for y in hist_years
if not pd.isna(pillar_scores_history.get(y, np.nan))
]
if len(hist_scores) >= 3:
trend = _detect_series_trend(hist_scores)
if trend == "improving_consistent":
s3_en = f"This pillar has shown consistent improvement since {hist_years[0]}."
s3_id = f"Pilar ini menunjukkan perbaikan yang konsisten sejak {hist_years[0]}."
elif trend == "improving_slowing":
s3_en = f"While the pillar improved since {hist_years[0]}, the pace has slowed in recent years."
s3_id = f"Meskipun pilar ini membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir."
elif trend == "deteriorating":
s3_en = f"This pillar has shown a declining trend since {hist_years[0]}, requiring targeted intervention."
s3_id = f"Pilar ini menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah."
elif trend == "fluctuating":
s3_en = f"Performance in this pillar has been inconsistent since {hist_years[0]}, with no clear trend."
s3_id = f"Performa pilar ini tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas."
else:
s3_en = ""
s3_id = ""
if s3_en:
sentences_en.append(s3_en)
sentences_id.append(s3_id)
if not country_pillar_all.empty:
gap_trend = _detect_country_gap(
country_pillar_all[country_pillar_all["year"] <= year],
"pillar_country_score_1_100"
)
if gap_trend == "widening":
s4_en = "Country disparities within this pillar have widened over time."
s4_id = "Kesenjangan antar negara dalam pilar ini semakin melebar seiring waktu."
elif gap_trend == "narrowing":
s4_en = "Country disparities within this pillar have narrowed, indicating more balanced progress."
s4_id = "Kesenjangan antar negara dalam pilar ini menyempit, mengindikasikan kemajuan yang lebih merata."
else:
s4_en = ""
s4_id = ""
if s4_en:
sentences_en.append(s4_en)
sentences_id.append(s4_id)
if top_country and bot_country and top_country != bot_country:
s5_en = (
f"{top_country} performed best in this pillar ({_fmt_score(top_country_score)}), "
f"while {bot_country} had the lowest score ({_fmt_score(bot_country_score)})."
)
s5_id = (
f"{top_country} memiliki performa terbaik dalam pilar ini ({_fmt_score(top_country_score)}), "
f"sementara {bot_country} memiliki skor terendah ({_fmt_score(bot_country_score)})."
)
sentences_en.append(s5_en)
sentences_id.append(s5_id)
if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1:
sorted_pillars = all_pillar_scores_year.sort_values("pillar_score_1_100", ascending=False)
strongest = sorted_pillars.iloc[0]
weakest = sorted_pillars.iloc[-1]
if strongest["pillar_name"] != pillar_name and weakest["pillar_name"] != pillar_name:
s6_en = (
f"Across all pillars in {year}, {strongest['pillar_name']} scored highest "
f"({_fmt_score(strongest['pillar_score_1_100'])}) and {weakest['pillar_name']} "
f"scored lowest ({_fmt_score(weakest['pillar_score_1_100'])})."
)
s6_id = (
f"Di antara semua pilar pada tahun {year}, {strongest['pillar_name']} mendapat skor "
f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest['pillar_name']} "
f"mendapat skor terendah ({_fmt_score(weakest['pillar_score_1_100'])})."
)
sentences_en.append(s6_en)
sentences_id.append(s6_id)
narrative_en = " ".join(s for s in sentences_en if s)
narrative_id = " ".join(s for s in sentences_id if s)
return narrative_en, narrative_id
# =============================================================================
# MAIN CLASS
# =============================================================================
class FoodSecurityAggregator:
def __init__(self, client: bigquery.Client):
self.client = client
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.propagate = False
self.load_metadata = {
"agg_pillar_composite": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
"agg_pillar_by_country": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
"agg_framework_by_country": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
"agg_framework_asean": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
"agg_narrative_overview": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
"agg_narrative_pillar": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
}
self.df = None
self.sdgs_start_year = None
self._ind_year_framework: pd.DataFrame = None
# =========================================================================
# STEP 1: Load data
# =========================================================================
def load_data(self):
self.logger.info("=" * 70)
self.logger.info("STEP 1: LOAD DATA from fs_asean_gold")
self.logger.info("=" * 70)
self.df = read_from_bigquery(
self.client, "fact_asean_food_security_selected", layer='gold'
)
self.logger.info(f" fact_asean_food_security_selected : {len(self.df):,} rows")
required_cols = {
"country_id", "country_name",
"indicator_id", "indicator_name", "direction",
"pillar_id", "pillar_name",
"time_id", "year", "value",
}
missing_cols = required_cols - set(self.df.columns)
if missing_cols:
raise ValueError(f"Kolom berikut tidak ditemukan: {missing_cols}")
n_null_dir = self.df["direction"].isna().sum()
if n_null_dir > 0:
self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'")
self.df["direction"] = self.df["direction"].fillna("positive")
# Pastikan kolom terjemahan Indonesia tersedia (bisa dari fact atau dibuat ulang)
if "indicator_name_id" not in self.df.columns:
self.df["indicator_name_id"] = self.df["indicator_name"].apply(translate_indicator)
self.logger.info(" [TRANSLATION] Kolom indicator_name_id dibuat dari mapping.")
if "pillar_name_id" not in self.df.columns:
self.df["pillar_name_id"] = self.df["pillar_name"].apply(translate_pillar)
self.logger.info(" [TRANSLATION] Kolom pillar_name_id dibuat dari mapping.")
self.logger.info(f" Rows : {len(self.df):,}")
self.logger.info(f" Countries : {self.df['country_id'].nunique()}")
self.logger.info(f" Indicators: {self.df['indicator_id'].nunique()}")
self.logger.info(
f" Years : {int(self.df['year'].min())} - {int(self.df['year'].max())}"
)
# =========================================================================
# STEP 1b: Detect sdgs_start_year + assign framework
# =========================================================================
def _detect_sdgs_start_year(self) -> int:
fies_rows = self.df[
self.df["indicator_name"].str.lower().str.strip().isin(_FIES_DETECTION_LOWER)
]
if not fies_rows.empty:
sdgs_start = int(fies_rows["year"].min())
self.logger.info(f" [FIES explicit] sdgs_start_year = {sdgs_start}")
return sdgs_start
ind_min_year = (
self.df.groupby("indicator_id")["year"]
.min().reset_index().rename(columns={"year": "min_year"})
)
unique_years = sorted(ind_min_year["min_year"].unique())
if len(unique_years) == 1:
return int(unique_years[0]) + 9999
gaps = [
(unique_years[i+1] - unique_years[i], unique_years[i], unique_years[i+1])
for i in range(len(unique_years) - 1)
]
gaps.sort(reverse=True)
_, y_before, y_after = gaps[0]
self.logger.info(f" [Fallback gap] sdgs_start_year = {y_after}")
return int(y_after)
def _assign_framework_labels(self):
self.logger.info("\n" + "=" * 70)
self.logger.info("STEP 1b: ASSIGN FRAMEWORK LABELS")
self.logger.info(f" sdgs_start_year = {self.sdgs_start_year}")
self.logger.info("=" * 70)
df = self.df.copy()
df["_is_sdg_kw"] = df["indicator_name"].str.lower().str.strip().isin(_SDG_ONLY_LOWER)
df["framework"] = "MDGs"
mask_sdgs = df["_is_sdg_kw"] & (df["year"] >= self.sdgs_start_year)
df.loc[mask_sdgs, "framework"] = "SDGs"
df = df.drop(columns=["_is_sdg_kw"])
self.df = df
self._ind_year_framework = (
self.df[["indicator_id", "year", "framework"]]
.drop_duplicates()
.reset_index(drop=True)
)
fw_dist = self.df["framework"].value_counts()
self.logger.info("\n Framework distribution (rows):")
for fw, cnt in fw_dist.items():
self.logger.info(f" {fw:<6}: {cnt:,} rows")
def _count_framework_indicators(self, year: int, framework: str) -> int:
mask = (
(self._ind_year_framework["year"] == year) &
(self._ind_year_framework["framework"] == framework)
)
return int(self._ind_year_framework.loc[mask, "indicator_id"].nunique())
# =========================================================================
# CORE HELPER: normalisasi raw value per indikator
# =========================================================================
def _get_norm_value_df(self) -> pd.DataFrame:
if "framework" not in self.df.columns:
raise ValueError("Kolom 'framework' tidak ada.")
norm_parts = []
for ind_id, grp in self.df.groupby("indicator_id"):
grp = grp.copy()
direction = str(grp["direction"].iloc[0])
do_invert = _should_invert(direction, self.logger, context=f"indicator_id={ind_id}")
valid_mask = grp["value"].notna()
n_valid = valid_mask.sum()
if n_valid < 2:
grp["norm_value"] = np.nan
norm_parts.append(grp)
continue
raw = grp.loc[valid_mask, "value"].values
v_min, v_max = raw.min(), raw.max()
normed = np.full(len(grp), np.nan)
if v_min == v_max:
normed[valid_mask.values] = 0.5
else:
normed[valid_mask.values] = (raw - v_min) / (v_max - v_min)
if do_invert:
normed = np.where(np.isnan(normed), np.nan, 1.0 - normed)
grp["norm_value"] = normed
norm_parts.append(grp)
return pd.concat(norm_parts, ignore_index=True)
# =========================================================================
# METADATA BUILDER
# =========================================================================
def _build_etl_metadata(
self,
table_name: str,
rows_loaded: int,
start_time: datetime,
end_time: datetime,
status: str,
error_msg: str = None,
) -> dict:
duration = (end_time - start_time).total_seconds() if (start_time and end_time) else 0.0
return {
"source_class" : "FoodSecurityAggregator",
"table_name" : table_name,
"execution_timestamp": start_time or end_time,
"duration_seconds" : round(duration, 4),
"rows_fetched" : rows_loaded,
"rows_transformed" : rows_loaded,
"rows_loaded" : rows_loaded,
"completeness_pct" : 100.0 if status == "success" else 0.0,
"config_snapshot" : json.dumps({
"layer" : "gold",
"write_disposition" : "WRITE_TRUNCATE",
"normalize_frameworks_jointly": NORMALIZE_FRAMEWORKS_JOINTLY,
"performance_threshold" : PERFORMANCE_THRESHOLD,
"status" : status,
}),
"validation_metrics" : json.dumps({
"status" : status,
"error_msg": error_msg or "",
}),
}
# =========================================================================
# STEP 2: agg_pillar_composite
# Kolom tambahan: pillar_name_id
# =========================================================================
def calc_pillar_composite(self) -> pd.DataFrame:
table_name = "agg_pillar_composite"
self.load_metadata[table_name]["start_time"] = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 2: {table_name} -> [Gold] fs_asean_gold")
self.logger.info("=" * 70)
try:
df_normed = self._get_norm_value_df()
df = (
df_normed
.groupby(["pillar_id", "pillar_name", "year"])
.agg(
pillar_norm =("norm_value", "mean"),
n_indicators=("indicator_id", "nunique"),
n_countries =("country_id", "nunique"),
)
.reset_index()
)
df["pillar_score_1_100"] = global_minmax(df["pillar_norm"])
df["rank_in_year"] = (
df.groupby("year")["pillar_score_1_100"]
.rank(method="min", ascending=False)
.astype(int)
)
df = add_yoy(df, ["pillar_id"], "pillar_score_1_100")
# Kolom terjemahan Indonesia
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["pillar_id"] = df["pillar_id"].astype(int)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
df["n_countries"] = safe_int(df["n_countries"], col_name="n_countries", logger=self.logger)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_norm"] = df["pillar_norm"].astype(float)
df["pillar_score_1_100"] = df["pillar_score_1_100"].astype(float)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
schema = [
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_norm", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("n_countries", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_score_1_100", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
write_disposition="WRITE_TRUNCATE", schema=schema
)
self._finalize(table_name, rows)
return df
except Exception as e:
self._fail(table_name, e)
raise
# =========================================================================
# STEP 3: agg_pillar_by_country
# Kolom tambahan: pillar_name_id
# =========================================================================
def calc_pillar_by_country(self) -> pd.DataFrame:
table_name = "agg_pillar_by_country"
self.load_metadata[table_name]["start_time"] = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 3: {table_name} -> [Gold] fs_asean_gold")
self.logger.info("=" * 70)
try:
df_normed = self._get_norm_value_df()
df = (
df_normed
.groupby(["country_id", "country_name", "pillar_id", "pillar_name", "year"])
.agg(pillar_country_norm=("norm_value", "mean"))
.reset_index()
)
df["pillar_country_score_1_100"] = global_minmax(df["pillar_country_norm"])
df["rank_in_pillar_year"] = (
df.groupby(["pillar_id", "year"])["pillar_country_score_1_100"]
.rank(method="min", ascending=False)
.astype(int)
)
df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100")
# Kolom terjemahan Indonesia
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["country_id"] = df["country_id"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["year"] = df["year"].astype(int)
df["rank_in_pillar_year"] = df["rank_in_pillar_year"].astype(int)
df["pillar_country_norm"] = df["pillar_country_norm"].astype(float)
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_country_norm", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("pillar_country_score_1_100", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("rank_in_pillar_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
write_disposition="WRITE_TRUNCATE", schema=schema
)
self._finalize(table_name, rows)
return df
except Exception as e:
self._fail(table_name, e)
raise
# =========================================================================
# STEP 4: agg_framework_by_country
# Tidak ada kolom pillar/indicator di tabel ini (sudah di level framework)
# =========================================================================
def _calc_country_composite_inmemory(self) -> pd.DataFrame:
df_normed = self._get_norm_value_df()
df = (
df_normed
.groupby(["country_id", "country_name", "year"])
.agg(
composite_score=("norm_value", "mean"),
n_indicators =("indicator_id", "nunique"),
)
.reset_index()
)
df["score_1_100"] = global_minmax(df["composite_score"])
df["rank_in_asean"] = (
df.groupby("year")["score_1_100"]
.rank(method="min", ascending=False)
.astype(int)
)
df = add_yoy(df, ["country_id"], "score_1_100")
df["country_id"] = df["country_id"].astype(int)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
df["composite_score"] = df["composite_score"].astype(float)
df["score_1_100"] = df["score_1_100"].astype(float)
df["rank_in_asean"] = df["rank_in_asean"].astype(int)
return df
def calc_framework_by_country(self) -> pd.DataFrame:
table_name = "agg_framework_by_country"
self.load_metadata[table_name]["start_time"] = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 4: {table_name} -> [Gold] fs_asean_gold")
self.logger.info("=" * 70)
try:
country_composite = self._calc_country_composite_inmemory()
df_normed = self._get_norm_value_df()
parts = []
agg_total = (
country_composite[[
"country_id", "country_name", "year",
"score_1_100", "n_indicators", "composite_score"
]]
.copy()
.rename(columns={
"score_1_100" : "framework_score_1_100",
"composite_score": "framework_norm",
})
)
agg_total["framework"] = "Total"
parts.append(agg_total)
pre_sdgs_rows = country_composite[country_composite["year"] < self.sdgs_start_year].copy()
if not pre_sdgs_rows.empty:
mdgs_pre = (
pre_sdgs_rows[[
"country_id", "country_name", "year",
"score_1_100", "n_indicators", "composite_score"
]]
.copy()
.rename(columns={
"score_1_100" : "framework_score_1_100",
"composite_score": "framework_norm",
})
)
mdgs_pre["framework"] = "MDGs"
parts.append(mdgs_pre)
mdgs_indicator_ids = set(
self._ind_year_framework[self._ind_year_framework["framework"] == "MDGs"]["indicator_id"]
)
if mdgs_indicator_ids:
df_mdgs_mixed = df_normed[
(df_normed["indicator_id"].isin(mdgs_indicator_ids)) &
(df_normed["year"] >= self.sdgs_start_year)
].copy()
if not df_mdgs_mixed.empty:
agg_mdgs_mixed = (
df_mdgs_mixed
.groupby(["country_id", "country_name", "year"])
.agg(
framework_norm=("norm_value", "mean"),
n_indicators =("indicator_id", "nunique"),
)
.reset_index()
)
if not NORMALIZE_FRAMEWORKS_JOINTLY:
agg_mdgs_mixed["framework_score_1_100"] = global_minmax(agg_mdgs_mixed["framework_norm"])
agg_mdgs_mixed["framework"] = "MDGs"
parts.append(agg_mdgs_mixed)
sdgs_indicator_ids = set(
self._ind_year_framework[self._ind_year_framework["framework"] == "SDGs"]["indicator_id"]
)
if sdgs_indicator_ids:
df_sdgs = df_normed[
(df_normed["indicator_id"].isin(sdgs_indicator_ids)) &
(df_normed["year"] >= self.sdgs_start_year)
].copy()
if not df_sdgs.empty:
agg_sdgs = (
df_sdgs
.groupby(["country_id", "country_name", "year"])
.agg(
framework_norm=("norm_value", "mean"),
n_indicators =("indicator_id", "nunique"),
)
.reset_index()
)
if not NORMALIZE_FRAMEWORKS_JOINTLY:
agg_sdgs["framework_score_1_100"] = global_minmax(agg_sdgs["framework_norm"])
agg_sdgs["framework"] = "SDGs"
parts.append(agg_sdgs)
df = pd.concat(parts, ignore_index=True)
if NORMALIZE_FRAMEWORKS_JOINTLY:
mixed_mask = (df["framework"].isin(["MDGs", "SDGs"])) & (df["year"] >= self.sdgs_start_year)
if mixed_mask.any():
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(df.loc[mixed_mask, "framework_norm"])
df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger)
df["rank_in_framework_year"] = (
df.groupby(["framework", "year"])["framework_score_1_100"]
.rank(method="min", ascending=False)
.astype(int)
)
df = add_yoy(df, ["country_id", "framework"], "framework_score_1_100")
df["country_id"] = df["country_id"].astype(int)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
df["rank_in_framework_year"] = safe_int(df["rank_in_framework_year"], col_name="rank_in_framework_year", logger=self.logger)
df["framework_norm"] = df["framework_norm"].astype(float)
df["framework_score_1_100"] = df["framework_score_1_100"].astype(float)
self._validate_mdgs_equals_total(df, level="country")
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("framework_norm", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("rank_in_framework_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
write_disposition="WRITE_TRUNCATE", schema=schema
)
self._finalize(table_name, rows)
return df
except Exception as e:
self._fail(table_name, e)
raise
# =========================================================================
# STEP 5: agg_framework_asean
# Tidak ada kolom pillar/indicator langsung di tabel ini
# =========================================================================
def calc_framework_asean(self) -> pd.DataFrame:
table_name = "agg_framework_asean"
self.load_metadata[table_name]["start_time"] = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 5: {table_name} -> [Gold] fs_asean_gold")
self.logger.info("=" * 70)
try:
df_normed = self._get_norm_value_df()
country_composite = self._calc_country_composite_inmemory()
country_norm = (
df_normed
.groupby(["country_id", "country_name", "year"])["norm_value"]
.mean().reset_index()
.rename(columns={"norm_value": "country_norm"})
)
asean_overall = (
country_norm.groupby("year")
.agg(
asean_norm =("country_norm", "mean"),
std_norm =("country_norm", "std"),
n_countries=("country_norm", "count"),
)
.reset_index()
)
asean_overall["asean_score_1_100"] = global_minmax(asean_overall["asean_norm"])
asean_comp = (
country_composite.groupby("year")["composite_score"]
.mean().reset_index()
.rename(columns={"composite_score": "asean_composite"})
)
asean_overall = asean_overall.merge(asean_comp, on="year", how="left")
parts = []
def _n_ind(year_val, framework_val):
return self._count_framework_indicators(year_val, framework_val)
total_cols = asean_overall[[
"year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"
]].copy().rename(columns={
"asean_score_1_100": "framework_score_1_100",
"asean_norm" : "framework_norm",
"n_countries" : "n_countries_with_data",
})
total_cols["n_indicators"] = total_cols["year"].apply(
lambda y: int(self._ind_year_framework[
self._ind_year_framework["year"] == y
]["indicator_id"].nunique())
)
total_cols["framework"] = "Total"
parts.append(total_cols)
pre_sdgs = asean_overall[asean_overall["year"] < self.sdgs_start_year].copy()
if not pre_sdgs.empty:
mdgs_pre = pre_sdgs[[
"year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"
]].copy().rename(columns={
"asean_score_1_100": "framework_score_1_100",
"asean_norm" : "framework_norm",
"n_countries" : "n_countries_with_data",
})
mdgs_pre["n_indicators"] = mdgs_pre["year"].apply(lambda y: _n_ind(y, "MDGs"))
mdgs_pre["framework"] = "MDGs"
parts.append(mdgs_pre)
mdgs_indicator_ids = set(
self._ind_year_framework[self._ind_year_framework["framework"] == "MDGs"]["indicator_id"]
)
if mdgs_indicator_ids:
df_mdgs_mixed = df_normed[
(df_normed["indicator_id"].isin(mdgs_indicator_ids)) &
(df_normed["year"] >= self.sdgs_start_year)
].copy()
if not df_mdgs_mixed.empty:
cn = (
df_mdgs_mixed
.groupby(["country_id", "year"])["norm_value"].mean()
.reset_index().rename(columns={"norm_value": "country_norm"})
)
asean_mdgs = cn.groupby("year").agg(
framework_norm =("country_norm", "mean"),
std_norm =("country_norm", "std"),
n_countries_with_data=("country_id", "count"),
).reset_index()
asean_mdgs["n_indicators"] = asean_mdgs["year"].apply(lambda y: _n_ind(y, "MDGs"))
if not NORMALIZE_FRAMEWORKS_JOINTLY:
asean_mdgs["framework_score_1_100"] = global_minmax(asean_mdgs["framework_norm"])
asean_mdgs["framework"] = "MDGs"
parts.append(asean_mdgs)
sdgs_indicator_ids = set(
self._ind_year_framework[self._ind_year_framework["framework"] == "SDGs"]["indicator_id"]
)
if sdgs_indicator_ids:
df_sdgs = df_normed[
(df_normed["indicator_id"].isin(sdgs_indicator_ids)) &
(df_normed["year"] >= self.sdgs_start_year)
].copy()
if not df_sdgs.empty:
cn = (
df_sdgs
.groupby(["country_id", "year"])["norm_value"].mean()
.reset_index().rename(columns={"norm_value": "country_norm"})
)
asean_sdgs = cn.groupby("year").agg(
framework_norm =("country_norm", "mean"),
std_norm =("country_norm", "std"),
n_countries_with_data=("country_id", "count"),
).reset_index()
asean_sdgs["n_indicators"] = asean_sdgs["year"].apply(lambda y: _n_ind(y, "SDGs"))
if not NORMALIZE_FRAMEWORKS_JOINTLY:
asean_sdgs["framework_score_1_100"] = global_minmax(asean_sdgs["framework_norm"])
asean_sdgs["framework"] = "SDGs"
parts.append(asean_sdgs)
df = pd.concat(parts, ignore_index=True)
if NORMALIZE_FRAMEWORKS_JOINTLY:
mixed_mask = (df["framework"].isin(["MDGs", "SDGs"])) & (df["year"] >= self.sdgs_start_year)
if mixed_mask.any():
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(df.loc[mixed_mask, "framework_norm"])
df = check_and_dedup(df, ["framework", "year"], context=table_name, logger=self.logger)
df = add_yoy(df, ["framework"], "framework_score_1_100")
df["performance_status"] = df["framework_score_1_100"].apply(_performance_status)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
df["n_countries_with_data"] = safe_int(df["n_countries_with_data"], col_name="n_countries_with_data", logger=self.logger)
for col in ["framework_norm", "std_norm", "framework_score_1_100"]:
df[col] = df[col].astype(float)
self._validate_mdgs_equals_total(df, level="asean")
schema = [
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("n_countries_with_data", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("framework_norm", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("std_norm", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("performance_status", "STRING", mode="REQUIRED"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
write_disposition="WRITE_TRUNCATE", schema=schema
)
self._finalize(table_name, rows)
return df
except Exception as e:
self._fail(table_name, e)
raise
# =========================================================================
# STEP 6: agg_narrative_overview
# Tidak ada kolom pillar/indicator di tabel ini
# =========================================================================
def calc_narrative_overview(
self,
df_framework_asean: pd.DataFrame,
df_framework_by_country: pd.DataFrame,
) -> pd.DataFrame:
table_name = "agg_narrative_overview"
self.load_metadata[table_name]["start_time"] = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID")
self.logger.info("=" * 70)
try:
asean_total = (
df_framework_asean[df_framework_asean["framework"] == "Total"]
.sort_values("year")
.reset_index(drop=True)
)
score_by_year = dict(zip(asean_total["year"].astype(int), asean_total["framework_score_1_100"].astype(float)))
status_by_year = dict(zip(asean_total["year"].astype(int), asean_total["performance_status"].astype(str)))
country_total = df_framework_by_country[df_framework_by_country["framework"] == "Total"].copy()
records = []
for _, row in asean_total.iterrows():
yr = int(row["year"])
score = float(row["framework_score_1_100"])
perf_status = str(row["performance_status"])
yoy = row["year_over_year_change"]
yoy_val = float(yoy) if pd.notna(yoy) else None
n_mdg = self._count_framework_indicators(yr, "MDGs")
n_sdg = self._count_framework_indicators(yr, "SDGs")
n_total_ind = int(
self._ind_year_framework[
self._ind_year_framework["year"] == yr
]["indicator_id"].nunique()
)
prev_score = score_by_year.get(yr - 1, None)
prev_status = status_by_year.get(yr - 1, "N/A")
yoy_pct = (
(yoy_val / prev_score * 100)
if (yoy_val is not None and prev_score is not None and prev_score != 0)
else None
)
yr_country = (
country_total[country_total["year"] == yr]
.sort_values("rank_in_framework_year")
.reset_index(drop=True)
)
ranking_list = []
for _, cr in yr_country.iterrows():
cr_yoy = cr.get("year_over_year_change", None)
ranking_list.append({
"rank": int(cr["rank_in_framework_year"]),
"country_name": str(cr["country_name"]),
"score": round(float(cr["framework_score_1_100"]), 2),
"yoy_change": round(float(cr_yoy), 2) if pd.notna(cr_yoy) else None,
})
country_ranking_json = json.dumps(ranking_list, ensure_ascii=False)
yr_country_yoy = yr_country.dropna(subset=["year_over_year_change"])
if not yr_country_yoy.empty:
best_idx = yr_country_yoy["year_over_year_change"].idxmax()
worst_idx = yr_country_yoy["year_over_year_change"].idxmin()
most_improved_country = str(yr_country_yoy.loc[best_idx, "country_name"])
most_improved_delta = round(float(yr_country_yoy.loc[best_idx, "year_over_year_change"]), 2)
most_declined_country = str(yr_country_yoy.loc[worst_idx, "country_name"])
most_declined_delta = round(float(yr_country_yoy.loc[worst_idx, "year_over_year_change"]), 2)
else:
most_improved_country = most_declined_country = None
most_improved_delta = most_declined_delta = None
country_scores_all = country_total[["year", "country_id", "framework_score_1_100"]].copy()
narrative_en, narrative_id = _build_overview_narrative(
year = yr,
score = score,
performance_status = perf_status,
yoy_val = yoy_val,
n_mdg = n_mdg,
n_sdg = n_sdg,
ranking_list = ranking_list,
most_improved_country = most_improved_country,
most_improved_delta = most_improved_delta,
most_declined_country = most_declined_country,
most_declined_delta = most_declined_delta,
historical_scores = score_by_year,
country_scores_all = country_scores_all,
)
records.append({
"year": yr,
"n_mdg_indicators": n_mdg,
"n_sdg_indicators": n_sdg,
"n_total_indicators": n_total_ind,
"asean_total_score": round(score, 2),
"performance_status": perf_status,
"yoy_change": yoy_val,
"yoy_change_pct": round(yoy_pct, 2) if yoy_pct is not None else None,
"country_ranking_json": country_ranking_json,
"most_improved_country": most_improved_country,
"most_improved_delta": most_improved_delta,
"most_declined_country": most_declined_country,
"most_declined_delta": most_declined_delta,
"narrative_en": narrative_en,
"narrative_id": narrative_id,
})
df = pd.DataFrame(records)
df["year"] = df["year"].astype(int)
df["n_mdg_indicators"] = df["n_mdg_indicators"].astype(int)
df["n_sdg_indicators"] = df["n_sdg_indicators"].astype(int)
df["n_total_indicators"] = df["n_total_indicators"].astype(int)
df["asean_total_score"] = df["asean_total_score"].astype(float)
df["performance_status"] = df["performance_status"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
for col in ["yoy_change", "yoy_change_pct", "most_improved_delta", "most_declined_delta"]:
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
self.logger.info("\n Sample narrative_en (year 1):")
self.logger.info(f" {df.iloc[0]['narrative_en'][:300]}")
self.logger.info("\n Sample narrative_id (year 1):")
self.logger.info(f" {df.iloc[0]['narrative_id'][:300]}")
schema = [
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("n_mdg_indicators", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("n_sdg_indicators", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("n_total_indicators", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("asean_total_score", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("performance_status", "STRING", mode="REQUIRED"),
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("yoy_change_pct", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("country_ranking_json", "STRING", mode="REQUIRED"),
bigquery.SchemaField("most_improved_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("most_improved_delta", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("most_declined_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("most_declined_delta", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"),
bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
write_disposition="WRITE_TRUNCATE", schema=schema,
)
self._finalize(table_name, rows)
return df
except Exception as e:
self._fail(table_name, e)
raise
# =========================================================================
# STEP 7: agg_narrative_pillar
# Kolom tambahan: pillar_name_id
# =========================================================================
def calc_narrative_pillar(
self,
df_pillar_composite: pd.DataFrame,
df_pillar_by_country: pd.DataFrame,
) -> pd.DataFrame:
table_name = "agg_narrative_pillar"
self.load_metadata[table_name]["start_time"] = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 7: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID")
self.logger.info("=" * 70)
try:
records = []
years = sorted(df_pillar_composite["year"].unique())
# Precompute history per pillar
pillar_history = {}
for p_id, grp in df_pillar_composite.groupby("pillar_id"):
pillar_history[p_id] = dict(
zip(grp["year"].astype(int), grp["pillar_score_1_100"].astype(float))
)
for yr in years:
yr_pillars = (
df_pillar_composite[df_pillar_composite["year"] == yr]
.sort_values("rank_in_year")
.reset_index(drop=True)
)
yr_country_pillar = df_pillar_by_country[df_pillar_by_country["year"] == yr]
for _, prow in yr_pillars.iterrows():
p_id = int(prow["pillar_id"])
p_name = str(prow["pillar_name"])
p_score = float(prow["pillar_score_1_100"])
p_rank = int(prow["rank_in_year"])
p_yoy = prow["year_over_year_change"]
p_yoy_val = float(p_yoy) if pd.notna(p_yoy) else None
# Terjemahan Indonesia nama pillar
p_name_id = translate_pillar(p_name)
p_country = (
yr_country_pillar[yr_country_pillar["pillar_id"] == p_id]
.sort_values("rank_in_pillar_year")
.reset_index(drop=True)
)
if not p_country.empty:
top_country = str(p_country.iloc[0]["country_name"])
top_country_score = round(float(p_country.iloc[0]["pillar_country_score_1_100"]), 2)
bot_country = str(p_country.iloc[-1]["country_name"])
bot_country_score = round(float(p_country.iloc[-1]["pillar_country_score_1_100"]), 2)
else:
top_country = bot_country = None
top_country_score = bot_country_score = None
hist_up_to_yr = {
y: s for y, s in pillar_history.get(p_id, {}).items() if y <= yr
}
country_pillar_all = df_pillar_by_country[
df_pillar_by_country["pillar_id"] == p_id
][["year", "country_id", "pillar_country_score_1_100"]].copy()
narrative_en, narrative_id = _build_pillar_narrative(
year = yr,
pillar_name = p_name,
pillar_score = p_score,
rank_in_year = p_rank,
n_pillars = len(yr_pillars),
yoy_val = p_yoy_val,
top_country = top_country,
top_country_score = top_country_score,
bot_country = bot_country,
bot_country_score = bot_country_score,
pillar_scores_history = hist_up_to_yr,
all_pillar_scores_year= yr_pillars[["pillar_name", "pillar_score_1_100"]].copy(),
country_pillar_all = country_pillar_all,
)
records.append({
"year": yr,
"pillar_id": p_id,
"pillar_name": p_name,
"pillar_name_id": p_name_id,
"pillar_score": round(p_score, 2),
"rank_in_year": p_rank,
"yoy_change": p_yoy_val,
"top_country": top_country,
"top_country_score": top_country_score,
"bottom_country": bot_country,
"bottom_country_score": bot_country_score,
"narrative_en": narrative_en,
"narrative_id": narrative_id,
})
df = pd.DataFrame(records)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
self.logger.info("\n Sample narrative_en (first row):")
self.logger.info(f" {df.iloc[0]['narrative_en'][:300]}")
self.logger.info("\n Sample narrative_id (first row):")
self.logger.info(f" {df.iloc[0]['narrative_id'][:300]}")
schema = [
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_score", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("top_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("top_country_score", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("bottom_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"),
bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
write_disposition="WRITE_TRUNCATE", schema=schema,
)
self._finalize(table_name, rows)
return df
except Exception as e:
self._fail(table_name, e)
raise
# =========================================================================
# HELPERS
# =========================================================================
def _validate_mdgs_equals_total(self, df: pd.DataFrame, level: str = ""):
self.logger.info(f"\n Validasi MDGs < {self.sdgs_start_year} == Total [{level}]:")
group_by = ["year"] if level.startswith("asean") else ["country_id", "year"]
mdgs_pre = df[(df["framework"] == "MDGs") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "mdgs_score"})
total_pre = df[(df["framework"] == "Total") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "total_score"})
if mdgs_pre.empty and total_pre.empty:
self.logger.info(f" -> Tidak ada data pre-{self.sdgs_start_year} (skip)")
return
if mdgs_pre.empty or total_pre.empty:
self.logger.warning(f" -> [WARNING] Salah satu kosong")
return
check = mdgs_pre.merge(total_pre, on=group_by)
max_diff = (check["mdgs_score"] - check["total_score"]).abs().max()
status = "OK (identik)" if max_diff < 0.01 else f"MISMATCH! max_diff={max_diff:.6f}"
self.logger.info(f" -> {status} (n_checked={len(check)})")
def _finalize(self, table_name: str, rows_loaded: int):
end_time = datetime.now()
start_time = self.load_metadata[table_name].get("start_time")
self.load_metadata[table_name].update({
"rows_loaded": rows_loaded,
"status" : "success",
"end_time" : end_time,
})
log_update(self.client, "DW", table_name, "full_load", rows_loaded)
try:
save_etl_metadata(
self.client,
self._build_etl_metadata(
table_name = table_name,
rows_loaded = rows_loaded,
start_time = start_time,
end_time = end_time,
status = "success",
)
)
except Exception as meta_err:
self.logger.warning(f" [METADATA WARNING] {table_name}: {meta_err}")
self.logger.info(f" [OK] {table_name}: {rows_loaded:,} rows -> [Gold] fs_asean_gold")
def _fail(self, table_name: str, error: Exception):
end_time = datetime.now()
start_time = self.load_metadata[table_name].get("start_time")
error_msg = str(error)
self.load_metadata[table_name].update({"status": "failed", "end_time": end_time})
log_update(self.client, "DW", table_name, "full_load", 0, "failed", error_msg)
try:
save_etl_metadata(
self.client,
self._build_etl_metadata(
table_name = table_name,
rows_loaded = 0,
start_time = start_time,
end_time = end_time,
status = "failed",
error_msg = error_msg,
)
)
except Exception as meta_err:
self.logger.warning(f" [METADATA WARNING] {table_name}: {meta_err}")
self.logger.error(f" [FAIL] {table_name}: {error_msg}")
# =========================================================================
# RUN
# =========================================================================
def run(self):
start = datetime.now()
self.logger.info("\n" + "=" * 70)
self.logger.info("FOOD SECURITY AGGREGATION — 6 TABLES -> fs_asean_gold")
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}")
self.logger.info(f" Narrative style : interpretive, plain text, bilingual EN/ID")
self.logger.info("=" * 70)
self.load_data()
self.sdgs_start_year = self._detect_sdgs_start_year()
self._assign_framework_labels()
df_pillar_composite = self.calc_pillar_composite()
df_pillar_by_country = self.calc_pillar_by_country()
df_framework_by_country = self.calc_framework_by_country()
df_framework_asean = self.calc_framework_asean()
self.calc_narrative_overview(
df_framework_asean = df_framework_asean,
df_framework_by_country = df_framework_by_country,
)
self.calc_narrative_pillar(
df_pillar_composite = df_pillar_composite,
df_pillar_by_country = df_pillar_by_country,
)
duration = (datetime.now() - start).total_seconds()
total_rows = sum(m["rows_loaded"] for m in self.load_metadata.values())
self.logger.info("\n" + "=" * 70)
self.logger.info("SELESAI")
self.logger.info("=" * 70)
self.logger.info(f" Durasi : {duration:.2f}s")
self.logger.info(f" Total rows : {total_rows:,}")
for tbl, meta in self.load_metadata.items():
icon = "[OK]" if meta["status"] == "success" else "[FAIL]"
self.logger.info(f" {icon} {tbl:<35} {meta['rows_loaded']:>10,}")
# =============================================================================
# AIRFLOW TASK
# =============================================================================
def run_aggregation():
from scripts.bigquery_config import get_bigquery_client
client = get_bigquery_client()
agg = FoodSecurityAggregator(client)
agg.run()
total = sum(m["rows_loaded"] for m in agg.load_metadata.values())
print(f"Aggregation completed: {total:,} total rows loaded")
# =============================================================================
# MAIN
# =============================================================================
if __name__ == "__main__":
import io
if _sys.stdout.encoding and _sys.stdout.encoding.lower() not in ("utf-8", "utf8"):
_sys.stdout = io.TextIOWrapper(_sys.stdout.buffer, encoding="utf-8", errors="replace")
if _sys.stderr.encoding and _sys.stderr.encoding.lower() not in ("utf-8", "utf8"):
_sys.stderr = io.TextIOWrapper(_sys.stderr.buffer, encoding="utf-8", errors="replace")
print("=" * 70)
print("FOOD SECURITY AGGREGATION -> fs_asean_gold")
print(f" NORMALIZE_FRAMEWORKS_JOINTLY : {NORMALIZE_FRAMEWORKS_JOINTLY}")
print(f" PERFORMANCE_THRESHOLD : {PERFORMANCE_THRESHOLD}")
print("=" * 70)
logger = setup_logging()
for handler in logger.handlers:
handler.__class__ = _SafeStreamHandler
client = get_bigquery_client()
agg = FoodSecurityAggregator(client)
agg.run()
print("\n" + "=" * 70)
print("[OK] SELESAI")
print("=" * 70)
Reference in New Issue View Git Blame Copy Permalink