1623 lines
73 KiB
Python
1623 lines
73 KiB
Python
"""
|
|
BIGQUERY ANALYSIS LAYER - FOOD SECURITY AGGREGATION
|
|
Semua agregasi pakai norm_value dari _get_norm_value_df()
|
|
UPDATED: Simpan 6 tabel ke fs_asean_gold (layer='gold'):
|
|
- agg_pillar_composite
|
|
- agg_pillar_by_country
|
|
- agg_framework_by_country
|
|
- agg_framework_asean (+ kolom performance_status: 'Good'/'Bad', threshold=60)
|
|
- agg_narrative_overview
|
|
- agg_narrative_pillar
|
|
|
|
SOURCE TABLE: fact_asean_food_security_selected (sudah include nama + ID)
|
|
|
|
n_indicators logic (sesuai agg_indicator_norm):
|
|
- Setiap tahun dihitung dari indikator yang benar-benar hadir di tahun tsb.
|
|
- Framework MDGs/SDGs per tahun mengikuti SDG_ONLY_KEYWORDS:
|
|
* Indikator tidak di SDG_ONLY -> selalu MDGs
|
|
* Indikator di SDG_ONLY + year >= sdgs_start_year -> SDGs
|
|
* Indikator di SDG_ONLY + year < sdgs_start_year -> MDGs
|
|
- Sehingga n_indicators MDGs dan SDGs bisa berbeda antar tahun.
|
|
"""
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
from datetime import datetime
|
|
import logging
|
|
import json
|
|
import sys as _sys
|
|
|
|
from scripts.bigquery_config import get_bigquery_client
|
|
from scripts.bigquery_helpers import (
|
|
log_update,
|
|
load_to_bigquery,
|
|
read_from_bigquery,
|
|
setup_logging,
|
|
save_etl_metadata,
|
|
)
|
|
from google.cloud import bigquery
|
|
|
|
|
|
# =============================================================================
|
|
# KONSTANTA GLOBAL
|
|
# =============================================================================
|
|
|
|
DIRECTION_INVERT_KEYWORDS = frozenset({
|
|
"negative", "lower_better", "lower_is_better", "inverse", "neg",
|
|
})
|
|
DIRECTION_POSITIVE_KEYWORDS = frozenset({
|
|
"positive", "higher_better", "higher_is_better",
|
|
})
|
|
|
|
NORMALIZE_FRAMEWORKS_JOINTLY = False
|
|
|
|
# Threshold performance_status di agg_framework_asean
|
|
PERFORMANCE_THRESHOLD = 60.0 # score >= 60 -> "Good", < 60 -> "Bad"
|
|
|
|
# SDG_ONLY_KEYWORDS (sama persis dengan bigquery_aggraget_fact_selected_layer.py)
|
|
SDG_ONLY_KEYWORDS: frozenset = frozenset([
|
|
"prevalence of undernourishment (percent) (3-year average)",
|
|
"number of people undernourished (million) (3-year average)",
|
|
"prevalence of severe food insecurity in the total population (percent) (3-year average)",
|
|
"prevalence of severe food insecurity in the male adult population (percent) (3-year average)",
|
|
"prevalence of severe food insecurity in the female adult population (percent) (3-year average)",
|
|
"prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)",
|
|
"prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)",
|
|
"prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)",
|
|
"number of severely food insecure people (million) (3-year average)",
|
|
"number of severely food insecure male adults (million) (3-year average)",
|
|
"number of severely food insecure female adults (million) (3-year average)",
|
|
"number of moderately or severely food insecure people (million) (3-year average)",
|
|
"number of moderately or severely food insecure male adults (million) (3-year average)",
|
|
"number of moderately or severely food insecure female adults (million) (3-year average)",
|
|
"percentage of children under 5 years of age who are stunted (modelled estimates) (percent)",
|
|
"number of children under 5 years of age who are stunted (modeled estimates) (million)",
|
|
"percentage of children under 5 years affected by wasting (percent)",
|
|
"number of children under 5 years affected by wasting (million)",
|
|
"percentage of children under 5 years of age who are overweight (modelled estimates) (percent)",
|
|
"number of children under 5 years of age who are overweight (modeled estimates) (million)",
|
|
"prevalence of anemia among women of reproductive age (15-49 years) (percent)",
|
|
"number of women of reproductive age (15-49 years) affected by anemia (million)",
|
|
])
|
|
_SDG_ONLY_LOWER: frozenset = frozenset(k.lower() for k in SDG_ONLY_KEYWORDS)
|
|
|
|
_FIES_DETECTION_LOWER: frozenset = frozenset([
|
|
"prevalence of severe food insecurity in the total population (percent) (3-year average)",
|
|
"prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)",
|
|
"number of severely food insecure people (million) (3-year average)",
|
|
"number of moderately or severely food insecure people (million) (3-year average)",
|
|
])
|
|
|
|
|
|
# =============================================================================
|
|
# Windows CP1252 safe logging
|
|
# =============================================================================
|
|
|
|
class _SafeStreamHandler(logging.StreamHandler):
|
|
def emit(self, record):
|
|
try:
|
|
super().emit(record)
|
|
except UnicodeEncodeError:
|
|
try:
|
|
msg = self.format(record)
|
|
self.stream.write(
|
|
msg.encode("utf-8", errors="replace").decode("ascii", errors="replace")
|
|
+ self.terminator
|
|
)
|
|
self.flush()
|
|
except Exception:
|
|
self.handleError(record)
|
|
|
|
|
|
# =============================================================================
|
|
# HELPERS
|
|
# =============================================================================
|
|
|
|
def _should_invert(direction: str, logger=None, context: str = "") -> bool:
|
|
d = str(direction).lower().strip()
|
|
if d in DIRECTION_INVERT_KEYWORDS:
|
|
return True
|
|
if d in DIRECTION_POSITIVE_KEYWORDS:
|
|
return False
|
|
if logger:
|
|
logger.warning(
|
|
f" [DIRECTION WARNING] Unknown direction '{direction}' "
|
|
f"{'(' + context + ')' if context else ''}. Defaulting to positive (no invert)."
|
|
)
|
|
return False
|
|
|
|
|
|
def global_minmax(series: pd.Series, lo: float = 1.0, hi: float = 100.0) -> pd.Series:
|
|
values = series.dropna().values
|
|
if len(values) == 0:
|
|
return pd.Series(np.nan, index=series.index)
|
|
v_min, v_max = values.min(), values.max()
|
|
if v_min == v_max:
|
|
return pd.Series((lo + hi) / 2.0, index=series.index)
|
|
result = np.full(len(series), np.nan)
|
|
not_nan = series.notna()
|
|
raw = series[not_nan].values
|
|
result[not_nan.values] = lo + (raw - v_min) / (v_max - v_min) * (hi - lo)
|
|
return pd.Series(result, index=series.index)
|
|
|
|
|
|
def add_yoy(df: pd.DataFrame, group_cols: list, score_col: str) -> pd.DataFrame:
|
|
df = df.sort_values(group_cols + ["year"]).reset_index(drop=True)
|
|
if group_cols:
|
|
df["year_over_year_change"] = df.groupby(group_cols)[score_col].diff()
|
|
else:
|
|
df["year_over_year_change"] = df[score_col].diff()
|
|
return df
|
|
|
|
|
|
def safe_int(series: pd.Series, fill: int = 0, col_name: str = "", logger=None) -> pd.Series:
|
|
n_nan = series.isna().sum()
|
|
if n_nan > 0 and logger:
|
|
logger.warning(
|
|
f" [NaN WARNING] Kolom '{col_name}' punya {n_nan} NaN -> di-fill dengan {fill}"
|
|
)
|
|
return series.fillna(fill).astype(int)
|
|
|
|
|
|
def check_and_dedup(df: pd.DataFrame, key_cols: list, context: str = "", logger=None) -> pd.DataFrame:
|
|
dupes = df.duplicated(subset=key_cols, keep=False)
|
|
if dupes.any():
|
|
n_dupes = dupes.sum()
|
|
if logger:
|
|
logger.warning(
|
|
f" [DEDUP WARNING] {context}: {n_dupes} duplikat rows pada {key_cols}. "
|
|
f"Di-aggregate dengan mean."
|
|
)
|
|
numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
|
|
agg_dict = {
|
|
c: ("mean" if c in numeric_cols else "first")
|
|
for c in df.columns if c not in key_cols
|
|
}
|
|
df = df.groupby(key_cols, as_index=False).agg(agg_dict)
|
|
return df
|
|
|
|
|
|
def _performance_status(score) -> str:
|
|
"""Classify score into 'Good' or 'Bad' based on PERFORMANCE_THRESHOLD."""
|
|
if score is None or (isinstance(score, float) and np.isnan(score)):
|
|
return "N/A"
|
|
return "Good" if score >= PERFORMANCE_THRESHOLD else "Bad"
|
|
|
|
|
|
# =============================================================================
|
|
# NARRATIVE HELPERS
|
|
# =============================================================================
|
|
|
|
def _fmt_score(score) -> str:
|
|
if score is None or (isinstance(score, float) and np.isnan(score)):
|
|
return "N/A"
|
|
return f"{score:.2f}"
|
|
|
|
|
|
def _fmt_delta(delta) -> str:
|
|
if delta is None or (isinstance(delta, float) and np.isnan(delta)):
|
|
return "N/A"
|
|
sign = "+" if delta >= 0 else ""
|
|
return f"{sign}{delta:.2f}"
|
|
|
|
|
|
def _build_overview_narrative(
|
|
year: int,
|
|
n_mdg: int,
|
|
n_sdg: int,
|
|
n_total_ind: int,
|
|
score: float,
|
|
performance_status: str,
|
|
yoy_val,
|
|
yoy_pct,
|
|
prev_year: int,
|
|
prev_score,
|
|
prev_performance_status: str,
|
|
ranking_list: list,
|
|
most_improved_country,
|
|
most_improved_delta,
|
|
most_declined_country,
|
|
most_declined_delta,
|
|
) -> str:
|
|
# Sentence 1: indicator breakdown
|
|
parts_ind = []
|
|
if n_mdg > 0:
|
|
parts_ind.append(f"{n_mdg} MDG indicator{'s' if n_mdg > 1 else ''}")
|
|
if n_sdg > 0:
|
|
parts_ind.append(f"{n_sdg} SDG indicator{'s' if n_sdg > 1 else ''}")
|
|
|
|
if parts_ind:
|
|
ind_detail = " and ".join(parts_ind)
|
|
sent1 = (
|
|
f"In {year}, the ASEAN food security assessment incorporated a total of "
|
|
f"{n_total_ind} indicator{'s' if n_total_ind != 1 else ''}, "
|
|
f"consisting of {ind_detail}."
|
|
)
|
|
else:
|
|
sent1 = (
|
|
f"In {year}, the ASEAN food security assessment incorporated "
|
|
f"{n_total_ind} indicator{'s' if n_total_ind != 1 else ''}."
|
|
)
|
|
|
|
# Sentence 2: score + performance status + YoY
|
|
status_phrase = (
|
|
f"classified as \"{performance_status}\" performance "
|
|
f"(threshold: {PERFORMANCE_THRESHOLD:.0f})"
|
|
)
|
|
if yoy_val is not None and prev_score is not None:
|
|
direction_word = "increasing" if yoy_val >= 0 else "decreasing"
|
|
pct_clause = ""
|
|
if yoy_pct is not None:
|
|
abs_pct = abs(yoy_pct)
|
|
trend_word = "improvement" if yoy_val >= 0 else "decline"
|
|
pct_clause = f", representing a {abs_pct:.2f}% {trend_word} year-over-year"
|
|
|
|
status_change = ""
|
|
if prev_performance_status not in ("N/A", None) and prev_performance_status != performance_status:
|
|
status_change = (
|
|
f" This marks a shift from \"{prev_performance_status}\" in {prev_year} "
|
|
f"to \"{performance_status}\" in {year}."
|
|
)
|
|
|
|
sent2 = (
|
|
f"The ASEAN overall score (Total framework) reached {_fmt_score(score)}, "
|
|
f"{status_phrase}, {direction_word} by {abs(yoy_val):.2f} points compared to "
|
|
f"{prev_year} ({_fmt_score(prev_score)}, \"{prev_performance_status}\"){pct_clause}.{status_change}"
|
|
)
|
|
else:
|
|
sent2 = (
|
|
f"The ASEAN overall score (Total framework) reached {_fmt_score(score)} in {year}, "
|
|
f"{status_phrase}. No prior-year data is available for year-over-year comparison."
|
|
)
|
|
|
|
# Sentence 3: country ranking
|
|
sent3 = ""
|
|
if ranking_list:
|
|
first = ranking_list[0]
|
|
last = ranking_list[-1]
|
|
middle = ranking_list[1:-1]
|
|
|
|
if len(ranking_list) == 1:
|
|
sent3 = (
|
|
f"In terms of country performance, {first['country_name']} was the only "
|
|
f"country assessed, scoring {_fmt_score(first['score'])} in {year}."
|
|
)
|
|
elif len(ranking_list) == 2:
|
|
sent3 = (
|
|
f"In terms of country performance, {first['country_name']} led the region "
|
|
f"with a score of {_fmt_score(first['score'])}, while "
|
|
f"{last['country_name']} recorded the lowest score of "
|
|
f"{_fmt_score(last['score'])} in {year}."
|
|
)
|
|
else:
|
|
middle_parts = [
|
|
f"{c['country_name']} ({_fmt_score(c['score'])})" for c in middle
|
|
]
|
|
if len(middle_parts) == 1:
|
|
middle_str = middle_parts[0]
|
|
else:
|
|
middle_str = ", ".join(middle_parts[:-1]) + f", and {middle_parts[-1]}"
|
|
sent3 = (
|
|
f"In terms of country performance, {first['country_name']} led the region "
|
|
f"with a score of {_fmt_score(first['score'])}, followed by {middle_str}. "
|
|
f"At the other end, {last['country_name']} recorded the lowest score "
|
|
f"of {_fmt_score(last['score'])} in {year}."
|
|
)
|
|
|
|
# Sentence 4: most improved / declined country
|
|
sent4_parts = []
|
|
if most_improved_country and most_improved_delta is not None:
|
|
sent4_parts.append(
|
|
f"the most notable improvement was seen in {most_improved_country}, "
|
|
f"which gained {_fmt_delta(most_improved_delta)} points from the previous year"
|
|
)
|
|
if most_declined_country and most_declined_delta is not None:
|
|
if most_declined_delta < 0:
|
|
sent4_parts.append(
|
|
f"while {most_declined_country} experienced the largest decline "
|
|
f"of {_fmt_delta(most_declined_delta)} points"
|
|
)
|
|
else:
|
|
sent4_parts.append(
|
|
f"while {most_declined_country} recorded the smallest gain "
|
|
f"of {_fmt_delta(most_declined_delta)} points"
|
|
)
|
|
|
|
sent4 = ""
|
|
if sent4_parts:
|
|
sent4 = ", ".join(sent4_parts) + "."
|
|
sent4 = sent4[0].upper() + sent4[1:]
|
|
|
|
return " ".join(s for s in [sent1, sent2, sent3, sent4] if s)
|
|
|
|
|
|
def _build_pillar_narrative(
|
|
year: int,
|
|
pillar_name: str,
|
|
pillar_score: float,
|
|
rank_in_year: int,
|
|
n_pillars: int,
|
|
yoy_val,
|
|
top_country,
|
|
top_country_score,
|
|
bot_country,
|
|
bot_country_score,
|
|
strongest_pillar,
|
|
strongest_score,
|
|
weakest_pillar,
|
|
weakest_score,
|
|
most_improved_pillar,
|
|
most_improved_delta,
|
|
most_declined_pillar,
|
|
most_declined_delta,
|
|
) -> str:
|
|
rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th")
|
|
sent1 = (
|
|
f"In {year}, the {pillar_name} pillar scored {_fmt_score(pillar_score)}, "
|
|
f"ranking {rank_in_year}{rank_suffix} out of {n_pillars} pillars assessed across ASEAN."
|
|
)
|
|
|
|
sent2 = ""
|
|
if strongest_pillar and weakest_pillar:
|
|
if strongest_pillar == pillar_name:
|
|
sent2 = (
|
|
f"This made {pillar_name} the strongest performing pillar in {year}, "
|
|
f"compared to the weakest pillar, {weakest_pillar}, "
|
|
f"which scored {_fmt_score(weakest_score)}."
|
|
)
|
|
elif weakest_pillar == pillar_name:
|
|
sent2 = (
|
|
f"This made {pillar_name} the weakest performing pillar in {year}, "
|
|
f"compared to the strongest pillar, {strongest_pillar}, "
|
|
f"which scored {_fmt_score(strongest_score)}."
|
|
)
|
|
else:
|
|
sent2 = (
|
|
f"Across all pillars in {year}, {strongest_pillar} was the strongest "
|
|
f"(score: {_fmt_score(strongest_score)}), while {weakest_pillar} "
|
|
f"was the weakest (score: {_fmt_score(weakest_score)})."
|
|
)
|
|
|
|
sent3 = ""
|
|
if top_country and bot_country:
|
|
if top_country != bot_country:
|
|
sent3 = (
|
|
f"Within the {pillar_name} pillar, {top_country} led with a score of "
|
|
f"{_fmt_score(top_country_score)}, while {bot_country} recorded the lowest "
|
|
f"score of {_fmt_score(bot_country_score)}."
|
|
)
|
|
else:
|
|
sent3 = (
|
|
f"Within the {pillar_name} pillar, {top_country} was the only country "
|
|
f"with available data, scoring {_fmt_score(top_country_score)}."
|
|
)
|
|
|
|
if yoy_val is not None:
|
|
direction_word = "improved" if yoy_val >= 0 else "declined"
|
|
sent4 = (
|
|
f"Compared to the previous year, the {pillar_name} pillar "
|
|
f"{direction_word} by {abs(yoy_val):.2f} points"
|
|
)
|
|
else:
|
|
sent4 = (
|
|
f"No prior-year data is available to calculate year-over-year change "
|
|
f"for the {pillar_name} pillar in {year}"
|
|
)
|
|
|
|
if (most_improved_pillar and most_improved_delta is not None
|
|
and most_declined_pillar and most_declined_delta is not None
|
|
and most_improved_pillar != most_declined_pillar):
|
|
sent4 += (
|
|
f". Across all pillars, {most_improved_pillar} showed the greatest improvement "
|
|
f"({_fmt_delta(most_improved_delta)} pts), while {most_declined_pillar} "
|
|
f"recorded the largest decline ({_fmt_delta(most_declined_delta)} pts)"
|
|
)
|
|
|
|
sent4 += "."
|
|
sent4 = sent4[0].upper() + sent4[1:]
|
|
|
|
return " ".join(s for s in [sent1, sent2, sent3, sent4] if s)
|
|
|
|
|
|
# =============================================================================
|
|
# MAIN CLASS
|
|
# =============================================================================
|
|
|
|
class FoodSecurityAggregator:
|
|
|
|
def __init__(self, client: bigquery.Client):
|
|
self.client = client
|
|
self.logger = logging.getLogger(self.__class__.__name__)
|
|
self.logger.propagate = False
|
|
|
|
self.load_metadata = {
|
|
"agg_pillar_composite": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
|
|
"agg_pillar_by_country": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
|
|
"agg_framework_by_country": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
|
|
"agg_framework_asean": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
|
|
"agg_narrative_overview": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
|
|
"agg_narrative_pillar": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None},
|
|
}
|
|
|
|
self.df = None
|
|
self.sdgs_start_year = None
|
|
|
|
# Lookup: (indicator_id, year) -> framework label
|
|
self._ind_year_framework: pd.DataFrame = None
|
|
|
|
# =========================================================================
|
|
# STEP 1: Load data
|
|
# =========================================================================
|
|
|
|
def load_data(self):
|
|
self.logger.info("=" * 70)
|
|
self.logger.info("STEP 1: LOAD DATA from fs_asean_gold")
|
|
self.logger.info("=" * 70)
|
|
|
|
self.df = read_from_bigquery(
|
|
self.client, "fact_asean_food_security_selected", layer='gold'
|
|
)
|
|
self.logger.info(f" fact_asean_food_security_selected : {len(self.df):,} rows")
|
|
|
|
required_cols = {
|
|
"country_id", "country_name",
|
|
"indicator_id", "indicator_name", "direction",
|
|
"pillar_id", "pillar_name",
|
|
"time_id", "year", "value",
|
|
}
|
|
missing_cols = required_cols - set(self.df.columns)
|
|
if missing_cols:
|
|
raise ValueError(
|
|
f"Kolom berikut tidak ditemukan di fact_asean_food_security_selected: {missing_cols}"
|
|
)
|
|
|
|
n_null_dir = self.df["direction"].isna().sum()
|
|
if n_null_dir > 0:
|
|
self.logger.warning(
|
|
f" [DIRECTION] {n_null_dir} rows dengan direction NULL -> diisi 'positive'"
|
|
)
|
|
self.df["direction"] = self.df["direction"].fillna("positive")
|
|
|
|
dir_dist = self.df.drop_duplicates("indicator_id")["direction"].value_counts()
|
|
self.logger.info(f"\n Distribusi direction per indikator:")
|
|
for d, cnt in dir_dist.items():
|
|
tag = "INVERT" if _should_invert(d, self.logger, "load_data check") else "normal"
|
|
self.logger.info(f" {d:<25} : {cnt:>3} indikator [{tag}]")
|
|
|
|
self.logger.info(f"\n Rows loaded : {len(self.df):,}")
|
|
self.logger.info(f" Negara : {self.df['country_id'].nunique()}")
|
|
self.logger.info(f" Indikator : {self.df['indicator_id'].nunique()}")
|
|
self.logger.info(
|
|
f" Tahun : {int(self.df['year'].min())} - {int(self.df['year'].max())}"
|
|
)
|
|
|
|
# =========================================================================
|
|
# STEP 1b: Detect sdgs_start_year + assign framework per (indicator, year)
|
|
# =========================================================================
|
|
|
|
def _detect_sdgs_start_year(self) -> int:
|
|
fies_rows = self.df[
|
|
self.df["indicator_name"].str.lower().str.strip().isin(_FIES_DETECTION_LOWER)
|
|
]
|
|
if not fies_rows.empty:
|
|
sdgs_start = int(fies_rows["year"].min())
|
|
self.logger.info(f" [FIES explicit] sdgs_start_year = {sdgs_start}")
|
|
return sdgs_start
|
|
|
|
ind_min_year = (
|
|
self.df.groupby("indicator_id")["year"]
|
|
.min().reset_index().rename(columns={"year": "min_year"})
|
|
)
|
|
unique_years = sorted(ind_min_year["min_year"].unique())
|
|
if len(unique_years) == 1:
|
|
self.logger.info(" [Fallback] Hanya 1 cluster -> semua MDGs")
|
|
return int(unique_years[0]) + 9999
|
|
|
|
gaps = [
|
|
(unique_years[i+1] - unique_years[i], unique_years[i], unique_years[i+1])
|
|
for i in range(len(unique_years) - 1)
|
|
]
|
|
gaps.sort(reverse=True)
|
|
_, y_before, y_after = gaps[0]
|
|
self.logger.info(f" [Fallback gap] sdgs_start_year = {y_after} (gap {y_before}->{y_after})")
|
|
return int(y_after)
|
|
|
|
def _assign_framework_labels(self):
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info("STEP 1b: ASSIGN FRAMEWORK LABELS (per indicator per year)")
|
|
self.logger.info(f" sdgs_start_year = {self.sdgs_start_year}")
|
|
self.logger.info("=" * 70)
|
|
|
|
df = self.df.copy()
|
|
df["_is_sdg_kw"] = df["indicator_name"].str.lower().str.strip().isin(_SDG_ONLY_LOWER)
|
|
df["framework"] = "MDGs"
|
|
mask_sdgs = df["_is_sdg_kw"] & (df["year"] >= self.sdgs_start_year)
|
|
df.loc[mask_sdgs, "framework"] = "SDGs"
|
|
df = df.drop(columns=["_is_sdg_kw"])
|
|
self.df = df
|
|
|
|
self._ind_year_framework = (
|
|
self.df[["indicator_id", "year", "framework"]]
|
|
.drop_duplicates()
|
|
.reset_index(drop=True)
|
|
)
|
|
|
|
fw_dist = self.df["framework"].value_counts()
|
|
self.logger.info("\n Framework distribution (rows):")
|
|
for fw, cnt in fw_dist.items():
|
|
self.logger.info(f" {fw:<6}: {cnt:,} rows")
|
|
|
|
ind_fw_yr = (
|
|
self._ind_year_framework
|
|
.groupby(["year", "framework"])["indicator_id"]
|
|
.nunique()
|
|
.reset_index()
|
|
.rename(columns={"indicator_id": "n_indicators"})
|
|
.sort_values(["year", "framework"])
|
|
)
|
|
self.logger.info(f"\n {'Year':<6} {'Framework':<8} {'n_indicators'}")
|
|
self.logger.info(" " + "-" * 30)
|
|
for _, r in ind_fw_yr.iterrows():
|
|
self.logger.info(
|
|
f" {int(r['year']):<6} {r['framework']:<8} {int(r['n_indicators'])}"
|
|
)
|
|
|
|
def _count_framework_indicators(self, year: int, framework: str) -> int:
|
|
mask = (
|
|
(self._ind_year_framework["year"] == year) &
|
|
(self._ind_year_framework["framework"] == framework)
|
|
)
|
|
return int(self._ind_year_framework.loc[mask, "indicator_id"].nunique())
|
|
|
|
# =========================================================================
|
|
# CORE HELPER: normalisasi raw value per indikator
|
|
# =========================================================================
|
|
|
|
def _get_norm_value_df(self) -> pd.DataFrame:
|
|
if "framework" not in self.df.columns:
|
|
raise ValueError(
|
|
"Kolom 'framework' tidak ada. Pastikan _assign_framework_labels() dipanggil lebih dulu."
|
|
)
|
|
|
|
norm_parts = []
|
|
for ind_id, grp in self.df.groupby("indicator_id"):
|
|
grp = grp.copy()
|
|
direction = str(grp["direction"].iloc[0])
|
|
do_invert = _should_invert(direction, self.logger, context=f"indicator_id={ind_id}")
|
|
valid_mask = grp["value"].notna()
|
|
n_valid = valid_mask.sum()
|
|
|
|
if n_valid < 2:
|
|
grp["norm_value"] = np.nan
|
|
norm_parts.append(grp)
|
|
continue
|
|
|
|
raw = grp.loc[valid_mask, "value"].values
|
|
v_min, v_max = raw.min(), raw.max()
|
|
normed = np.full(len(grp), np.nan)
|
|
if v_min == v_max:
|
|
normed[valid_mask.values] = 0.5
|
|
else:
|
|
normed[valid_mask.values] = (raw - v_min) / (v_max - v_min)
|
|
|
|
if do_invert:
|
|
normed = np.where(np.isnan(normed), np.nan, 1.0 - normed)
|
|
|
|
grp["norm_value"] = normed
|
|
norm_parts.append(grp)
|
|
|
|
return pd.concat(norm_parts, ignore_index=True)
|
|
|
|
# =========================================================================
|
|
# METADATA BUILDER
|
|
# Menyesuaikan dengan signature: save_etl_metadata(client, metadata: dict)
|
|
# dan skema etl_metadata: source_class, table_name, execution_timestamp,
|
|
# duration_seconds, rows_fetched, rows_transformed, rows_loaded,
|
|
# completeness_pct, config_snapshot, validation_metrics
|
|
# =========================================================================
|
|
|
|
def _build_etl_metadata(
|
|
self,
|
|
table_name: str,
|
|
rows_loaded: int,
|
|
start_time: datetime,
|
|
end_time: datetime,
|
|
status: str,
|
|
error_msg: str = None,
|
|
) -> dict:
|
|
duration = (end_time - start_time).total_seconds() if (start_time and end_time) else 0.0
|
|
return {
|
|
"source_class" : "FoodSecurityAggregator",
|
|
"table_name" : table_name,
|
|
"execution_timestamp": start_time or end_time,
|
|
"duration_seconds" : round(duration, 4),
|
|
"rows_fetched" : rows_loaded,
|
|
"rows_transformed" : rows_loaded,
|
|
"rows_loaded" : rows_loaded,
|
|
"completeness_pct" : 100.0 if status == "success" else 0.0,
|
|
"config_snapshot" : json.dumps({
|
|
"layer" : "gold",
|
|
"write_disposition" : "WRITE_TRUNCATE",
|
|
"normalize_frameworks_jointly": NORMALIZE_FRAMEWORKS_JOINTLY,
|
|
"performance_threshold" : PERFORMANCE_THRESHOLD,
|
|
"status" : status,
|
|
}),
|
|
"validation_metrics" : json.dumps({
|
|
"status" : status,
|
|
"error_msg": error_msg or "",
|
|
}),
|
|
}
|
|
|
|
# =========================================================================
|
|
# STEP 2: agg_pillar_composite
|
|
# =========================================================================
|
|
|
|
def calc_pillar_composite(self) -> pd.DataFrame:
|
|
table_name = "agg_pillar_composite"
|
|
self.load_metadata[table_name]["start_time"] = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info(f"STEP 2: {table_name} -> [Gold] fs_asean_gold")
|
|
self.logger.info("=" * 70)
|
|
|
|
try:
|
|
df_normed = self._get_norm_value_df()
|
|
|
|
df = (
|
|
df_normed
|
|
.groupby(["pillar_id", "pillar_name", "year"])
|
|
.agg(
|
|
pillar_norm =("norm_value", "mean"),
|
|
n_indicators=("indicator_id", "nunique"),
|
|
n_countries =("country_id", "nunique"),
|
|
)
|
|
.reset_index()
|
|
)
|
|
|
|
df["pillar_score_1_100"] = global_minmax(df["pillar_norm"])
|
|
df["rank_in_year"] = (
|
|
df.groupby("year")["pillar_score_1_100"]
|
|
.rank(method="min", ascending=False)
|
|
.astype(int)
|
|
)
|
|
df = add_yoy(df, ["pillar_id"], "pillar_score_1_100")
|
|
|
|
df["pillar_id"] = df["pillar_id"].astype(int)
|
|
df["year"] = df["year"].astype(int)
|
|
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
|
|
df["n_countries"] = safe_int(df["n_countries"], col_name="n_countries", logger=self.logger)
|
|
df["rank_in_year"] = df["rank_in_year"].astype(int)
|
|
df["pillar_norm"] = df["pillar_norm"].astype(float)
|
|
df["pillar_score_1_100"] = df["pillar_score_1_100"].astype(float)
|
|
|
|
schema = [
|
|
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_norm", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_countries", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_score_1_100", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
|
|
]
|
|
rows = load_to_bigquery(
|
|
self.client, df, table_name, layer='gold',
|
|
write_disposition="WRITE_TRUNCATE", schema=schema
|
|
)
|
|
self._finalize(table_name, rows)
|
|
return df
|
|
|
|
except Exception as e:
|
|
self._fail(table_name, e)
|
|
raise
|
|
|
|
# =========================================================================
|
|
# STEP 3: agg_pillar_by_country
|
|
# =========================================================================
|
|
|
|
def calc_pillar_by_country(self) -> pd.DataFrame:
|
|
table_name = "agg_pillar_by_country"
|
|
self.load_metadata[table_name]["start_time"] = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info(f"STEP 3: {table_name} -> [Gold] fs_asean_gold")
|
|
self.logger.info("=" * 70)
|
|
|
|
try:
|
|
df_normed = self._get_norm_value_df()
|
|
|
|
df = (
|
|
df_normed
|
|
.groupby(["country_id", "country_name", "pillar_id", "pillar_name", "year"])
|
|
.agg(pillar_country_norm=("norm_value", "mean"))
|
|
.reset_index()
|
|
)
|
|
|
|
df["pillar_country_score_1_100"] = global_minmax(df["pillar_country_norm"])
|
|
df["rank_in_pillar_year"] = (
|
|
df.groupby(["pillar_id", "year"])["pillar_country_score_1_100"]
|
|
.rank(method="min", ascending=False)
|
|
.astype(int)
|
|
)
|
|
df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100")
|
|
|
|
df["country_id"] = df["country_id"].astype(int)
|
|
df["pillar_id"] = df["pillar_id"].astype(int)
|
|
df["year"] = df["year"].astype(int)
|
|
df["rank_in_pillar_year"] = df["rank_in_pillar_year"].astype(int)
|
|
df["pillar_country_norm"] = df["pillar_country_norm"].astype(float)
|
|
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
|
|
|
|
schema = [
|
|
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_country_norm", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_country_score_1_100", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("rank_in_pillar_year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
|
|
]
|
|
rows = load_to_bigquery(
|
|
self.client, df, table_name, layer='gold',
|
|
write_disposition="WRITE_TRUNCATE", schema=schema
|
|
)
|
|
self._finalize(table_name, rows)
|
|
return df
|
|
|
|
except Exception as e:
|
|
self._fail(table_name, e)
|
|
raise
|
|
|
|
# =========================================================================
|
|
# STEP 4: agg_framework_by_country
|
|
# =========================================================================
|
|
|
|
def _calc_country_composite_inmemory(self) -> pd.DataFrame:
|
|
df_normed = self._get_norm_value_df()
|
|
df = (
|
|
df_normed
|
|
.groupby(["country_id", "country_name", "year"])
|
|
.agg(
|
|
composite_score=("norm_value", "mean"),
|
|
n_indicators =("indicator_id", "nunique"),
|
|
)
|
|
.reset_index()
|
|
)
|
|
df["score_1_100"] = global_minmax(df["composite_score"])
|
|
df["rank_in_asean"] = (
|
|
df.groupby("year")["score_1_100"]
|
|
.rank(method="min", ascending=False)
|
|
.astype(int)
|
|
)
|
|
df = add_yoy(df, ["country_id"], "score_1_100")
|
|
df["country_id"] = df["country_id"].astype(int)
|
|
df["year"] = df["year"].astype(int)
|
|
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
|
|
df["composite_score"] = df["composite_score"].astype(float)
|
|
df["score_1_100"] = df["score_1_100"].astype(float)
|
|
df["rank_in_asean"] = df["rank_in_asean"].astype(int)
|
|
return df
|
|
|
|
def calc_framework_by_country(self) -> pd.DataFrame:
|
|
table_name = "agg_framework_by_country"
|
|
self.load_metadata[table_name]["start_time"] = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info(f"STEP 4: {table_name} -> [Gold] fs_asean_gold")
|
|
self.logger.info("=" * 70)
|
|
|
|
try:
|
|
country_composite = self._calc_country_composite_inmemory()
|
|
df_normed = self._get_norm_value_df()
|
|
parts = []
|
|
|
|
# TOTAL
|
|
agg_total = (
|
|
country_composite[[
|
|
"country_id", "country_name", "year",
|
|
"score_1_100", "n_indicators", "composite_score"
|
|
]]
|
|
.copy()
|
|
.rename(columns={
|
|
"score_1_100" : "framework_score_1_100",
|
|
"composite_score": "framework_norm",
|
|
})
|
|
)
|
|
agg_total["framework"] = "Total"
|
|
parts.append(agg_total)
|
|
|
|
# MDGs pre-SDGs
|
|
pre_sdgs_rows = country_composite[country_composite["year"] < self.sdgs_start_year].copy()
|
|
if not pre_sdgs_rows.empty:
|
|
mdgs_pre = (
|
|
pre_sdgs_rows[[
|
|
"country_id", "country_name", "year",
|
|
"score_1_100", "n_indicators", "composite_score"
|
|
]]
|
|
.copy()
|
|
.rename(columns={
|
|
"score_1_100" : "framework_score_1_100",
|
|
"composite_score": "framework_norm",
|
|
})
|
|
)
|
|
mdgs_pre["framework"] = "MDGs"
|
|
parts.append(mdgs_pre)
|
|
|
|
# MDGs mixed (year >= sdgs_start_year, hanya indikator MDGs)
|
|
mdgs_indicator_ids = set(
|
|
self._ind_year_framework[self._ind_year_framework["framework"] == "MDGs"]["indicator_id"]
|
|
)
|
|
if mdgs_indicator_ids:
|
|
df_mdgs_mixed = df_normed[
|
|
(df_normed["indicator_id"].isin(mdgs_indicator_ids)) &
|
|
(df_normed["year"] >= self.sdgs_start_year)
|
|
].copy()
|
|
if not df_mdgs_mixed.empty:
|
|
agg_mdgs_mixed = (
|
|
df_mdgs_mixed
|
|
.groupby(["country_id", "country_name", "year"])
|
|
.agg(
|
|
framework_norm=("norm_value", "mean"),
|
|
n_indicators =("indicator_id", "nunique"),
|
|
)
|
|
.reset_index()
|
|
)
|
|
if not NORMALIZE_FRAMEWORKS_JOINTLY:
|
|
agg_mdgs_mixed["framework_score_1_100"] = global_minmax(agg_mdgs_mixed["framework_norm"])
|
|
agg_mdgs_mixed["framework"] = "MDGs"
|
|
parts.append(agg_mdgs_mixed)
|
|
|
|
# SDGs
|
|
sdgs_indicator_ids = set(
|
|
self._ind_year_framework[self._ind_year_framework["framework"] == "SDGs"]["indicator_id"]
|
|
)
|
|
if sdgs_indicator_ids:
|
|
df_sdgs = df_normed[
|
|
(df_normed["indicator_id"].isin(sdgs_indicator_ids)) &
|
|
(df_normed["year"] >= self.sdgs_start_year)
|
|
].copy()
|
|
if not df_sdgs.empty:
|
|
agg_sdgs = (
|
|
df_sdgs
|
|
.groupby(["country_id", "country_name", "year"])
|
|
.agg(
|
|
framework_norm=("norm_value", "mean"),
|
|
n_indicators =("indicator_id", "nunique"),
|
|
)
|
|
.reset_index()
|
|
)
|
|
if not NORMALIZE_FRAMEWORKS_JOINTLY:
|
|
agg_sdgs["framework_score_1_100"] = global_minmax(agg_sdgs["framework_norm"])
|
|
agg_sdgs["framework"] = "SDGs"
|
|
parts.append(agg_sdgs)
|
|
|
|
df = pd.concat(parts, ignore_index=True)
|
|
|
|
if NORMALIZE_FRAMEWORKS_JOINTLY:
|
|
mixed_mask = (df["framework"].isin(["MDGs", "SDGs"])) & (df["year"] >= self.sdgs_start_year)
|
|
if mixed_mask.any():
|
|
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(df.loc[mixed_mask, "framework_norm"])
|
|
|
|
df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger)
|
|
df["rank_in_framework_year"] = (
|
|
df.groupby(["framework", "year"])["framework_score_1_100"]
|
|
.rank(method="min", ascending=False)
|
|
.astype(int)
|
|
)
|
|
df = add_yoy(df, ["country_id", "framework"], "framework_score_1_100")
|
|
|
|
df["country_id"] = df["country_id"].astype(int)
|
|
df["year"] = df["year"].astype(int)
|
|
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
|
|
df["rank_in_framework_year"] = safe_int(df["rank_in_framework_year"], col_name="rank_in_framework_year", logger=self.logger)
|
|
df["framework_norm"] = df["framework_norm"].astype(float)
|
|
df["framework_score_1_100"] = df["framework_score_1_100"].astype(float)
|
|
|
|
self._validate_mdgs_equals_total(df, level="country")
|
|
|
|
schema = [
|
|
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("framework_norm", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("rank_in_framework_year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
|
|
]
|
|
rows = load_to_bigquery(
|
|
self.client, df, table_name, layer='gold',
|
|
write_disposition="WRITE_TRUNCATE", schema=schema
|
|
)
|
|
self._finalize(table_name, rows)
|
|
return df
|
|
|
|
except Exception as e:
|
|
self._fail(table_name, e)
|
|
raise
|
|
|
|
# =========================================================================
|
|
# STEP 5: agg_framework_asean (+ performance_status)
|
|
# =========================================================================
|
|
|
|
def calc_framework_asean(self) -> pd.DataFrame:
|
|
table_name = "agg_framework_asean"
|
|
self.load_metadata[table_name]["start_time"] = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info(f"STEP 5: {table_name} -> [Gold] fs_asean_gold")
|
|
self.logger.info(f" performance_status threshold: {PERFORMANCE_THRESHOLD}")
|
|
self.logger.info("=" * 70)
|
|
|
|
try:
|
|
df_normed = self._get_norm_value_df()
|
|
country_composite = self._calc_country_composite_inmemory()
|
|
|
|
country_norm = (
|
|
df_normed
|
|
.groupby(["country_id", "country_name", "year"])["norm_value"]
|
|
.mean().reset_index()
|
|
.rename(columns={"norm_value": "country_norm"})
|
|
)
|
|
asean_overall = (
|
|
country_norm.groupby("year")
|
|
.agg(
|
|
asean_norm =("country_norm", "mean"),
|
|
std_norm =("country_norm", "std"),
|
|
n_countries=("country_norm", "count"),
|
|
)
|
|
.reset_index()
|
|
)
|
|
asean_overall["asean_score_1_100"] = global_minmax(asean_overall["asean_norm"])
|
|
asean_comp = (
|
|
country_composite.groupby("year")["composite_score"]
|
|
.mean().reset_index()
|
|
.rename(columns={"composite_score": "asean_composite"})
|
|
)
|
|
asean_overall = asean_overall.merge(asean_comp, on="year", how="left")
|
|
|
|
parts = []
|
|
|
|
def _n_ind(year_val, framework_val):
|
|
return self._count_framework_indicators(year_val, framework_val)
|
|
|
|
# TOTAL
|
|
total_cols = asean_overall[[
|
|
"year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"
|
|
]].copy().rename(columns={
|
|
"asean_score_1_100": "framework_score_1_100",
|
|
"asean_norm" : "framework_norm",
|
|
"n_countries" : "n_countries_with_data",
|
|
})
|
|
total_cols["n_indicators"] = total_cols["year"].apply(
|
|
lambda y: int(self._ind_year_framework[
|
|
self._ind_year_framework["year"] == y
|
|
]["indicator_id"].nunique())
|
|
)
|
|
total_cols["framework"] = "Total"
|
|
parts.append(total_cols)
|
|
|
|
# MDGs pre-SDGs
|
|
pre_sdgs = asean_overall[asean_overall["year"] < self.sdgs_start_year].copy()
|
|
if not pre_sdgs.empty:
|
|
mdgs_pre = pre_sdgs[[
|
|
"year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"
|
|
]].copy().rename(columns={
|
|
"asean_score_1_100": "framework_score_1_100",
|
|
"asean_norm" : "framework_norm",
|
|
"n_countries" : "n_countries_with_data",
|
|
})
|
|
mdgs_pre["n_indicators"] = mdgs_pre["year"].apply(lambda y: _n_ind(y, "MDGs"))
|
|
mdgs_pre["framework"] = "MDGs"
|
|
parts.append(mdgs_pre)
|
|
|
|
# MDGs mixed
|
|
mdgs_indicator_ids = set(
|
|
self._ind_year_framework[self._ind_year_framework["framework"] == "MDGs"]["indicator_id"]
|
|
)
|
|
if mdgs_indicator_ids:
|
|
df_mdgs_mixed = df_normed[
|
|
(df_normed["indicator_id"].isin(mdgs_indicator_ids)) &
|
|
(df_normed["year"] >= self.sdgs_start_year)
|
|
].copy()
|
|
if not df_mdgs_mixed.empty:
|
|
cn = (
|
|
df_mdgs_mixed
|
|
.groupby(["country_id", "year"])["norm_value"].mean()
|
|
.reset_index().rename(columns={"norm_value": "country_norm"})
|
|
)
|
|
asean_mdgs = cn.groupby("year").agg(
|
|
framework_norm =("country_norm", "mean"),
|
|
std_norm =("country_norm", "std"),
|
|
n_countries_with_data=("country_id", "count"),
|
|
).reset_index()
|
|
asean_mdgs["n_indicators"] = asean_mdgs["year"].apply(lambda y: _n_ind(y, "MDGs"))
|
|
if not NORMALIZE_FRAMEWORKS_JOINTLY:
|
|
asean_mdgs["framework_score_1_100"] = global_minmax(asean_mdgs["framework_norm"])
|
|
asean_mdgs["framework"] = "MDGs"
|
|
parts.append(asean_mdgs)
|
|
|
|
# SDGs
|
|
sdgs_indicator_ids = set(
|
|
self._ind_year_framework[self._ind_year_framework["framework"] == "SDGs"]["indicator_id"]
|
|
)
|
|
if sdgs_indicator_ids:
|
|
df_sdgs = df_normed[
|
|
(df_normed["indicator_id"].isin(sdgs_indicator_ids)) &
|
|
(df_normed["year"] >= self.sdgs_start_year)
|
|
].copy()
|
|
if not df_sdgs.empty:
|
|
cn = (
|
|
df_sdgs
|
|
.groupby(["country_id", "year"])["norm_value"].mean()
|
|
.reset_index().rename(columns={"norm_value": "country_norm"})
|
|
)
|
|
asean_sdgs = cn.groupby("year").agg(
|
|
framework_norm =("country_norm", "mean"),
|
|
std_norm =("country_norm", "std"),
|
|
n_countries_with_data=("country_id", "count"),
|
|
).reset_index()
|
|
asean_sdgs["n_indicators"] = asean_sdgs["year"].apply(lambda y: _n_ind(y, "SDGs"))
|
|
if not NORMALIZE_FRAMEWORKS_JOINTLY:
|
|
asean_sdgs["framework_score_1_100"] = global_minmax(asean_sdgs["framework_norm"])
|
|
asean_sdgs["framework"] = "SDGs"
|
|
parts.append(asean_sdgs)
|
|
|
|
df = pd.concat(parts, ignore_index=True)
|
|
|
|
if NORMALIZE_FRAMEWORKS_JOINTLY:
|
|
mixed_mask = (df["framework"].isin(["MDGs", "SDGs"])) & (df["year"] >= self.sdgs_start_year)
|
|
if mixed_mask.any():
|
|
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(df.loc[mixed_mask, "framework_norm"])
|
|
|
|
df = check_and_dedup(df, ["framework", "year"], context=table_name, logger=self.logger)
|
|
df = add_yoy(df, ["framework"], "framework_score_1_100")
|
|
|
|
df["performance_status"] = df["framework_score_1_100"].apply(_performance_status)
|
|
df["year"] = df["year"].astype(int)
|
|
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
|
|
df["n_countries_with_data"] = safe_int(df["n_countries_with_data"], col_name="n_countries_with_data", logger=self.logger)
|
|
for col in ["framework_norm", "std_norm", "framework_score_1_100"]:
|
|
df[col] = df[col].astype(float)
|
|
df["performance_status"] = df["performance_status"].astype(str)
|
|
|
|
self._validate_mdgs_equals_total(df, level="asean")
|
|
|
|
self.logger.info(f"\n performance_status summary (threshold={PERFORMANCE_THRESHOLD}):")
|
|
for fw in df["framework"].unique():
|
|
sub = df[df["framework"] == fw].sort_values("year")
|
|
for _, r in sub.iterrows():
|
|
self.logger.info(
|
|
f" {fw:<8} {int(r['year'])}: "
|
|
f"score={r['framework_score_1_100']:.2f} "
|
|
f"n_ind={int(r['n_indicators'])} "
|
|
f"-> {r['performance_status']}"
|
|
)
|
|
|
|
schema = [
|
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_countries_with_data", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("framework_norm", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("std_norm", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("performance_status", "STRING", mode="REQUIRED"),
|
|
]
|
|
rows = load_to_bigquery(
|
|
self.client, df, table_name, layer='gold',
|
|
write_disposition="WRITE_TRUNCATE", schema=schema
|
|
)
|
|
self._finalize(table_name, rows)
|
|
return df
|
|
|
|
except Exception as e:
|
|
self._fail(table_name, e)
|
|
raise
|
|
|
|
# =========================================================================
|
|
# STEP 6: agg_narrative_overview
|
|
# =========================================================================
|
|
|
|
def calc_narrative_overview(
|
|
self,
|
|
df_framework_asean: pd.DataFrame,
|
|
df_framework_by_country: pd.DataFrame,
|
|
) -> pd.DataFrame:
|
|
table_name = "agg_narrative_overview"
|
|
self.load_metadata[table_name]["start_time"] = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold")
|
|
self.logger.info("=" * 70)
|
|
|
|
try:
|
|
asean_total = (
|
|
df_framework_asean[df_framework_asean["framework"] == "Total"]
|
|
.sort_values("year")
|
|
.reset_index(drop=True)
|
|
)
|
|
|
|
score_by_year = dict(zip(asean_total["year"].astype(int), asean_total["framework_score_1_100"].astype(float)))
|
|
status_by_year = dict(zip(asean_total["year"].astype(int), asean_total["performance_status"].astype(str)))
|
|
country_total = df_framework_by_country[df_framework_by_country["framework"] == "Total"].copy()
|
|
|
|
records = []
|
|
|
|
for _, row in asean_total.iterrows():
|
|
yr = int(row["year"])
|
|
score = float(row["framework_score_1_100"])
|
|
perf_status = str(row["performance_status"])
|
|
yoy = row["year_over_year_change"]
|
|
yoy_val = float(yoy) if pd.notna(yoy) else None
|
|
|
|
n_mdg = self._count_framework_indicators(yr, "MDGs")
|
|
n_sdg = self._count_framework_indicators(yr, "SDGs")
|
|
n_total_ind = int(
|
|
self._ind_year_framework[
|
|
self._ind_year_framework["year"] == yr
|
|
]["indicator_id"].nunique()
|
|
)
|
|
|
|
prev_score = score_by_year.get(yr - 1, None)
|
|
prev_status = status_by_year.get(yr - 1, "N/A")
|
|
yoy_pct = (
|
|
(yoy_val / prev_score * 100)
|
|
if (yoy_val is not None and prev_score is not None and prev_score != 0)
|
|
else None
|
|
)
|
|
|
|
yr_country = (
|
|
country_total[country_total["year"] == yr]
|
|
.sort_values("rank_in_framework_year")
|
|
.reset_index(drop=True)
|
|
)
|
|
|
|
ranking_list = []
|
|
for _, cr in yr_country.iterrows():
|
|
cr_yoy = cr.get("year_over_year_change", None)
|
|
ranking_list.append({
|
|
"rank": int(cr["rank_in_framework_year"]),
|
|
"country_name": str(cr["country_name"]),
|
|
"score": round(float(cr["framework_score_1_100"]), 2),
|
|
"yoy_change": round(float(cr_yoy), 2) if pd.notna(cr_yoy) else None,
|
|
})
|
|
country_ranking_json = json.dumps(ranking_list, ensure_ascii=False)
|
|
|
|
yr_country_yoy = yr_country.dropna(subset=["year_over_year_change"])
|
|
if not yr_country_yoy.empty:
|
|
best_idx = yr_country_yoy["year_over_year_change"].idxmax()
|
|
worst_idx = yr_country_yoy["year_over_year_change"].idxmin()
|
|
most_improved_country = str(yr_country_yoy.loc[best_idx, "country_name"])
|
|
most_improved_delta = round(float(yr_country_yoy.loc[best_idx, "year_over_year_change"]), 2)
|
|
most_declined_country = str(yr_country_yoy.loc[worst_idx, "country_name"])
|
|
most_declined_delta = round(float(yr_country_yoy.loc[worst_idx, "year_over_year_change"]), 2)
|
|
else:
|
|
most_improved_country = most_declined_country = None
|
|
most_improved_delta = most_declined_delta = None
|
|
|
|
narrative = _build_overview_narrative(
|
|
year = yr,
|
|
n_mdg = n_mdg,
|
|
n_sdg = n_sdg,
|
|
n_total_ind = n_total_ind,
|
|
score = score,
|
|
performance_status = perf_status,
|
|
yoy_val = yoy_val,
|
|
yoy_pct = yoy_pct,
|
|
prev_year = yr - 1,
|
|
prev_score = prev_score,
|
|
prev_performance_status = prev_status,
|
|
ranking_list = ranking_list,
|
|
most_improved_country = most_improved_country,
|
|
most_improved_delta = most_improved_delta,
|
|
most_declined_country = most_declined_country,
|
|
most_declined_delta = most_declined_delta,
|
|
)
|
|
|
|
records.append({
|
|
"year": yr,
|
|
"n_mdg_indicators": n_mdg,
|
|
"n_sdg_indicators": n_sdg,
|
|
"n_total_indicators": n_total_ind,
|
|
"asean_total_score": round(score, 2),
|
|
"performance_status": perf_status,
|
|
"yoy_change": yoy_val,
|
|
"yoy_change_pct": round(yoy_pct, 2) if yoy_pct is not None else None,
|
|
"country_ranking_json": country_ranking_json,
|
|
"most_improved_country": most_improved_country,
|
|
"most_improved_delta": most_improved_delta,
|
|
"most_declined_country": most_declined_country,
|
|
"most_declined_delta": most_declined_delta,
|
|
"narrative_overview": narrative,
|
|
})
|
|
|
|
df = pd.DataFrame(records)
|
|
df["year"] = df["year"].astype(int)
|
|
df["n_mdg_indicators"] = df["n_mdg_indicators"].astype(int)
|
|
df["n_sdg_indicators"] = df["n_sdg_indicators"].astype(int)
|
|
df["n_total_indicators"] = df["n_total_indicators"].astype(int)
|
|
df["asean_total_score"] = df["asean_total_score"].astype(float)
|
|
df["performance_status"] = df["performance_status"].astype(str)
|
|
for col in ["yoy_change", "yoy_change_pct", "most_improved_delta", "most_declined_delta"]:
|
|
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
|
|
|
|
schema = [
|
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_mdg_indicators", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_sdg_indicators", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("n_total_indicators", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("asean_total_score", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("performance_status", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("yoy_change_pct", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("country_ranking_json", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("most_improved_country", "STRING", mode="NULLABLE"),
|
|
bigquery.SchemaField("most_improved_delta", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("most_declined_country", "STRING", mode="NULLABLE"),
|
|
bigquery.SchemaField("most_declined_delta", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("narrative_overview", "STRING", mode="REQUIRED"),
|
|
]
|
|
rows = load_to_bigquery(
|
|
self.client, df, table_name, layer='gold',
|
|
write_disposition="WRITE_TRUNCATE", schema=schema,
|
|
)
|
|
self._finalize(table_name, rows)
|
|
return df
|
|
|
|
except Exception as e:
|
|
self._fail(table_name, e)
|
|
raise
|
|
|
|
# =========================================================================
|
|
# STEP 7: agg_narrative_pillar
|
|
# =========================================================================
|
|
|
|
def calc_narrative_pillar(
|
|
self,
|
|
df_pillar_composite: pd.DataFrame,
|
|
df_pillar_by_country: pd.DataFrame,
|
|
) -> pd.DataFrame:
|
|
table_name = "agg_narrative_pillar"
|
|
self.load_metadata[table_name]["start_time"] = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info(f"STEP 7: {table_name} -> [Gold] fs_asean_gold")
|
|
self.logger.info("=" * 70)
|
|
|
|
try:
|
|
records = []
|
|
years = sorted(df_pillar_composite["year"].unique())
|
|
|
|
for yr in years:
|
|
yr_pillars = (
|
|
df_pillar_composite[df_pillar_composite["year"] == yr]
|
|
.sort_values("rank_in_year")
|
|
.reset_index(drop=True)
|
|
)
|
|
yr_country_pillar = df_pillar_by_country[df_pillar_by_country["year"] == yr]
|
|
|
|
strongest_pillar = yr_pillars.iloc[0] if len(yr_pillars) > 0 else None
|
|
weakest_pillar = yr_pillars.iloc[-1] if len(yr_pillars) > 0 else None
|
|
|
|
yr_pillars_yoy = yr_pillars.dropna(subset=["year_over_year_change"])
|
|
if not yr_pillars_yoy.empty:
|
|
best_p_idx = yr_pillars_yoy["year_over_year_change"].idxmax()
|
|
worst_p_idx = yr_pillars_yoy["year_over_year_change"].idxmin()
|
|
most_improved_pillar = str(yr_pillars_yoy.loc[best_p_idx, "pillar_name"])
|
|
most_improved_delta = round(float(yr_pillars_yoy.loc[best_p_idx, "year_over_year_change"]), 2)
|
|
most_declined_pillar = str(yr_pillars_yoy.loc[worst_p_idx, "pillar_name"])
|
|
most_declined_delta = round(float(yr_pillars_yoy.loc[worst_p_idx, "year_over_year_change"]), 2)
|
|
else:
|
|
most_improved_pillar = most_declined_pillar = None
|
|
most_improved_delta = most_declined_delta = None
|
|
|
|
for _, prow in yr_pillars.iterrows():
|
|
p_id = int(prow["pillar_id"])
|
|
p_name = str(prow["pillar_name"])
|
|
p_score = float(prow["pillar_score_1_100"])
|
|
p_rank = int(prow["rank_in_year"])
|
|
p_yoy = prow["year_over_year_change"]
|
|
p_yoy_val = float(p_yoy) if pd.notna(p_yoy) else None
|
|
|
|
p_country = (
|
|
yr_country_pillar[yr_country_pillar["pillar_id"] == p_id]
|
|
.sort_values("rank_in_pillar_year")
|
|
.reset_index(drop=True)
|
|
)
|
|
if not p_country.empty:
|
|
top_country = str(p_country.iloc[0]["country_name"])
|
|
top_country_score = round(float(p_country.iloc[0]["pillar_country_score_1_100"]), 2)
|
|
bot_country = str(p_country.iloc[-1]["country_name"])
|
|
bot_country_score = round(float(p_country.iloc[-1]["pillar_country_score_1_100"]), 2)
|
|
else:
|
|
top_country = bot_country = None
|
|
top_country_score = bot_country_score = None
|
|
|
|
narrative = _build_pillar_narrative(
|
|
year = yr,
|
|
pillar_name = p_name,
|
|
pillar_score = p_score,
|
|
rank_in_year = p_rank,
|
|
n_pillars = len(yr_pillars),
|
|
yoy_val = p_yoy_val,
|
|
top_country = top_country,
|
|
top_country_score = top_country_score,
|
|
bot_country = bot_country,
|
|
bot_country_score = bot_country_score,
|
|
strongest_pillar = str(strongest_pillar["pillar_name"]) if strongest_pillar is not None else None,
|
|
strongest_score = round(float(strongest_pillar["pillar_score_1_100"]), 2) if strongest_pillar is not None else None,
|
|
weakest_pillar = str(weakest_pillar["pillar_name"]) if weakest_pillar is not None else None,
|
|
weakest_score = round(float(weakest_pillar["pillar_score_1_100"]), 2) if weakest_pillar is not None else None,
|
|
most_improved_pillar = most_improved_pillar,
|
|
most_improved_delta = most_improved_delta,
|
|
most_declined_pillar = most_declined_pillar,
|
|
most_declined_delta = most_declined_delta,
|
|
)
|
|
|
|
records.append({
|
|
"year": yr,
|
|
"pillar_id": p_id,
|
|
"pillar_name": p_name,
|
|
"pillar_score": round(p_score, 2),
|
|
"rank_in_year": p_rank,
|
|
"yoy_change": p_yoy_val,
|
|
"top_country": top_country,
|
|
"top_country_score": top_country_score,
|
|
"bottom_country": bot_country,
|
|
"bottom_country_score": bot_country_score,
|
|
"narrative_pillar": narrative,
|
|
})
|
|
|
|
df = pd.DataFrame(records)
|
|
df["year"] = df["year"].astype(int)
|
|
df["pillar_id"] = df["pillar_id"].astype(int)
|
|
df["rank_in_year"] = df["rank_in_year"].astype(int)
|
|
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
|
|
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
|
|
|
|
schema = [
|
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
|
|
bigquery.SchemaField("pillar_score", "FLOAT", mode="REQUIRED"),
|
|
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
|
|
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("top_country", "STRING", mode="NULLABLE"),
|
|
bigquery.SchemaField("top_country_score", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("bottom_country", "STRING", mode="NULLABLE"),
|
|
bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"),
|
|
bigquery.SchemaField("narrative_pillar", "STRING", mode="REQUIRED"),
|
|
]
|
|
rows = load_to_bigquery(
|
|
self.client, df, table_name, layer='gold',
|
|
write_disposition="WRITE_TRUNCATE", schema=schema,
|
|
)
|
|
self._finalize(table_name, rows)
|
|
return df
|
|
|
|
except Exception as e:
|
|
self._fail(table_name, e)
|
|
raise
|
|
|
|
# =========================================================================
|
|
# HELPERS
|
|
# =========================================================================
|
|
|
|
def _validate_mdgs_equals_total(self, df: pd.DataFrame, level: str = ""):
|
|
self.logger.info(f"\n Validasi MDGs < {self.sdgs_start_year} == Total [{level}]:")
|
|
group_by = ["year"] if level.startswith("asean") else ["country_id", "year"]
|
|
mdgs_pre = df[(df["framework"] == "MDGs") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "mdgs_score"})
|
|
total_pre = df[(df["framework"] == "Total") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "total_score"})
|
|
if mdgs_pre.empty and total_pre.empty:
|
|
self.logger.info(f" -> Tidak ada data pre-{self.sdgs_start_year} (skip)")
|
|
return
|
|
if mdgs_pre.empty or total_pre.empty:
|
|
self.logger.warning(f" -> [WARNING] Salah satu kosong: MDGs={len(mdgs_pre)}, Total={len(total_pre)}")
|
|
return
|
|
check = mdgs_pre.merge(total_pre, on=group_by)
|
|
max_diff = (check["mdgs_score"] - check["total_score"]).abs().max()
|
|
status = "OK (identik)" if max_diff < 0.01 else f"MISMATCH! max_diff={max_diff:.6f}"
|
|
self.logger.info(f" -> {status} (n_checked={len(check)})")
|
|
|
|
def _build_etl_metadata(
|
|
self,
|
|
table_name: str,
|
|
rows_loaded: int,
|
|
start_time: datetime,
|
|
end_time: datetime,
|
|
status: str,
|
|
error_msg: str = None,
|
|
) -> dict:
|
|
"""
|
|
Susun dict metadata sesuai signature save_etl_metadata(client, metadata: dict)
|
|
dan kolom skema etl_metadata di bigquery_helpers.py:
|
|
source_class, table_name, execution_timestamp, duration_seconds,
|
|
rows_fetched, rows_transformed, rows_loaded, completeness_pct,
|
|
config_snapshot, validation_metrics
|
|
"""
|
|
duration = (end_time - start_time).total_seconds() if (start_time and end_time) else 0.0
|
|
return {
|
|
"source_class" : "FoodSecurityAggregator",
|
|
"table_name" : table_name,
|
|
"execution_timestamp": start_time or end_time,
|
|
"duration_seconds" : round(duration, 4),
|
|
"rows_fetched" : rows_loaded,
|
|
"rows_transformed" : rows_loaded,
|
|
"rows_loaded" : rows_loaded,
|
|
"completeness_pct" : 100.0 if status == "success" else 0.0,
|
|
"config_snapshot" : json.dumps({
|
|
"layer" : "gold",
|
|
"write_disposition" : "WRITE_TRUNCATE",
|
|
"normalize_frameworks_jointly": NORMALIZE_FRAMEWORKS_JOINTLY,
|
|
"performance_threshold" : PERFORMANCE_THRESHOLD,
|
|
"status" : status,
|
|
}),
|
|
"validation_metrics" : json.dumps({
|
|
"status" : status,
|
|
"error_msg": error_msg or "",
|
|
}),
|
|
}
|
|
|
|
def _finalize(self, table_name: str, rows_loaded: int):
|
|
"""
|
|
Tandai tabel sukses. Catat ke etl_logs dan etl_metadata.
|
|
Pemanggilan: save_etl_metadata(client, metadata_dict)
|
|
"""
|
|
end_time = datetime.now()
|
|
start_time = self.load_metadata[table_name].get("start_time")
|
|
|
|
self.load_metadata[table_name].update({
|
|
"rows_loaded": rows_loaded,
|
|
"status" : "success",
|
|
"end_time" : end_time,
|
|
})
|
|
|
|
log_update(self.client, "DW", table_name, "full_load", rows_loaded)
|
|
|
|
try:
|
|
save_etl_metadata(
|
|
self.client,
|
|
self._build_etl_metadata(
|
|
table_name = table_name,
|
|
rows_loaded = rows_loaded,
|
|
start_time = start_time,
|
|
end_time = end_time,
|
|
status = "success",
|
|
)
|
|
)
|
|
except Exception as meta_err:
|
|
# Error metadata tidak boleh menghentikan pipeline
|
|
self.logger.warning(
|
|
f" [METADATA WARNING] Gagal simpan etl_metadata untuk {table_name}: {meta_err}"
|
|
)
|
|
|
|
self.logger.info(f" [OK] {table_name}: {rows_loaded:,} rows -> [Gold] fs_asean_gold")
|
|
|
|
def _fail(self, table_name: str, error: Exception):
|
|
"""
|
|
Tandai tabel gagal. Catat ke etl_logs dan etl_metadata.
|
|
Pemanggilan: save_etl_metadata(client, metadata_dict)
|
|
"""
|
|
end_time = datetime.now()
|
|
start_time = self.load_metadata[table_name].get("start_time")
|
|
error_msg = str(error)
|
|
|
|
self.load_metadata[table_name].update({
|
|
"status" : "failed",
|
|
"end_time": end_time,
|
|
})
|
|
|
|
log_update(self.client, "DW", table_name, "full_load", 0, "failed", error_msg)
|
|
|
|
try:
|
|
save_etl_metadata(
|
|
self.client,
|
|
self._build_etl_metadata(
|
|
table_name = table_name,
|
|
rows_loaded = 0,
|
|
start_time = start_time,
|
|
end_time = end_time,
|
|
status = "failed",
|
|
error_msg = error_msg,
|
|
)
|
|
)
|
|
except Exception as meta_err:
|
|
self.logger.warning(
|
|
f" [METADATA WARNING] Gagal simpan etl_metadata untuk {table_name}: {meta_err}"
|
|
)
|
|
|
|
self.logger.error(f" [FAIL] {table_name}: {error_msg}")
|
|
|
|
# =========================================================================
|
|
# RUN
|
|
# =========================================================================
|
|
|
|
def run(self):
|
|
start = datetime.now()
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info("FOOD SECURITY AGGREGATION — 6 TABLES -> fs_asean_gold")
|
|
self.logger.info(" Source : fact_asean_food_security_selected")
|
|
self.logger.info(" Outputs : agg_pillar_composite | agg_pillar_by_country")
|
|
self.logger.info(" agg_framework_by_country | agg_framework_asean")
|
|
self.logger.info(" agg_narrative_overview | agg_narrative_pillar")
|
|
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD} (Good/Bad)")
|
|
self.logger.info("=" * 70)
|
|
|
|
self.load_data()
|
|
self.sdgs_start_year = self._detect_sdgs_start_year()
|
|
self._assign_framework_labels()
|
|
|
|
df_pillar_composite = self.calc_pillar_composite()
|
|
df_pillar_by_country = self.calc_pillar_by_country()
|
|
df_framework_by_country = self.calc_framework_by_country()
|
|
df_framework_asean = self.calc_framework_asean()
|
|
|
|
self.calc_narrative_overview(
|
|
df_framework_asean = df_framework_asean,
|
|
df_framework_by_country = df_framework_by_country,
|
|
)
|
|
self.calc_narrative_pillar(
|
|
df_pillar_composite = df_pillar_composite,
|
|
df_pillar_by_country = df_pillar_by_country,
|
|
)
|
|
|
|
duration = (datetime.now() - start).total_seconds()
|
|
total_rows = sum(m["rows_loaded"] for m in self.load_metadata.values())
|
|
|
|
self.logger.info("\n" + "=" * 70)
|
|
self.logger.info("SELESAI")
|
|
self.logger.info("=" * 70)
|
|
self.logger.info(f" Durasi : {duration:.2f}s")
|
|
self.logger.info(f" Total rows : {total_rows:,}")
|
|
for tbl, meta in self.load_metadata.items():
|
|
icon = "[OK]" if meta["status"] == "success" else "[FAIL]"
|
|
self.logger.info(f" {icon} {tbl:<35} {meta['rows_loaded']:>10,}")
|
|
|
|
|
|
# =============================================================================
|
|
# AIRFLOW TASK
|
|
# =============================================================================
|
|
|
|
def run_aggregation():
|
|
"""
|
|
Airflow task: Hitung semua agregasi dari fact_asean_food_security_selected.
|
|
Dipanggil setelah analytical_layer_to_gold selesai.
|
|
"""
|
|
from scripts.bigquery_config import get_bigquery_client
|
|
client = get_bigquery_client()
|
|
agg = FoodSecurityAggregator(client)
|
|
agg.run()
|
|
total = sum(m["rows_loaded"] for m in agg.load_metadata.values())
|
|
print(f"Aggregation completed: {total:,} total rows loaded")
|
|
|
|
|
|
# =============================================================================
|
|
# MAIN
|
|
# =============================================================================
|
|
|
|
if __name__ == "__main__":
|
|
import io
|
|
|
|
if _sys.stdout.encoding and _sys.stdout.encoding.lower() not in ("utf-8", "utf8"):
|
|
_sys.stdout = io.TextIOWrapper(_sys.stdout.buffer, encoding="utf-8", errors="replace")
|
|
if _sys.stderr.encoding and _sys.stderr.encoding.lower() not in ("utf-8", "utf8"):
|
|
_sys.stderr = io.TextIOWrapper(_sys.stderr.buffer, encoding="utf-8", errors="replace")
|
|
|
|
print("=" * 70)
|
|
print("FOOD SECURITY AGGREGATION -> fs_asean_gold")
|
|
print(f" Source : fact_asean_food_security_selected")
|
|
print(f" NORMALIZE_FRAMEWORKS_JOINTLY : {NORMALIZE_FRAMEWORKS_JOINTLY}")
|
|
print(f" PERFORMANCE_THRESHOLD : {PERFORMANCE_THRESHOLD}")
|
|
print("=" * 70)
|
|
|
|
logger = setup_logging()
|
|
for handler in logger.handlers:
|
|
handler.__class__ = _SafeStreamHandler
|
|
|
|
client = get_bigquery_client()
|
|
agg = FoodSecurityAggregator(client)
|
|
agg.run()
|
|
|
|
print("\n" + "=" * 70)
|
|
print("[OK] SELESAI")
|
|
print("=" * 70) |