""" BIGQUERY ANALYSIS LAYER - FOOD SECURITY AGGREGATION Semua agregasi pakai norm_value dari _get_norm_value_df() PERBAIKAN (vs versi sebelumnya): ───────────────────────────────────────────────────────────────────────────── 1. NORMALIZE_FRAMEWORKS_JOINTLY dihapus. Setelah perbaikan di analytical_layer, norm_value_1_100 sudah dihitung SEKALI per indikator dari seluruh data (semua tahun, semua negara). Tidak ada lagi rescaling ulang per-framework di layer ini. Semua framework (MDGs, SDGs, Total) menggunakan norm_value yang SAMA sebagai basis, sehingga skor mereka berada pada skala yang setara. 2. _get_norm_value_df() DISEDERHANAKAN. Fungsi ini sekarang hanya membaca kolom norm_value_1_100 yang sudah ada di fact_asean_food_security_selected (hasil dari analytical_layer), kemudian memetakan ke skala 0-1 untuk keperluan agregasi internal. TIDAK ada lagi normalisasi ulang per indikator di sini. 3. global_minmax() TETAP DIGUNAKAN untuk mengubah rata-rata norm (0-1) menjadi skor 1-100 di level agregasi (pillar / country / asean). Ini adalah rescaling level AGREGAT (bukan level indikator), sehingga masih valid dan tidak menimbulkan bias komparabilitas. 4. Framework MDGs dan SDGs sekarang comparable: - Jika skor SDGs < skor MDGs → memang karena indikator SDGs mengukur dimensi deprivasi yang lebih dalam (substantif), bukan artefak teknis. - Log diagnostik ditambahkan untuk memverifikasi ini. 5. Kolom 'condition' (good/moderate/bad) TETAP dengan threshold yang sama. Simpan 6 tabel ke fs_asean_gold (layer='gold'): - agg_pillar_composite - agg_pillar_by_country - agg_framework_by_country - agg_framework_asean - agg_narrative_overview - agg_narrative_pillar SOURCE TABLE: fact_asean_food_security_selected """ import pandas as pd import numpy as np from datetime import datetime import logging import json import sys as _sys from scripts.bigquery_config import get_bigquery_client from scripts.bigquery_helpers import ( log_update, load_to_bigquery, read_from_bigquery, setup_logging, save_etl_metadata, ) from google.cloud import bigquery # ============================================================================= # KONSTANTA GLOBAL # ============================================================================= DIRECTION_INVERT_KEYWORDS = frozenset({ "negative", "lower_better", "lower_is_better", "inverse", "neg", }) DIRECTION_POSITIVE_KEYWORDS = frozenset({ "positive", "higher_better", "higher_is_better", }) # Threshold kondisi — fixed absolute, skala 1-100 THRESHOLD_BAD = 40.0 THRESHOLD_GOOD = 60.0 def assign_condition(score) -> str: """ Assign kondisi berdasarkan score skala 1-100 (direction-aware, nilai tinggi = lebih baik). Returns: 'good' / 'moderate' / 'bad' / None jika NaN """ if score is None or (isinstance(score, float) and np.isnan(score)): return None if score > THRESHOLD_GOOD: return 'good' if score < THRESHOLD_BAD: return 'bad' return 'moderate' # ============================================================================= # Windows CP1252 safe logging # ============================================================================= class _SafeStreamHandler(logging.StreamHandler): def emit(self, record): try: super().emit(record) except UnicodeEncodeError: try: msg = self.format(record) self.stream.write( msg.encode("utf-8", errors="replace").decode("ascii", errors="replace") + self.terminator ) self.flush() except Exception: self.handleError(record) # ============================================================================= # HELPERS # ============================================================================= def _should_invert(direction: str, logger=None, context: str = "") -> bool: d = str(direction).lower().strip() if d in DIRECTION_INVERT_KEYWORDS: return True if d in DIRECTION_POSITIVE_KEYWORDS: return False if logger: logger.warning( f" [DIRECTION WARNING] Unknown direction '{direction}' " f"{'(' + context + ')' if context else ''}. Defaulting to positive (no invert)." ) return False def global_minmax(series: pd.Series, lo: float = 1.0, hi: float = 100.0) -> pd.Series: """ Rescale series ke rentang [lo, hi]. Digunakan untuk mengubah norm agregat (0-1) menjadi skor 1-100 di level pillar / country / asean. Bukan untuk normalisasi indikator mentah. """ values = series.dropna().values if len(values) == 0: return pd.Series(np.nan, index=series.index) v_min, v_max = values.min(), values.max() if v_min == v_max: return pd.Series((lo + hi) / 2.0, index=series.index) result = np.full(len(series), np.nan) not_nan = series.notna() raw = series[not_nan].values result[not_nan.values] = lo + (raw - v_min) / (v_max - v_min) * (hi - lo) return pd.Series(result, index=series.index) def add_yoy(df: pd.DataFrame, group_cols: list, score_col: str) -> pd.DataFrame: df = df.sort_values(group_cols + ["year"]).reset_index(drop=True) if group_cols: df["year_over_year_change"] = df.groupby(group_cols)[score_col].diff() else: df["year_over_year_change"] = df[score_col].diff() return df def safe_int( series: pd.Series, fill: int = 0, col_name: str = "", logger=None ) -> pd.Series: n_nan = series.isna().sum() if n_nan > 0 and logger: logger.warning( f" [NaN WARNING] Kolom '{col_name}' punya {n_nan} NaN -> di-fill dengan {fill}" ) return series.fillna(fill).astype(int) def check_and_dedup( df: pd.DataFrame, key_cols: list, context: str = "", logger=None ) -> pd.DataFrame: dupes = df.duplicated(subset=key_cols, keep=False) if dupes.any(): n_dupes = dupes.sum() if logger: logger.warning( f" [DEDUP WARNING] {context}: {n_dupes} duplikat rows pada {key_cols}. " f"Di-aggregate dengan mean." ) numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist() agg_dict = { c: ("mean" if c in numeric_cols else "first") for c in df.columns if c not in key_cols } df = df.groupby(key_cols, as_index=False).agg(agg_dict) return df def add_condition_column(df: pd.DataFrame, score_col: str) -> pd.DataFrame: df['condition'] = df[score_col].apply(assign_condition) return df def log_condition_summary(df: pd.DataFrame, context: str, logger) -> None: dist = df['condition'].value_counts() logger.info( f" Condition distribution ({context}): " + " | ".join(f"{c}: {n:,}" for c, n in dist.items()) ) # ============================================================================= # NARRATIVE BUILDER FUNCTIONS (tidak berubah) # ============================================================================= def _fmt_score(score) -> str: if score is None or (isinstance(score, float) and np.isnan(score)): return "N/A" return f"{score:.2f}" def _fmt_delta(delta) -> str: if delta is None or (isinstance(delta, float) and np.isnan(delta)): return "N/A" sign = "+" if delta >= 0 else "" return f"{sign}{delta:.2f}" def _build_overview_narrative( year, n_mdg, n_sdg, n_total_ind, score, yoy_val, yoy_pct, prev_year, prev_score, ranking_list, most_improved_country, most_improved_delta, most_declined_country, most_declined_delta, ) -> str: parts_ind = [] if n_mdg > 0: parts_ind.append(f"{n_mdg} MDG indicator{'s' if n_mdg > 1 else ''}") if n_sdg > 0: parts_ind.append(f"{n_sdg} SDG indicator{'s' if n_sdg > 1 else ''}") if parts_ind: ind_detail = " and ".join(parts_ind) sent1 = ( f"In {year}, the ASEAN food security assessment incorporated a total of " f"{n_total_ind} indicator{'s' if n_total_ind != 1 else ''}, " f"consisting of {ind_detail}." ) else: sent1 = ( f"In {year}, the ASEAN food security assessment incorporated " f"{n_total_ind} indicator{'s' if n_total_ind != 1 else ''}." ) if yoy_val is not None and prev_score is not None: direction_word = "increasing" if yoy_val >= 0 else "decreasing" pct_clause = "" if yoy_pct is not None: abs_pct = abs(yoy_pct) trend_word = "improvement" if yoy_val >= 0 else "decline" pct_clause = f", which represents a {abs_pct:.2f}% {trend_word} year-over-year" sent2 = ( f"The ASEAN overall score (Total framework) reached {_fmt_score(score)}, " f"{direction_word} by {abs(yoy_val):.2f} points compared to the previous year " f"({_fmt_score(prev_score)} in {prev_year}){pct_clause}." ) else: sent2 = ( f"The ASEAN overall score (Total framework) reached {_fmt_score(score)} in {year}; " f"no prior-year data is available for year-over-year comparison." ) sent3 = "" if ranking_list: first = ranking_list[0] last = ranking_list[-1] middle = ranking_list[1:-1] if len(ranking_list) == 1: sent3 = ( f"In terms of country performance, {first['country_name']} was the only " f"country assessed, scoring {_fmt_score(first['score'])} in {year}." ) elif len(ranking_list) == 2: sent3 = ( f"In terms of country performance, {first['country_name']} led the region " f"with a score of {_fmt_score(first['score'])}, while " f"{last['country_name']} recorded the lowest score of " f"{_fmt_score(last['score'])} in {year}." ) else: middle_parts = [f"{c['country_name']} ({_fmt_score(c['score'])})" for c in middle] middle_str = ( middle_parts[0] if len(middle_parts) == 1 else ", ".join(middle_parts[:-1]) + f", and {middle_parts[-1]}" ) sent3 = ( f"In terms of country performance, {first['country_name']} led the region " f"with a score of {_fmt_score(first['score'])}, followed by {middle_str}. " f"At the other end, {last['country_name']} recorded the lowest score " f"of {_fmt_score(last['score'])} in {year}." ) sent4_parts = [] if most_improved_country and most_improved_delta is not None: sent4_parts.append( f"the most notable improvement was seen in {most_improved_country}, " f"which gained {_fmt_delta(most_improved_delta)} points from the previous year" ) if most_declined_country and most_declined_delta is not None: if most_declined_delta < 0: sent4_parts.append( f"while {most_declined_country} experienced the largest decline " f"of {_fmt_delta(most_declined_delta)} points" ) else: sent4_parts.append( f"while {most_declined_country} recorded the smallest gain " f"of {_fmt_delta(most_declined_delta)} points" ) sent4 = "" if sent4_parts: sent4 = ", ".join(sent4_parts) + "." sent4 = sent4[0].upper() + sent4[1:] return " ".join(s for s in [sent1, sent2, sent3, sent4] if s) def _build_pillar_narrative( year, pillar_name, pillar_score, rank_in_year, n_pillars, yoy_val, top_country, top_country_score, bot_country, bot_country_score, strongest_pillar, strongest_score, weakest_pillar, weakest_score, most_improved_pillar, most_improved_delta, most_declined_pillar, most_declined_delta, ) -> str: rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th") sent1 = ( f"In {year}, the {pillar_name} pillar scored {_fmt_score(pillar_score)}, " f"ranking {rank_in_year}{rank_suffix} out of {n_pillars} pillars assessed across ASEAN." ) sent2 = "" if strongest_pillar and weakest_pillar: if strongest_pillar == pillar_name: sent2 = ( f"This made {pillar_name} the strongest performing pillar in {year}, " f"compared to the weakest pillar, {weakest_pillar}, " f"which scored {_fmt_score(weakest_score)}." ) elif weakest_pillar == pillar_name: sent2 = ( f"This made {pillar_name} the weakest performing pillar in {year}, " f"compared to the strongest pillar, {strongest_pillar}, " f"which scored {_fmt_score(strongest_score)}." ) else: sent2 = ( f"Across all pillars in {year}, {strongest_pillar} was the strongest " f"(score: {_fmt_score(strongest_score)}), while {weakest_pillar} " f"was the weakest (score: {_fmt_score(weakest_score)})." ) sent3 = "" if top_country and bot_country: if top_country != bot_country: sent3 = ( f"Within the {pillar_name} pillar, {top_country} led with a score of " f"{_fmt_score(top_country_score)}, while {bot_country} recorded the lowest " f"score of {_fmt_score(bot_country_score)}." ) else: sent3 = ( f"Within the {pillar_name} pillar, {top_country} was the only country " f"with available data, scoring {_fmt_score(top_country_score)}." ) if yoy_val is not None: direction_word = "improved" if yoy_val >= 0 else "declined" sent4 = ( f"Compared to the previous year, the {pillar_name} pillar " f"{direction_word} by {abs(yoy_val):.2f} points" ) else: sent4 = ( f"No prior-year data is available to calculate year-over-year change " f"for the {pillar_name} pillar in {year}" ) if (most_improved_pillar and most_improved_delta is not None and most_declined_pillar and most_declined_delta is not None and most_improved_pillar != most_declined_pillar): sent4 += ( f". Across all pillars, {most_improved_pillar} showed the greatest improvement " f"({_fmt_delta(most_improved_delta)} pts), while {most_declined_pillar} " f"recorded the largest decline ({_fmt_delta(most_declined_delta)} pts)" ) sent4 += "." sent4 = sent4[0].upper() + sent4[1:] return " ".join(s for s in [sent1, sent2, sent3, sent4] if s) # ============================================================================= # MAIN CLASS # ============================================================================= class FoodSecurityAggregator: def __init__(self, client: bigquery.Client): self.client = client self.logger = logging.getLogger(self.__class__.__name__) self.logger.propagate = False self.load_metadata = { "agg_pillar_composite": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None}, "agg_pillar_by_country": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None}, "agg_framework_by_country": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None}, "agg_framework_asean": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None}, "agg_narrative_overview": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None}, "agg_narrative_pillar": {"rows_loaded": 0, "status": "pending", "start_time": None, "end_time": None}, } self.df = None self.dims = {} self.sdgs_start_year = None self.mdgs_indicator_ids = set() self.sdgs_indicator_ids = set() # ========================================================================= # STEP 1: Load data # ========================================================================= def load_data(self): self.logger.info("=" * 70) self.logger.info("STEP 1: LOAD DATA from fs_asean_gold") self.logger.info("=" * 70) self.df = read_from_bigquery( self.client, "fact_asean_food_security_selected", layer='gold' ) self.logger.info(f" fact_asean_food_security_selected : {len(self.df):,} rows") required_cols = { "country_id", "country_name", "indicator_id", "indicator_name", "direction", "framework", "pillar_id", "pillar_name", "time_id", "year", "value", # PERBAIKAN: norm_value_1_100 wajib ada (hasil analytical_layer) "norm_value_1_100", } missing_cols = required_cols - set(self.df.columns) if missing_cols: raise ValueError( f"Kolom berikut tidak ditemukan: {missing_cols}\n" f"Pastikan pipeline dijalankan berurutan:\n" f" 1. bigquery_cleaned_layer.py\n" f" 2. bigquery_dimensional_model.py\n" f" 3. bigquery_analytical_layer.py ← harus dijalankan dulu\n" f" 4. bigquery_analysis_layer.py (file ini)" ) self.df["direction"] = self.df["direction"].fillna("positive") self.df["framework"] = self.df["framework"].fillna("MDGs") self.df["norm_value_1_100"] = self.df["norm_value_1_100"].astype(float) dir_dist = self.df.drop_duplicates("indicator_id")["direction"].value_counts() self.logger.info(f"\n Distribusi direction per indikator:") for d, cnt in dir_dist.items(): tag = "INVERT" if _should_invert(d, self.logger, "load_data") else "normal" self.logger.info(f" {d:<25} : {cnt:>3} [{tag}]") fw_dist = self.df.drop_duplicates("indicator_id")["framework"].value_counts() self.logger.info(f"\n Distribusi framework per indikator:") for fw, cnt in fw_dist.items(): self.logger.info(f" {fw:<10} : {cnt:>3}") self.logger.info( f"\n Rows: {len(self.df):,} | Negara: {self.df['country_id'].nunique()} | " f"Indikator: {self.df['indicator_id'].nunique()} | " f"Tahun: {int(self.df['year'].min())}-{int(self.df['year'].max())}" ) # Diagnostik: cek komparabilitas norm antar framework self._log_norm_comparability_diagnostics() def _log_norm_comparability_diagnostics(self): """ Log diagnostik untuk memverifikasi bahwa norm_value_1_100 sudah comparable antar framework setelah perbaikan di analytical_layer. """ self.logger.info(f"\n [DIAGNOSTIK] Komparabilitas norm_value_1_100 antar framework:") self.logger.info(f" {'─'*60}") fw_stats = ( self.df.groupby('framework')['norm_value_1_100'] .agg(['mean', 'median', 'std', 'min', 'max']) .round(2) ) for fw, row in fw_stats.iterrows(): self.logger.info( f" {fw:<8} mean={row['mean']:>6.2f} median={row['median']:>6.2f} " f"std={row['std']:>5.2f} range=[{row['min']:.2f},{row['max']:.2f}]" ) mdgs_mean = self.df[self.df['framework'] == 'MDGs']['norm_value_1_100'].mean() sdgs_mean = self.df[self.df['framework'] == 'SDGs']['norm_value_1_100'].mean() gap = mdgs_mean - sdgs_mean if abs(gap) > 15: self.logger.info( f"\n [INFO] Gap MDGs-SDGs = {gap:.2f} poin." f"\n Ini adalah perbedaan SUBSTANTIF (bukan artefak normalisasi):" f"\n Indikator SDGs mengukur deprivasi yang lebih dalam" f"\n (FIES, stunting, wasting, anaemia) vs indikator MDGs." f"\n Gap ini valid untuk dilaporkan sebagai temuan analisis." ) else: self.logger.info( f"\n [OK] Gap MDGs-SDGs = {gap:.2f} poin — dalam batas wajar." ) # ========================================================================= # STEP 1b: Klasifikasi indikator # ========================================================================= def _classify_indicators(self): self.logger.info("\n" + "=" * 70) self.logger.info("STEP 1b: KLASIFIKASI INDIKATOR -> MDGs / SDGs") self.logger.info("=" * 70) self.mdgs_indicator_ids = set( self.df[self.df["framework"] == "MDGs"]["indicator_id"].unique().tolist() ) self.sdgs_indicator_ids = set( self.df[self.df["framework"] == "SDGs"]["indicator_id"].unique().tolist() ) _PROXY_KW = frozenset(['food insecurity', 'anemia', 'anaemia']) proxy_mask = ( (self.df["framework"] == "SDGs") & self.df["indicator_name"].str.lower().apply( lambda n: any(kw in n for kw in _PROXY_KW) ) ) df_proxy = self.df[proxy_mask] if not df_proxy.empty: self.sdgs_start_year = int(df_proxy["year"].min()) self.logger.info( f"\n sdgs_start_year = {self.sdgs_start_year} " f"(dari proxy FIES/anaemia di tabel)" ) else: sdgs_rows = self.df[self.df["framework"] == "SDGs"] if not sdgs_rows.empty: self.sdgs_start_year = int(sdgs_rows["year"].min()) self.logger.warning( f" [WARN] Proxy tidak ditemukan, fallback ke min(year) SDGs: " f"{self.sdgs_start_year}" ) else: self.sdgs_start_year = int(self.df["year"].max()) + 1 self.logger.warning( f" [WARN] Tidak ada SDGs. sdgs_start_year = {self.sdgs_start_year}" ) self.logger.info(f" MDGs : {len(self.mdgs_indicator_ids)} indikator") self.logger.info(f" SDGs : {len(self.sdgs_indicator_ids)} indikator") for fw in ["MDGs", "SDGs"]: fw_inds = ( self.df[self.df["framework"] == fw] .drop_duplicates("indicator_id")[["indicator_id", "indicator_name"]] .sort_values("indicator_name") ) self.logger.info(f"\n {fw} indicators ({len(fw_inds)}):") for _, row in fw_inds.iterrows(): self.logger.info(f" [{int(row['indicator_id'])}] {row['indicator_name']}") # ========================================================================= # CORE HELPER: _get_norm_value_df() # ========================================================================= # PERBAIKAN: # Fungsi ini TIDAK lagi melakukan normalisasi ulang per indikator. # Kolom norm_value_1_100 sudah dihitung sekali di analytical_layer # dengan referensi global (semua tahun, semua negara, per indikator). # # Yang dilakukan di sini hanya: # 1. Membaca norm_value_1_100 dari df # 2. Mengubah skala 1-100 → 0-1 (untuk keperluan rata-rata agregat) # dengan rumus linear: norm_0_1 = (norm_1_100 - 1) / 99 # # Rescaling agregat (0-1 → 1-100) tetap dilakukan via global_minmax() # di masing-masing fungsi calc_* untuk menghasilkan skor level pillar/country/asean. # ========================================================================= def _get_norm_value_df(self) -> pd.DataFrame: """ Mengembalikan df dengan kolom 'norm_value' (skala 0-1) yang diturunkan dari norm_value_1_100 (sudah ada di source, dihitung di analytical_layer). Transformasi: norm_value = (norm_value_1_100 - 1) / 99 Ini adalah transformasi LINEAR — tidak mengubah urutan relatif antar indikator, negara, atau tahun. Komparabilitas lintas framework tetap terjaga. """ df = self.df.copy() # Konversi 1-100 → 0-1 secara linear df["norm_value"] = np.where( df["norm_value_1_100"].notna(), (df["norm_value_1_100"] - 1.0) / 99.0, np.nan ) n_null = df["norm_value"].isna().sum() n_valid = df["norm_value"].notna().sum() self.logger.debug( f" _get_norm_value_df: {n_valid:,} valid | {n_null:,} null " f"(dari norm_value_1_100 analytical_layer)" ) return df # ========================================================================= # STEP 2: agg_pillar_composite # ========================================================================= def calc_pillar_composite(self) -> pd.DataFrame: table_name = "agg_pillar_composite" self.load_metadata[table_name]["start_time"] = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 2: {table_name}") self.logger.info("=" * 70) df_normed = self._get_norm_value_df() df = ( df_normed .groupby(["pillar_id", "pillar_name", "year"]) .agg( pillar_norm =("norm_value", "mean"), n_indicators =("indicator_id", "nunique"), n_countries =("country_id", "nunique"), ) .reset_index() ) df["pillar_score_1_100"] = global_minmax(df["pillar_norm"]) df["rank_in_year"] = ( df.groupby("year")["pillar_score_1_100"] .rank(method="min", ascending=False) .astype(int) ) df = add_yoy(df, ["pillar_id"], "pillar_score_1_100") df = add_condition_column(df, "pillar_score_1_100") log_condition_summary(df, table_name, self.logger) df["pillar_id"] = df["pillar_id"].astype(int) df["year"] = df["year"].astype(int) df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger) df["n_countries"] = safe_int(df["n_countries"], col_name="n_countries", logger=self.logger) df["rank_in_year"] = df["rank_in_year"].astype(int) df["pillar_norm"] = df["pillar_norm"].astype(float) df["pillar_score_1_100"] = df["pillar_score_1_100"].astype(float) schema = [ bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_norm", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("n_countries", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_score_1_100", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("condition", "STRING", mode="NULLABLE"), ] rows = load_to_bigquery( self.client, df, table_name, layer='gold', write_disposition="WRITE_TRUNCATE", schema=schema ) self._finalize(table_name, rows) return df # ========================================================================= # STEP 3: agg_pillar_by_country # ========================================================================= def calc_pillar_by_country(self) -> pd.DataFrame: table_name = "agg_pillar_by_country" self.load_metadata[table_name]["start_time"] = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 3: {table_name}") self.logger.info("=" * 70) df_normed = self._get_norm_value_df() df = ( df_normed .groupby(["country_id", "country_name", "pillar_id", "pillar_name", "year"]) .agg(pillar_country_norm=("norm_value", "mean")) .reset_index() ) df["pillar_country_score_1_100"] = global_minmax(df["pillar_country_norm"]) df["rank_in_pillar_year"] = ( df.groupby(["pillar_id", "year"])["pillar_country_score_1_100"] .rank(method="min", ascending=False) .astype(int) ) df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100") df = add_condition_column(df, "pillar_country_score_1_100") log_condition_summary(df, table_name, self.logger) df["country_id"] = df["country_id"].astype(int) df["pillar_id"] = df["pillar_id"].astype(int) df["year"] = df["year"].astype(int) df["rank_in_pillar_year"] = df["rank_in_pillar_year"].astype(int) df["pillar_country_norm"] = df["pillar_country_norm"].astype(float) df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float) schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_country_norm", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("pillar_country_score_1_100", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("rank_in_pillar_year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("condition", "STRING", mode="NULLABLE"), ] rows = load_to_bigquery( self.client, df, table_name, layer='gold', write_disposition="WRITE_TRUNCATE", schema=schema ) self._finalize(table_name, rows) return df # ========================================================================= # STEP 4: agg_framework_by_country # ========================================================================= # PERBAIKAN: # - Flag NORMALIZE_FRAMEWORKS_JOINTLY dihapus. # - Tidak ada lagi rescaling ulang per-framework di sini. # - Semua framework (Total, MDGs, SDGs) menggunakan norm_value yang SAMA # sebagai basis (sudah comparable dari analytical_layer). # - global_minmax() hanya digunakan SEKALI untuk mengubah norm agregat # (rata-rata norm_value per country-framework-year) menjadi skor 1-100 # di level country-framework, menggunakan SATU POOL DATA BERSAMA. # - Dengan ini, perbandingan skor MDGs vs SDGs per negara adalah valid. # ========================================================================= def _calc_country_composite_inmemory(self) -> pd.DataFrame: df_normed = self._get_norm_value_df() df = ( df_normed .groupby(["country_id", "country_name", "year"]) .agg( composite_score=("norm_value", "mean"), n_indicators =("indicator_id", "nunique"), ) .reset_index() ) df["score_1_100"] = global_minmax(df["composite_score"]) df["rank_in_asean"] = ( df.groupby("year")["score_1_100"] .rank(method="min", ascending=False) .astype(int) ) df = add_yoy(df, ["country_id"], "score_1_100") df["country_id"] = df["country_id"].astype(int) df["year"] = df["year"].astype(int) df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger) df["composite_score"] = df["composite_score"].astype(float) df["score_1_100"] = df["score_1_100"].astype(float) df["rank_in_asean"] = df["rank_in_asean"].astype(int) return df def calc_framework_by_country(self) -> pd.DataFrame: table_name = "agg_framework_by_country" self.load_metadata[table_name]["start_time"] = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 4: {table_name}") self.logger.info("=" * 70) self.logger.info( " [PERBAIKAN] Semua framework di-aggregate dari norm_value yang SAMA." "\n Tidak ada rescaling per-framework. Skor MDGs dan SDGs comparable." ) country_composite = self._calc_country_composite_inmemory() df_normed = self._get_norm_value_df() parts = [] # ── Layer TOTAL ─────────────────────────────────────────────────────── agg_total = ( country_composite[[ "country_id", "country_name", "year", "score_1_100", "n_indicators", "composite_score" ]] .copy() .rename(columns={ "score_1_100" : "framework_score_1_100", "composite_score": "framework_norm" }) ) agg_total["framework"] = "Total" parts.append(agg_total) # ── Layer MDGs pre-SDGs (tahun sebelum sdgs_start_year) ────────────── pre_sdgs_rows = country_composite[ country_composite["year"] < self.sdgs_start_year ].copy() if not pre_sdgs_rows.empty: mdgs_pre = ( pre_sdgs_rows[[ "country_id", "country_name", "year", "score_1_100", "n_indicators", "composite_score" ]] .copy() .rename(columns={ "score_1_100" : "framework_score_1_100", "composite_score": "framework_norm" }) ) mdgs_pre["framework"] = "MDGs" parts.append(mdgs_pre) # ── Layer MDGs mixed (setelah SDGs masuk, hanya indikator MDGs) ────── if self.mdgs_indicator_ids: df_mdgs_mixed = df_normed[ (df_normed["indicator_id"].isin(self.mdgs_indicator_ids)) & (df_normed["year"] >= self.sdgs_start_year) ].copy() if not df_mdgs_mixed.empty: agg_mdgs_mixed = ( df_mdgs_mixed .groupby(["country_id", "country_name", "year"]) .agg( framework_norm=("norm_value", "mean"), n_indicators =("indicator_id", "nunique") ) .reset_index() ) # PERBAIKAN: rescale dari POOL GABUNGAN bersama SDGs (lihat bawah) agg_mdgs_mixed["framework"] = "MDGs" parts.append(agg_mdgs_mixed) # ── Layer SDGs (hanya indikator SDGs, mulai sdgs_start_year) ───────── if self.sdgs_indicator_ids: df_sdgs = df_normed[ (df_normed["indicator_id"].isin(self.sdgs_indicator_ids)) & (df_normed["year"] >= self.sdgs_start_year) ].copy() if not df_sdgs.empty: agg_sdgs = ( df_sdgs .groupby(["country_id", "country_name", "year"]) .agg( framework_norm=("norm_value", "mean"), n_indicators =("indicator_id", "nunique") ) .reset_index() ) agg_sdgs["framework"] = "SDGs" parts.append(agg_sdgs) df = pd.concat(parts, ignore_index=True) # PERBAIKAN: Rescale framework_score_1_100 dari SATU POOL BERSAMA # untuk semua framework (MDGs mixed + SDGs) sekaligus. # Ini memastikan skor 60 di MDGs dan skor 60 di SDGs memiliki makna # yang sama: posisi relatif yang sama dalam distribusi gabungan. mixed_mask = df["framework"].isin(["MDGs", "SDGs"]) mixed_pre_mask = (df["framework"] == "MDGs") & (df["year"] < self.sdgs_start_year) # Rescale pre-SDGs MDGs dari pool Total (sudah dihitung) # → sudah ada di agg_total (framework_score_1_100 = dari country_composite) # Rescale MDGs mixed + SDGs dari SATU POOL BERSAMA post_sdgs_mask = mixed_mask & ~mixed_pre_mask & df["framework_norm"].notna() if post_sdgs_mask.any(): df.loc[post_sdgs_mask, "framework_score_1_100"] = global_minmax( df.loc[post_sdgs_mask, "framework_norm"] ) df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger) # Pastikan kolom framework_score_1_100 ada untuk semua baris if "framework_score_1_100" not in df.columns: df["framework_score_1_100"] = np.nan df["rank_in_framework_year"] = ( df.groupby(["framework", "year"])["framework_score_1_100"] .rank(method="min", ascending=False) .astype(int) ) df = add_yoy(df, ["country_id", "framework"], "framework_score_1_100") df = add_condition_column(df, "framework_score_1_100") log_condition_summary(df, table_name, self.logger) # Log diagnostik: bandingkan skor MDGs vs SDGs self._log_framework_score_diagnostics(df, table_name) df["country_id"] = df["country_id"].astype(int) df["year"] = df["year"].astype(int) df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger) df["rank_in_framework_year"] = safe_int(df["rank_in_framework_year"], col_name="rank_in_framework_year", logger=self.logger) df["framework_norm"] = df["framework_norm"].astype(float) df["framework_score_1_100"] = df["framework_score_1_100"].astype(float) self._validate_mdgs_equals_total(df, level="country") schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("framework", "STRING", mode="REQUIRED"), bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("framework_norm", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("rank_in_framework_year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("condition", "STRING", mode="NULLABLE"), ] rows = load_to_bigquery( self.client, df, table_name, layer='gold', write_disposition="WRITE_TRUNCATE", schema=schema ) self._finalize(table_name, rows) return df # ========================================================================= # STEP 5: agg_framework_asean # ========================================================================= # PERBAIKAN: Sama dengan framework_by_country — tidak ada rescaling terpisah # per framework. MDGs mixed dan SDGs di-rescale dari satu pool bersama. # ========================================================================= def calc_framework_asean(self) -> pd.DataFrame: table_name = "agg_framework_asean" self.load_metadata[table_name]["start_time"] = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 5: {table_name}") self.logger.info("=" * 70) self.logger.info( " [PERBAIKAN] MDGs mixed + SDGs di-rescale dari SATU POOL BERSAMA." "\n Skor ASEAN MDGs dan SDGs sekarang comparable." ) df_normed = self._get_norm_value_df() country_composite = self._calc_country_composite_inmemory() country_norm = ( df_normed .groupby(["country_id", "country_name", "year"])["norm_value"] .mean().reset_index() .rename(columns={"norm_value": "country_norm"}) ) asean_overall = ( country_norm.groupby("year") .agg( asean_norm =("country_norm", "mean"), std_norm =("country_norm", "std"), n_countries =("country_norm", "count") ) .reset_index() ) asean_overall["asean_score_1_100"] = global_minmax(asean_overall["asean_norm"]) parts = [] # ── Layer TOTAL ─────────────────────────────────────────────────────── total_cols = asean_overall[["year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"]].copy() total_cols = total_cols.rename(columns={ "asean_score_1_100": "framework_score_1_100", "asean_norm" : "framework_norm", "n_countries" : "n_countries_with_data", }) n_ind_total = df_normed.groupby("year")["indicator_id"].nunique().reset_index().rename(columns={"indicator_id": "n_indicators"}) total_cols = total_cols.merge(n_ind_total, on="year", how="left") total_cols["framework"] = "Total" parts.append(total_cols) # ── Layer MDGs pre-SDGs ─────────────────────────────────────────────── pre_sdgs = asean_overall[asean_overall["year"] < self.sdgs_start_year].copy() if not pre_sdgs.empty: mdgs_pre = pre_sdgs[["year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"]].copy() mdgs_pre = mdgs_pre.rename(columns={ "asean_score_1_100": "framework_score_1_100", "asean_norm" : "framework_norm", "n_countries" : "n_countries_with_data", }) n_ind_pre = ( df_normed[df_normed["year"] < self.sdgs_start_year] .groupby("year")["indicator_id"].nunique() .reset_index().rename(columns={"indicator_id": "n_indicators"}) ) mdgs_pre = mdgs_pre.merge(n_ind_pre, on="year", how="left") mdgs_pre["framework"] = "MDGs" parts.append(mdgs_pre) # ── Siapkan MDGs mixed dan SDGs untuk rescaling BERSAMA ─────────────── mixed_parts = [] if self.mdgs_indicator_ids: df_mdgs_mixed = df_normed[ (df_normed["indicator_id"].isin(self.mdgs_indicator_ids)) & (df_normed["year"] >= self.sdgs_start_year) ].copy() if not df_mdgs_mixed.empty: cn = ( df_mdgs_mixed.groupby(["country_id", "year"])["norm_value"].mean() .reset_index().rename(columns={"norm_value": "country_norm"}) ) asean_mdgs = cn.groupby("year").agg( framework_norm =("country_norm", "mean"), std_norm =("country_norm", "std"), n_countries_with_data =("country_id", "count"), ).reset_index() n_ind_mdgs = df_mdgs_mixed.groupby("year")["indicator_id"].nunique().reset_index().rename(columns={"indicator_id": "n_indicators"}) asean_mdgs = asean_mdgs.merge(n_ind_mdgs, on="year", how="left") asean_mdgs["framework"] = "MDGs" mixed_parts.append(asean_mdgs) if self.sdgs_indicator_ids: df_sdgs = df_normed[ (df_normed["indicator_id"].isin(self.sdgs_indicator_ids)) & (df_normed["year"] >= self.sdgs_start_year) ].copy() if not df_sdgs.empty: cn = ( df_sdgs.groupby(["country_id", "year"])["norm_value"].mean() .reset_index().rename(columns={"norm_value": "country_norm"}) ) asean_sdgs = cn.groupby("year").agg( framework_norm =("country_norm", "mean"), std_norm =("country_norm", "std"), n_countries_with_data =("country_id", "count"), ).reset_index() n_ind_sdgs = df_sdgs.groupby("year")["indicator_id"].nunique().reset_index().rename(columns={"indicator_id": "n_indicators"}) asean_sdgs = asean_sdgs.merge(n_ind_sdgs, on="year", how="left") asean_sdgs["framework"] = "SDGs" mixed_parts.append(asean_sdgs) # PERBAIKAN: Rescale MDGs mixed + SDGs dari SATU POOL BERSAMA if mixed_parts: df_mixed = pd.concat(mixed_parts, ignore_index=True) df_mixed["framework_score_1_100"] = global_minmax(df_mixed["framework_norm"]) parts.append(df_mixed) df = pd.concat(parts, ignore_index=True) df = check_and_dedup(df, ["framework", "year"], context=table_name, logger=self.logger) df = add_yoy(df, ["framework"], "framework_score_1_100") df = add_condition_column(df, "framework_score_1_100") log_condition_summary(df, table_name, self.logger) # Log diagnostik: bandingkan skor ASEAN MDGs vs SDGs self._log_framework_score_diagnostics(df, table_name) df["year"] = df["year"].astype(int) df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger) df["n_countries_with_data"] = safe_int(df["n_countries_with_data"], col_name="n_countries_with_data", logger=self.logger) for col in ["framework_norm", "std_norm", "framework_score_1_100"]: df[col] = df[col].astype(float) self._validate_mdgs_equals_total(df, level="asean") schema = [ bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("framework", "STRING", mode="REQUIRED"), bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("n_countries_with_data", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("framework_norm", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("std_norm", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("condition", "STRING", mode="NULLABLE"), ] rows = load_to_bigquery( self.client, df, table_name, layer='gold', write_disposition="WRITE_TRUNCATE", schema=schema ) self._finalize(table_name, rows) return df # ========================================================================= # STEP 6 & 7: Narrative (tidak ada perubahan) # ========================================================================= def calc_narrative_overview(self, df_framework_asean, df_framework_by_country): table_name = "agg_narrative_overview" self.load_metadata[table_name]["start_time"] = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 6: {table_name}") self.logger.info("=" * 70) asean_total = df_framework_asean[df_framework_asean["framework"] == "Total"].sort_values("year").reset_index(drop=True) score_by_year = dict(zip(asean_total["year"].astype(int), asean_total["framework_score_1_100"].astype(float))) country_total = df_framework_by_country[df_framework_by_country["framework"] == "Total"].copy() ind_year = self.df.drop_duplicates(subset=["indicator_id", "year", "framework"]) records = [] for _, row in asean_total.iterrows(): yr = int(row["year"]) score = float(row["framework_score_1_100"]) yoy = row["year_over_year_change"] yoy_val = float(yoy) if pd.notna(yoy) else None yr_ind = ind_year[ind_year["year"] == yr] n_mdg = int(yr_ind[yr_ind["framework"] == "MDGs"]["indicator_id"].nunique()) n_sdg = int(yr_ind[yr_ind["framework"] == "SDGs"]["indicator_id"].nunique()) n_total_ind = int(yr_ind["indicator_id"].nunique()) prev_score = score_by_year.get(yr - 1, None) yoy_pct = ((yoy_val / prev_score * 100) if (yoy_val is not None and prev_score and prev_score != 0) else None) yr_country = country_total[country_total["year"] == yr].sort_values("rank_in_framework_year").reset_index(drop=True) ranking_list = [] for _, cr in yr_country.iterrows(): cr_yoy = cr.get("year_over_year_change", None) ranking_list.append({ "rank" : int(cr["rank_in_framework_year"]), "country_name": str(cr["country_name"]), "score" : round(float(cr["framework_score_1_100"]), 2), "yoy_change" : round(float(cr_yoy), 2) if pd.notna(cr_yoy) else None, }) yr_country_yoy = yr_country.dropna(subset=["year_over_year_change"]) if not yr_country_yoy.empty: best_idx = yr_country_yoy["year_over_year_change"].idxmax() worst_idx = yr_country_yoy["year_over_year_change"].idxmin() most_improved_country = str(yr_country_yoy.loc[best_idx, "country_name"]) most_improved_delta = round(float(yr_country_yoy.loc[best_idx, "year_over_year_change"]), 2) most_declined_country = str(yr_country_yoy.loc[worst_idx, "country_name"]) most_declined_delta = round(float(yr_country_yoy.loc[worst_idx, "year_over_year_change"]), 2) else: most_improved_country = most_declined_country = None most_improved_delta = most_declined_delta = None narrative = _build_overview_narrative( year=yr, n_mdg=n_mdg, n_sdg=n_sdg, n_total_ind=n_total_ind, score=score, yoy_val=yoy_val, yoy_pct=yoy_pct, prev_year=yr-1, prev_score=prev_score, ranking_list=ranking_list, most_improved_country=most_improved_country, most_improved_delta=most_improved_delta, most_declined_country=most_declined_country, most_declined_delta=most_declined_delta, ) records.append({ "year" : yr, "n_mdg_indicators" : n_mdg, "n_sdg_indicators" : n_sdg, "n_total_indicators" : n_total_ind, "asean_total_score" : round(score, 2), "yoy_change" : yoy_val, "yoy_change_pct" : round(yoy_pct, 2) if yoy_pct is not None else None, "country_ranking_json" : json.dumps(ranking_list, ensure_ascii=False), "most_improved_country": most_improved_country, "most_improved_delta" : most_improved_delta, "most_declined_country": most_declined_country, "most_declined_delta" : most_declined_delta, "narrative_overview" : narrative, }) df = pd.DataFrame(records) df["year"] = df["year"].astype(int) df["n_mdg_indicators"] = df["n_mdg_indicators"].astype(int) df["n_sdg_indicators"] = df["n_sdg_indicators"].astype(int) df["n_total_indicators"] = df["n_total_indicators"].astype(int) df["asean_total_score"] = df["asean_total_score"].astype(float) for col in ["yoy_change", "yoy_change_pct", "most_improved_delta", "most_declined_delta"]: df[col] = pd.to_numeric(df[col], errors="coerce").astype(float) schema = [ bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("n_mdg_indicators", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("n_sdg_indicators", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("n_total_indicators", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("asean_total_score", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("yoy_change_pct", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("country_ranking_json", "STRING", mode="REQUIRED"), bigquery.SchemaField("most_improved_country", "STRING", mode="NULLABLE"), bigquery.SchemaField("most_improved_delta", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("most_declined_country", "STRING", mode="NULLABLE"), bigquery.SchemaField("most_declined_delta", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("narrative_overview", "STRING", mode="REQUIRED"), ] rows = load_to_bigquery(self.client, df, table_name, layer='gold', write_disposition="WRITE_TRUNCATE", schema=schema) self._finalize(table_name, rows) return df def calc_narrative_pillar(self, df_pillar_composite, df_pillar_by_country): table_name = "agg_narrative_pillar" self.load_metadata[table_name]["start_time"] = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 7: {table_name}") self.logger.info("=" * 70) records = [] for yr in sorted(df_pillar_composite["year"].unique()): yr_pillars = df_pillar_composite[df_pillar_composite["year"] == yr].sort_values("rank_in_year").reset_index(drop=True) yr_country_pillar = df_pillar_by_country[df_pillar_by_country["year"] == yr] strongest_pillar = yr_pillars.iloc[0] if len(yr_pillars) > 0 else None weakest_pillar = yr_pillars.iloc[-1] if len(yr_pillars) > 0 else None yr_pillars_yoy = yr_pillars.dropna(subset=["year_over_year_change"]) if not yr_pillars_yoy.empty: best_p_idx = yr_pillars_yoy["year_over_year_change"].idxmax() worst_p_idx = yr_pillars_yoy["year_over_year_change"].idxmin() most_improved_pillar = str(yr_pillars_yoy.loc[best_p_idx, "pillar_name"]) most_improved_delta = round(float(yr_pillars_yoy.loc[best_p_idx, "year_over_year_change"]), 2) most_declined_pillar = str(yr_pillars_yoy.loc[worst_p_idx, "pillar_name"]) most_declined_delta = round(float(yr_pillars_yoy.loc[worst_p_idx, "year_over_year_change"]), 2) else: most_improved_pillar = most_declined_pillar = None most_improved_delta = most_declined_delta = None for _, prow in yr_pillars.iterrows(): p_id = int(prow["pillar_id"]) p_country = yr_country_pillar[yr_country_pillar["pillar_id"] == p_id].sort_values("rank_in_pillar_year").reset_index(drop=True) top_country = bot_country = None top_country_score = bot_country_score = None if not p_country.empty: top_country = str(p_country.iloc[0]["country_name"]) top_country_score = round(float(p_country.iloc[0]["pillar_country_score_1_100"]), 2) bot_country = str(p_country.iloc[-1]["country_name"]) bot_country_score = round(float(p_country.iloc[-1]["pillar_country_score_1_100"]), 2) p_yoy = prow["year_over_year_change"] narrative = _build_pillar_narrative( year=yr, pillar_name=str(prow["pillar_name"]), pillar_score=float(prow["pillar_score_1_100"]), rank_in_year=int(prow["rank_in_year"]), n_pillars=len(yr_pillars), yoy_val=float(p_yoy) if pd.notna(p_yoy) else None, top_country=top_country, top_country_score=top_country_score, bot_country=bot_country, bot_country_score=bot_country_score, strongest_pillar=str(strongest_pillar["pillar_name"]) if strongest_pillar is not None else None, strongest_score=round(float(strongest_pillar["pillar_score_1_100"]), 2) if strongest_pillar is not None else None, weakest_pillar=str(weakest_pillar["pillar_name"]) if weakest_pillar is not None else None, weakest_score=round(float(weakest_pillar["pillar_score_1_100"]), 2) if weakest_pillar is not None else None, most_improved_pillar=most_improved_pillar, most_improved_delta=most_improved_delta, most_declined_pillar=most_declined_pillar, most_declined_delta=most_declined_delta, ) records.append({ "year" : yr, "pillar_id" : p_id, "pillar_name" : str(prow["pillar_name"]), "pillar_score" : round(float(prow["pillar_score_1_100"]), 2), "rank_in_year" : int(prow["rank_in_year"]), "yoy_change" : float(p_yoy) if pd.notna(p_yoy) else None, "top_country" : top_country, "top_country_score" : top_country_score, "bottom_country" : bot_country, "bottom_country_score": bot_country_score, "narrative_pillar" : narrative, }) df = pd.DataFrame(records) df["year"] = df["year"].astype(int) df["pillar_id"] = df["pillar_id"].astype(int) df["rank_in_year"] = df["rank_in_year"].astype(int) for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]: df[col] = pd.to_numeric(df[col], errors="coerce").astype(float) schema = [ bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("pillar_score", "FLOAT", mode="REQUIRED"), bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("top_country", "STRING", mode="NULLABLE"), bigquery.SchemaField("top_country_score", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("bottom_country", "STRING", mode="NULLABLE"), bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("narrative_pillar", "STRING", mode="REQUIRED"), ] rows = load_to_bigquery(self.client, df, table_name, layer='gold', write_disposition="WRITE_TRUNCATE", schema=schema) self._finalize(table_name, rows) return df # ========================================================================= # DIAGNOSTIK & VALIDASI # ========================================================================= def _log_framework_score_diagnostics(self, df: pd.DataFrame, context: str): """ Log perbandingan rata-rata skor per framework. Setelah perbaikan, gap antar framework mencerminkan perbedaan substantif, bukan artefak normalisasi. """ self.logger.info(f"\n [DIAGNOSTIK] Rata-rata skor per framework ({context}):") fw_means = df.groupby("framework")["framework_score_1_100"].agg(['mean', 'min', 'max']).round(2) for fw, row in fw_means.iterrows(): self.logger.info( f" {fw:<8} mean={row['mean']:>6.2f} " f"range=[{row['min']:.2f}, {row['max']:.2f}]" ) if "MDGs" in fw_means.index and "SDGs" in fw_means.index: gap = fw_means.loc["MDGs", "mean"] - fw_means.loc["SDGs", "mean"] self.logger.info( f"\n Gap MDGs-SDGs = {gap:.2f} poin" + ( " → SUBSTANTIF (indikator SDGs mengukur deprivasi lebih dalam)" if abs(gap) > 10 else " → dalam batas wajar" ) ) def _validate_mdgs_equals_total(self, df: pd.DataFrame, level: str = ""): self.logger.info(f"\n Validasi MDGs < {self.sdgs_start_year} == Total [{level}]:") group_by = ["year"] if level.startswith("asean") else ["country_id", "year"] mdgs_pre = df[(df["framework"] == "MDGs") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "mdgs_score"}) total_pre = df[(df["framework"] == "Total") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "total_score"}) if mdgs_pre.empty and total_pre.empty: self.logger.info(f" -> Tidak ada data pre-{self.sdgs_start_year} (skip)") return if mdgs_pre.empty or total_pre.empty: self.logger.warning(f" -> [WARNING] Salah satu kosong: MDGs={len(mdgs_pre)}, Total={len(total_pre)}") return check = mdgs_pre.merge(total_pre, on=group_by) max_diff = (check["mdgs_score"] - check["total_score"]).abs().max() status = "OK (identik)" if max_diff < 0.01 else f"MISMATCH! max_diff={max_diff:.6f}" self.logger.info(f" -> {status} (n_checked={len(check)})") def _finalize(self, table_name: str, rows_loaded: int): self.load_metadata[table_name].update({"rows_loaded": rows_loaded, "status": "success", "end_time": datetime.now()}) log_update(self.client, "DW", table_name, "full_load", rows_loaded) self.logger.info(f" {table_name}: {rows_loaded:,} rows -> [Gold] fs_asean_gold") def _fail(self, table_name: str, error: Exception): self.load_metadata[table_name].update({"status": "failed", "end_time": datetime.now()}) self.logger.error(f" [FAIL] {table_name}: {error}") log_update(self.client, "DW", table_name, "full_load", 0, "failed", str(error)) # ========================================================================= # RUN # ========================================================================= def run(self): start = datetime.now() self.logger.info("\n" + "=" * 70) self.logger.info("FOOD SECURITY AGGREGATION — 6 TABLES -> fs_asean_gold") self.logger.info(f" Condition threshold: bad<{THRESHOLD_BAD}, good>{THRESHOLD_GOOD}") self.logger.info( " NORMALISASI: norm_value dari analytical_layer (satu referensi global)." "\n Tidak ada rescaling per-framework. MDGs dan SDGs comparable." ) self.logger.info("=" * 70) self.load_data() self._classify_indicators() df_pillar_composite = self.calc_pillar_composite() df_pillar_by_country = self.calc_pillar_by_country() df_framework_by_country = self.calc_framework_by_country() df_framework_asean = self.calc_framework_asean() self.calc_narrative_overview(df_framework_asean=df_framework_asean, df_framework_by_country=df_framework_by_country) self.calc_narrative_pillar(df_pillar_composite=df_pillar_composite, df_pillar_by_country=df_pillar_by_country) duration = (datetime.now() - start).total_seconds() total_rows = sum(m["rows_loaded"] for m in self.load_metadata.values()) self.logger.info("\n" + "=" * 70) self.logger.info("SELESAI") self.logger.info("=" * 70) self.logger.info(f" Durasi : {duration:.2f}s") self.logger.info(f" Total rows : {total_rows:,}") for tbl, meta in self.load_metadata.items(): icon = "OK" if meta["status"] == "success" else "FAIL" self.logger.info(f" [{icon}] {tbl:<35} {meta['rows_loaded']:>10,}") # ============================================================================= # AIRFLOW & MAIN # ============================================================================= def run_aggregation(): from scripts.bigquery_config import get_bigquery_client client = get_bigquery_client() agg = FoodSecurityAggregator(client) agg.run() total = sum(m["rows_loaded"] for m in agg.load_metadata.values()) print(f"Aggregation completed: {total:,} total rows loaded") if __name__ == "__main__": import io if _sys.stdout.encoding and _sys.stdout.encoding.lower() not in ("utf-8", "utf8"): _sys.stdout = io.TextIOWrapper(_sys.stdout.buffer, encoding="utf-8", errors="replace") if _sys.stderr.encoding and _sys.stderr.encoding.lower() not in ("utf-8", "utf8"): _sys.stderr = io.TextIOWrapper(_sys.stderr.buffer, encoding="utf-8", errors="replace") print("=" * 70) print("FOOD SECURITY AGGREGATION -> fs_asean_gold") print(f"Condition threshold: bad<{THRESHOLD_BAD}, moderate {THRESHOLD_BAD}-{THRESHOLD_GOOD}, good>{THRESHOLD_GOOD}") print("NORMALISASI: satu referensi global per indikator (dari analytical_layer).") print("Tidak ada rescaling per-framework. MDGs dan SDGs comparable.") print("=" * 70) logger = setup_logging() for handler in logger.handlers: handler.__class__ = _SafeStreamHandler client = get_bigquery_client() agg = FoodSecurityAggregator(client) agg.run() print("\n" + "=" * 70) print("[OK] SELESAI") print("=" * 70)