diff --git a/scripts/bigquery_aggregate_layer.py b/scripts/bigquery_aggregate_layer.py index cbb97aa..07b02c2 100644 --- a/scripts/bigquery_aggregate_layer.py +++ b/scripts/bigquery_aggregate_layer.py @@ -6,8 +6,8 @@ UPDATED: Simpan 6 tabel ke fs_asean_gold (layer='gold'): - agg_pillar_by_country - agg_framework_by_country - agg_framework_asean - - agg_narrative_overview (NEW) - - agg_narrative_pillar (NEW) + - agg_narrative_overview + - agg_narrative_pillar """ import pandas as pd @@ -928,13 +928,21 @@ class FoodSecurityAggregator: self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold") self.logger.info("=" * 70) - # ASEAN-level Total framework rows, sorted by year + # ASEAN-level Total framework rows only, sorted by year + # PENTING: filter framework='Total' dulu sebelum apapun asean_total = ( df_framework_asean[df_framework_asean["framework"] == "Total"] .sort_values("year") .reset_index(drop=True) ) + # Buat lookup score per tahun untuk ambil prev_score yang akurat + # Tidak mengandalkan score - yoy_val karena floating point bisa drift + score_by_year = dict(zip( + asean_total["year"].astype(int), + asean_total["framework_score_1_100"].astype(float), + )) + # Country-level Total framework rows (ranking + YoY per country) country_total = ( df_framework_by_country[df_framework_by_country["framework"] == "Total"] @@ -953,16 +961,19 @@ class FoodSecurityAggregator: yoy_val = float(yoy) if pd.notna(yoy) else None # -- Indicator counts per framework for this year --------------- - yr_ind = ind_year[ind_year["year"] == yr] - n_mdg = int(yr_ind[yr_ind["framework"] == "MDGs"]["indicator_id"].nunique()) - n_sdg = int(yr_ind[yr_ind["framework"] == "SDGs"]["indicator_id"].nunique()) + yr_ind = ind_year[ind_year["year"] == yr] + n_mdg = int(yr_ind[yr_ind["framework"] == "MDGs"]["indicator_id"].nunique()) + n_sdg = int(yr_ind[yr_ind["framework"] == "SDGs"]["indicator_id"].nunique()) n_total_ind = int(yr_ind["indicator_id"].nunique()) + # -- prev_score diambil langsung dari lookup, bukan score - yoy_val + # Ini memastikan nilai konsisten 100% dengan tabel agg_framework_asean + prev_score = score_by_year.get(yr - 1, None) + # -- YoY % ----------------------------------------------------- - prev_score = score - yoy_val if yoy_val is not None else None - yoy_pct = ( + yoy_pct = ( (yoy_val / prev_score * 100) - if (yoy_val is not None and prev_score and prev_score != 0) + if (yoy_val is not None and prev_score is not None and prev_score != 0) else None ) @@ -1306,7 +1317,7 @@ if __name__ == "__main__": _sys.stderr = io.TextIOWrapper(_sys.stderr.buffer, encoding="utf-8", errors="replace") print("=" * 70) - print("FOOD SECURITY AGGREGATION v9.0 — 6 TABLES -> fs_asean_gold") + print("FOOD SECURITY AGGREGATION-> fs_asean_gold") print(f" NORMALIZE_FRAMEWORKS_JOINTLY = {NORMALIZE_FRAMEWORKS_JOINTLY}") print("=" * 70)