izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

agg narrative v2

Browse Source
This commit is contained in:
Debby
2026-03-26 19:15:39 +07:00
parent 1d732167f5
commit e25ae0dfe7
1 changed files with 21 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
scripts/bigquery_aggregate_layer.py
View File

@@ -6,8 +6,8 @@ UPDATED: Simpan 6 tabel ke fs_asean_gold (layer='gold'):
- agg_pillar_by_country
- agg_framework_by_country
- agg_framework_asean
- agg_narrative_overview (NEW)
- agg_narrative_pillar (NEW)
- agg_narrative_overview
- agg_narrative_pillar
"""
import pandas as pd
@@ -928,13 +928,21 @@ class FoodSecurityAggregator:
self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold")
self.logger.info("=" * 70)
# ASEAN-level Total framework rows, sorted by year
# ASEAN-level Total framework rows only, sorted by year
# PENTING: filter framework='Total' dulu sebelum apapun
asean_total = (
df_framework_asean[df_framework_asean["framework"] == "Total"]
.sort_values("year")
.reset_index(drop=True)
)
# Buat lookup score per tahun untuk ambil prev_score yang akurat
# Tidak mengandalkan score - yoy_val karena floating point bisa drift
score_by_year = dict(zip(
asean_total["year"].astype(int),
asean_total["framework_score_1_100"].astype(float),
))
# Country-level Total framework rows (ranking + YoY per country)
country_total = (
df_framework_by_country[df_framework_by_country["framework"] == "Total"]
@@ -953,16 +961,19 @@ class FoodSecurityAggregator:
yoy_val = float(yoy) if pd.notna(yoy) else None
# -- Indicator counts per framework for this year ---------------
yr_ind = ind_year[ind_year["year"] == yr]
n_mdg = int(yr_ind[yr_ind["framework"] == "MDGs"]["indicator_id"].nunique())
n_sdg = int(yr_ind[yr_ind["framework"] == "SDGs"]["indicator_id"].nunique())
yr_ind = ind_year[ind_year["year"] == yr]
n_mdg = int(yr_ind[yr_ind["framework"] == "MDGs"]["indicator_id"].nunique())
n_sdg = int(yr_ind[yr_ind["framework"] == "SDGs"]["indicator_id"].nunique())
n_total_ind = int(yr_ind["indicator_id"].nunique())
# -- prev_score diambil langsung dari lookup, bukan score - yoy_val
# Ini memastikan nilai konsisten 100% dengan tabel agg_framework_asean
prev_score = score_by_year.get(yr - 1, None)
# -- YoY % -----------------------------------------------------
prev_score = score - yoy_val if yoy_val is not None else None
yoy_pct = (
yoy_pct = (
(yoy_val / prev_score * 100)
if (yoy_val is not None and prev_score and prev_score != 0)
if (yoy_val is not None and prev_score is not None and prev_score != 0)
else None
)
@@ -1306,7 +1317,7 @@ if __name__ == "__main__":
_sys.stderr = io.TextIOWrapper(_sys.stderr.buffer, encoding="utf-8", errors="replace")
print("=" * 70)
print("FOOD SECURITY AGGREGATION v9.0 — 6 TABLES -> fs_asean_gold")
print("FOOD SECURITY AGGREGATION-> fs_asean_gold")
print(f" NORMALIZE_FRAMEWORKS_JOINTLY = {NORMALIZE_FRAMEWORKS_JOINTLY}")
print("=" * 70)