izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SDGS MDGS indicator

Browse Source
This commit is contained in:
Debby
2026-03-28 19:15:24 +07:00
parent 0ffdf40430
commit dc981aacab
4 changed files with 812 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

384
scripts/bigquery_aggregate_layer.py
View File

@@ -1,7 +1,14 @@
"""
BIGQUERY ANALYSIS LAYER - FOOD SECURITY AGGREGATION
Semua agregasi pakai norm_value dari _get_norm_value_df()
UPDATED: Simpan 6 tabel ke fs_asean_gold (layer='gold'):
UPDATED:
- _classify_indicators() membaca kolom 'framework' langsung dari
fact_asean_food_security_selected (bukan heuristik gap min_year).
- Kolom 'framework' sudah ditanam sejak bigquery_cleaned_layer.py
berdasarkan daftar eksplisit SDG Goal 2 (2030 Agenda, versi Maret 2020).
Simpan 6 tabel ke fs_asean_gold (layer='gold'):
- agg_pillar_composite
- agg_pillar_by_country
- agg_framework_by_country
@@ -9,7 +16,8 @@ UPDATED: Simpan 6 tabel ke fs_asean_gold (layer='gold'):
- agg_narrative_overview
- agg_narrative_pillar
SOURCE TABLE: fact_asean_food_security_selected (sudah include nama + ID)
SOURCE TABLE: fact_asean_food_security_selected
(sudah include country_name, indicator_name, pillar_name, direction, framework)
"""
import pandas as pd
@@ -106,7 +114,9 @@ def add_yoy(df: pd.DataFrame, group_cols: list, score_col: str) -> pd.DataFrame:
return df
def safe_int(series: pd.Series, fill: int = 0, col_name: str = "", logger=None) -> pd.Series:
def safe_int(
series: pd.Series, fill: int = 0, col_name: str = "", logger=None
) -> pd.Series:
n_nan = series.isna().sum()
if n_nan > 0 and logger:
logger.warning(
@@ -115,7 +125,9 @@ def safe_int(series: pd.Series, fill: int = 0, col_name: str = "", logger=None)
return series.fillna(fill).astype(int)
def check_and_dedup(df: pd.DataFrame, key_cols: list, context: str = "", logger=None) -> pd.DataFrame:
def check_and_dedup(
df: pd.DataFrame, key_cols: list, context: str = "", logger=None
) -> pd.DataFrame:
dupes = df.duplicated(subset=key_cols, keep=False)
if dupes.any():
n_dupes = dupes.sum()
@@ -134,18 +146,16 @@ def check_and_dedup(df: pd.DataFrame, key_cols: list, context: str = "", logger=
# =============================================================================
# NARRATIVE BUILDER FUNCTIONS (pure functions, easy to unit-test)
# NARRATIVE BUILDER FUNCTIONS
# =============================================================================
def _fmt_score(score) -> str:
"""Format score to 2 decimal places."""
if score is None or (isinstance(score, float) and np.isnan(score)):
return "N/A"
return f"{score:.2f}"
def _fmt_delta(delta) -> str:
"""Format YoY delta with sign and 2 decimal places."""
if delta is None or (isinstance(delta, float) and np.isnan(delta)):
return "N/A"
sign = "+" if delta >= 0 else ""
@@ -339,9 +349,9 @@ def _build_pillar_narrative(
f"for the {pillar_name} pillar in {year}"
)
if most_improved_pillar and most_improved_delta is not None \
and most_declined_pillar and most_declined_delta is not None \
and most_improved_pillar != most_declined_pillar:
if (most_improved_pillar and most_improved_delta is not None
and most_declined_pillar and most_declined_delta is not None
and most_improved_pillar != most_declined_pillar):
sent4 += (
f". Across all pillars, {most_improved_pillar} showed the greatest improvement "
f"({_fmt_delta(most_improved_delta)} pts), while {most_declined_pillar} "
@@ -390,20 +400,14 @@ class FoodSecurityAggregator:
self.logger.info("STEP 1: LOAD DATA from fs_asean_gold")
self.logger.info("=" * 70)
# -----------------------------------------------------------------------
# CHANGED: sumber tabel -> fact_asean_food_security_selected
# Tabel ini sudah include: country_name, indicator_name, pillar_name,
# direction, year -> tidak perlu join ke dim_* lagi
# -----------------------------------------------------------------------
self.df = read_from_bigquery(
self.client, "fact_asean_food_security_selected", layer='gold'
)
self.logger.info(f" fact_asean_food_security_selected : {len(self.df):,} rows")
# Validasi kolom wajib yang harus sudah ada di tabel baru
required_cols = {
"country_id", "country_name",
"indicator_id", "indicator_name", "direction",
"indicator_id", "indicator_name", "direction", "framework",
"pillar_id", "pillar_name",
"time_id", "year",
"value",
@@ -412,14 +416,14 @@ class FoodSecurityAggregator:
if missing_cols:
raise ValueError(
f"Kolom berikut tidak ditemukan di fact_asean_food_security_selected: "
f"{missing_cols}"
f"{missing_cols}\n"
f"Pastikan pipeline dijalankan berurutan:\n"
f" 1. bigquery_cleaned_layer.py\n"
f" 2. bigquery_dimensional_model.py\n"
f" 3. bigquery_analytical_layer.py\n"
f" 4. bigquery_analysis_layer.py (file ini)"
)
# -----------------------------------------------------------------------
# Tidak perlu join ke dim_* lagi karena semua nama sudah ada.
# Hanya load dim_indicator untuk keperluan fallback / referensi direction
# jika ada NULL yang perlu di-fill.
# -----------------------------------------------------------------------
n_null_dir = self.df["direction"].isna().sum()
if n_null_dir > 0:
self.logger.warning(
@@ -427,12 +431,24 @@ class FoodSecurityAggregator:
)
self.df["direction"] = self.df["direction"].fillna("positive")
n_null_fw = self.df["framework"].isna().sum()
if n_null_fw > 0:
self.logger.warning(
f" [FRAMEWORK] {n_null_fw} rows dengan framework NULL -> diisi 'MDGs'"
)
self.df["framework"] = self.df["framework"].fillna("MDGs")
dir_dist = self.df.drop_duplicates("indicator_id")["direction"].value_counts()
self.logger.info(f"\n Distribusi direction per indikator:")
for d, cnt in dir_dist.items():
tag = "INVERT" if _should_invert(d, self.logger, "load_data check") else "normal"
self.logger.info(f" {d:<25} : {cnt:>3} indikator [{tag}]")
fw_dist = self.df.drop_duplicates("indicator_id")["framework"].value_counts()
self.logger.info(f"\n Distribusi framework per indikator:")
for fw, cnt in fw_dist.items():
self.logger.info(f" {fw:<10} : {cnt:>3} indikator")
self.logger.info(f"\n Rows loaded : {len(self.df):,}")
self.logger.info(f" Negara : {self.df['country_id'].nunique()}")
self.logger.info(f" Indikator : {self.df['indicator_id'].nunique()}")
@@ -445,57 +461,66 @@ class FoodSecurityAggregator:
# =========================================================================
def _classify_indicators(self):
"""
Klasifikasi indikator ke MDGs / SDGs.
UPDATED: Membaca kolom 'framework' langsung dari tabel
fact_asean_food_security_selected — tidak lagi menggunakan heuristik
gap detection berdasarkan min_year. Klasifikasi eksplisit sudah dilakukan
di bigquery_cleaned_layer.py berdasarkan daftar resmi SDG Goal 2.
sdgs_start_year dihitung dari tahun minimum data SDG yang tersedia,
bukan dari asumsi threshold hardcoded.
"""
self.logger.info("\n" + "=" * 70)
self.logger.info("STEP 1b: KLASIFIKASI INDIKATOR -> MDGs / SDGs")
self.logger.info("=" * 70)
ind_min_year = (
self.df.groupby("indicator_id")["year"]
.min().reset_index()
.rename(columns={"year": "min_year"})
)
unique_years = sorted(ind_min_year["min_year"].unique())
self.logger.info(f"\n Unique min_year per indikator: {unique_years}")
if len(unique_years) == 1:
gap_threshold = unique_years[0] + 1
self.logger.info(" Hanya 1 cluster -> semua = MDGs")
else:
gaps = [
(unique_years[i+1] - unique_years[i], unique_years[i], unique_years[i+1])
for i in range(len(unique_years) - 1)
]
gaps.sort(reverse=True)
largest_gap_size, y_before, y_after = gaps[0]
gap_threshold = y_after
self.logger.info(f" Gap terbesar: {y_before} -> {y_after} (selisih {largest_gap_size})")
ind_min_year["framework"] = ind_min_year["min_year"].apply(
lambda y: "MDGs" if int(y) < gap_threshold else "SDGs"
)
sdgs_rows = ind_min_year[ind_min_year["framework"] == "SDGs"]
self.sdgs_start_year = (
int(sdgs_rows["min_year"].min()) if not sdgs_rows.empty
else int(self.df["year"].max()) + 1
)
self.logger.info(f" sdgs_start_year: {self.sdgs_start_year}")
if "framework" not in self.df.columns:
raise ValueError(
"Kolom 'framework' tidak ditemukan di fact_asean_food_security_selected.\n"
"Pastikan pipeline dijalankan berurutan:\n"
" 1. bigquery_cleaned_layer.py (assign_framework)\n"
" 2. bigquery_dimensional_model.py (dim_indicator + framework)\n"
" 3. bigquery_analytical_layer.py (propagasi ke fact_selected)\n"
" 4. bigquery_analysis_layer.py (file ini)"
)
# Baca langsung dari kolom — tidak ada gap detection / heuristik
self.mdgs_indicator_ids = set(
ind_min_year[ind_min_year["framework"] == "MDGs"]["indicator_id"].tolist()
self.df[self.df["framework"] == "MDGs"]["indicator_id"].unique().tolist()
)
self.sdgs_indicator_ids = set(
ind_min_year[ind_min_year["framework"] == "SDGs"]["indicator_id"].tolist()
self.df[self.df["framework"] == "SDGs"]["indicator_id"].unique().tolist()
)
self.logger.info(f" MDGs: {len(self.mdgs_indicator_ids)} indicators")
self.logger.info(f" SDGs: {len(self.sdgs_indicator_ids)} indicators")
# sdgs_start_year: tahun pertama kemunculan data SDG di dataset
# Digunakan untuk memisahkan era pre-SDG (MDGs only) dan era campuran (MDGs + SDGs)
sdgs_rows = self.df[self.df["framework"] == "SDGs"]
if not sdgs_rows.empty:
self.sdgs_start_year = int(sdgs_rows["year"].min())
else:
# Tidak ada SDG sama sekali — set ke tahun setelah akhir data
self.sdgs_start_year = int(self.df["year"].max()) + 1
self.logger.warning(
f" [WARN] Tidak ada indikator SDGs. sdgs_start_year = {self.sdgs_start_year}"
)
self.df = self.df.merge(
ind_min_year[["indicator_id", "framework"]], on="indicator_id", how="left"
)
self.logger.info(f"\n Sumber klasifikasi : kolom 'framework' dari tabel")
self.logger.info(f" MDGs : {len(self.mdgs_indicator_ids)} indikator")
self.logger.info(f" SDGs : {len(self.sdgs_indicator_ids)} indikator")
self.logger.info(f" sdgs_start_year : {self.sdgs_start_year} (dari data aktual)")
# Log detail per framework untuk verifikasi
for fw in ["MDGs", "SDGs"]:
fw_inds = (
self.df[self.df["framework"] == fw]
.drop_duplicates("indicator_id")[["indicator_id", "indicator_name"]]
.sort_values("indicator_name")
)
self.logger.info(f"\n {fw} indicators ({len(fw_inds)}):")
for _, row in fw_inds.iterrows():
self.logger.info(f" [{int(row['indicator_id'])}] {row['indicator_name']}")
# =========================================================================
# CORE HELPER: normalisasi raw value per indikator
@@ -520,9 +545,9 @@ class FoodSecurityAggregator:
norm_parts.append(grp)
continue
raw = grp.loc[valid_mask, "value"].values
v_min, v_max = raw.min(), raw.max()
normed = np.full(len(grp), np.nan)
raw = grp.loc[valid_mask, "value"].values
v_min, v_max = raw.min(), raw.max()
normed = np.full(len(grp), np.nan)
if v_min == v_max:
normed[valid_mask.values] = 0.5
else:
@@ -553,9 +578,9 @@ class FoodSecurityAggregator:
df_normed
.groupby(["pillar_id", "pillar_name", "year"])
.agg(
pillar_norm =("norm_value", "mean"),
n_indicators=("indicator_id", "nunique"),
n_countries =("country_id", "nunique"),
pillar_norm =("norm_value", "mean"),
n_indicators =("indicator_id", "nunique"),
n_countries =("country_id", "nunique"),
)
.reset_index()
)
@@ -696,13 +721,18 @@ class FoodSecurityAggregator:
"score_1_100", "n_indicators", "composite_score"
]]
.copy()
.rename(columns={"score_1_100": "framework_score_1_100", "composite_score": "framework_norm"})
.rename(columns={
"score_1_100" : "framework_score_1_100",
"composite_score": "framework_norm"
})
)
agg_total["framework"] = "Total"
parts.append(agg_total)
# Layer MDGs — Era pre-SDGs = Total
pre_sdgs_rows = country_composite[country_composite["year"] < self.sdgs_start_year].copy()
pre_sdgs_rows = country_composite[
country_composite["year"] < self.sdgs_start_year
].copy()
if not pre_sdgs_rows.empty:
mdgs_pre = (
pre_sdgs_rows[[
@@ -710,12 +740,15 @@ class FoodSecurityAggregator:
"score_1_100", "n_indicators", "composite_score"
]]
.copy()
.rename(columns={"score_1_100": "framework_score_1_100", "composite_score": "framework_norm"})
.rename(columns={
"score_1_100" : "framework_score_1_100",
"composite_score": "framework_norm"
})
)
mdgs_pre["framework"] = "MDGs"
parts.append(mdgs_pre)
# Layer MDGs — Era mixed
# Layer MDGs — Era mixed (setelah SDGs masuk)
if self.mdgs_indicator_ids:
df_mdgs_mixed = df_normed[
(df_normed["indicator_id"].isin(self.mdgs_indicator_ids)) &
@@ -725,11 +758,16 @@ class FoodSecurityAggregator:
agg_mdgs_mixed = (
df_mdgs_mixed
.groupby(["country_id", "country_name", "year"])
.agg(framework_norm=("norm_value", "mean"), n_indicators=("indicator_id", "nunique"))
.agg(
framework_norm=("norm_value", "mean"),
n_indicators =("indicator_id", "nunique")
)
.reset_index()
)
if not NORMALIZE_FRAMEWORKS_JOINTLY:
agg_mdgs_mixed["framework_score_1_100"] = global_minmax(agg_mdgs_mixed["framework_norm"])
agg_mdgs_mixed["framework_score_1_100"] = global_minmax(
agg_mdgs_mixed["framework_norm"]
)
agg_mdgs_mixed["framework"] = "MDGs"
parts.append(agg_mdgs_mixed)
@@ -743,22 +781,34 @@ class FoodSecurityAggregator:
agg_sdgs = (
df_sdgs
.groupby(["country_id", "country_name", "year"])
.agg(framework_norm=("norm_value", "mean"), n_indicators=("indicator_id", "nunique"))
.agg(
framework_norm=("norm_value", "mean"),
n_indicators =("indicator_id", "nunique")
)
.reset_index()
)
if not NORMALIZE_FRAMEWORKS_JOINTLY:
agg_sdgs["framework_score_1_100"] = global_minmax(agg_sdgs["framework_norm"])
agg_sdgs["framework_score_1_100"] = global_minmax(
agg_sdgs["framework_norm"]
)
agg_sdgs["framework"] = "SDGs"
parts.append(agg_sdgs)
df = pd.concat(parts, ignore_index=True)
if NORMALIZE_FRAMEWORKS_JOINTLY:
mixed_mask = (df["framework"].isin(["MDGs", "SDGs"])) & (df["year"] >= self.sdgs_start_year)
mixed_mask = (
(df["framework"].isin(["MDGs", "SDGs"])) &
(df["year"] >= self.sdgs_start_year)
)
if mixed_mask.any():
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(df.loc[mixed_mask, "framework_norm"])
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(
df.loc[mixed_mask, "framework_norm"]
)
df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger)
df = check_and_dedup(
df, ["country_id", "framework", "year"], context=table_name, logger=self.logger
)
df["rank_in_framework_year"] = (
df.groupby(["framework", "year"])["framework_score_1_100"]
.rank(method="min", ascending=False)
@@ -808,51 +858,62 @@ class FoodSecurityAggregator:
country_composite = self._calc_country_composite_inmemory()
country_norm = (
df_normed.groupby(["country_id", "country_name", "year"])["norm_value"]
.mean().reset_index().rename(columns={"norm_value": "country_norm"})
df_normed
.groupby(["country_id", "country_name", "year"])["norm_value"]
.mean().reset_index()
.rename(columns={"norm_value": "country_norm"})
)
asean_overall = (
country_norm.groupby("year")
.agg(asean_norm=("country_norm", "mean"), std_norm=("country_norm", "std"),
n_countries=("country_norm", "count"))
.agg(
asean_norm =("country_norm", "mean"),
std_norm =("country_norm", "std"),
n_countries =("country_norm", "count")
)
.reset_index()
)
asean_overall["asean_score_1_100"] = global_minmax(asean_overall["asean_norm"])
asean_comp = (
country_composite.groupby("year")["composite_score"]
.mean().reset_index().rename(columns={"composite_score": "asean_composite"})
.mean().reset_index()
.rename(columns={"composite_score": "asean_composite"})
)
asean_overall = asean_overall.merge(asean_comp, on="year", how="left")
parts = []
# Layer TOTAL
total_cols = asean_overall[["year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"]].copy()
total_cols = total_cols.rename(columns={
total_cols = asean_overall[[
"year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"
]].copy().rename(columns={
"asean_score_1_100": "framework_score_1_100",
"asean_norm": "framework_norm",
"n_countries": "n_countries_with_data",
"asean_norm" : "framework_norm",
"n_countries" : "n_countries_with_data",
})
n_ind_total = df_normed.groupby("year")["indicator_id"].nunique().reset_index().rename(columns={"indicator_id": "n_indicators"})
total_cols = total_cols.merge(n_ind_total, on="year", how="left")
n_ind_total = (
df_normed.groupby("year")["indicator_id"].nunique()
.reset_index().rename(columns={"indicator_id": "n_indicators"})
)
total_cols = total_cols.merge(n_ind_total, on="year", how="left")
total_cols["framework"] = "Total"
parts.append(total_cols)
# Layer MDGs — pre-SDGs = Total
pre_sdgs = asean_overall[asean_overall["year"] < self.sdgs_start_year].copy()
if not pre_sdgs.empty:
mdgs_pre = pre_sdgs[["year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"]].copy()
mdgs_pre = mdgs_pre.rename(columns={
mdgs_pre = pre_sdgs[[
"year", "asean_score_1_100", "asean_norm", "std_norm", "n_countries"
]].copy().rename(columns={
"asean_score_1_100": "framework_score_1_100",
"asean_norm": "framework_norm",
"n_countries": "n_countries_with_data",
"asean_norm" : "framework_norm",
"n_countries" : "n_countries_with_data",
})
n_ind_pre = (
df_normed[df_normed["year"] < self.sdgs_start_year]
.groupby("year")["indicator_id"].nunique()
.reset_index().rename(columns={"indicator_id": "n_indicators"})
)
mdgs_pre = mdgs_pre.merge(n_ind_pre, on="year", how="left")
mdgs_pre = mdgs_pre.merge(n_ind_pre, on="year", how="left")
mdgs_pre["framework"] = "MDGs"
parts.append(mdgs_pre)
@@ -863,16 +924,25 @@ class FoodSecurityAggregator:
(df_normed["year"] >= self.sdgs_start_year)
].copy()
if not df_mdgs_mixed.empty:
cn = df_mdgs_mixed.groupby(["country_id", "year"])["norm_value"].mean().reset_index().rename(columns={"norm_value": "country_norm"})
cn = (
df_mdgs_mixed
.groupby(["country_id", "year"])["norm_value"].mean()
.reset_index().rename(columns={"norm_value": "country_norm"})
)
asean_mdgs = cn.groupby("year").agg(
framework_norm=("country_norm", "mean"),
std_norm=("country_norm", "std"),
n_countries_with_data=("country_id", "count"),
framework_norm =("country_norm", "mean"),
std_norm =("country_norm", "std"),
n_countries_with_data =("country_id", "count"),
).reset_index()
n_ind_mdgs = df_mdgs_mixed.groupby("year")["indicator_id"].nunique().reset_index().rename(columns={"indicator_id": "n_indicators"})
n_ind_mdgs = (
df_mdgs_mixed.groupby("year")["indicator_id"].nunique()
.reset_index().rename(columns={"indicator_id": "n_indicators"})
)
asean_mdgs = asean_mdgs.merge(n_ind_mdgs, on="year", how="left")
if not NORMALIZE_FRAMEWORKS_JOINTLY:
asean_mdgs["framework_score_1_100"] = global_minmax(asean_mdgs["framework_norm"])
asean_mdgs["framework_score_1_100"] = global_minmax(
asean_mdgs["framework_norm"]
)
asean_mdgs["framework"] = "MDGs"
parts.append(asean_mdgs)
@@ -883,27 +953,43 @@ class FoodSecurityAggregator:
(df_normed["year"] >= self.sdgs_start_year)
].copy()
if not df_sdgs.empty:
cn = df_sdgs.groupby(["country_id", "year"])["norm_value"].mean().reset_index().rename(columns={"norm_value": "country_norm"})
cn = (
df_sdgs
.groupby(["country_id", "year"])["norm_value"].mean()
.reset_index().rename(columns={"norm_value": "country_norm"})
)
asean_sdgs = cn.groupby("year").agg(
framework_norm=("country_norm", "mean"),
std_norm=("country_norm", "std"),
n_countries_with_data=("country_id", "count"),
framework_norm =("country_norm", "mean"),
std_norm =("country_norm", "std"),
n_countries_with_data =("country_id", "count"),
).reset_index()
n_ind_sdgs = df_sdgs.groupby("year")["indicator_id"].nunique().reset_index().rename(columns={"indicator_id": "n_indicators"})
n_ind_sdgs = (
df_sdgs.groupby("year")["indicator_id"].nunique()
.reset_index().rename(columns={"indicator_id": "n_indicators"})
)
asean_sdgs = asean_sdgs.merge(n_ind_sdgs, on="year", how="left")
if not NORMALIZE_FRAMEWORKS_JOINTLY:
asean_sdgs["framework_score_1_100"] = global_minmax(asean_sdgs["framework_norm"])
asean_sdgs["framework_score_1_100"] = global_minmax(
asean_sdgs["framework_norm"]
)
asean_sdgs["framework"] = "SDGs"
parts.append(asean_sdgs)
df = pd.concat(parts, ignore_index=True)
if NORMALIZE_FRAMEWORKS_JOINTLY:
mixed_mask = (df["framework"].isin(["MDGs", "SDGs"])) & (df["year"] >= self.sdgs_start_year)
mixed_mask = (
(df["framework"].isin(["MDGs", "SDGs"])) &
(df["year"] >= self.sdgs_start_year)
)
if mixed_mask.any():
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(df.loc[mixed_mask, "framework_norm"])
df.loc[mixed_mask, "framework_score_1_100"] = global_minmax(
df.loc[mixed_mask, "framework_norm"]
)
df = check_and_dedup(df, ["framework", "year"], context=table_name, logger=self.logger)
df = check_and_dedup(
df, ["framework", "year"], context=table_name, logger=self.logger
)
df = add_yoy(df, ["framework"], "framework_score_1_100")
df["year"] = df["year"].astype(int)
@@ -962,6 +1048,7 @@ class FoodSecurityAggregator:
.copy()
)
# Gunakan kolom framework dari self.df untuk hitung MDG/SDG per tahun
ind_year = self.df.drop_duplicates(subset=["indicator_id", "year", "framework"])
records = []
@@ -995,10 +1082,10 @@ class FoodSecurityAggregator:
for _, cr in yr_country.iterrows():
cr_yoy = cr.get("year_over_year_change", None)
ranking_list.append({
"rank": int(cr["rank_in_framework_year"]),
"rank" : int(cr["rank_in_framework_year"]),
"country_name": str(cr["country_name"]),
"score": round(float(cr["framework_score_1_100"]), 2),
"yoy_change": round(float(cr_yoy), 2) if pd.notna(cr_yoy) else None,
"score" : round(float(cr["framework_score_1_100"]), 2),
"yoy_change" : round(float(cr_yoy), 2) if pd.notna(cr_yoy) else None,
})
country_ranking_json = json.dumps(ranking_list, ensure_ascii=False)
@@ -1032,19 +1119,19 @@ class FoodSecurityAggregator:
)
records.append({
"year": yr,
"n_mdg_indicators": n_mdg,
"n_sdg_indicators": n_sdg,
"n_total_indicators": n_total_ind,
"asean_total_score": round(score, 2),
"yoy_change": yoy_val,
"yoy_change_pct": round(yoy_pct, 2) if yoy_pct is not None else None,
"country_ranking_json": country_ranking_json,
"year" : yr,
"n_mdg_indicators" : n_mdg,
"n_sdg_indicators" : n_sdg,
"n_total_indicators" : n_total_ind,
"asean_total_score" : round(score, 2),
"yoy_change" : yoy_val,
"yoy_change_pct" : round(yoy_pct, 2) if yoy_pct is not None else None,
"country_ranking_json" : country_ranking_json,
"most_improved_country": most_improved_country,
"most_improved_delta": most_improved_delta,
"most_improved_delta" : most_improved_delta,
"most_declined_country": most_declined_country,
"most_declined_delta": most_declined_delta,
"narrative_overview": narrative,
"most_declined_delta" : most_declined_delta,
"narrative_overview" : narrative,
})
df = pd.DataFrame(records)
@@ -1109,8 +1196,8 @@ class FoodSecurityAggregator:
yr_pillars_yoy = yr_pillars.dropna(subset=["year_over_year_change"])
if not yr_pillars_yoy.empty:
best_p_idx = yr_pillars_yoy["year_over_year_change"].idxmax()
worst_p_idx = yr_pillars_yoy["year_over_year_change"].idxmin()
best_p_idx = yr_pillars_yoy["year_over_year_change"].idxmax()
worst_p_idx = yr_pillars_yoy["year_over_year_change"].idxmin()
most_improved_pillar = str(yr_pillars_yoy.loc[best_p_idx, "pillar_name"])
most_improved_delta = round(float(yr_pillars_yoy.loc[best_p_idx, "year_over_year_change"]), 2)
most_declined_pillar = str(yr_pillars_yoy.loc[worst_p_idx, "pillar_name"])
@@ -1163,17 +1250,17 @@ class FoodSecurityAggregator:
)
records.append({
"year": yr,
"pillar_id": p_id,
"pillar_name": p_name,
"pillar_score": round(p_score, 2),
"rank_in_year": p_rank,
"yoy_change": p_yoy_val,
"top_country": top_country,
"top_country_score": top_country_score,
"bottom_country": bot_country,
"year" : yr,
"pillar_id" : p_id,
"pillar_name" : p_name,
"pillar_score" : round(p_score, 2),
"rank_in_year" : p_rank,
"yoy_change" : p_yoy_val,
"top_country" : top_country,
"top_country_score" : top_country_score,
"bottom_country" : bot_country,
"bottom_country_score": bot_country_score,
"narrative_pillar": narrative,
"narrative_pillar" : narrative,
})
df = pd.DataFrame(records)
@@ -1210,13 +1297,19 @@ class FoodSecurityAggregator:
def _validate_mdgs_equals_total(self, df: pd.DataFrame, level: str = ""):
self.logger.info(f"\n Validasi MDGs < {self.sdgs_start_year} == Total [{level}]:")
group_by = ["year"] if level.startswith("asean") else ["country_id", "year"]
mdgs_pre = df[(df["framework"] == "MDGs") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "mdgs_score"})
total_pre = df[(df["framework"] == "Total") & (df["year"] < self.sdgs_start_year)][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "total_score"})
mdgs_pre = df[
(df["framework"] == "MDGs") & (df["year"] < self.sdgs_start_year)
][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "mdgs_score"})
total_pre = df[
(df["framework"] == "Total") & (df["year"] < self.sdgs_start_year)
][group_by + ["framework_score_1_100"]].rename(columns={"framework_score_1_100": "total_score"})
if mdgs_pre.empty and total_pre.empty:
self.logger.info(f" -> Tidak ada data pre-{self.sdgs_start_year} (skip)")
return
if mdgs_pre.empty or total_pre.empty:
self.logger.warning(f" -> [WARNING] Salah satu kosong: MDGs={len(mdgs_pre)}, Total={len(total_pre)}")
self.logger.warning(
f" -> [WARNING] Salah satu kosong: MDGs={len(mdgs_pre)}, Total={len(total_pre)}"
)
return
check = mdgs_pre.merge(total_pre, on=group_by)
max_diff = (check["mdgs_score"] - check["total_score"]).abs().max()
@@ -1228,8 +1321,8 @@ class FoodSecurityAggregator:
"rows_loaded": rows_loaded, "status": "success", "end_time": datetime.now(),
})
log_update(self.client, "DW", table_name, "full_load", rows_loaded)
self.logger.info(f" ✓ {table_name}: {rows_loaded:,} rows → [Gold] fs_asean_gold")
self.logger.info(f" Metadata → [AUDIT] etl_logs")
self.logger.info(f" {table_name}: {rows_loaded:,} rows -> [Gold] fs_asean_gold")
self.logger.info(f" Metadata -> [AUDIT] etl_logs")
def _fail(self, table_name: str, error: Exception):
self.load_metadata[table_name].update({"status": "failed", "end_time": datetime.now()})
@@ -1248,6 +1341,8 @@ class FoodSecurityAggregator:
self.logger.info(" Outputs : agg_pillar_composite | agg_pillar_by_country")
self.logger.info(" agg_framework_by_country| agg_framework_asean")
self.logger.info(" agg_narrative_overview | agg_narrative_pillar")
self.logger.info(" NOTE : framework (MDGs/SDGs) dibaca dari kolom tabel,")
self.logger.info(" bukan heuristik gap min_year")
self.logger.info("=" * 70)
self.load_data()
@@ -1276,8 +1371,8 @@ class FoodSecurityAggregator:
self.logger.info(f" Durasi : {duration:.2f}s")
self.logger.info(f" Total rows : {total_rows:,}")
for tbl, meta in self.load_metadata.items():
icon = "✓" if meta["status"] == "success" else "✗"
self.logger.info(f" {icon} {tbl:<35} {meta['rows_loaded']:>10,}")
icon = "OK" if meta["status"] == "success" else "FAIL"
self.logger.info(f" [{icon}] {tbl:<35} {meta['rows_loaded']:>10,}")
# =============================================================================
@@ -1312,6 +1407,7 @@ if __name__ == "__main__":
print("=" * 70)
print("FOOD SECURITY AGGREGATION -> fs_asean_gold")
print(f" Source : fact_asean_food_security_selected")
print(f" Framework classification : dari kolom tabel (bukan heuristik)")
print(f" NORMALIZE_FRAMEWORKS_JOINTLY : {NORMALIZE_FRAMEWORKS_JOINTLY}")
print("=" * 70)