add a condition for pillar
This commit is contained in:
@@ -20,6 +20,32 @@ Narrative style:
|
|||||||
- Plain text, tanpa markdown bold (**)
|
- Plain text, tanpa markdown bold (**)
|
||||||
- Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata
|
- Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata
|
||||||
- Bilingual: narrative_en (Inggris) + narrative_id (Indonesia)
|
- Bilingual: narrative_en (Inggris) + narrative_id (Indonesia)
|
||||||
|
|
||||||
|
KONDISI PILAR (pillar_condition_en / pillar_condition_id):
|
||||||
|
Kolom tambahan di agg_pillar_by_country untuk mendeskripsikan kondisi
|
||||||
|
tiap pilar per negara per tahun secara kontekstual dan kuantitatif.
|
||||||
|
|
||||||
|
Landasan teori:
|
||||||
|
1. FAO & CFS (1996 World Food Summit; CFS Reform Document 2009):
|
||||||
|
Definisi 4 pilar ketahanan pangan dan makna substantif masing-masing.
|
||||||
|
Referensi: FAO (2009). "Declaration of the World Summit on Food Security."
|
||||||
|
CFS (2012). "Global Strategic Framework for Food Security & Nutrition."
|
||||||
|
2. GFSI — Economist Impact (2022):
|
||||||
|
Threshold klasifikasi skor 0-100:
|
||||||
|
>= 75 : "Good" environment -> label "Secure / Aman"
|
||||||
|
>= 60 : above threshold -> label "Adequate / Memadai"
|
||||||
|
>= 40 : "Moderate" env -> label "Moderate / Sedang"
|
||||||
|
>= 20 : below moderate -> label "At Risk / Berisiko"
|
||||||
|
< 20 : severe -> label "Critical / Kritis"
|
||||||
|
Referensi: Economist Impact (2022). "Global Food Security Index 2022."
|
||||||
|
3. IPC — Integrated Food Security Phase Classification (2019):
|
||||||
|
Klasifikasi bertingkat per pilar: dari "Moderate Risk" hingga "Critical".
|
||||||
|
Referensi: IPC (2019). "IPC Technical Manual Version 3.0."
|
||||||
|
4. FAO SOFI (2023/2024):
|
||||||
|
Konteks kondisi per pilar: availability (supply/stok), access (keterjangkauan),
|
||||||
|
utilization (nutrisi/sanitasi), stability (kerentanan terhadap guncangan).
|
||||||
|
Referensi: FAO et al. (2024). "The State of Food Security and Nutrition
|
||||||
|
in the World 2024."
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
@@ -157,6 +183,173 @@ def translate_pillar(name: str) -> str:
|
|||||||
return PILLAR_TRANSLATION_ID.get(name, name)
|
return PILLAR_TRANSLATION_ID.get(name, name)
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# PILLAR CONDITION CLASSIFIER
|
||||||
|
# =============================================================================
|
||||||
|
#
|
||||||
|
# Landasan teori (lihat docstring modul di atas untuk referensi lengkap):
|
||||||
|
#
|
||||||
|
# Tier skor (skala 1-100, mengacu GFSI 2022 + IPC Phase Classification):
|
||||||
|
# >= 75 : Secure / Aman — performa tinggi, kondisi baik
|
||||||
|
# >= 60 : Adequate / Memadai — di atas threshold, masih ada ruang
|
||||||
|
# >= 40 : Moderate / Sedang — tantangan nyata, perlu perhatian
|
||||||
|
# >= 20 : At Risk / Berisiko — kondisi lemah, butuh intervensi
|
||||||
|
# < 20 : Critical / Kritis — sangat buruk, tindakan segera
|
||||||
|
#
|
||||||
|
# Label kontekstual per pilar mengacu definisi FAO/CFS empat pilar:
|
||||||
|
# Food Availability : ketersediaan pasokan (produksi, stok, impor)
|
||||||
|
# Food Access : keterjangkauan ekonomi & fisik terhadap pangan
|
||||||
|
# Food Utilization : pemanfaatan biologis (gizi, sanitasi, kesehatan)
|
||||||
|
# Food Stability : konsistensi tiga pilar di atas dari waktu ke waktu
|
||||||
|
# Food Other : indikator multidimensi / suplemen
|
||||||
|
#
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
# Tier thresholds (urut dari tertinggi)
|
||||||
|
_CONDITION_TIERS = [
|
||||||
|
# (min_score, base_label_en, base_label_id)
|
||||||
|
(75, "Secure", "Aman"),
|
||||||
|
(60, "Adequate", "Memadai"),
|
||||||
|
(40, "Moderate", "Sedang"),
|
||||||
|
(20, "At Risk", "Berisiko"),
|
||||||
|
( 0, "Critical", "Kritis"),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Konteks kondisi per pilar per tier (EN, ID)
|
||||||
|
# Mengacu makna substantif pilar (FAO SOFI 2024; FSC Handbook 2020;
|
||||||
|
# IPC Technical Manual 2019).
|
||||||
|
_PILLAR_CONTEXT: dict = {
|
||||||
|
# ---- Food Availability ----
|
||||||
|
"Food Availability": {
|
||||||
|
"Secure" : ("Food supply is abundant and well-distributed",
|
||||||
|
"Pasokan pangan berlimpah dan terdistribusi merata"),
|
||||||
|
"Adequate" : ("Food supply is sufficient with minor gaps",
|
||||||
|
"Pasokan pangan cukup dengan kesenjangan minor"),
|
||||||
|
"Moderate" : ("Food supply shows signs of strain",
|
||||||
|
"Pasokan pangan menunjukkan tanda-tanda tekanan"),
|
||||||
|
"At Risk" : ("Food supply is insufficient; stocks are dwindling",
|
||||||
|
"Pasokan pangan tidak mencukupi; stok mulai menipis"),
|
||||||
|
"Critical" : ("Severe food supply deficit; stocks critically low",
|
||||||
|
"Defisit pasokan pangan parah; stok dalam kondisi kritis"),
|
||||||
|
},
|
||||||
|
# ---- Food Access ----
|
||||||
|
"Food Access": {
|
||||||
|
"Secure" : ("Food is economically and physically accessible to all",
|
||||||
|
"Pangan terjangkau secara ekonomi dan fisik bagi semua"),
|
||||||
|
"Adequate" : ("Food access is generally good with limited barriers",
|
||||||
|
"Akses pangan umumnya baik dengan hambatan terbatas"),
|
||||||
|
"Moderate" : ("Portions of the population face access constraints",
|
||||||
|
"Sebagian penduduk menghadapi kendala akses pangan"),
|
||||||
|
"At Risk" : ("Significant affordability or physical access barriers",
|
||||||
|
"Hambatan keterjangkauan atau akses fisik yang signifikan"),
|
||||||
|
"Critical" : ("Widespread inability to access sufficient food",
|
||||||
|
"Ketidakmampuan meluas dalam mengakses pangan yang cukup"),
|
||||||
|
},
|
||||||
|
# ---- Food Utilization ----
|
||||||
|
"Food Utilization": {
|
||||||
|
"Secure" : ("Dietary quality, nutrition, and sanitation are strong",
|
||||||
|
"Kualitas gizi, nutrisi, dan sanitasi dalam kondisi baik"),
|
||||||
|
"Adequate" : ("Nutrition and sanitation are adequate; minor deficiencies",
|
||||||
|
"Gizi dan sanitasi memadai; kekurangan minor masih ada"),
|
||||||
|
"Moderate" : ("Nutritional gaps or sanitation issues are evident",
|
||||||
|
"Kesenjangan gizi atau masalah sanitasi mulai terlihat"),
|
||||||
|
"At Risk" : ("Significant nutritional deficiencies or poor sanitation",
|
||||||
|
"Kekurangan gizi atau sanitasi buruk yang signifikan"),
|
||||||
|
"Critical" : ("Severe malnutrition and/or critical sanitation deficits",
|
||||||
|
"Malnutrisi parah dan/atau defisit sanitasi yang kritis"),
|
||||||
|
},
|
||||||
|
# ---- Food Stability ----
|
||||||
|
"Food Stability": {
|
||||||
|
"Secure" : ("Food security is consistently maintained over time",
|
||||||
|
"Ketahanan pangan terjaga konsisten dari waktu ke waktu"),
|
||||||
|
"Adequate" : ("Stability is generally good with manageable risks",
|
||||||
|
"Stabilitas umumnya baik dengan risiko yang masih terkelola"),
|
||||||
|
"Moderate" : ("Periodic shocks or vulnerabilities affect stability",
|
||||||
|
"Guncangan periodik atau kerentanan memengaruhi stabilitas"),
|
||||||
|
"At Risk" : ("Frequent disruptions threaten food security continuity",
|
||||||
|
"Gangguan berulang mengancam kesinambungan ketahanan pangan"),
|
||||||
|
"Critical" : ("Sustained instability; food security is highly fragile",
|
||||||
|
"Ketidakstabilan berkelanjutan; ketahanan pangan sangat rapuh"),
|
||||||
|
},
|
||||||
|
# ---- Food Other / Indikator Tambahan ----
|
||||||
|
"Food Other": {
|
||||||
|
"Secure" : ("Supplementary indicators reflect strong food system",
|
||||||
|
"Indikator tambahan mencerminkan sistem pangan yang kuat"),
|
||||||
|
"Adequate" : ("Supplementary indicators are at acceptable levels",
|
||||||
|
"Indikator tambahan berada pada level yang dapat diterima"),
|
||||||
|
"Moderate" : ("Supplementary indicators signal emerging challenges",
|
||||||
|
"Indikator tambahan memberi sinyal tantangan yang muncul"),
|
||||||
|
"At Risk" : ("Supplementary indicators show concerning levels",
|
||||||
|
"Indikator tambahan menunjukkan level yang mengkhawatirkan"),
|
||||||
|
"Critical" : ("Supplementary indicators reflect systemic food system failure",
|
||||||
|
"Indikator tambahan mencerminkan kegagalan sistemik pangan"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fallback jika pillar_name tidak dikenali
|
||||||
|
_PILLAR_CONTEXT_FALLBACK: dict = {
|
||||||
|
"Secure" : ("Performance is high across food security indicators",
|
||||||
|
"Performa tinggi pada indikator ketahanan pangan"),
|
||||||
|
"Adequate" : ("Performance is adequate across food security indicators",
|
||||||
|
"Performa memadai pada indikator ketahanan pangan"),
|
||||||
|
"Moderate" : ("Performance shows moderate challenges",
|
||||||
|
"Performa menunjukkan tantangan yang moderat"),
|
||||||
|
"At Risk" : ("Performance indicates vulnerability in food security",
|
||||||
|
"Performa mengindikasikan kerentanan ketahanan pangan"),
|
||||||
|
"Critical" : ("Performance is critically low; urgent action needed",
|
||||||
|
"Performa sangat rendah; tindakan segera diperlukan"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_pillar_condition(pillar_name: str, score: float) -> tuple:
|
||||||
|
"""
|
||||||
|
Mengembalikan (condition_en, condition_id) berdasarkan skor dan nama pilar.
|
||||||
|
|
||||||
|
Tier mengacu GFSI 2022 (Economist Impact) + IPC Phase Classification (2019):
|
||||||
|
>= 75 -> Secure / Aman
|
||||||
|
>= 60 -> Adequate / Memadai
|
||||||
|
>= 40 -> Moderate / Sedang
|
||||||
|
>= 20 -> At Risk / Berisiko
|
||||||
|
< 20 -> Critical / Kritis
|
||||||
|
|
||||||
|
Deskripsi kontekstual mengacu FAO/CFS definisi 4 pilar (World Food Summit
|
||||||
|
1996; CFS 2009) dan FAO SOFI 2024.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pillar_name : Nama pilar dalam bahasa Inggris (e.g. "Food Availability").
|
||||||
|
score : Skor ternormalisasi skala 1-100.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple (condition_en: str, condition_id: str)
|
||||||
|
"""
|
||||||
|
if score is None or (isinstance(score, float) and np.isnan(score)):
|
||||||
|
return ("N/A", "N/A")
|
||||||
|
|
||||||
|
# Tentukan tier
|
||||||
|
tier_label_en = _CONDITION_TIERS[-1][1] # default: Critical
|
||||||
|
tier_label_id = _CONDITION_TIERS[-1][2]
|
||||||
|
for min_score, lbl_en, lbl_id in _CONDITION_TIERS:
|
||||||
|
if score >= min_score:
|
||||||
|
tier_label_en = lbl_en
|
||||||
|
tier_label_id = lbl_id
|
||||||
|
break
|
||||||
|
|
||||||
|
# Ambil konteks per pilar
|
||||||
|
ctx = _PILLAR_CONTEXT.get(pillar_name, None)
|
||||||
|
if ctx:
|
||||||
|
ctx_en, ctx_id = ctx.get(
|
||||||
|
tier_label_en,
|
||||||
|
_PILLAR_CONTEXT_FALLBACK.get(tier_label_en, ("", ""))
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
ctx_en, ctx_id = _PILLAR_CONTEXT_FALLBACK.get(tier_label_en, ("", ""))
|
||||||
|
|
||||||
|
# Format akhir: "TIER — Context"
|
||||||
|
condition_en = f"{tier_label_en} — {ctx_en}"
|
||||||
|
condition_id = f"{tier_label_id} — {ctx_id}"
|
||||||
|
return condition_en, condition_id
|
||||||
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# WINDOWS CP1252 SAFE LOGGING
|
# WINDOWS CP1252 SAFE LOGGING
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -334,8 +527,6 @@ def _find_anomaly_year(values_by_year: dict) -> tuple:
|
|||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# NARRATIVE BUILDER — PILLAR
|
# NARRATIVE BUILDER — PILLAR
|
||||||
# Digunakan untuk SEMUA baris: per negara dan ASEAN aggregate.
|
|
||||||
# Jika is_asean=True, narasi tidak menyebut "country" melainkan "ASEAN region".
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
def _build_pillar_narrative(
|
def _build_pillar_narrative(
|
||||||
@@ -418,7 +609,6 @@ def _build_pillar_narrative(
|
|||||||
sentences_en.append(s3_en)
|
sentences_en.append(s3_en)
|
||||||
sentences_id.append(s3_id)
|
sentences_id.append(s3_id)
|
||||||
|
|
||||||
# Gap antar negara hanya relevan untuk ASEAN narrative
|
|
||||||
if is_asean and not country_pillar_all.empty:
|
if is_asean and not country_pillar_all.empty:
|
||||||
gap_trend = _detect_country_gap(
|
gap_trend = _detect_country_gap(
|
||||||
country_pillar_all[country_pillar_all["year"] <= year],
|
country_pillar_all[country_pillar_all["year"] <= year],
|
||||||
@@ -437,7 +627,6 @@ def _build_pillar_narrative(
|
|||||||
sentences_en.append(s4_en)
|
sentences_en.append(s4_en)
|
||||||
sentences_id.append(s4_id)
|
sentences_id.append(s4_id)
|
||||||
|
|
||||||
# Top/bottom hanya ditampilkan untuk baris ASEAN
|
|
||||||
if is_asean and top_country and bot_country and top_country != bot_country:
|
if is_asean and top_country and bot_country and top_country != bot_country:
|
||||||
top_country_id = translate_country(top_country)
|
top_country_id = translate_country(top_country)
|
||||||
bot_country_id = translate_country(bot_country)
|
bot_country_id = translate_country(bot_country)
|
||||||
@@ -453,7 +642,6 @@ def _build_pillar_narrative(
|
|||||||
sentences_en.append(s5_en)
|
sentences_en.append(s5_en)
|
||||||
sentences_id.append(s5_id)
|
sentences_id.append(s5_id)
|
||||||
|
|
||||||
# Perbandingan antar pilar
|
|
||||||
if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1:
|
if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1:
|
||||||
sorted_pillars = all_pillar_scores_year.sort_values("pillar_country_score_1_100", ascending=False)
|
sorted_pillars = all_pillar_scores_year.sort_values("pillar_country_score_1_100", ascending=False)
|
||||||
strongest = sorted_pillars.iloc[0]
|
strongest = sorted_pillars.iloc[0]
|
||||||
@@ -530,7 +718,6 @@ class FoodSecurityAggregator:
|
|||||||
self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'")
|
self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'")
|
||||||
self.df["direction"] = self.df["direction"].fillna("positive")
|
self.df["direction"] = self.df["direction"].fillna("positive")
|
||||||
|
|
||||||
# Rename pillar_name: add 'Food ' prefix, remove Sustainability
|
|
||||||
PILLAR_RENAME_MAP = {
|
PILLAR_RENAME_MAP = {
|
||||||
'Availability' : 'Food Availability',
|
'Availability' : 'Food Availability',
|
||||||
'Access' : 'Food Access',
|
'Access' : 'Food Access',
|
||||||
@@ -542,7 +729,6 @@ class FoodSecurityAggregator:
|
|||||||
}
|
}
|
||||||
self.df["pillar_name"] = self.df["pillar_name"].replace(PILLAR_RENAME_MAP)
|
self.df["pillar_name"] = self.df["pillar_name"].replace(PILLAR_RENAME_MAP)
|
||||||
|
|
||||||
# Kolom terjemahan Indonesia
|
|
||||||
if "country_name_id" not in self.df.columns:
|
if "country_name_id" not in self.df.columns:
|
||||||
self.df["country_name_id"] = self.df["country_name"].apply(translate_country)
|
self.df["country_name_id"] = self.df["country_name"].apply(translate_country)
|
||||||
if "pillar_name_id" not in self.df.columns:
|
if "pillar_name_id" not in self.df.columns:
|
||||||
@@ -686,6 +872,12 @@ class FoodSecurityAggregator:
|
|||||||
"asean_country_id" : ASEAN_COUNTRY_ID,
|
"asean_country_id" : ASEAN_COUNTRY_ID,
|
||||||
"pillar_change" : "Sustainability renamed to Food Other, all pillars prefixed with Food",
|
"pillar_change" : "Sustainability renamed to Food Other, all pillars prefixed with Food",
|
||||||
"architecture" : "ASEAN merged into country tables (country_id=0)",
|
"architecture" : "ASEAN merged into country tables (country_id=0)",
|
||||||
|
"condition_column" : "pillar_condition_en/id added to agg_pillar_by_country",
|
||||||
|
"condition_reference" : (
|
||||||
|
"GFSI 2022 (Economist Impact) score tiers >= 75/60/40/20; "
|
||||||
|
"IPC Technical Manual 2019; FAO/CFS 4-pillar framework 1996/2009; "
|
||||||
|
"FAO SOFI 2024"
|
||||||
|
),
|
||||||
}),
|
}),
|
||||||
"validation_metrics" : json.dumps({
|
"validation_metrics" : json.dumps({
|
||||||
"status" : status,
|
"status" : status,
|
||||||
@@ -698,11 +890,6 @@ class FoodSecurityAggregator:
|
|||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
||||||
def _build_asean_pillar_rows(self, df_normed: pd.DataFrame) -> pd.DataFrame:
|
def _build_asean_pillar_rows(self, df_normed: pd.DataFrame) -> pd.DataFrame:
|
||||||
"""
|
|
||||||
Hitung rata-rata ASEAN per pillar per year dari norm_value semua negara,
|
|
||||||
kemudian scale ulang ke 1-100 dalam konteks SELURUH tabel (negara + ASEAN).
|
|
||||||
Return DataFrame dengan format sama seperti baris per-negara.
|
|
||||||
"""
|
|
||||||
asean_agg = (
|
asean_agg = (
|
||||||
df_normed
|
df_normed
|
||||||
.groupby(["pillar_id", "pillar_name", "year"])
|
.groupby(["pillar_id", "pillar_name", "year"])
|
||||||
@@ -716,7 +903,7 @@ class FoodSecurityAggregator:
|
|||||||
return asean_agg
|
return asean_agg
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# STEP 2: agg_pillar_by_country (termasuk ASEAN)
|
# STEP 2: agg_pillar_by_country (termasuk ASEAN + kolom kondisi)
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
||||||
def calc_pillar_by_country(self) -> pd.DataFrame:
|
def calc_pillar_by_country(self) -> pd.DataFrame:
|
||||||
@@ -725,6 +912,7 @@ class FoodSecurityAggregator:
|
|||||||
self.logger.info("\n" + "=" * 70)
|
self.logger.info("\n" + "=" * 70)
|
||||||
self.logger.info(f"STEP 2: {table_name} -> [Gold] fs_asean_gold")
|
self.logger.info(f"STEP 2: {table_name} -> [Gold] fs_asean_gold")
|
||||||
self.logger.info(" Termasuk baris ASEAN (country_id=0) untuk filter Looker Studio")
|
self.logger.info(" Termasuk baris ASEAN (country_id=0) untuk filter Looker Studio")
|
||||||
|
self.logger.info(" Kolom baru: pillar_condition_en, pillar_condition_id")
|
||||||
self.logger.info("=" * 70)
|
self.logger.info("=" * 70)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -746,10 +934,27 @@ class FoodSecurityAggregator:
|
|||||||
# Gabung
|
# Gabung
|
||||||
df = pd.concat([df_countries, df_asean], ignore_index=True)
|
df = pd.concat([df_countries, df_asean], ignore_index=True)
|
||||||
|
|
||||||
# Scale 1-100 secara BERSAMA (negara + ASEAN dalam satu ruang skala)
|
# Scale 1-100 secara BERSAMA
|
||||||
df["pillar_country_score_1_100"] = global_minmax(df["pillar_country_norm"])
|
df["pillar_country_score_1_100"] = global_minmax(df["pillar_country_norm"])
|
||||||
|
|
||||||
# Rank hanya di antara negara asli (ASEAN tidak di-rank melawan dirinya sendiri)
|
# ---------------------------------------------------------------
|
||||||
|
# TAMBAHAN: kolom kondisi pilar
|
||||||
|
# Dibangkitkan SETELAH score_1_100 tersedia, sehingga tier
|
||||||
|
# langsung mencerminkan skor dalam skala akhir 1-100.
|
||||||
|
# Referensi tier: GFSI 2022 (Economist Impact); IPC 2019;
|
||||||
|
# FAO/CFS 1996/2009; FAO SOFI 2024.
|
||||||
|
# ---------------------------------------------------------------
|
||||||
|
conditions = df.apply(
|
||||||
|
lambda row: get_pillar_condition(
|
||||||
|
row["pillar_name"],
|
||||||
|
row["pillar_country_score_1_100"]
|
||||||
|
),
|
||||||
|
axis=1
|
||||||
|
)
|
||||||
|
df["pillar_condition_en"] = conditions.apply(lambda x: x[0])
|
||||||
|
df["pillar_condition_id"] = conditions.apply(lambda x: x[1])
|
||||||
|
|
||||||
|
# Rank hanya di antara negara asli
|
||||||
country_only = df[df["country_id"] != ASEAN_COUNTRY_ID].copy()
|
country_only = df[df["country_id"] != ASEAN_COUNTRY_ID].copy()
|
||||||
country_only["rank_in_pillar_year"] = (
|
country_only["rank_in_pillar_year"] = (
|
||||||
country_only.groupby(["pillar_id", "year"])["pillar_country_score_1_100"]
|
country_only.groupby(["pillar_id", "year"])["pillar_country_score_1_100"]
|
||||||
@@ -771,12 +976,20 @@ class FoodSecurityAggregator:
|
|||||||
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
|
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
|
||||||
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
|
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
|
||||||
df["country_name_id"] = df["country_name_id"].astype(str)
|
df["country_name_id"] = df["country_name_id"].astype(str)
|
||||||
|
df["pillar_condition_en"] = df["pillar_condition_en"].astype(str)
|
||||||
|
df["pillar_condition_id"] = df["pillar_condition_id"].astype(str)
|
||||||
|
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
f" Total rows: {len(df):,} "
|
f" Total rows: {len(df):,} "
|
||||||
f"({len(df_countries):,} country + {len(asean_only):,} ASEAN)"
|
f"({len(df_countries):,} country + {len(asean_only):,} ASEAN)"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Log distribusi kondisi untuk QA
|
||||||
|
self.logger.info("\n Distribusi pillar_condition_en (sample):")
|
||||||
|
cond_dist = df["pillar_condition_en"].value_counts().head(10)
|
||||||
|
for cond, cnt in cond_dist.items():
|
||||||
|
self.logger.info(f" {cnt:>6,} {cond}")
|
||||||
|
|
||||||
schema = [
|
schema = [
|
||||||
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
|
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
|
||||||
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
|
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
|
||||||
@@ -789,6 +1002,10 @@ class FoodSecurityAggregator:
|
|||||||
bigquery.SchemaField("pillar_country_score_1_100", "FLOAT", mode="REQUIRED"),
|
bigquery.SchemaField("pillar_country_score_1_100", "FLOAT", mode="REQUIRED"),
|
||||||
bigquery.SchemaField("rank_in_pillar_year", "INTEGER", mode="REQUIRED"),
|
bigquery.SchemaField("rank_in_pillar_year", "INTEGER", mode="REQUIRED"),
|
||||||
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
|
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
|
||||||
|
# --- KOLOM KONDISI BARU ---
|
||||||
|
# Tier skor (GFSI 2022) + konteks substantif per pilar (FAO/CFS; IPC 2019)
|
||||||
|
bigquery.SchemaField("pillar_condition_en", "STRING", mode="REQUIRED"),
|
||||||
|
bigquery.SchemaField("pillar_condition_id", "STRING", mode="REQUIRED"),
|
||||||
]
|
]
|
||||||
rows = load_to_bigquery(
|
rows = load_to_bigquery(
|
||||||
self.client, df, table_name, layer='gold',
|
self.client, df, table_name, layer='gold',
|
||||||
@@ -927,7 +1144,7 @@ class FoodSecurityAggregator:
|
|||||||
|
|
||||||
df_countries = pd.concat(parts, ignore_index=True)
|
df_countries = pd.concat(parts, ignore_index=True)
|
||||||
|
|
||||||
# ---- ASEAN aggregate (rata-rata dari semua negara per framework per year) ----
|
# ---- ASEAN aggregate ----
|
||||||
asean_parts = []
|
asean_parts = []
|
||||||
for fw in df_countries["framework"].unique():
|
for fw in df_countries["framework"].unique():
|
||||||
fw_df = df_countries[
|
fw_df = df_countries[
|
||||||
@@ -958,7 +1175,6 @@ class FoodSecurityAggregator:
|
|||||||
|
|
||||||
df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger)
|
df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger)
|
||||||
|
|
||||||
# Rank hanya di antara negara asli
|
|
||||||
country_mask = df["country_id"] != ASEAN_COUNTRY_ID
|
country_mask = df["country_id"] != ASEAN_COUNTRY_ID
|
||||||
df.loc[country_mask, "rank_in_framework_year"] = (
|
df.loc[country_mask, "rank_in_framework_year"] = (
|
||||||
df[country_mask]
|
df[country_mask]
|
||||||
@@ -1014,7 +1230,6 @@ class FoodSecurityAggregator:
|
|||||||
self.logger.info("\n" + "=" * 70)
|
self.logger.info("\n" + "=" * 70)
|
||||||
self.logger.info(f"STEP 4: {table_name} -> [Gold] fs_asean_gold")
|
self.logger.info(f"STEP 4: {table_name} -> [Gold] fs_asean_gold")
|
||||||
self.logger.info(" Termasuk baris ASEAN (country_id=0)")
|
self.logger.info(" Termasuk baris ASEAN (country_id=0)")
|
||||||
self.logger.info(" Filter country_name='ASEAN' untuk overview regional")
|
|
||||||
self.logger.info("=" * 70)
|
self.logger.info("=" * 70)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -1022,7 +1237,6 @@ class FoodSecurityAggregator:
|
|||||||
years = sorted(df_pillar_by_country["year"].unique())
|
years = sorted(df_pillar_by_country["year"].unique())
|
||||||
pillars = df_pillar_by_country["pillar_id"].unique()
|
pillars = df_pillar_by_country["pillar_id"].unique()
|
||||||
|
|
||||||
# Precompute history per country x pillar
|
|
||||||
history = {}
|
history = {}
|
||||||
for (c_id, p_id), grp in df_pillar_by_country.groupby(["country_id", "pillar_id"]):
|
for (c_id, p_id), grp in df_pillar_by_country.groupby(["country_id", "pillar_id"]):
|
||||||
history[(c_id, p_id)] = dict(
|
history[(c_id, p_id)] = dict(
|
||||||
@@ -1031,8 +1245,6 @@ class FoodSecurityAggregator:
|
|||||||
|
|
||||||
for yr in years:
|
for yr in years:
|
||||||
yr_df = df_pillar_by_country[df_pillar_by_country["year"] == yr]
|
yr_df = df_pillar_by_country[df_pillar_by_country["year"] == yr]
|
||||||
|
|
||||||
# Semua negara asli untuk referensi top/bottom dalam narasi ASEAN
|
|
||||||
country_only_yr = yr_df[yr_df["country_id"] != ASEAN_COUNTRY_ID]
|
country_only_yr = yr_df[yr_df["country_id"] != ASEAN_COUNTRY_ID]
|
||||||
|
|
||||||
for p_id in pillars:
|
for p_id in pillars:
|
||||||
@@ -1044,11 +1256,9 @@ class FoodSecurityAggregator:
|
|||||||
p_name = str(p_name_row["pillar_name"])
|
p_name = str(p_name_row["pillar_name"])
|
||||||
n_pillars = len(pillars)
|
n_pillars = len(pillars)
|
||||||
|
|
||||||
# Ranking di antara semua pillar (gunakan skor ASEAN untuk rank antar pillar)
|
|
||||||
asean_yr_all_pillars = yr_df[yr_df["country_id"] == ASEAN_COUNTRY_ID]
|
asean_yr_all_pillars = yr_df[yr_df["country_id"] == ASEAN_COUNTRY_ID]
|
||||||
asean_sorted = asean_yr_all_pillars.sort_values("pillar_country_score_1_100", ascending=False).reset_index(drop=True)
|
asean_sorted = asean_yr_all_pillars.sort_values("pillar_country_score_1_100", ascending=False).reset_index(drop=True)
|
||||||
|
|
||||||
# Top/bottom di antara negara asli (untuk narasi ASEAN)
|
|
||||||
country_pillar_yr = country_only_yr[country_only_yr["pillar_id"] == p_id]
|
country_pillar_yr = country_only_yr[country_only_yr["pillar_id"] == p_id]
|
||||||
if not country_pillar_yr.empty:
|
if not country_pillar_yr.empty:
|
||||||
top_row = country_pillar_yr.loc[country_pillar_yr["pillar_country_score_1_100"].idxmax()]
|
top_row = country_pillar_yr.loc[country_pillar_yr["pillar_country_score_1_100"].idxmax()]
|
||||||
@@ -1061,7 +1271,6 @@ class FoodSecurityAggregator:
|
|||||||
top_country = bot_country = None
|
top_country = bot_country = None
|
||||||
top_score = bot_score = None
|
top_score = bot_score = None
|
||||||
|
|
||||||
# Iterasi setiap baris (negara + ASEAN) pada pillar ini
|
|
||||||
for _, row in yr_pillar_all.iterrows():
|
for _, row in yr_pillar_all.iterrows():
|
||||||
c_id = int(row["country_id"])
|
c_id = int(row["country_id"])
|
||||||
c_name = str(row["country_name"])
|
c_name = str(row["country_name"])
|
||||||
@@ -1072,22 +1281,15 @@ class FoodSecurityAggregator:
|
|||||||
p_name_id = translate_pillar(p_name)
|
p_name_id = translate_pillar(p_name)
|
||||||
is_asean = (c_id == ASEAN_COUNTRY_ID)
|
is_asean = (c_id == ASEAN_COUNTRY_ID)
|
||||||
|
|
||||||
# Rank pilar ini dalam konteks yang sesuai
|
|
||||||
if is_asean:
|
if is_asean:
|
||||||
# ASEAN: rank pilar ini di antara semua pilar ASEAN tahun ini
|
|
||||||
rank_sorted = asean_sorted.reset_index(drop=True)
|
rank_sorted = asean_sorted.reset_index(drop=True)
|
||||||
p_rank = int(rank_sorted[rank_sorted["pillar_id"] == p_id].index[0]) + 1 if p_id in rank_sorted["pillar_id"].values else 0
|
p_rank = int(rank_sorted[rank_sorted["pillar_id"] == p_id].index[0]) + 1 if p_id in rank_sorted["pillar_id"].values else 0
|
||||||
else:
|
else:
|
||||||
# Negara: rank pillar ini di antara semua pillar negara ini
|
|
||||||
country_all_pillars = yr_df[yr_df["country_id"] == c_id].sort_values("pillar_country_score_1_100", ascending=False).reset_index(drop=True)
|
country_all_pillars = yr_df[yr_df["country_id"] == c_id].sort_values("pillar_country_score_1_100", ascending=False).reset_index(drop=True)
|
||||||
p_rank = int(country_all_pillars[country_all_pillars["pillar_id"] == p_id].index[0]) + 1 if p_id in country_all_pillars["pillar_id"].values else 0
|
p_rank = int(country_all_pillars[country_all_pillars["pillar_id"] == p_id].index[0]) + 1 if p_id in country_all_pillars["pillar_id"].values else 0
|
||||||
|
|
||||||
hist_up = {y: s for y, s in history.get((c_id, p_id), {}).items() if y <= yr}
|
hist_up = {y: s for y, s in history.get((c_id, p_id), {}).items() if y <= yr}
|
||||||
|
|
||||||
# all_pillar_scores_year untuk perbandingan lintas pilar
|
|
||||||
all_pillar_yr = yr_df[yr_df["country_id"] == c_id][["pillar_name", "pillar_country_score_1_100"]].copy()
|
all_pillar_yr = yr_df[yr_df["country_id"] == c_id][["pillar_name", "pillar_country_score_1_100"]].copy()
|
||||||
|
|
||||||
# country_pillar_all untuk gap trend (hanya relevan untuk ASEAN)
|
|
||||||
cpa = df_pillar_by_country[
|
cpa = df_pillar_by_country[
|
||||||
(df_pillar_by_country["pillar_id"] == p_id) &
|
(df_pillar_by_country["pillar_id"] == p_id) &
|
||||||
(df_pillar_by_country["country_id"] != ASEAN_COUNTRY_ID)
|
(df_pillar_by_country["country_id"] != ASEAN_COUNTRY_ID)
|
||||||
@@ -1110,6 +1312,10 @@ class FoodSecurityAggregator:
|
|||||||
is_asean = is_asean,
|
is_asean = is_asean,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Ambil kondisi dari kolom yang sudah dihitung di df_pillar_by_country
|
||||||
|
cond_en = str(row.get("pillar_condition_en", "N/A"))
|
||||||
|
cond_id = str(row.get("pillar_condition_id", "N/A"))
|
||||||
|
|
||||||
records.append({
|
records.append({
|
||||||
"year": yr,
|
"year": yr,
|
||||||
"country_id": c_id,
|
"country_id": c_id,
|
||||||
@@ -1128,6 +1334,8 @@ class FoodSecurityAggregator:
|
|||||||
"bottom_country_id": translate_country(bot_country) if (is_asean and bot_country) else None,
|
"bottom_country_id": translate_country(bot_country) if (is_asean and bot_country) else None,
|
||||||
"bottom_country_score": bot_score if is_asean else None,
|
"bottom_country_score": bot_score if is_asean else None,
|
||||||
"is_asean_aggregate": is_asean,
|
"is_asean_aggregate": is_asean,
|
||||||
|
"pillar_condition_en": cond_en,
|
||||||
|
"pillar_condition_id": cond_id,
|
||||||
"narrative_en": narrative_en,
|
"narrative_en": narrative_en,
|
||||||
"narrative_id": narrative_id,
|
"narrative_id": narrative_id,
|
||||||
})
|
})
|
||||||
@@ -1140,6 +1348,8 @@ class FoodSecurityAggregator:
|
|||||||
df["is_asean_aggregate"] = df["is_asean_aggregate"].astype(bool)
|
df["is_asean_aggregate"] = df["is_asean_aggregate"].astype(bool)
|
||||||
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
|
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
|
||||||
df["country_name_id"] = df["country_name_id"].astype(str)
|
df["country_name_id"] = df["country_name_id"].astype(str)
|
||||||
|
df["pillar_condition_en"] = df["pillar_condition_en"].astype(str)
|
||||||
|
df["pillar_condition_id"] = df["pillar_condition_id"].astype(str)
|
||||||
df["narrative_en"] = df["narrative_en"].astype(str)
|
df["narrative_en"] = df["narrative_en"].astype(str)
|
||||||
df["narrative_id"] = df["narrative_id"].astype(str)
|
df["narrative_id"] = df["narrative_id"].astype(str)
|
||||||
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
|
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
|
||||||
@@ -1148,10 +1358,6 @@ class FoodSecurityAggregator:
|
|||||||
self.logger.info(f"\n Total rows: {len(df):,}")
|
self.logger.info(f"\n Total rows: {len(df):,}")
|
||||||
self.logger.info(f" ASEAN rows: {df['is_asean_aggregate'].sum():,}")
|
self.logger.info(f" ASEAN rows: {df['is_asean_aggregate'].sum():,}")
|
||||||
self.logger.info(f" Country rows: {(~df['is_asean_aggregate']).sum():,}")
|
self.logger.info(f" Country rows: {(~df['is_asean_aggregate']).sum():,}")
|
||||||
self.logger.info("\n Sample ASEAN narrative_en (first):")
|
|
||||||
asean_sample = df[df["is_asean_aggregate"]].head(1)
|
|
||||||
if not asean_sample.empty:
|
|
||||||
self.logger.info(f" {asean_sample.iloc[0]['narrative_en'][:300]}")
|
|
||||||
|
|
||||||
schema = [
|
schema = [
|
||||||
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
||||||
@@ -1171,6 +1377,8 @@ class FoodSecurityAggregator:
|
|||||||
bigquery.SchemaField("bottom_country_id", "STRING", mode="NULLABLE"),
|
bigquery.SchemaField("bottom_country_id", "STRING", mode="NULLABLE"),
|
||||||
bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"),
|
bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"),
|
||||||
bigquery.SchemaField("is_asean_aggregate", "BOOL", mode="REQUIRED"),
|
bigquery.SchemaField("is_asean_aggregate", "BOOL", mode="REQUIRED"),
|
||||||
|
bigquery.SchemaField("pillar_condition_en", "STRING", mode="REQUIRED"),
|
||||||
|
bigquery.SchemaField("pillar_condition_id", "STRING", mode="REQUIRED"),
|
||||||
bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"),
|
bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"),
|
||||||
bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"),
|
bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"),
|
||||||
]
|
]
|
||||||
@@ -1244,11 +1452,12 @@ class FoodSecurityAggregator:
|
|||||||
self.logger.info("\n" + "=" * 70)
|
self.logger.info("\n" + "=" * 70)
|
||||||
self.logger.info("FOOD SECURITY AGGREGATION — 3 TABLES -> fs_asean_gold")
|
self.logger.info("FOOD SECURITY AGGREGATION — 3 TABLES -> fs_asean_gold")
|
||||||
self.logger.info(" ASEAN aggregate DIGABUNG ke tabel yang sama (country_id=0)")
|
self.logger.info(" ASEAN aggregate DIGABUNG ke tabel yang sama (country_id=0)")
|
||||||
self.logger.info(" Tabel dihapus: agg_pillar_composite, agg_framework_asean,")
|
self.logger.info(" Kolom baru : pillar_condition_en, pillar_condition_id")
|
||||||
self.logger.info(" agg_narrative_overview")
|
|
||||||
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}")
|
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}")
|
||||||
|
self.logger.info(f" Condition tiers (GFSI 2022): >=75 Secure | >=60 Adequate |")
|
||||||
|
self.logger.info(f" >=40 Moderate | >=20 At Risk | <20 Critical")
|
||||||
self.logger.info(f" Narrative style : interpretive, plain text, bilingual EN/ID")
|
self.logger.info(f" Narrative style : interpretive, plain text, bilingual EN/ID")
|
||||||
self.logger.info(f" Sustainability : renamed to 'Food Other' (EN) / 'Indikator Tambahan' (ID)")
|
self.logger.info(f" Sustainability : renamed to 'Food Other' / 'Indikator Tambahan'")
|
||||||
self.logger.info("=" * 70)
|
self.logger.info("=" * 70)
|
||||||
|
|
||||||
self.load_data()
|
self.load_data()
|
||||||
@@ -1302,6 +1511,7 @@ if __name__ == "__main__":
|
|||||||
print(f" NORMALIZE_FRAMEWORKS_JOINTLY : {NORMALIZE_FRAMEWORKS_JOINTLY}")
|
print(f" NORMALIZE_FRAMEWORKS_JOINTLY : {NORMALIZE_FRAMEWORKS_JOINTLY}")
|
||||||
print(f" PERFORMANCE_THRESHOLD : {PERFORMANCE_THRESHOLD}")
|
print(f" PERFORMANCE_THRESHOLD : {PERFORMANCE_THRESHOLD}")
|
||||||
print(f" ASEAN_COUNTRY_ID : {ASEAN_COUNTRY_ID}")
|
print(f" ASEAN_COUNTRY_ID : {ASEAN_COUNTRY_ID}")
|
||||||
|
print(f" Condition tiers (GFSI 2022) : >=75 Secure | >=60 Adequate | >=40 Moderate | >=20 At Risk | <20 Critical")
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
|
|
||||||
logger = setup_logging()
|
logger = setup_logging()
|
||||||
|
|||||||
Reference in New Issue
Block a user