izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

indonesian version column

Browse Source
This commit is contained in:
Debby
2026-05-19 10:09:48 +07:00
parent 4bab746779
commit cfb0df3a15
3 changed files with 701 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
scripts/bigquery_aggregate_layer.py
View File

@@ -14,6 +14,12 @@ Narrative style:
- Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata
- Bilingual: narrative_en (Inggris) + narrative_id (Indonesia)
- Granularity: per tahun (Overview & Pillar)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
- agg_pillar_composite : + pillar_name_id
- agg_pillar_by_country : + pillar_name_id
- agg_framework_by_country : (framework tidak diterjemahkan, sudah singkat)
- agg_narrative_pillar : + pillar_name_id
"""
import pandas as pd
@@ -82,6 +88,176 @@ _FIES_DETECTION_LOWER: frozenset = frozenset([
])
# =============================================================================
# TRANSLATION DICTIONARIES
# =============================================================================
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
}
INDICATOR_TRANSLATION_ID: dict = {
# -------------------------------------------------------------------------
# AVAILABILITY
# -------------------------------------------------------------------------
"Average dietary energy supply adequacy (percent) (3-year average)":
"Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)",
"Average value of food production (constant 2014-2016 thousand US$) (3-year average)":
"Nilai rata-rata produksi pangan (ribu US$ konstan 2014-2016) (rata-rata 3 tahun)",
"Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
"Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)",
"Average protein supply (g/cap/day) (3-year average)":
"Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)",
"Average supply of protein of animal origin (g/cap/day) (3-year average)":
"Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)",
"Cereal import dependency ratio (percent) (3-year average)":
"Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
"Percent of arable land equipped for irrigation (percent) (3-year average)":
"Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)",
"Crop production index (2014-2016 = 100)":
"Indeks produksi tanaman pangan (2014-2016 = 100)",
"Livestock production index (2014-2016 = 100)":
"Indeks produksi peternakan (2014-2016 = 100)",
"Value of food imports over total merchandise exports (percent) (3-year average)":
"Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)",
"Food production variability (constant 2014-2016 thousand US$ per capita)":
"Variabilitas produksi pangan (ribu US$ konstan 2014-2016 per kapita)",
"Food supply variability (kcal/cap/day)":
"Variabilitas pasokan pangan (kkal/kapita/hari)",
# -------------------------------------------------------------------------
# ACCESS
# -------------------------------------------------------------------------
"Gross domestic product per capita, PPP (constant 2017 international $)":
"Produk domestik bruto per kapita, PPP (internasional konstan 2017 US$)",
"Domestic food price level index (2015 = 1.00)":
"Indeks tingkat harga pangan domestik (2015 = 1,00)",
"Domestic food price volatility index":
"Indeks volatilitas harga pangan domestik",
"Prevalence of undernourishment (percent) (3-year average)":
"Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)",
"Number of people undernourished (million) (3-year average)":
"Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)",
"Depth of the food deficit (kcal/capita/day) (3-year average)":
"Kedalaman defisit pangan (kkal/kapita/hari) (rata-rata 3 tahun)",
"Percentage of population using at least basic drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum dasar (persen)",
"Percentage of population using safely managed drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)",
"Percentage of population using at least basic sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)",
"Percentage of population using safely managed sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)",
"Access to electricity (percent of rural population)":
"Akses listrik (persen penduduk pedesaan)",
"Proportion of population with access to electricity (percent)":
"Proporsi penduduk dengan akses listrik (persen)",
"Road infrastructure index":
"Indeks infrastruktur jalan",
"Rail lines density (total route-km per 100 square km of land area)":
"Kepadatan jalur kereta api (total rute-km per 100 km2 lahan)",
"Gross national income per capita (Atlas method, current US$)":
"Pendapatan nasional bruto per kapita (metode Atlas, US$ terkini)",
"Food Insecurity Experience Scale (FIES)":
"Skala Pengalaman Ketidakamanan Pangan (FIES)",
# -------------------------------------------------------------------------
# UTILIZATION
# -------------------------------------------------------------------------
"Prevalence of severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Number of severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)",
"Number of children under 5 years of age who are stunted (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)",
"Percentage of children under 5 years affected by wasting (percent)":
"Persentase anak di bawah 5 tahun yang mengalami wasting (persen)",
"Number of children under 5 years affected by wasting (million)":
"Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)",
"Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)",
"Number of children under 5 years of age who are overweight (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)",
"Prevalence of anemia among women of reproductive age (15-49 years) (percent)":
"Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)",
"Number of women of reproductive age (15-49 years) affected by anemia (million)":
"Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)",
"Prevalence of obesity in the adult population (18 years and older) (percent)":
"Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)",
"Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)":
"Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)",
"Minimum dietary diversity for women (MDD-W) (percent)":
"Keragaman pola makan minimum untuk perempuan (MDD-W) (persen)",
# -------------------------------------------------------------------------
# STABILITY
# -------------------------------------------------------------------------
"Cereal import dependency ratio (percent)":
"Rasio ketergantungan impor sereal (persen)",
"Political stability and absence of violence/terrorism (index)":
"Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)",
"Domestic food price volatility":
"Volatilitas harga pangan domestik",
"Per capita food supply variability (kcal/cap/day)":
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
"Percentage of arable land equipped for irrigation (percent)":
"Persentase lahan pertanian yang dilengkapi irigasi (persen)",
"GDP per capita growth (annual %)":
"Pertumbuhan PDB per kapita (% tahunan)",
"GDP growth (annual %)":
"Pertumbuhan PDB (% tahunan)",
}
def translate_indicator(name: str) -> str:
"""Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return INDICATOR_TRANSLATION_ID.get(name, name)
def translate_pillar(name: str) -> str:
"""Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return PILLAR_TRANSLATION_ID.get(name, name)
# =============================================================================
# WINDOWS CP1252 SAFE LOGGING
# =============================================================================
@@ -194,10 +370,6 @@ def _fmt_delta(delta) -> str:
# =============================================================================
def _detect_series_trend(scores: list) -> str:
"""
Deteksi tren dari list skor berurutan.
Return: 'improving_consistent' | 'improving_slowing' | 'deteriorating' | 'fluctuating'
"""
if len(scores) < 3:
return "insufficient"
@@ -220,10 +392,6 @@ def _detect_series_trend(scores: list) -> str:
def _detect_country_gap(scores_by_country_year: pd.DataFrame, score_col: str) -> str:
"""
Deteksi apakah std antar negara melebar atau menyempit dari waktu ke waktu.
scores_by_country_year: df dengan kolom [year, country_id, score_col]
"""
std_by_year = (
scores_by_country_year.groupby("year")[score_col]
.std().dropna()
@@ -242,11 +410,6 @@ def _detect_country_gap(scores_by_country_year: pd.DataFrame, score_col: str) ->
def _find_anomaly_year(values_by_year: dict) -> tuple:
"""
Cari tahun dengan perubahan YoY paling ekstrem.
values_by_year: {year: score}
Return: (year, 'drop' | 'rise') atau (None, None)
"""
years = sorted(values_by_year.keys())
deltas = {}
for i in range(1, len(years)):
@@ -285,17 +448,12 @@ def _build_overview_narrative(
most_improved_delta,
most_declined_country,
most_declined_delta,
historical_scores: dict, # {year: score} semua tahun sebelumnya
country_scores_all: pd.DataFrame, # df [year, country_name, framework_score_1_100]
historical_scores: dict,
country_scores_all: pd.DataFrame,
) -> tuple:
"""
Narasi overview per tahun — interpretatif, plain text, bilingual.
Return: (narrative_en, narrative_id)
"""
sentences_en = []
sentences_id = []
# ---- 1. Status tahun ini vs threshold ----
perf_word_en = "good" if performance_status == "Good" else "below target"
perf_word_id = "baik" if performance_status == "Good" else "di bawah target"
@@ -312,7 +470,6 @@ def _build_overview_narrative(
sentences_en.append(s1_en)
sentences_id.append(s1_id)
# ---- 2. Kondisi YoY tahun ini ----
if yoy_val is not None and not pd.isna(yoy_val):
if abs(yoy_val) < 0.5:
s2_en = f"The score was relatively stable compared to the previous year."
@@ -326,7 +483,6 @@ def _build_overview_narrative(
sentences_en.append(s2_en)
sentences_id.append(s2_id)
# ---- 3. Tren historis (baca dari semua data yang ada) ----
hist_years = sorted(historical_scores.keys())
hist_scores = [historical_scores[y] for y in hist_years if not pd.isna(historical_scores.get(y, np.nan))]
@@ -352,7 +508,6 @@ def _build_overview_narrative(
sentences_en.append(s3_en)
sentences_id.append(s3_id)
# ---- 4. Gap antar negara ----
if not country_scores_all.empty:
gap_trend = _detect_country_gap(
country_scores_all[country_scores_all["year"] <= year],
@@ -375,7 +530,6 @@ def _build_overview_narrative(
sentences_en.append(s4_en)
sentences_id.append(s4_id)
# ---- 5. Top dan bottom country tahun ini ----
if ranking_list and len(ranking_list) >= 2:
top = ranking_list[0]
bottom = ranking_list[-1]
@@ -392,7 +546,6 @@ def _build_overview_narrative(
sentences_en.append(s5_en)
sentences_id.append(s5_id)
# ---- 6. Most improved / declined country ----
if most_improved_country and most_declined_country:
if most_improved_country != most_declined_country:
s6_en = (
@@ -430,19 +583,14 @@ def _build_pillar_narrative(
top_country_score,
bot_country: str,
bot_country_score,
pillar_scores_history: dict, # {year: score} untuk pilar ini
all_pillar_scores_year: pd.DataFrame, # df [pillar_name, pillar_score_1_100] tahun ini
country_pillar_all: pd.DataFrame, # df [year, country_id, pillar_country_score_1_100] pilar ini
pillar_scores_history: dict,
all_pillar_scores_year: pd.DataFrame,
country_pillar_all: pd.DataFrame,
) -> tuple:
"""
Narasi pillar per tahun — interpretatif, plain text, bilingual.
Return: (narrative_en, narrative_id)
"""
sentences_en = []
sentences_id = []
# ---- 1. Posisi pilar tahun ini ----
rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th")
rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th")
perf_word_en = "good" if pillar_score >= PERFORMANCE_THRESHOLD else "below target"
perf_word_id = "baik" if pillar_score >= PERFORMANCE_THRESHOLD else "di bawah target"
@@ -457,7 +605,6 @@ def _build_pillar_narrative(
sentences_en.append(s1_en)
sentences_id.append(s1_id)
# ---- 2. YoY pilar ini ----
if yoy_val is not None and not pd.isna(yoy_val):
if abs(yoy_val) < 0.5:
s2_en = "Performance was relatively stable compared to the previous year."
@@ -471,7 +618,6 @@ def _build_pillar_narrative(
sentences_en.append(s2_en)
sentences_id.append(s2_id)
# ---- 3. Tren historis pilar ini ----
hist_years = sorted(pillar_scores_history.keys())
hist_scores = [
pillar_scores_history[y]
@@ -501,7 +647,6 @@ def _build_pillar_narrative(
sentences_en.append(s3_en)
sentences_id.append(s3_id)
# ---- 4. Gap antar negara dalam pilar ini ----
if not country_pillar_all.empty:
gap_trend = _detect_country_gap(
country_pillar_all[country_pillar_all["year"] <= year],
@@ -521,7 +666,6 @@ def _build_pillar_narrative(
sentences_en.append(s4_en)
sentences_id.append(s4_id)
# ---- 5. Top/bottom country dalam pilar ini ----
if top_country and bot_country and top_country != bot_country:
s5_en = (
f"{top_country} performed best in this pillar ({_fmt_score(top_country_score)}), "
@@ -534,7 +678,6 @@ def _build_pillar_narrative(
sentences_en.append(s5_en)
sentences_id.append(s5_id)
# ---- 6. Posisi relatif pilar ini vs pilar lain ----
if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1:
sorted_pillars = all_pillar_scores_year.sort_values("pillar_score_1_100", ascending=False)
strongest = sorted_pillars.iloc[0]
@@ -605,15 +748,21 @@ class FoodSecurityAggregator:
}
missing_cols = required_cols - set(self.df.columns)
if missing_cols:
raise ValueError(
f"Kolom berikut tidak ditemukan: {missing_cols}"
)
raise ValueError(f"Kolom berikut tidak ditemukan: {missing_cols}")
n_null_dir = self.df["direction"].isna().sum()
if n_null_dir > 0:
self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'")
self.df["direction"] = self.df["direction"].fillna("positive")
# Pastikan kolom terjemahan Indonesia tersedia (bisa dari fact atau dibuat ulang)
if "indicator_name_id" not in self.df.columns:
self.df["indicator_name_id"] = self.df["indicator_name"].apply(translate_indicator)
self.logger.info(" [TRANSLATION] Kolom indicator_name_id dibuat dari mapping.")
if "pillar_name_id" not in self.df.columns:
self.df["pillar_name_id"] = self.df["pillar_name"].apply(translate_pillar)
self.logger.info(" [TRANSLATION] Kolom pillar_name_id dibuat dari mapping.")
self.logger.info(f" Rows : {len(self.df):,}")
self.logger.info(f" Countries : {self.df['country_id'].nunique()}")
self.logger.info(f" Indicators: {self.df['indicator_id'].nunique()}")
@@ -758,6 +907,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 2: agg_pillar_composite
# Kolom tambahan: pillar_name_id
# =========================================================================
def calc_pillar_composite(self) -> pd.DataFrame:
@@ -789,6 +939,9 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["pillar_id"], "pillar_score_1_100")
# Kolom terjemahan Indonesia
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["pillar_id"] = df["pillar_id"].astype(int)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
@@ -796,10 +949,12 @@ class FoodSecurityAggregator:
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_norm"] = df["pillar_norm"].astype(float)
df["pillar_score_1_100"] = df["pillar_score_1_100"].astype(float)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
schema = [
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_norm", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
@@ -821,6 +976,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 3: agg_pillar_by_country
# Kolom tambahan: pillar_name_id
# =========================================================================
def calc_pillar_by_country(self) -> pd.DataFrame:
@@ -848,18 +1004,23 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100")
# Kolom terjemahan Indonesia
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["country_id"] = df["country_id"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["year"] = df["year"].astype(int)
df["rank_in_pillar_year"] = df["rank_in_pillar_year"].astype(int)
df["pillar_country_norm"] = df["pillar_country_norm"].astype(float)
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_country_norm", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("pillar_country_score_1_100", "FLOAT", mode="REQUIRED"),
@@ -879,6 +1040,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 4: agg_framework_by_country
# Tidak ada kolom pillar/indicator di tabel ini (sudah di level framework)
# =========================================================================
def _calc_country_composite_inmemory(self) -> pd.DataFrame:
@@ -1043,6 +1205,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 5: agg_framework_asean
# Tidak ada kolom pillar/indicator langsung di tabel ini
# =========================================================================
def calc_framework_asean(self) -> pd.DataFrame:
@@ -1205,6 +1368,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 6: agg_narrative_overview
# Tidak ada kolom pillar/indicator di tabel ini
# =========================================================================
def calc_narrative_overview(
@@ -1284,7 +1448,6 @@ class FoodSecurityAggregator:
most_improved_country = most_declined_country = None
most_improved_delta = most_declined_delta = None
# Semua data skor negara untuk gap analysis
country_scores_all = country_total[["year", "country_id", "framework_score_1_100"]].copy()
narrative_en, narrative_id = _build_overview_narrative(
@@ -1368,6 +1531,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 7: agg_narrative_pillar
# Kolom tambahan: pillar_name_id
# =========================================================================
def calc_narrative_pillar(
@@ -1409,6 +1573,9 @@ class FoodSecurityAggregator:
p_yoy = prow["year_over_year_change"]
p_yoy_val = float(p_yoy) if pd.notna(p_yoy) else None
# Terjemahan Indonesia nama pillar
p_name_id = translate_pillar(p_name)
p_country = (
yr_country_pillar[yr_country_pillar["pillar_id"] == p_id]
.sort_values("rank_in_pillar_year")
@@ -1423,12 +1590,10 @@ class FoodSecurityAggregator:
top_country = bot_country = None
top_country_score = bot_country_score = None
# Data historis hanya sampai tahun ini
hist_up_to_yr = {
y: s for y, s in pillar_history.get(p_id, {}).items() if y <= yr
}
# Data negara-pilar ini semua tahun (untuk gap analysis)
country_pillar_all = df_pillar_by_country[
df_pillar_by_country["pillar_id"] == p_id
][["year", "country_id", "pillar_country_score_1_100"]].copy()
@@ -1453,6 +1618,7 @@ class FoodSecurityAggregator:
"year": yr,
"pillar_id": p_id,
"pillar_name": p_name,
"pillar_name_id": p_name_id,
"pillar_score": round(p_score, 2),
"rank_in_year": p_rank,
"yoy_change": p_yoy_val,
@@ -1465,11 +1631,12 @@ class FoodSecurityAggregator:
})
df = pd.DataFrame(records)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
@@ -1482,6 +1649,7 @@ class FoodSecurityAggregator:
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_score", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),