indonesian version column
This commit is contained in:
Debby
@@ -4,6 +4,12 @@ Tabel 1: agg_indicator_norm -> fs_asean_gold
|
||||
Tabel 2: agg_narrative_indicator -> fs_asean_gold
|
||||
|
||||
=============================================================================
|
||||
PERUBAHAN:
|
||||
- Ditambahkan kolom indicator_name_id : nama indikator dalam Bahasa Indonesia
|
||||
- Ditambahkan kolom pillar_name_id : nama pilar dalam Bahasa Indonesia
|
||||
- Kedua kolom ikut tersimpan di BigQuery (schema + DataFrame output)
|
||||
=============================================================================
|
||||
|
||||
agg_indicator_norm
|
||||
=============================================================================
|
||||
Tujuan:
|
||||
@@ -30,8 +36,9 @@ Performance Label Logic:
|
||||
|
||||
Output Schema (agg_indicator_norm):
|
||||
year, country_id, country_name,
|
||||
indicator_id, indicator_name, unit, direction,
|
||||
pillar_id, pillar_name,
|
||||
indicator_id, indicator_name, indicator_name_id,
|
||||
unit, direction,
|
||||
pillar_id, pillar_name, pillar_name_id,
|
||||
framework,
|
||||
value,
|
||||
norm_value,
|
||||
@@ -53,8 +60,10 @@ Granularity:
|
||||
indicator_id (all years, all ASEAN countries)
|
||||
|
||||
Output Schema (agg_narrative_indicator):
|
||||
indicator_id, indicator_name, unit, direction,
|
||||
pillar_name, framework,
|
||||
indicator_id, indicator_name, indicator_name_id,
|
||||
unit, direction,
|
||||
pillar_name, pillar_name_id,
|
||||
framework,
|
||||
year_min, year_max, n_countries,
|
||||
avg_value_first, avg_value_last,
|
||||
avg_norm_score_1_100,
|
||||
@@ -83,6 +92,128 @@ from scripts.bigquery_helpers import (
|
||||
from google.cloud import bigquery
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MAPPING BAHASA INDONESIA
|
||||
# =============================================================================
|
||||
|
||||
# Mapping nama pilar (Inggris -> Indonesia)
|
||||
PILLAR_NAME_ID_MAP: dict = {
|
||||
"Availability" : "Ketersediaan",
|
||||
"Access" : "Akses",
|
||||
"Utilization" : "Pemanfaatan",
|
||||
"Stability" : "Stabilitas",
|
||||
"availability" : "Ketersediaan",
|
||||
"access" : "Akses",
|
||||
"utilization" : "Pemanfaatan",
|
||||
"stability" : "Stabilitas",
|
||||
}
|
||||
|
||||
# Mapping nama indikator (Inggris -> Indonesia)
|
||||
# Kunci: indicator_name lowercase stripped
|
||||
INDICATOR_NAME_ID_MAP: dict = {
|
||||
# --- Availability / Ketersediaan ---
|
||||
"prevalence of undernourishment (percent) (3-year average)":
|
||||
"Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)",
|
||||
"number of people undernourished (million) (3-year average)":
|
||||
"Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)",
|
||||
"prevalence of severe food insecurity in the total population (percent) (3-year average)":
|
||||
"Prevalensi ketidaktahanan pangan berat pada total populasi (persen) (rata-rata 3 tahun)",
|
||||
"prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
|
||||
"Prevalensi ketidaktahanan pangan berat pada populasi dewasa laki-laki (persen) (rata-rata 3 tahun)",
|
||||
"prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
|
||||
"Prevalensi ketidaktahanan pangan berat pada populasi dewasa perempuan (persen) (rata-rata 3 tahun)",
|
||||
"prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
|
||||
"Prevalensi ketidaktahanan pangan sedang atau berat pada total populasi (persen) (rata-rata 3 tahun)",
|
||||
"prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
|
||||
"Prevalensi ketidaktahanan pangan sedang atau berat pada populasi dewasa laki-laki (persen) (rata-rata 3 tahun)",
|
||||
"prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
|
||||
"Prevalensi ketidaktahanan pangan sedang atau berat pada populasi dewasa perempuan (persen) (rata-rata 3 tahun)",
|
||||
"number of severely food insecure people (million) (3-year average)":
|
||||
"Jumlah penduduk mengalami ketidaktahanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
|
||||
"number of severely food insecure male adults (million) (3-year average)":
|
||||
"Jumlah dewasa laki-laki mengalami ketidaktahanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
|
||||
"number of severely food insecure female adults (million) (3-year average)":
|
||||
"Jumlah dewasa perempuan mengalami ketidaktahanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
|
||||
"number of moderately or severely food insecure people (million) (3-year average)":
|
||||
"Jumlah penduduk mengalami ketidaktahanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
|
||||
"number of moderately or severely food insecure male adults (million) (3-year average)":
|
||||
"Jumlah dewasa laki-laki mengalami ketidaktahanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
|
||||
"number of moderately or severely food insecure female adults (million) (3-year average)":
|
||||
"Jumlah dewasa perempuan mengalami ketidaktahanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
|
||||
# --- Utilization / Pemanfaatan ---
|
||||
"percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
|
||||
"Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)",
|
||||
"number of children under 5 years of age who are stunted (modeled estimates) (million)":
|
||||
"Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)",
|
||||
"percentage of children under 5 years affected by wasting (percent)":
|
||||
"Persentase anak di bawah 5 tahun yang mengalami wasting (persen)",
|
||||
"number of children under 5 years affected by wasting (million)":
|
||||
"Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)",
|
||||
"percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
|
||||
"Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)",
|
||||
"number of children under 5 years of age who are overweight (modeled estimates) (million)":
|
||||
"Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)",
|
||||
"prevalence of anemia among women of reproductive age (15-49 years) (percent)":
|
||||
"Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)",
|
||||
"number of women of reproductive age (15-49 years) affected by anemia (million)":
|
||||
"Jumlah perempuan usia reproduksi (15-49 tahun) yang mengalami anemia (juta jiwa)",
|
||||
# --- Access / Akses ---
|
||||
"gdp per capita (current us$)":
|
||||
"PDB per kapita (US$ saat ini)",
|
||||
"gdp per capita, ppp (current international $)":
|
||||
"PDB per kapita, PPP (internasional $ saat ini)",
|
||||
"food consumer price index (cpi)":
|
||||
"Indeks Harga Konsumen (IHK) pangan",
|
||||
"per capita food supply variability (kcal/cap/day)":
|
||||
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
|
||||
"percentage of population using at least basic drinking water services":
|
||||
"Persentase penduduk yang menggunakan layanan air minum dasar",
|
||||
"percentage of population using at least basic sanitation services":
|
||||
"Persentase penduduk yang menggunakan layanan sanitasi dasar",
|
||||
"prevalence of obesity in the adult population (18 years and older)":
|
||||
"Prevalensi obesitas pada populasi dewasa (18 tahun ke atas)",
|
||||
"prevalence of overweight in the adult population (18 years and older)":
|
||||
"Prevalensi kelebihan berat badan pada populasi dewasa (18 tahun ke atas)",
|
||||
"minimum dietary energy requirement (mder) (kcal/cap/day)":
|
||||
"Kebutuhan energi pangan minimum (KEPM) (kkal/kapita/hari)",
|
||||
"average dietary energy supply adequacy (percent) (3-year average)":
|
||||
"Kecukupan rata-rata pasokan energi pangan (persen) (rata-rata 3 tahun)",
|
||||
"average protein supply (g/cap/day) (3-year average)":
|
||||
"Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)",
|
||||
"average supply of protein of animal origin (g/cap/day) (3-year average)":
|
||||
"Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)",
|
||||
# --- Stability / Stabilitas ---
|
||||
"political stability and absence of violence/terrorism":
|
||||
"Stabilitas politik dan ketiadaan kekerasan/terorisme",
|
||||
"domestic food price volatility index":
|
||||
"Indeks volatilitas harga pangan domestik",
|
||||
"per capita food supply variability (kcal/capita/day)":
|
||||
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
|
||||
"cereal import dependency ratio (percent) (3-year average)":
|
||||
"Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
|
||||
"value of food imports in total merchandise exports (percent) (3-year average)":
|
||||
"Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)",
|
||||
"share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
|
||||
"Pangsa pasokan energi pangan dari sereal, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)",
|
||||
}
|
||||
|
||||
|
||||
def get_indicator_name_id(indicator_name: str) -> str:
|
||||
"""Kembalikan terjemahan Bahasa Indonesia untuk nama indikator."""
|
||||
return INDICATOR_NAME_ID_MAP.get(
|
||||
str(indicator_name).lower().strip(),
|
||||
str(indicator_name), # fallback: kembalikan nama asli jika tidak ada mapping
|
||||
)
|
||||
|
||||
|
||||
def get_pillar_name_id(pillar_name: str) -> str:
|
||||
"""Kembalikan terjemahan Bahasa Indonesia untuk nama pilar."""
|
||||
return PILLAR_NAME_ID_MAP.get(
|
||||
str(pillar_name).strip(),
|
||||
str(pillar_name), # fallback: kembalikan nama asli jika tidak ada mapping
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# SDG-ONLY KEYWORD SET
|
||||
# =============================================================================
|
||||
@@ -190,55 +321,42 @@ def _is_lower_better(direction: str) -> bool:
|
||||
# =============================================================================
|
||||
|
||||
def _detect_trend(scores_by_year: pd.Series, lower_better: bool) -> str:
|
||||
"""
|
||||
Deteksi tren: improving_consistent, improving_slowing, fluctuating, deteriorating.
|
||||
scores_by_year: Series dengan index=year, value=avg_score (sudah direction-aware).
|
||||
"""
|
||||
if len(scores_by_year) < 3:
|
||||
return "insufficient_data"
|
||||
|
||||
years = sorted(scores_by_year.index)
|
||||
vals = [scores_by_year[y] for y in years if not pd.isna(scores_by_year.get(y, np.nan))]
|
||||
years = sorted(scores_by_year.index)
|
||||
vals = [scores_by_year[y] for y in years if not pd.isna(scores_by_year.get(y, np.nan))]
|
||||
|
||||
if len(vals) < 3:
|
||||
return "insufficient_data"
|
||||
|
||||
# Hitung slope keseluruhan
|
||||
x = np.arange(len(vals))
|
||||
slope = np.polyfit(x, vals, 1)[0]
|
||||
x = np.arange(len(vals))
|
||||
slope = np.polyfit(x, vals, 1)[0]
|
||||
|
||||
# Slope positif = skor naik = baik untuk higher_better, buruk untuk lower_better
|
||||
improving = (slope > 0 and not lower_better) or (slope < 0 and lower_better)
|
||||
|
||||
# Hitung apakah laju melambat: bandingkan slope paruh pertama vs paruh kedua
|
||||
mid = len(vals) // 2
|
||||
first_half = vals[:mid]
|
||||
mid = len(vals) // 2
|
||||
first_half = vals[:mid]
|
||||
second_half = vals[mid:]
|
||||
slope1 = np.polyfit(np.arange(len(first_half)), first_half, 1)[0] if len(first_half) > 1 else 0
|
||||
slope2 = np.polyfit(np.arange(len(second_half)), second_half, 1)[0] if len(second_half) > 1 else 0
|
||||
|
||||
# Koefisien variasi untuk cek fluktuasi
|
||||
cv = np.std(vals) / (np.mean(vals) + 1e-9)
|
||||
|
||||
if cv > 0.25:
|
||||
return "fluctuating"
|
||||
|
||||
if improving:
|
||||
# Cek apakah melambat
|
||||
if lower_better:
|
||||
slowing = slope2 > slope1 # slope negatif mengecil artinya melambat
|
||||
slowing = slope2 > slope1
|
||||
else:
|
||||
slowing = slope2 < slope1 # slope positif mengecil artinya melambat
|
||||
slowing = slope2 < slope1
|
||||
return "improving_slowing" if slowing else "improving_consistent"
|
||||
else:
|
||||
return "deteriorating"
|
||||
|
||||
|
||||
def _detect_gap_trend(df_ind: pd.DataFrame, lower_better: bool) -> str:
|
||||
"""
|
||||
Deteksi apakah gap antar negara melebar, menyempit, atau stabil.
|
||||
df_ind: rows untuk 1 indikator, kolom: year, country_id, value
|
||||
"""
|
||||
std_by_year = (
|
||||
df_ind.groupby("year")["value"]
|
||||
.std()
|
||||
@@ -257,10 +375,6 @@ def _detect_gap_trend(df_ind: pd.DataFrame, lower_better: bool) -> str:
|
||||
|
||||
|
||||
def _detect_anomaly_year(scores_by_year: pd.Series) -> tuple:
|
||||
"""
|
||||
Deteksi tahun dengan perubahan paling ekstrem (naik atau turun tajam).
|
||||
Return: (anomaly_year, direction) atau (None, None)
|
||||
"""
|
||||
if len(scores_by_year) < 3:
|
||||
return None, None
|
||||
|
||||
@@ -290,10 +404,6 @@ def _detect_anomaly_year(scores_by_year: pd.Series) -> tuple:
|
||||
|
||||
|
||||
def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple:
|
||||
"""
|
||||
Cari negara yang paling konsisten terbaik dan terburuk.
|
||||
Return: (consistent_best, consistent_worst, is_consistent)
|
||||
"""
|
||||
country_avg = (
|
||||
df_ind.groupby("country_name")["value"]
|
||||
.mean()
|
||||
@@ -309,7 +419,6 @@ def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple:
|
||||
best = country_avg.idxmax()
|
||||
worst = country_avg.idxmin()
|
||||
|
||||
# Cek konsistensi: apakah negara terbaik selalu di atas rata-rata?
|
||||
asean_avg_by_year = df_ind.groupby("year")["value"].mean()
|
||||
country_by_year = df_ind[df_ind["country_name"] == best].set_index("year")["value"]
|
||||
|
||||
@@ -338,10 +447,6 @@ def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple:
|
||||
# =============================================================================
|
||||
|
||||
def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tuple:
|
||||
"""
|
||||
Bangun narasi interpretatif per indikator berdasarkan kondisi nyata data.
|
||||
Return: (narrative_en, narrative_id) — plain text tanpa markdown bold.
|
||||
"""
|
||||
ind_id = int(row["indicator_id"])
|
||||
ind_name = str(row["indicator_name"]).strip()
|
||||
unit = str(row["unit"]).strip() if row["unit"] else ""
|
||||
@@ -352,7 +457,6 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
|
||||
year_max = int(row["year_max"])
|
||||
lower_better = _is_lower_better(direction)
|
||||
|
||||
# Subset data untuk indikator ini
|
||||
df_ind = df_full[df_full["indicator_id"] == ind_id].copy()
|
||||
|
||||
if df_ind.empty:
|
||||
@@ -360,13 +464,12 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
|
||||
na_id = f"{ind_name} ({framework}, {pillar}): Data tidak cukup untuk dianalisis."
|
||||
return na_en, na_id
|
||||
|
||||
# ---- Hitung kondisi dari data ----
|
||||
asean_avg_by_year = (
|
||||
df_ind.groupby("year")["value"].mean().dropna()
|
||||
)
|
||||
|
||||
trend_label = _detect_trend(asean_avg_by_year, lower_better)
|
||||
gap_label = _detect_gap_trend(df_ind, lower_better)
|
||||
trend_label = _detect_trend(asean_avg_by_year, lower_better)
|
||||
gap_label = _detect_gap_trend(df_ind, lower_better)
|
||||
anomaly_year, anomaly_dir = _detect_anomaly_year(asean_avg_by_year)
|
||||
best_country, worst_country, is_consistent = _detect_consistency(df_ind, lower_better)
|
||||
|
||||
@@ -380,17 +483,14 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
|
||||
s = f"{v:,.1f}" if abs_v >= 1000 else (f"{v:.2f}" if abs_v >= 10 else f"{v:.3f}")
|
||||
return f"{s} {unit}".strip() if unit else s
|
||||
|
||||
# ---- Bangun kalimat EN ----
|
||||
sentences_en = []
|
||||
sentences_id = []
|
||||
|
||||
# Kalimat 1: konteks indikator
|
||||
s1_en = f"{ind_name} ({framework}, {pillar}, {year_min}-{year_max}):"
|
||||
s1_id = f"{ind_name} ({framework}, {pillar}, {year_min}-{year_max}):"
|
||||
sentences_en.append(s1_en)
|
||||
sentences_id.append(s1_id)
|
||||
|
||||
# Kalimat 2: tren keseluruhan
|
||||
trend_map_en = {
|
||||
"improving_consistent": f"Regional average improved consistently from {fmt(avg_first)} to {fmt(avg_last)}.",
|
||||
"improving_slowing": f"Regional average improved from {fmt(avg_first)} to {fmt(avg_last)}, though the pace slowed in recent years.",
|
||||
@@ -408,7 +508,6 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
|
||||
sentences_en.append(trend_map_en.get(trend_label, ""))
|
||||
sentences_id.append(trend_map_id.get(trend_label, ""))
|
||||
|
||||
# Kalimat 3: gap antar negara
|
||||
if gap_label == "widening":
|
||||
sentences_en.append("Disparity among ASEAN countries has widened over time, indicating unequal progress.")
|
||||
sentences_id.append("Kesenjangan antar negara ASEAN melebar seiring waktu, menunjukkan kemajuan yang tidak merata.")
|
||||
@@ -419,7 +518,6 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
|
||||
sentences_en.append("The gap among ASEAN countries remained relatively stable throughout the period.")
|
||||
sentences_id.append("Kesenjangan antar negara ASEAN relatif stabil sepanjang periode.")
|
||||
|
||||
# Kalimat 4: anomali
|
||||
if anomaly_year is not None:
|
||||
if anomaly_dir == "drop":
|
||||
sentences_en.append(f"A notable decline was recorded in {anomaly_year}, which stood out from the overall pattern.")
|
||||
@@ -428,7 +526,6 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
|
||||
sentences_en.append(f"A sharp improvement was observed in {anomaly_year}, standing out from the overall pattern.")
|
||||
sentences_id.append(f"Peningkatan tajam tercatat pada tahun {anomaly_year}, yang menyimpang dari pola keseluruhan.")
|
||||
|
||||
# Kalimat 5: konsistensi negara terbaik/terburuk
|
||||
if best_country and worst_country:
|
||||
if is_consistent:
|
||||
sentences_en.append(
|
||||
@@ -581,6 +678,50 @@ class IndicatorNormAggregator:
|
||||
f" Merge OK. Rows: {after:,} | Rows dengan unit kosong: {n_empty}"
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# STEP 3b: Tambah kolom nama Bahasa Indonesia
|
||||
# =========================================================================
|
||||
|
||||
def _add_indonesia_name_columns(self):
|
||||
self.logger.info("\n" + "=" * 80)
|
||||
self.logger.info("STEP 3b: ADD BAHASA INDONESIA NAME COLUMNS")
|
||||
self.logger.info("=" * 80)
|
||||
|
||||
self.df["indicator_name_id"] = (
|
||||
self.df["indicator_name"]
|
||||
.apply(get_indicator_name_id)
|
||||
.astype(str)
|
||||
)
|
||||
self.df["pillar_name_id"] = (
|
||||
self.df["pillar_name"]
|
||||
.apply(get_pillar_name_id)
|
||||
.astype(str)
|
||||
)
|
||||
|
||||
n_indicator_mapped = (self.df["indicator_name_id"] != self.df["indicator_name"]).sum()
|
||||
n_pillar_mapped = (self.df["pillar_name_id"] != self.df["pillar_name"]).sum()
|
||||
self.logger.info(f" indicator_name_id mapped rows : {n_indicator_mapped:,}")
|
||||
self.logger.info(f" pillar_name_id mapped rows : {n_pillar_mapped:,}")
|
||||
|
||||
# Log sample mapping
|
||||
sample_ind = (
|
||||
self.df[["indicator_name", "indicator_name_id"]]
|
||||
.drop_duplicates()
|
||||
.head(5)
|
||||
)
|
||||
self.logger.info("\n Sample indicator mapping (EN -> ID):")
|
||||
for _, r in sample_ind.iterrows():
|
||||
self.logger.info(f" EN: {r['indicator_name'][:55]}")
|
||||
self.logger.info(f" ID: {r['indicator_name_id'][:55]}")
|
||||
|
||||
sample_pil = (
|
||||
self.df[["pillar_name", "pillar_name_id"]]
|
||||
.drop_duplicates()
|
||||
)
|
||||
self.logger.info("\n Pillar mapping (EN -> ID):")
|
||||
for _, r in sample_pil.iterrows():
|
||||
self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}")
|
||||
|
||||
# =========================================================================
|
||||
# STEP 4: Deteksi sdgs_start_year
|
||||
# =========================================================================
|
||||
@@ -783,8 +924,10 @@ class IndicatorNormAggregator:
|
||||
|
||||
out = df[[
|
||||
"year", "country_id", "country_name",
|
||||
"indicator_id", "indicator_name", "unit", "direction",
|
||||
"pillar_id", "pillar_name", "framework",
|
||||
"indicator_id", "indicator_name", "indicator_name_id",
|
||||
"unit", "direction",
|
||||
"pillar_id", "pillar_name", "pillar_name_id",
|
||||
"framework",
|
||||
"value", "norm_value", "norm_score_1_100",
|
||||
"yoy_value", "yoy_norm_value", "performance",
|
||||
]].copy()
|
||||
@@ -793,22 +936,24 @@ class IndicatorNormAggregator:
|
||||
["year", "country_name", "pillar_name", "indicator_name"]
|
||||
).reset_index(drop=True)
|
||||
|
||||
out["year"] = out["year"].astype(int)
|
||||
out["country_id"] = out["country_id"].astype(int)
|
||||
out["country_name"] = out["country_name"].astype(str)
|
||||
out["indicator_id"] = out["indicator_id"].astype(int)
|
||||
out["indicator_name"] = out["indicator_name"].astype(str)
|
||||
out["unit"] = out["unit"].astype(str)
|
||||
out["direction"] = out["direction"].astype(str)
|
||||
out["pillar_id"] = out["pillar_id"].astype(int)
|
||||
out["pillar_name"] = out["pillar_name"].astype(str)
|
||||
out["framework"] = out["framework"].astype(str)
|
||||
out["value"] = out["value"].astype(float)
|
||||
out["norm_value"] = out["norm_value"].astype(float)
|
||||
out["norm_score_1_100"] = out["norm_score_1_100"].astype(float)
|
||||
out["yoy_value"] = pd.to_numeric(out["yoy_value"], errors="coerce").astype(float)
|
||||
out["yoy_norm_value"] = pd.to_numeric(out["yoy_norm_value"], errors="coerce").astype(float)
|
||||
out["performance"] = out["performance"].astype(str).replace("nan", pd.NA).astype("string")
|
||||
out["year"] = out["year"].astype(int)
|
||||
out["country_id"] = out["country_id"].astype(int)
|
||||
out["country_name"] = out["country_name"].astype(str)
|
||||
out["indicator_id"] = out["indicator_id"].astype(int)
|
||||
out["indicator_name"] = out["indicator_name"].astype(str)
|
||||
out["indicator_name_id"] = out["indicator_name_id"].astype(str)
|
||||
out["unit"] = out["unit"].astype(str)
|
||||
out["direction"] = out["direction"].astype(str)
|
||||
out["pillar_id"] = out["pillar_id"].astype(int)
|
||||
out["pillar_name"] = out["pillar_name"].astype(str)
|
||||
out["pillar_name_id"] = out["pillar_name_id"].astype(str)
|
||||
out["framework"] = out["framework"].astype(str)
|
||||
out["value"] = out["value"].astype(float)
|
||||
out["norm_value"] = out["norm_value"].astype(float)
|
||||
out["norm_score_1_100"] = out["norm_score_1_100"].astype(float)
|
||||
out["yoy_value"] = pd.to_numeric(out["yoy_value"], errors="coerce").astype(float)
|
||||
out["yoy_norm_value"] = pd.to_numeric(out["yoy_norm_value"], errors="coerce").astype(float)
|
||||
out["performance"] = out["performance"].astype(str).replace("nan", pd.NA).astype("string")
|
||||
|
||||
self.logger.info(f" Total rows : {len(out):,}")
|
||||
self.logger.info(f" Countries : {out['country_id'].nunique()}")
|
||||
@@ -816,22 +961,24 @@ class IndicatorNormAggregator:
|
||||
self.logger.info(f" Years : {int(out['year'].min())} - {int(out['year'].max())}")
|
||||
|
||||
schema = [
|
||||
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("unit", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("direction", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("value", "FLOAT", mode="REQUIRED"),
|
||||
bigquery.SchemaField("norm_value", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("norm_score_1_100", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("yoy_value", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("yoy_norm_value", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("performance", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_name_id", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("unit", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("direction", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_name_id", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("value", "FLOAT", mode="REQUIRED"),
|
||||
bigquery.SchemaField("norm_value", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("norm_score_1_100", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("yoy_value", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("yoy_norm_value", "FLOAT", mode="NULLABLE"),
|
||||
bigquery.SchemaField("performance", "STRING", mode="NULLABLE"),
|
||||
]
|
||||
|
||||
rows_loaded = load_to_bigquery(
|
||||
@@ -860,6 +1007,7 @@ class IndicatorNormAggregator:
|
||||
"yoy_columns" : ["yoy_value", "yoy_norm_value"],
|
||||
"performance_threshold": _PERFORMANCE_THRESHOLD,
|
||||
"unit_source" : "dim_indicator",
|
||||
"added_columns" : ["indicator_name_id", "pillar_name_id"],
|
||||
}),
|
||||
"validation_metrics" : json.dumps({
|
||||
"total_rows" : rows_loaded,
|
||||
@@ -1022,9 +1170,14 @@ class IndicatorNormAggregator:
|
||||
})
|
||||
df_country_stats = pd.DataFrame(country_stats)
|
||||
|
||||
# Dim cols
|
||||
dim_cols = ["indicator_name", "unit", "direction", "pillar_name", "framework"]
|
||||
df_dim = df[["indicator_id"] + dim_cols].drop_duplicates(subset=["indicator_id"])
|
||||
# Dim cols — sertakan kolom Indonesia
|
||||
dim_cols = [
|
||||
"indicator_name", "indicator_name_id",
|
||||
"unit", "direction",
|
||||
"pillar_name", "pillar_name_id",
|
||||
"framework",
|
||||
]
|
||||
df_dim = df[["indicator_id"] + dim_cols].drop_duplicates(subset=["indicator_id"])
|
||||
|
||||
# Merge semua
|
||||
df_agg = (
|
||||
@@ -1043,7 +1196,7 @@ class IndicatorNormAggregator:
|
||||
df_agg.loc[has_score & (df_agg["avg_norm_score_1_100"] >= _PERFORMANCE_THRESHOLD), "performance"] = "Good"
|
||||
df_agg.loc[has_score & (df_agg["avg_norm_score_1_100"] < _PERFORMANCE_THRESHOLD), "performance"] = "Bad"
|
||||
|
||||
# ---- Build narrative (bilingual, interpretatif, plain text) ----
|
||||
# ---- Build narrative ----
|
||||
self.logger.info("\n--- BUILD NARRATIVE (interpretatif, plain text, bilingual EN/ID) ---")
|
||||
narratives_en = []
|
||||
narratives_id = []
|
||||
@@ -1064,8 +1217,10 @@ class IndicatorNormAggregator:
|
||||
|
||||
# ---- Save ----
|
||||
out = df_agg[[
|
||||
"indicator_id", "indicator_name", "unit", "direction",
|
||||
"pillar_name", "framework",
|
||||
"indicator_id", "indicator_name", "indicator_name_id",
|
||||
"unit", "direction",
|
||||
"pillar_name", "pillar_name_id",
|
||||
"framework",
|
||||
"year_min", "year_max", "n_countries",
|
||||
"avg_value_first", "avg_value_last",
|
||||
"avg_norm_score_1_100", "performance",
|
||||
@@ -1079,9 +1234,11 @@ class IndicatorNormAggregator:
|
||||
|
||||
out["indicator_id"] = out["indicator_id"].astype(int)
|
||||
out["indicator_name"] = out["indicator_name"].astype(str)
|
||||
out["indicator_name_id"] = out["indicator_name_id"].astype(str)
|
||||
out["unit"] = out["unit"].fillna("").astype(str)
|
||||
out["direction"] = out["direction"].astype(str)
|
||||
out["pillar_name"] = out["pillar_name"].astype(str)
|
||||
out["pillar_name_id"] = out["pillar_name_id"].astype(str)
|
||||
out["framework"] = out["framework"].astype(str)
|
||||
out["year_min"] = out["year_min"].astype(int)
|
||||
out["year_max"] = out["year_max"].astype(int)
|
||||
@@ -1102,9 +1259,11 @@ class IndicatorNormAggregator:
|
||||
schema = [
|
||||
bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("indicator_name_id", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("unit", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("direction", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("pillar_name_id", "STRING", mode="NULLABLE"),
|
||||
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("year_min", "INTEGER", mode="REQUIRED"),
|
||||
bigquery.SchemaField("year_max", "INTEGER", mode="REQUIRED"),
|
||||
@@ -1149,6 +1308,7 @@ class IndicatorNormAggregator:
|
||||
"narrative_dimensions" : ["trend", "gap_trend", "anomaly", "country_consistency"],
|
||||
"performance_threshold": _PERFORMANCE_THRESHOLD,
|
||||
"layer" : "gold",
|
||||
"added_columns" : ["indicator_name_id", "pillar_name_id"],
|
||||
}),
|
||||
"validation_metrics" : json.dumps({
|
||||
"total_rows" : rows_loaded,
|
||||
@@ -1172,11 +1332,13 @@ class IndicatorNormAggregator:
|
||||
self.logger.info(" Dim : dim_indicator (unit)")
|
||||
self.logger.info(" Output : agg_indicator_norm -> fs_asean_gold")
|
||||
self.logger.info(" agg_narrative_indicator -> fs_asean_gold")
|
||||
self.logger.info(" Added : indicator_name_id, pillar_name_id (Bahasa Indonesia)")
|
||||
self.logger.info("=" * 80)
|
||||
|
||||
self.load_data()
|
||||
self.load_units()
|
||||
self._merge_unit()
|
||||
self._add_indonesia_name_columns() # <-- BARU
|
||||
self.sdgs_start_year = self._detect_sdgs_start_year()
|
||||
self._assign_framework()
|
||||
df_normed = self._compute_norm_values()
|
||||
|
||||
Reference in New Issue
Block a user