izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

country name indo version

Browse Source
This commit is contained in:
Debby
2026-06-07 08:09:14 +07:00
parent 512c08bff3
commit ca1e0d3949
3 changed files with 344 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bigquery_aggregate_layer.py
+143 -55
View File
@@ -13,13 +13,16 @@ Narrative style:
- Plain text, tanpa markdown bold (**)
- Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata
- Bilingual: narrative_en (Inggris) + narrative_id (Indonesia)
* narrative_en : nama negara & pilar dalam Bahasa Inggris
* narrative_id : nama negara & pilar dalam Bahasa Indonesia [FIXED]
- Granularity: per tahun (Overview & Pillar)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
- agg_pillar_composite : + pillar_name_id
- agg_pillar_by_country : + pillar_name_id
- agg_framework_by_country : (framework tidak diterjemahkan, sudah singkat)
- agg_narrative_pillar : + pillar_name_id
FIXED & ADDED:
- "Access" -> "Akses" di semua mapping pilar (bukan "Keterjangkauan")
- Tambah COUNTRY_NAME_ID_MAP untuk terjemahan nama 10 negara ASEAN
- narrative_id menggunakan nama negara & pilar dalam Bahasa Indonesia
- Tambah kolom country_name_id di tabel yang menyimpan country_name
- Kolom pillar_name_id di semua tabel pilar
"""
import pandas as pd
@@ -92,23 +95,40 @@ _FIES_DETECTION_LOWER: frozenset = frozenset([
# TRANSLATION DICTIONARIES
# =============================================================================
# Nama negara ASEAN -> Bahasa Indonesia [BARU]
COUNTRY_NAME_ID_MAP: dict = {
"Brunei Darussalam" : "Brunei Darussalam",
"Cambodia" : "Kamboja",
"Indonesia" : "Indonesia",
"Lao People's Democratic Republic" : "Laos",
"Lao PDR" : "Laos",
"Malaysia" : "Malaysia",
"Myanmar" : "Myanmar",
"Philippines" : "Filipina",
"Singapore" : "Singapura",
"Thailand" : "Thailand",
"Timor-Leste" : "Timor-Leste",
"Viet Nam" : "Vietnam",
"Vietnam" : "Vietnam",
}
# Nama pilar -> Bahasa Indonesia
# FIXED: "Access" -> "Akses" (bukan "Keterjangkauan")
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability" : "Keberlanjutan",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability" : "Keberlanjutan",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Availability" : "Ketersediaan",
"Access" : "Akses",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability" : "Keberlanjutan",
"availability" : "Ketersediaan",
"access" : "Akses",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability" : "Keberlanjutan",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Akses Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Food Sustainability": "Keberlanjutan Pangan",
}
@@ -247,6 +267,13 @@ INDICATOR_TRANSLATION_ID: dict = {
}
def translate_country(name: str) -> str:
"""Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return COUNTRY_NAME_ID_MAP.get(name.strip(), name)
def translate_indicator(name: str) -> str:
"""Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
@@ -437,6 +464,7 @@ def _find_anomaly_year(values_by_year: dict) -> tuple:
# =============================================================================
# NARRATIVE BUILDER — OVERVIEW (per tahun)
# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia
# =============================================================================
def _build_overview_narrative(
@@ -533,23 +561,34 @@ def _build_overview_narrative(
sentences_en.append(s4_en)
sentences_id.append(s4_id)
# Ranking: EN pakai nama Inggris, ID pakai nama Indonesia
if ranking_list and len(ranking_list) >= 2:
top = ranking_list[0]
bottom = ranking_list[-1]
s5_en = (
f"In {year}, {top['country_name']} led the region with a score of "
f"{_fmt_score(top['score'])}, while {bottom['country_name']} ranked last "
top_name_en = top['country_name']
top_name_id = translate_country(top_name_en)
bottom_name_en = bottom['country_name']
bottom_name_id = translate_country(bottom_name_en)
s5_en = (
f"In {year}, {top_name_en} led the region with a score of "
f"{_fmt_score(top['score'])}, while {bottom_name_en} ranked last "
f"at {_fmt_score(bottom['score'])}."
)
s5_id = (
f"Pada tahun {year}, {top['country_name']} memimpin kawasan dengan skor "
f"{_fmt_score(top['score'])}, sementara {bottom['country_name']} berada di "
f"Pada tahun {year}, {top_name_id} memimpin kawasan dengan skor "
f"{_fmt_score(top['score'])}, sementara {bottom_name_id} berada di "
f"posisi terbawah dengan skor {_fmt_score(bottom['score'])}."
)
sentences_en.append(s5_en)
sentences_id.append(s5_id)
# Peningkatan/penurunan: EN pakai nama Inggris, ID pakai nama Indonesia
if most_improved_country and most_declined_country:
improved_name_id = translate_country(most_improved_country)
declined_name_id = translate_country(most_declined_country)
if most_improved_country != most_declined_country:
s6_en = (
f"{most_improved_country} showed the biggest improvement "
@@ -558,9 +597,9 @@ def _build_overview_narrative(
f"({_fmt_delta(most_declined_delta)} pts)."
)
s6_id = (
f"{most_improved_country} mencatat peningkatan terbesar "
f"{improved_name_id} mencatat peningkatan terbesar "
f"({_fmt_delta(most_improved_delta)} poin), "
f"sementara {most_declined_country} mengalami penurunan terbesar "
f"sementara {declined_name_id} mengalami penurunan terbesar "
f"({_fmt_delta(most_declined_delta)} poin)."
)
sentences_en.append(s6_en)
@@ -573,6 +612,7 @@ def _build_overview_narrative(
# =============================================================================
# NARRATIVE BUILDER — PILLAR (per tahun per pilar)
# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia
# =============================================================================
def _build_pillar_narrative(
@@ -593,6 +633,9 @@ def _build_pillar_narrative(
sentences_en = []
sentences_id = []
# Terjemahan nama pilar
pillar_name_id = translate_pillar(pillar_name)
rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th")
perf_word_en = "good" if pillar_score >= PERFORMANCE_THRESHOLD else "below target"
perf_word_id = "baik" if pillar_score >= PERFORMANCE_THRESHOLD else "di bawah target"
@@ -602,7 +645,7 @@ def _build_pillar_narrative(
f"{n_pillars} pillars with a score of {_fmt_score(pillar_score)} ({perf_word_en})."
)
s1_id = (
f"Pada tahun {year}, pilar {pillar_name} menempati peringkat {rank_in_year} dari "
f"Pada tahun {year}, pilar {pillar_name_id} menempati peringkat {rank_in_year} dari "
f"{n_pillars} pilar dengan skor {_fmt_score(pillar_score)} ({perf_word_id})."
)
sentences_en.append(s1_en)
@@ -632,16 +675,16 @@ def _build_pillar_narrative(
trend = _detect_series_trend(hist_scores)
if trend == "improving_consistent":
s3_en = f"This pillar has shown consistent improvement since {hist_years[0]}."
s3_id = f"Pilar ini menunjukkan perbaikan yang konsisten sejak {hist_years[0]}."
s3_id = f"Pilar {pillar_name_id} menunjukkan perbaikan yang konsisten sejak {hist_years[0]}."
elif trend == "improving_slowing":
s3_en = f"While the pillar improved since {hist_years[0]}, the pace has slowed in recent years."
s3_id = f"Meskipun pilar ini membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir."
s3_id = f"Meskipun pilar {pillar_name_id} membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir."
elif trend == "deteriorating":
s3_en = f"This pillar has shown a declining trend since {hist_years[0]}, requiring targeted intervention."
s3_id = f"Pilar ini menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah."
s3_id = f"Pilar {pillar_name_id} menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah."
elif trend == "fluctuating":
s3_en = f"Performance in this pillar has been inconsistent since {hist_years[0]}, with no clear trend."
s3_id = f"Performa pilar ini tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas."
s3_id = f"Performa pilar {pillar_name_id} tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas."
else:
s3_en = ""
s3_id = ""
@@ -657,10 +700,10 @@ def _build_pillar_narrative(
)
if gap_trend == "widening":
s4_en = "Country disparities within this pillar have widened over time."
s4_id = "Kesenjangan antar negara dalam pilar ini semakin melebar seiring waktu."
s4_id = f"Kesenjangan antar negara dalam pilar {pillar_name_id} semakin melebar seiring waktu."
elif gap_trend == "narrowing":
s4_en = "Country disparities within this pillar have narrowed, indicating more balanced progress."
s4_id = "Kesenjangan antar negara dalam pilar ini menyempit, mengindikasikan kemajuan yang lebih merata."
s4_id = f"Kesenjangan antar negara dalam pilar {pillar_name_id} menyempit, mengindikasikan kemajuan yang lebih merata."
else:
s4_en = ""
s4_id = ""
@@ -669,32 +712,41 @@ def _build_pillar_narrative(
sentences_en.append(s4_en)
sentences_id.append(s4_id)
# Negara terbaik/terburuk: EN pakai nama Inggris, ID pakai nama Indonesia
if top_country and bot_country and top_country != bot_country:
top_country_id = translate_country(top_country)
bot_country_id = translate_country(bot_country)
s5_en = (
f"{top_country} performed best in this pillar ({_fmt_score(top_country_score)}), "
f"while {bot_country} had the lowest score ({_fmt_score(bot_country_score)})."
)
s5_id = (
f"{top_country} memiliki performa terbaik dalam pilar ini ({_fmt_score(top_country_score)}), "
f"sementara {bot_country} memiliki skor terendah ({_fmt_score(bot_country_score)})."
f"{top_country_id} memiliki performa terbaik dalam pilar {pillar_name_id} "
f"({_fmt_score(top_country_score)}), "
f"sementara {bot_country_id} memiliki skor terendah ({_fmt_score(bot_country_score)})."
)
sentences_en.append(s5_en)
sentences_id.append(s5_id)
# Perbandingan antar pilar: EN pakai nama Inggris, ID pakai nama Indonesia
if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1:
sorted_pillars = all_pillar_scores_year.sort_values("pillar_score_1_100", ascending=False)
strongest = sorted_pillars.iloc[0]
weakest = sorted_pillars.iloc[-1]
if strongest["pillar_name"] != pillar_name and weakest["pillar_name"] != pillar_name:
strongest_id = translate_pillar(strongest["pillar_name"])
weakest_id = translate_pillar(weakest["pillar_name"])
s6_en = (
f"Across all pillars in {year}, {strongest['pillar_name']} scored highest "
f"({_fmt_score(strongest['pillar_score_1_100'])}) and {weakest['pillar_name']} "
f"scored lowest ({_fmt_score(weakest['pillar_score_1_100'])})."
)
s6_id = (
f"Di antara semua pilar pada tahun {year}, {strongest['pillar_name']} mendapat skor "
f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest['pillar_name']} "
f"Di antara semua pilar pada tahun {year}, {strongest_id} mendapat skor "
f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest_id} "
f"mendapat skor terendah ({_fmt_score(weakest['pillar_score_1_100'])})."
)
sentences_en.append(s6_en)
@@ -758,7 +810,10 @@ class FoodSecurityAggregator:
self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'")
self.df["direction"] = self.df["direction"].fillna("positive")
# Pastikan kolom terjemahan Indonesia tersedia (bisa dari fact atau dibuat ulang)
# Pastikan kolom terjemahan Indonesia tersedia
if "country_name_id" not in self.df.columns:
self.df["country_name_id"] = self.df["country_name"].apply(translate_country)
self.logger.info(" [TRANSLATION] Kolom country_name_id dibuat dari mapping.")
if "indicator_name_id" not in self.df.columns:
self.df["indicator_name_id"] = self.df["indicator_name"].apply(translate_indicator)
self.logger.info(" [TRANSLATION] Kolom indicator_name_id dibuat dari mapping.")
@@ -766,7 +821,23 @@ class FoodSecurityAggregator:
self.df["pillar_name_id"] = self.df["pillar_name"].apply(translate_pillar)
self.logger.info(" [TRANSLATION] Kolom pillar_name_id dibuat dari mapping.")
self.logger.info(f" Rows : {len(self.df):,}")
# Log terjemahan negara
sample_ctr = (
self.df[["country_name", "country_name_id"]]
.drop_duplicates()
.sort_values("country_name")
)
self.logger.info("\n Terjemahan nama negara (EN -> ID):")
for _, r in sample_ctr.iterrows():
self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}")
# Log terjemahan pilar
sample_pil = self.df[["pillar_name", "pillar_name_id"]].drop_duplicates()
self.logger.info("\n Terjemahan nama pilar (EN -> ID):")
for _, r in sample_pil.iterrows():
self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}")
self.logger.info(f"\n Rows : {len(self.df):,}")
self.logger.info(f" Countries : {self.df['country_id'].nunique()}")
self.logger.info(f" Indicators: {self.df['indicator_id'].nunique()}")
self.logger.info(
@@ -942,7 +1013,7 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["pillar_id"], "pillar_score_1_100")
# Kolom terjemahan Indonesia
# Kolom terjemahan Indonesia — FIXED: "Access" -> "Akses"
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["pillar_id"] = df["pillar_id"].astype(int)
@@ -979,7 +1050,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 3: agg_pillar_by_country
# Kolom tambahan: pillar_name_id
# Kolom tambahan: pillar_name_id, country_name_id
# =========================================================================
def calc_pillar_by_country(self) -> pd.DataFrame:
@@ -1007,8 +1078,9 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100")
# Kolom terjemahan Indonesia
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
# Kolom terjemahan Indonesia — FIXED
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["country_name_id"] = df["country_name"].apply(translate_country)
df["country_id"] = df["country_id"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
@@ -1017,10 +1089,12 @@ class FoodSecurityAggregator:
df["pillar_country_norm"] = df["pillar_country_norm"].astype(float)
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["country_name_id"] = df["country_name_id"].astype(str)
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
@@ -1043,7 +1117,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 4: agg_framework_by_country
# Tidak ada kolom pillar/indicator di tabel ini (sudah di level framework)
# Tambah kolom: country_name_id
# =========================================================================
def _calc_country_composite_inmemory(self) -> pd.DataFrame:
@@ -1175,18 +1249,23 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["country_id", "framework"], "framework_score_1_100")
# Tambah kolom nama negara Indonesia
df["country_name_id"] = df["country_name"].apply(translate_country)
df["country_id"] = df["country_id"].astype(int)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
df["rank_in_framework_year"] = safe_int(df["rank_in_framework_year"], col_name="rank_in_framework_year", logger=self.logger)
df["framework_norm"] = df["framework_norm"].astype(float)
df["framework_score_1_100"] = df["framework_score_1_100"].astype(float)
df["country_name_id"] = df["country_name_id"].astype(str)
self._validate_mdgs_equals_total(df, level="country")
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
@@ -1208,7 +1287,6 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 5: agg_framework_asean
# Tidak ada kolom pillar/indicator langsung di tabel ini
# =========================================================================
def calc_framework_asean(self) -> pd.DataFrame:
@@ -1371,7 +1449,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 6: agg_narrative_overview
# Tidak ada kolom pillar/indicator di tabel ini
# FIXED: narrative_id pakai nama negara Indonesia
# =========================================================================
def calc_narrative_overview(
@@ -1384,6 +1462,7 @@ class FoodSecurityAggregator:
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID")
self.logger.info(" FIXED : narrative_id pakai nama negara Bahasa Indonesia")
self.logger.info("=" * 70)
try:
@@ -1535,6 +1614,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 7: agg_narrative_pillar
# Kolom tambahan: pillar_name_id
# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia
# =========================================================================
def calc_narrative_pillar(
@@ -1547,6 +1627,7 @@ class FoodSecurityAggregator:
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 7: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID")
self.logger.info(" FIXED : narrative_id pakai nama negara & pilar Bahasa Indonesia")
self.logger.info("=" * 70)
try:
@@ -1576,7 +1657,7 @@ class FoodSecurityAggregator:
p_yoy = prow["year_over_year_change"]
p_yoy_val = float(p_yoy) if pd.notna(p_yoy) else None
# Terjemahan Indonesia nama pillar
# Terjemahan Indonesia nama pillar — FIXED
p_name_id = translate_pillar(p_name)
p_country = (
@@ -1626,20 +1707,24 @@ class FoodSecurityAggregator:
"rank_in_year": p_rank,
"yoy_change": p_yoy_val,
"top_country": top_country,
"top_country_id": translate_country(top_country) if top_country else None,
"top_country_score": top_country_score,
"bottom_country": bot_country,
"bottom_country_id": translate_country(bot_country) if bot_country else None,
"bottom_country_score": bot_country_score,
"narrative_en": narrative_en,
"narrative_id": narrative_id,
})
df = pd.DataFrame(records)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
df["top_country_id"] = df["top_country_id"].astype(str).replace("None", pd.NA).astype("string")
df["bottom_country_id"] = df["bottom_country_id"].astype(str).replace("None", pd.NA).astype("string")
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
@@ -1657,8 +1742,10 @@ class FoodSecurityAggregator:
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("top_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("top_country_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("top_country_score", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("bottom_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("bottom_country_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"),
bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"),
@@ -1750,6 +1837,7 @@ class FoodSecurityAggregator:
self.logger.info("FOOD SECURITY AGGREGATION — 6 TABLES -> fs_asean_gold")
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}")
self.logger.info(f" Narrative style : interpretive, plain text, bilingual EN/ID")
self.logger.info(f" FIXED : 'Access' -> 'Akses', nama negara & pilar ID")
self.logger.info("=" * 70)
self.load_data()