From ca1e0d3949e28c0ef40714b409692116476b8c6a Mon Sep 17 00:00:00 2001 From: Debby Date: Sun, 7 Jun 2026 08:09:14 +0700 Subject: [PATCH] country name indo version --- .../bigquery_aggraget_fact_selected_layer.py | 177 +++++++++++----- scripts/bigquery_aggregate_layer.py | 198 +++++++++++++----- scripts/bigquery_analytical_layer.py | 87 ++++++-- 3 files changed, 344 insertions(+), 118 deletions(-) diff --git a/scripts/bigquery_aggraget_fact_selected_layer.py b/scripts/bigquery_aggraget_fact_selected_layer.py index 450e8b6..430a9bb 100644 --- a/scripts/bigquery_aggraget_fact_selected_layer.py +++ b/scripts/bigquery_aggraget_fact_selected_layer.py @@ -5,9 +5,12 @@ Tabel 2: agg_narrative_indicator -> fs_asean_gold ============================================================================= PERUBAHAN: + - Ditambahkan kolom country_name_id : nama negara dalam Bahasa Indonesia [BARU] - Ditambahkan kolom indicator_name_id : nama indikator dalam Bahasa Indonesia - Ditambahkan kolom pillar_name_id : nama pilar dalam Bahasa Indonesia - - Kedua kolom ikut tersimpan di BigQuery (schema + DataFrame output) + - Ketiga kolom ikut tersimpan di BigQuery (schema + DataFrame output) + - Narrative versi Indonesia menggunakan nama negara & pilar dalam Bahasa Indonesia + - FIXED: "Access" -> "Akses" (konsisten di semua mapping pilar) ============================================================================= agg_indicator_norm @@ -35,7 +38,7 @@ Performance Label Logic: - performance : "Good" jika norm_score_1_100 >= 60, "Bad" jika < 60, null jika null Output Schema (agg_indicator_norm): - year, country_id, country_name, + year, country_id, country_name, country_name_id, indicator_id, indicator_name, indicator_name_id, unit, direction, pillar_id, pillar_name, pillar_name_id, @@ -54,6 +57,8 @@ Tujuan: Menghasilkan narasi otomatis per indikator (granularity: indicator_id). Narasi membaca kondisi nyata dari data: tren, gap, anomali, konsistensi. Tersedia dalam dua bahasa: Inggris (narrative_en) dan Indonesia (narrative_id). + - narrative_en : menggunakan nama negara & pilar dalam Bahasa Inggris + - narrative_id : menggunakan nama negara & pilar dalam Bahasa Indonesia Tanpa markdown bold (**) agar aman ditampilkan di Looker Studio. Granularity: @@ -71,6 +76,7 @@ Output Schema (agg_narrative_indicator): n_yoy_total, n_yoy_positive, best_yoy_from, best_yoy_to, country_worst, country_best, + country_worst_id, country_best_id, narrative_en, narrative_id """ @@ -96,7 +102,25 @@ from google.cloud import bigquery # MAPPING BAHASA INDONESIA # ============================================================================= +# Mapping nama negara (Inggris -> Indonesia) +COUNTRY_NAME_ID_MAP: dict = { + "Brunei Darussalam" : "Brunei Darussalam", + "Cambodia" : "Kamboja", + "Indonesia" : "Indonesia", + "Lao People's Democratic Republic" : "Laos", + "Lao PDR" : "Laos", + "Malaysia" : "Malaysia", + "Myanmar" : "Myanmar", + "Philippines" : "Filipina", + "Singapore" : "Singapura", + "Thailand" : "Thailand", + "Timor-Leste" : "Timor-Leste", + "Viet Nam" : "Vietnam", + "Vietnam" : "Vietnam", +} + # Mapping nama pilar (Inggris -> Indonesia) +# FIXED: "Access" -> "Akses" (bukan "Keterjangkauan") PILLAR_NAME_ID_MAP: dict = { "Availability" : "Ketersediaan", "Access" : "Akses", @@ -189,8 +213,6 @@ INDICATOR_NAME_ID_MAP: dict = { "Stabilitas politik dan ketiadaan kekerasan/terorisme", "domestic food price volatility index": "Indeks volatilitas harga pangan domestik", - "per capita food supply variability (kcal/capita/day)": - "Variabilitas pasokan pangan per kapita (kkal/kapita/hari)", "cereal import dependency ratio (percent) (3-year average)": "Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)", "value of food imports in total merchandise exports (percent) (3-year average)": @@ -200,11 +222,19 @@ INDICATOR_NAME_ID_MAP: dict = { } +def get_country_name_id(country_name: str) -> str: + """Kembalikan terjemahan Bahasa Indonesia untuk nama negara.""" + return COUNTRY_NAME_ID_MAP.get( + str(country_name).strip(), + str(country_name), # fallback: kembalikan nama asli + ) + + def get_indicator_name_id(indicator_name: str) -> str: """Kembalikan terjemahan Bahasa Indonesia untuk nama indikator.""" return INDICATOR_NAME_ID_MAP.get( str(indicator_name).lower().strip(), - str(indicator_name), # fallback: kembalikan nama asli jika tidak ada mapping + str(indicator_name), # fallback: kembalikan nama asli ) @@ -212,7 +242,7 @@ def get_pillar_name_id(pillar_name: str) -> str: """Kembalikan terjemahan Bahasa Indonesia untuk nama pilar.""" return PILLAR_NAME_ID_MAP.get( str(pillar_name).strip(), - str(pillar_name), # fallback: kembalikan nama asli jika tidak ada mapping + str(pillar_name), # fallback: kembalikan nama asli ) @@ -406,6 +436,11 @@ def _detect_anomaly_year(scores_by_year: pd.Series) -> tuple: def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple: + """ + Mengembalikan (best_country_en, worst_country_en, is_consistent). + Nama negara dikembalikan dalam Bahasa Inggris; penerjemahan dilakukan + di layer narrative builder. + """ country_avg = ( df_ind.groupby("country_name")["value"] .mean() @@ -446,34 +481,42 @@ def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple: # ============================================================================= # NARRATIVE BUILDER — plain text, no markdown, bilingual +# FIXED: narrative_id menggunakan nama negara & pilar dalam Bahasa Indonesia # ============================================================================= def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tuple: - ind_id = int(row["indicator_id"]) - ind_name = str(row["indicator_name"]).strip() - unit = str(row["unit"]).strip() if row["unit"] else "" - direction = str(row["direction"]).strip() - pillar = str(row["pillar_name"]).strip() - framework = str(row["framework"]).strip() - year_min = int(row["year_min"]) - year_max = int(row["year_max"]) + ind_id = int(row["indicator_id"]) + ind_name_en = str(row["indicator_name"]).strip() + ind_name_id = str(row.get("indicator_name_id", ind_name_en)).strip() + unit = str(row["unit"]).strip() if row["unit"] else "" + direction = str(row["direction"]).strip() + pillar_en = str(row["pillar_name"]).strip() + pillar_id_ = get_pillar_name_id(pillar_en) # nama pilar dalam Bahasa Indonesia + framework = str(row["framework"]).strip() + year_min = int(row["year_min"]) + year_max = int(row["year_max"]) lower_better = _is_lower_better(direction) df_ind = df_full[df_full["indicator_id"] == ind_id].copy() if df_ind.empty: - na_en = f"{ind_name} ({framework}, {pillar}): Insufficient data for analysis." - na_id = f"{ind_name} ({framework}, {pillar}): Data tidak cukup untuk dianalisis." + na_en = f"{ind_name_en} ({framework}, {pillar_en}): Insufficient data for analysis." + na_id = f"{ind_name_id} ({framework}, {pillar_id_}): Data tidak cukup untuk dianalisis." return na_en, na_id asean_avg_by_year = ( df_ind.groupby("year")["value"].mean().dropna() ) - trend_label = _detect_trend(asean_avg_by_year, lower_better) - gap_label = _detect_gap_trend(df_ind, lower_better) + trend_label = _detect_trend(asean_avg_by_year, lower_better) + gap_label = _detect_gap_trend(df_ind, lower_better) anomaly_year, anomaly_dir = _detect_anomaly_year(asean_avg_by_year) - best_country, worst_country, is_consistent = _detect_consistency(df_ind, lower_better) + # best_country & worst_country -> nama dalam Bahasa Inggris (dari data) + best_country_en, worst_country_en, is_consistent = _detect_consistency(df_ind, lower_better) + + # Terjemahan nama negara ke Bahasa Indonesia + best_country_id = get_country_name_id(best_country_en) if best_country_en else None + worst_country_id = get_country_name_id(worst_country_en) if worst_country_en else None avg_first = row.get("avg_value_first", np.nan) avg_last = row.get("avg_value_last", np.nan) @@ -488,8 +531,9 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup sentences_en = [] sentences_id = [] - s1_en = f"{ind_name} ({framework}, {pillar}, {year_min}-{year_max}):" - s1_id = f"{ind_name} ({framework}, {pillar}, {year_min}-{year_max}):" + # Header: EN menggunakan nama Inggris, ID menggunakan nama Indonesia + s1_en = f"{ind_name_en} ({framework}, {pillar_en}, {year_min}-{year_max}):" + s1_id = f"{ind_name_id} ({framework}, {pillar_id_}, {year_min}-{year_max}):" sentences_en.append(s1_en) sentences_id.append(s1_id) @@ -528,24 +572,25 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup sentences_en.append(f"A sharp improvement was observed in {anomaly_year}, standing out from the overall pattern.") sentences_id.append(f"Peningkatan tajam tercatat pada tahun {anomaly_year}, yang menyimpang dari pola keseluruhan.") - if best_country and worst_country: + # Kalimat tentang negara: EN pakai nama Inggris, ID pakai nama Indonesia + if best_country_en and worst_country_en: if is_consistent: sentences_en.append( - f"{best_country} consistently performed above the regional average, " - f"while {worst_country} consistently lagged behind." + f"{best_country_en} consistently performed above the regional average, " + f"while {worst_country_en} consistently lagged behind." ) sentences_id.append( - f"{best_country} secara konsisten berada di atas rata-rata regional, " - f"sementara {worst_country} secara konsisten tertinggal." + f"{best_country_id} secara konsisten berada di atas rata-rata regional, " + f"sementara {worst_country_id} secara konsisten tertinggal." ) else: sentences_en.append( - f"Overall, {best_country} showed the best performance, " - f"while {worst_country} had the weakest results across the period." + f"Overall, {best_country_en} showed the best performance, " + f"while {worst_country_en} had the weakest results across the period." ) sentences_id.append( - f"Secara keseluruhan, {best_country} menunjukkan performa terbaik, " - f"sementara {worst_country} memiliki hasil terlemah sepanjang periode." + f"Secara keseluruhan, {best_country_id} menunjukkan performa terbaik, " + f"sementara {worst_country_id} memiliki hasil terlemah sepanjang periode." ) narrative_en = " ".join(s for s in sentences_en if s) @@ -689,23 +734,54 @@ class IndicatorNormAggregator: self.logger.info("STEP 3b: ADD BAHASA INDONESIA NAME COLUMNS") self.logger.info("=" * 80) + # Nama negara + self.df["country_name_id"] = ( + self.df["country_name"] + .apply(get_country_name_id) + .astype(str) + ) + + # Nama indikator self.df["indicator_name_id"] = ( self.df["indicator_name"] .apply(get_indicator_name_id) .astype(str) ) + + # Nama pilar self.df["pillar_name_id"] = ( self.df["pillar_name"] .apply(get_pillar_name_id) .astype(str) ) + n_country_mapped = (self.df["country_name_id"] != self.df["country_name"]).sum() n_indicator_mapped = (self.df["indicator_name_id"] != self.df["indicator_name"]).sum() n_pillar_mapped = (self.df["pillar_name_id"] != self.df["pillar_name"]).sum() + self.logger.info(f" country_name_id mapped rows : {n_country_mapped:,}") self.logger.info(f" indicator_name_id mapped rows : {n_indicator_mapped:,}") self.logger.info(f" pillar_name_id mapped rows : {n_pillar_mapped:,}") - # Log sample mapping + # Log sample negara + sample_ctr = ( + self.df[["country_name", "country_name_id"]] + .drop_duplicates() + .sort_values("country_name") + ) + self.logger.info("\n Terjemahan nama negara (EN -> ID):") + for _, r in sample_ctr.iterrows(): + self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}") + + # Log sample pilar + sample_pil = ( + self.df[["pillar_name", "pillar_name_id"]] + .drop_duplicates() + ) + self.logger.info("\n Pillar mapping (EN -> ID):") + for _, r in sample_pil.iterrows(): + self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}") + + # Log sample indikator sample_ind = ( self.df[["indicator_name", "indicator_name_id"]] .drop_duplicates() @@ -716,14 +792,6 @@ class IndicatorNormAggregator: self.logger.info(f" EN: {r['indicator_name'][:55]}") self.logger.info(f" ID: {r['indicator_name_id'][:55]}") - sample_pil = ( - self.df[["pillar_name", "pillar_name_id"]] - .drop_duplicates() - ) - self.logger.info("\n Pillar mapping (EN -> ID):") - for _, r in sample_pil.iterrows(): - self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}") - # ========================================================================= # STEP 4: Deteksi sdgs_start_year # ========================================================================= @@ -925,7 +993,7 @@ class IndicatorNormAggregator: self.logger.info("=" * 80) out = df[[ - "year", "country_id", "country_name", + "year", "country_id", "country_name", "country_name_id", "indicator_id", "indicator_name", "indicator_name_id", "unit", "direction", "pillar_id", "pillar_name", "pillar_name_id", @@ -941,6 +1009,7 @@ class IndicatorNormAggregator: out["year"] = out["year"].astype(int) out["country_id"] = out["country_id"].astype(int) out["country_name"] = out["country_name"].astype(str) + out["country_name_id"] = out["country_name_id"].astype(str) out["indicator_id"] = out["indicator_id"].astype(int) out["indicator_name"] = out["indicator_name"].astype(str) out["indicator_name_id"] = out["indicator_name_id"].astype(str) @@ -966,6 +1035,7 @@ class IndicatorNormAggregator: bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("country_name_id", "STRING", mode="NULLABLE"), bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("indicator_name_id", "STRING", mode="NULLABLE"), @@ -1009,7 +1079,7 @@ class IndicatorNormAggregator: "yoy_columns" : ["yoy_value", "yoy_norm_value"], "performance_threshold": _PERFORMANCE_THRESHOLD, "unit_source" : "dim_indicator", - "added_columns" : ["indicator_name_id", "pillar_name_id"], + "added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id"], }), "validation_metrics" : json.dumps({ "total_rows" : rows_loaded, @@ -1062,6 +1132,7 @@ class IndicatorNormAggregator: self.logger.info("STEP 12-17: agg_narrative_indicator") self.logger.info(" Granularity: per indicator_id (all years + all ASEAN countries)") self.logger.info(" Narrative : interpretatif, plain text, bilingual EN/ID") + self.logger.info(" FIXED : narrative_id pakai nama negara & pilar Bahasa Indonesia") self.logger.info("=" * 80) df = df_final.copy() @@ -1150,7 +1221,7 @@ class IndicatorNormAggregator: }) df_yoy_stats = pd.DataFrame(yoy_stats) - # Country best/worst + # Country best/worst (nama asli Bahasa Inggris) df_country_avg = ( df.groupby(["indicator_id", "country_id", "country_name"]) .agg(country_avg_value=("value", "mean")) @@ -1166,9 +1237,12 @@ class IndicatorNormAggregator: worst_row = grp.loc[grp["country_avg_value"].idxmin()] best_row = grp.loc[grp["country_avg_value"].idxmax()] country_stats.append({ - "indicator_id" : ind_id, - "country_worst": worst_row["country_name"], - "country_best" : best_row["country_name"], + "indicator_id" : ind_id, + "country_worst" : worst_row["country_name"], # nama Inggris + "country_best" : best_row["country_name"], # nama Inggris + # Tambahan: nama Indonesia untuk kedua negara + "country_worst_id": get_country_name_id(worst_row["country_name"]), + "country_best_id" : get_country_name_id(best_row["country_name"]), }) df_country_stats = pd.DataFrame(country_stats) @@ -1229,6 +1303,7 @@ class IndicatorNormAggregator: "n_yoy_total", "n_yoy_positive", "best_yoy_from", "best_yoy_to", "country_worst", "country_best", + "country_worst_id", "country_best_id", "narrative_en", "narrative_id", ]].copy() @@ -1255,6 +1330,8 @@ class IndicatorNormAggregator: out["best_yoy_to"] = pd.to_numeric(out["best_yoy_to"], errors="coerce").astype("Int64") out["country_worst"] = out["country_worst"].astype(str).replace("nan", pd.NA).astype("string") out["country_best"] = out["country_best"].astype(str).replace("nan", pd.NA).astype("string") + out["country_worst_id"] = out["country_worst_id"].astype(str).replace("nan", pd.NA).astype("string") + out["country_best_id"] = out["country_best_id"].astype(str).replace("nan", pd.NA).astype("string") out["narrative_en"] = out["narrative_en"].astype(str) out["narrative_id"] = out["narrative_id"].astype(str) @@ -1280,6 +1357,8 @@ class IndicatorNormAggregator: bigquery.SchemaField("best_yoy_to", "INTEGER", mode="NULLABLE"), bigquery.SchemaField("country_worst", "STRING", mode="NULLABLE"), bigquery.SchemaField("country_best", "STRING", mode="NULLABLE"), + bigquery.SchemaField("country_worst_id", "STRING", mode="NULLABLE"), + bigquery.SchemaField("country_best_id", "STRING", mode="NULLABLE"), bigquery.SchemaField("narrative_en", "STRING", mode="NULLABLE"), bigquery.SchemaField("narrative_id", "STRING", mode="NULLABLE"), ] @@ -1310,7 +1389,8 @@ class IndicatorNormAggregator: "narrative_dimensions" : ["trend", "gap_trend", "anomaly", "country_consistency"], "performance_threshold": _PERFORMANCE_THRESHOLD, "layer" : "gold", - "added_columns" : ["indicator_name_id", "pillar_name_id"], + "added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id", + "country_worst_id", "country_best_id"], }), "validation_metrics" : json.dumps({ "total_rows" : rows_loaded, @@ -1334,13 +1414,14 @@ class IndicatorNormAggregator: self.logger.info(" Dim : dim_indicator (unit)") self.logger.info(" Output : agg_indicator_norm -> fs_asean_gold") self.logger.info(" agg_narrative_indicator -> fs_asean_gold") - self.logger.info(" Added : indicator_name_id, pillar_name_id (Bahasa Indonesia)") + self.logger.info(" Added : country_name_id, indicator_name_id, pillar_name_id (Bahasa Indonesia)") + self.logger.info(" FIXED : 'Access' -> 'Akses', narrative_id pakai nama ID") self.logger.info("=" * 80) self.load_data() self.load_units() self._merge_unit() - self._add_indonesia_name_columns() # <-- BARU + self._add_indonesia_name_columns() self.sdgs_start_year = self._detect_sdgs_start_year() self._assign_framework() df_normed = self._compute_norm_values() diff --git a/scripts/bigquery_aggregate_layer.py b/scripts/bigquery_aggregate_layer.py index 61a666a..6ec56fc 100644 --- a/scripts/bigquery_aggregate_layer.py +++ b/scripts/bigquery_aggregate_layer.py @@ -13,13 +13,16 @@ Narrative style: - Plain text, tanpa markdown bold (**) - Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata - Bilingual: narrative_en (Inggris) + narrative_id (Indonesia) + * narrative_en : nama negara & pilar dalam Bahasa Inggris + * narrative_id : nama negara & pilar dalam Bahasa Indonesia [FIXED] - Granularity: per tahun (Overview & Pillar) -ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) - - agg_pillar_composite : + pillar_name_id - - agg_pillar_by_country : + pillar_name_id - - agg_framework_by_country : (framework tidak diterjemahkan, sudah singkat) - - agg_narrative_pillar : + pillar_name_id +FIXED & ADDED: + - "Access" -> "Akses" di semua mapping pilar (bukan "Keterjangkauan") + - Tambah COUNTRY_NAME_ID_MAP untuk terjemahan nama 10 negara ASEAN + - narrative_id menggunakan nama negara & pilar dalam Bahasa Indonesia + - Tambah kolom country_name_id di tabel yang menyimpan country_name + - Kolom pillar_name_id di semua tabel pilar """ import pandas as pd @@ -92,23 +95,40 @@ _FIES_DETECTION_LOWER: frozenset = frozenset([ # TRANSLATION DICTIONARIES # ============================================================================= +# Nama negara ASEAN -> Bahasa Indonesia [BARU] +COUNTRY_NAME_ID_MAP: dict = { + "Brunei Darussalam" : "Brunei Darussalam", + "Cambodia" : "Kamboja", + "Indonesia" : "Indonesia", + "Lao People's Democratic Republic" : "Laos", + "Lao PDR" : "Laos", + "Malaysia" : "Malaysia", + "Myanmar" : "Myanmar", + "Philippines" : "Filipina", + "Singapore" : "Singapura", + "Thailand" : "Thailand", + "Timor-Leste" : "Timor-Leste", + "Viet Nam" : "Vietnam", + "Vietnam" : "Vietnam", +} + +# Nama pilar -> Bahasa Indonesia +# FIXED: "Access" -> "Akses" (bukan "Keterjangkauan") PILLAR_TRANSLATION_ID: dict = { - # 4 pilar utama Food Security - "Availability" : "Ketersediaan", - "Access" : "Keterjangkauan", - "Utilization" : "Pemanfaatan", - "Stability" : "Stabilitas", - "Sustainability" : "Keberlanjutan", - # Variasi penulisan yang mungkin muncul - "availability" : "Ketersediaan", - "access" : "Keterjangkauan", - "utilization" : "Pemanfaatan", - "stability" : "Stabilitas", - "sustainability" : "Keberlanjutan", - "Food Availability" : "Ketersediaan Pangan", - "Food Access" : "Keterjangkauan Pangan", - "Food Utilization" : "Pemanfaatan Pangan", - "Food Stability" : "Stabilitas Pangan", + "Availability" : "Ketersediaan", + "Access" : "Akses", + "Utilization" : "Pemanfaatan", + "Stability" : "Stabilitas", + "Sustainability" : "Keberlanjutan", + "availability" : "Ketersediaan", + "access" : "Akses", + "utilization" : "Pemanfaatan", + "stability" : "Stabilitas", + "sustainability" : "Keberlanjutan", + "Food Availability" : "Ketersediaan Pangan", + "Food Access" : "Akses Pangan", + "Food Utilization" : "Pemanfaatan Pangan", + "Food Stability" : "Stabilitas Pangan", "Food Sustainability": "Keberlanjutan Pangan", } @@ -247,6 +267,13 @@ INDICATOR_TRANSLATION_ID: dict = { } +def translate_country(name: str) -> str: + """Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli.""" + if not name: + return name + return COUNTRY_NAME_ID_MAP.get(name.strip(), name) + + def translate_indicator(name: str) -> str: """Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli.""" if not name: @@ -437,6 +464,7 @@ def _find_anomaly_year(values_by_year: dict) -> tuple: # ============================================================================= # NARRATIVE BUILDER — OVERVIEW (per tahun) +# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia # ============================================================================= def _build_overview_narrative( @@ -533,23 +561,34 @@ def _build_overview_narrative( sentences_en.append(s4_en) sentences_id.append(s4_id) + # Ranking: EN pakai nama Inggris, ID pakai nama Indonesia if ranking_list and len(ranking_list) >= 2: top = ranking_list[0] bottom = ranking_list[-1] - s5_en = ( - f"In {year}, {top['country_name']} led the region with a score of " - f"{_fmt_score(top['score'])}, while {bottom['country_name']} ranked last " + + top_name_en = top['country_name'] + top_name_id = translate_country(top_name_en) + bottom_name_en = bottom['country_name'] + bottom_name_id = translate_country(bottom_name_en) + + s5_en = ( + f"In {year}, {top_name_en} led the region with a score of " + f"{_fmt_score(top['score'])}, while {bottom_name_en} ranked last " f"at {_fmt_score(bottom['score'])}." ) s5_id = ( - f"Pada tahun {year}, {top['country_name']} memimpin kawasan dengan skor " - f"{_fmt_score(top['score'])}, sementara {bottom['country_name']} berada di " + f"Pada tahun {year}, {top_name_id} memimpin kawasan dengan skor " + f"{_fmt_score(top['score'])}, sementara {bottom_name_id} berada di " f"posisi terbawah dengan skor {_fmt_score(bottom['score'])}." ) sentences_en.append(s5_en) sentences_id.append(s5_id) + # Peningkatan/penurunan: EN pakai nama Inggris, ID pakai nama Indonesia if most_improved_country and most_declined_country: + improved_name_id = translate_country(most_improved_country) + declined_name_id = translate_country(most_declined_country) + if most_improved_country != most_declined_country: s6_en = ( f"{most_improved_country} showed the biggest improvement " @@ -558,9 +597,9 @@ def _build_overview_narrative( f"({_fmt_delta(most_declined_delta)} pts)." ) s6_id = ( - f"{most_improved_country} mencatat peningkatan terbesar " + f"{improved_name_id} mencatat peningkatan terbesar " f"({_fmt_delta(most_improved_delta)} poin), " - f"sementara {most_declined_country} mengalami penurunan terbesar " + f"sementara {declined_name_id} mengalami penurunan terbesar " f"({_fmt_delta(most_declined_delta)} poin)." ) sentences_en.append(s6_en) @@ -573,6 +612,7 @@ def _build_overview_narrative( # ============================================================================= # NARRATIVE BUILDER — PILLAR (per tahun per pilar) +# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia # ============================================================================= def _build_pillar_narrative( @@ -593,6 +633,9 @@ def _build_pillar_narrative( sentences_en = [] sentences_id = [] + # Terjemahan nama pilar + pillar_name_id = translate_pillar(pillar_name) + rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th") perf_word_en = "good" if pillar_score >= PERFORMANCE_THRESHOLD else "below target" perf_word_id = "baik" if pillar_score >= PERFORMANCE_THRESHOLD else "di bawah target" @@ -602,7 +645,7 @@ def _build_pillar_narrative( f"{n_pillars} pillars with a score of {_fmt_score(pillar_score)} ({perf_word_en})." ) s1_id = ( - f"Pada tahun {year}, pilar {pillar_name} menempati peringkat {rank_in_year} dari " + f"Pada tahun {year}, pilar {pillar_name_id} menempati peringkat {rank_in_year} dari " f"{n_pillars} pilar dengan skor {_fmt_score(pillar_score)} ({perf_word_id})." ) sentences_en.append(s1_en) @@ -632,16 +675,16 @@ def _build_pillar_narrative( trend = _detect_series_trend(hist_scores) if trend == "improving_consistent": s3_en = f"This pillar has shown consistent improvement since {hist_years[0]}." - s3_id = f"Pilar ini menunjukkan perbaikan yang konsisten sejak {hist_years[0]}." + s3_id = f"Pilar {pillar_name_id} menunjukkan perbaikan yang konsisten sejak {hist_years[0]}." elif trend == "improving_slowing": s3_en = f"While the pillar improved since {hist_years[0]}, the pace has slowed in recent years." - s3_id = f"Meskipun pilar ini membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir." + s3_id = f"Meskipun pilar {pillar_name_id} membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir." elif trend == "deteriorating": s3_en = f"This pillar has shown a declining trend since {hist_years[0]}, requiring targeted intervention." - s3_id = f"Pilar ini menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah." + s3_id = f"Pilar {pillar_name_id} menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah." elif trend == "fluctuating": s3_en = f"Performance in this pillar has been inconsistent since {hist_years[0]}, with no clear trend." - s3_id = f"Performa pilar ini tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas." + s3_id = f"Performa pilar {pillar_name_id} tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas." else: s3_en = "" s3_id = "" @@ -657,10 +700,10 @@ def _build_pillar_narrative( ) if gap_trend == "widening": s4_en = "Country disparities within this pillar have widened over time." - s4_id = "Kesenjangan antar negara dalam pilar ini semakin melebar seiring waktu." + s4_id = f"Kesenjangan antar negara dalam pilar {pillar_name_id} semakin melebar seiring waktu." elif gap_trend == "narrowing": s4_en = "Country disparities within this pillar have narrowed, indicating more balanced progress." - s4_id = "Kesenjangan antar negara dalam pilar ini menyempit, mengindikasikan kemajuan yang lebih merata." + s4_id = f"Kesenjangan antar negara dalam pilar {pillar_name_id} menyempit, mengindikasikan kemajuan yang lebih merata." else: s4_en = "" s4_id = "" @@ -669,32 +712,41 @@ def _build_pillar_narrative( sentences_en.append(s4_en) sentences_id.append(s4_id) + # Negara terbaik/terburuk: EN pakai nama Inggris, ID pakai nama Indonesia if top_country and bot_country and top_country != bot_country: + top_country_id = translate_country(top_country) + bot_country_id = translate_country(bot_country) + s5_en = ( f"{top_country} performed best in this pillar ({_fmt_score(top_country_score)}), " f"while {bot_country} had the lowest score ({_fmt_score(bot_country_score)})." ) s5_id = ( - f"{top_country} memiliki performa terbaik dalam pilar ini ({_fmt_score(top_country_score)}), " - f"sementara {bot_country} memiliki skor terendah ({_fmt_score(bot_country_score)})." + f"{top_country_id} memiliki performa terbaik dalam pilar {pillar_name_id} " + f"({_fmt_score(top_country_score)}), " + f"sementara {bot_country_id} memiliki skor terendah ({_fmt_score(bot_country_score)})." ) sentences_en.append(s5_en) sentences_id.append(s5_id) + # Perbandingan antar pilar: EN pakai nama Inggris, ID pakai nama Indonesia if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1: sorted_pillars = all_pillar_scores_year.sort_values("pillar_score_1_100", ascending=False) strongest = sorted_pillars.iloc[0] weakest = sorted_pillars.iloc[-1] if strongest["pillar_name"] != pillar_name and weakest["pillar_name"] != pillar_name: + strongest_id = translate_pillar(strongest["pillar_name"]) + weakest_id = translate_pillar(weakest["pillar_name"]) + s6_en = ( f"Across all pillars in {year}, {strongest['pillar_name']} scored highest " f"({_fmt_score(strongest['pillar_score_1_100'])}) and {weakest['pillar_name']} " f"scored lowest ({_fmt_score(weakest['pillar_score_1_100'])})." ) s6_id = ( - f"Di antara semua pilar pada tahun {year}, {strongest['pillar_name']} mendapat skor " - f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest['pillar_name']} " + f"Di antara semua pilar pada tahun {year}, {strongest_id} mendapat skor " + f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest_id} " f"mendapat skor terendah ({_fmt_score(weakest['pillar_score_1_100'])})." ) sentences_en.append(s6_en) @@ -758,7 +810,10 @@ class FoodSecurityAggregator: self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'") self.df["direction"] = self.df["direction"].fillna("positive") - # Pastikan kolom terjemahan Indonesia tersedia (bisa dari fact atau dibuat ulang) + # Pastikan kolom terjemahan Indonesia tersedia + if "country_name_id" not in self.df.columns: + self.df["country_name_id"] = self.df["country_name"].apply(translate_country) + self.logger.info(" [TRANSLATION] Kolom country_name_id dibuat dari mapping.") if "indicator_name_id" not in self.df.columns: self.df["indicator_name_id"] = self.df["indicator_name"].apply(translate_indicator) self.logger.info(" [TRANSLATION] Kolom indicator_name_id dibuat dari mapping.") @@ -766,7 +821,23 @@ class FoodSecurityAggregator: self.df["pillar_name_id"] = self.df["pillar_name"].apply(translate_pillar) self.logger.info(" [TRANSLATION] Kolom pillar_name_id dibuat dari mapping.") - self.logger.info(f" Rows : {len(self.df):,}") + # Log terjemahan negara + sample_ctr = ( + self.df[["country_name", "country_name_id"]] + .drop_duplicates() + .sort_values("country_name") + ) + self.logger.info("\n Terjemahan nama negara (EN -> ID):") + for _, r in sample_ctr.iterrows(): + self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}") + + # Log terjemahan pilar + sample_pil = self.df[["pillar_name", "pillar_name_id"]].drop_duplicates() + self.logger.info("\n Terjemahan nama pilar (EN -> ID):") + for _, r in sample_pil.iterrows(): + self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}") + + self.logger.info(f"\n Rows : {len(self.df):,}") self.logger.info(f" Countries : {self.df['country_id'].nunique()}") self.logger.info(f" Indicators: {self.df['indicator_id'].nunique()}") self.logger.info( @@ -942,7 +1013,7 @@ class FoodSecurityAggregator: ) df = add_yoy(df, ["pillar_id"], "pillar_score_1_100") - # Kolom terjemahan Indonesia + # Kolom terjemahan Indonesia — FIXED: "Access" -> "Akses" df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar) df["pillar_id"] = df["pillar_id"].astype(int) @@ -979,7 +1050,7 @@ class FoodSecurityAggregator: # ========================================================================= # STEP 3: agg_pillar_by_country - # Kolom tambahan: pillar_name_id + # Kolom tambahan: pillar_name_id, country_name_id # ========================================================================= def calc_pillar_by_country(self) -> pd.DataFrame: @@ -1007,8 +1078,9 @@ class FoodSecurityAggregator: ) df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100") - # Kolom terjemahan Indonesia - df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar) + # Kolom terjemahan Indonesia — FIXED + df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar) + df["country_name_id"] = df["country_name"].apply(translate_country) df["country_id"] = df["country_id"].astype(int) df["pillar_id"] = df["pillar_id"].astype(int) @@ -1017,10 +1089,12 @@ class FoodSecurityAggregator: df["pillar_country_norm"] = df["pillar_country_norm"].astype(float) df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float) df["pillar_name_id"] = df["pillar_name_id"].astype(str) + df["country_name_id"] = df["country_name_id"].astype(str) schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"), bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"), @@ -1043,7 +1117,7 @@ class FoodSecurityAggregator: # ========================================================================= # STEP 4: agg_framework_by_country - # Tidak ada kolom pillar/indicator di tabel ini (sudah di level framework) + # Tambah kolom: country_name_id # ========================================================================= def _calc_country_composite_inmemory(self) -> pd.DataFrame: @@ -1175,18 +1249,23 @@ class FoodSecurityAggregator: ) df = add_yoy(df, ["country_id", "framework"], "framework_score_1_100") + # Tambah kolom nama negara Indonesia + df["country_name_id"] = df["country_name"].apply(translate_country) + df["country_id"] = df["country_id"].astype(int) df["year"] = df["year"].astype(int) df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger) df["rank_in_framework_year"] = safe_int(df["rank_in_framework_year"], col_name="rank_in_framework_year", logger=self.logger) df["framework_norm"] = df["framework_norm"].astype(float) df["framework_score_1_100"] = df["framework_score_1_100"].astype(float) + df["country_name_id"] = df["country_name_id"].astype(str) self._validate_mdgs_equals_total(df, level="country") schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"), bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("framework", "STRING", mode="REQUIRED"), bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"), @@ -1208,7 +1287,6 @@ class FoodSecurityAggregator: # ========================================================================= # STEP 5: agg_framework_asean - # Tidak ada kolom pillar/indicator langsung di tabel ini # ========================================================================= def calc_framework_asean(self) -> pd.DataFrame: @@ -1371,7 +1449,7 @@ class FoodSecurityAggregator: # ========================================================================= # STEP 6: agg_narrative_overview - # Tidak ada kolom pillar/indicator di tabel ini + # FIXED: narrative_id pakai nama negara Indonesia # ========================================================================= def calc_narrative_overview( @@ -1384,6 +1462,7 @@ class FoodSecurityAggregator: self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold") self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID") + self.logger.info(" FIXED : narrative_id pakai nama negara Bahasa Indonesia") self.logger.info("=" * 70) try: @@ -1535,6 +1614,7 @@ class FoodSecurityAggregator: # ========================================================================= # STEP 7: agg_narrative_pillar # Kolom tambahan: pillar_name_id + # FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia # ========================================================================= def calc_narrative_pillar( @@ -1547,6 +1627,7 @@ class FoodSecurityAggregator: self.logger.info("\n" + "=" * 70) self.logger.info(f"STEP 7: {table_name} -> [Gold] fs_asean_gold") self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID") + self.logger.info(" FIXED : narrative_id pakai nama negara & pilar Bahasa Indonesia") self.logger.info("=" * 70) try: @@ -1576,7 +1657,7 @@ class FoodSecurityAggregator: p_yoy = prow["year_over_year_change"] p_yoy_val = float(p_yoy) if pd.notna(p_yoy) else None - # Terjemahan Indonesia nama pillar + # Terjemahan Indonesia nama pillar — FIXED p_name_id = translate_pillar(p_name) p_country = ( @@ -1626,20 +1707,24 @@ class FoodSecurityAggregator: "rank_in_year": p_rank, "yoy_change": p_yoy_val, "top_country": top_country, + "top_country_id": translate_country(top_country) if top_country else None, "top_country_score": top_country_score, "bottom_country": bot_country, + "bottom_country_id": translate_country(bot_country) if bot_country else None, "bottom_country_score": bot_country_score, "narrative_en": narrative_en, "narrative_id": narrative_id, }) df = pd.DataFrame(records) - df["year"] = df["year"].astype(int) - df["pillar_id"] = df["pillar_id"].astype(int) - df["rank_in_year"] = df["rank_in_year"].astype(int) - df["pillar_name_id"] = df["pillar_name_id"].astype(str) - df["narrative_en"] = df["narrative_en"].astype(str) - df["narrative_id"] = df["narrative_id"].astype(str) + df["year"] = df["year"].astype(int) + df["pillar_id"] = df["pillar_id"].astype(int) + df["rank_in_year"] = df["rank_in_year"].astype(int) + df["pillar_name_id"] = df["pillar_name_id"].astype(str) + df["narrative_en"] = df["narrative_en"].astype(str) + df["narrative_id"] = df["narrative_id"].astype(str) + df["top_country_id"] = df["top_country_id"].astype(str).replace("None", pd.NA).astype("string") + df["bottom_country_id"] = df["bottom_country_id"].astype(str).replace("None", pd.NA).astype("string") for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]: df[col] = pd.to_numeric(df[col], errors="coerce").astype(float) @@ -1657,8 +1742,10 @@ class FoodSecurityAggregator: bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("top_country", "STRING", mode="NULLABLE"), + bigquery.SchemaField("top_country_id", "STRING", mode="NULLABLE"), bigquery.SchemaField("top_country_score", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("bottom_country", "STRING", mode="NULLABLE"), + bigquery.SchemaField("bottom_country_id", "STRING", mode="NULLABLE"), bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"), bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"), bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"), @@ -1750,6 +1837,7 @@ class FoodSecurityAggregator: self.logger.info("FOOD SECURITY AGGREGATION — 6 TABLES -> fs_asean_gold") self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}") self.logger.info(f" Narrative style : interpretive, plain text, bilingual EN/ID") + self.logger.info(f" FIXED : 'Access' -> 'Akses', nama negara & pilar ID") self.logger.info("=" * 70) self.load_data() diff --git a/scripts/bigquery_analytical_layer.py b/scripts/bigquery_analytical_layer.py index c68df06..98955fc 100644 --- a/scripts/bigquery_analytical_layer.py +++ b/scripts/bigquery_analytical_layer.py @@ -10,7 +10,13 @@ Filtering Order: 5. Filter indicators with consistent presence across FIXED countries 6. Save analytical table (dengan nama/label lengkap untuk Looker Studio) -ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) +ADDED: +- Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) +- Kolom country_name_id (terjemahan Bahasa Indonesia nama negara) + +FIXED: +- Nama pilar "Access" -> "Akses" (konsisten di semua mapping) +- Nama negara ASEAN dalam Bahasa Indonesia """ import pandas as pd @@ -40,21 +46,39 @@ from google.cloud import bigquery # TRANSLATION DICTIONARIES # ============================================================================= +COUNTRY_NAME_ID_MAP: dict = { + # Nama resmi -> Bahasa Indonesia + "Brunei Darussalam" : "Brunei Darussalam", + "Cambodia" : "Kamboja", + "Indonesia" : "Indonesia", + "Lao People's Democratic Republic" : "Laos", + "Lao PDR" : "Laos", + "Malaysia" : "Malaysia", + "Myanmar" : "Myanmar", + "Philippines" : "Filipina", + "Singapore" : "Singapura", + "Thailand" : "Thailand", + "Timor-Leste" : "Timor-Leste", + "Viet Nam" : "Vietnam", + "Vietnam" : "Vietnam", +} + PILLAR_TRANSLATION_ID: dict = { - # 4 pilar utama Food Security - "Availability" : "Ketersediaan", - "Access" : "Keterjangkauan", - "Utilization" : "Pemanfaatan", - "Stability" : "Stabilitas", - "Sustainability": "Keberlanjutan", - # Variasi penulisan yang mungkin muncul - "availability" : "Ketersediaan", - "access" : "Keterjangkauan", - "utilization" : "Pemanfaatan", - "stability" : "Stabilitas", - "sustainability": "Keberlanjutan", + # 4 pilar utama Food Security — "Access" -> "Akses" (FIXED, bukan "Keterjangkauan") + "Availability" : "Ketersediaan", + "Access" : "Akses", + "Utilization" : "Pemanfaatan", + "Stability" : "Stabilitas", + "Sustainability" : "Keberlanjutan", + # Variasi penulisan huruf kecil + "availability" : "Ketersediaan", + "access" : "Akses", + "utilization" : "Pemanfaatan", + "stability" : "Stabilitas", + "sustainability" : "Keberlanjutan", + # Variasi dengan prefix "Food" "Food Availability" : "Ketersediaan Pangan", - "Food Access" : "Keterjangkauan Pangan", + "Food Access" : "Akses Pangan", "Food Utilization" : "Pemanfaatan Pangan", "Food Stability" : "Stabilitas Pangan", "Food Sustainability": "Keberlanjutan Pangan", @@ -195,6 +219,13 @@ INDICATOR_TRANSLATION_ID: dict = { } +def translate_country(name: str) -> str: + """Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli.""" + if not name: + return name + return COUNTRY_NAME_ID_MAP.get(name.strip(), name) + + def translate_indicator(name: str) -> str: """Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli.""" if not name: @@ -226,6 +257,7 @@ class AnalyticalLayerLoader: Output: fact_asean_food_security_selected -> DW layer (Gold) -> fs_asean_gold Kolom tambahan: + - country_name_id : terjemahan Bahasa Indonesia dari country_name - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name - pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name """ @@ -625,12 +657,24 @@ class AnalyticalLayerLoader: # ------------------------------------------------------------------ # TAMBAHAN: kolom terjemahan Bahasa Indonesia + # country_name_id : terjemahan Bahasa Indonesia dari country_name [BARU] # indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name # pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name # ------------------------------------------------------------------ + analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country) analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator) analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar) + # Log negara yang belum punya terjemahan + no_trans_ctr = analytical_df[ + analytical_df['country_name_id'] == analytical_df['country_name'] + ]['country_name'].unique() + if len(no_trans_ctr) > 0: + self.logger.warning( + f" [TRANSLATION] {len(no_trans_ctr)} country/countries tidak ada di kamus " + f"(menggunakan nama asli): {list(no_trans_ctr)}" + ) + # Log indikator yang belum punya terjemahan (fallback ke nama asli) no_trans_ind = analytical_df[ analytical_df['indicator_name_id'] == analytical_df['indicator_name'] @@ -657,6 +701,7 @@ class AnalyticalLayerLoader: # Pastikan tipe data konsisten analytical_df['country_id'] = analytical_df['country_id'].astype(int) analytical_df['country_name'] = analytical_df['country_name'].astype(str) + analytical_df['country_name_id'] = analytical_df['country_name_id'].astype(str) analytical_df['indicator_id'] = analytical_df['indicator_id'].astype(int) analytical_df['indicator_name'] = analytical_df['indicator_name'].astype(str) analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].astype(str) @@ -671,10 +716,21 @@ class AnalyticalLayerLoader: self.logger.info(f" Kolom yang disimpan: {list(analytical_df.columns)}") self.logger.info(f" Total rows: {len(analytical_df):,}") + # Log sample terjemahan negara + sample_ctr = ( + analytical_df[['country_name', 'country_name_id']] + .drop_duplicates() + .sort_values('country_name') + ) + self.logger.info("\n Terjemahan nama negara (EN -> ID):") + for _, r in sample_ctr.iterrows(): + self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}") + # Schema BigQuery schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"), bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("indicator_name_id", "STRING", mode="REQUIRED"), @@ -710,7 +766,8 @@ class AnalyticalLayerLoader: 'fixed_countries': len(self.selected_country_ids), 'no_gaps' : True, 'layer' : 'gold', - 'columns' : 'id + name + name_id (Looker Studio ready)' + 'columns' : 'id + name + name_id (Looker Studio ready)', + 'added_columns' : ['country_name_id', 'indicator_name_id', 'pillar_name_id'], }), 'validation_metrics' : json.dumps({ 'fixed_countries' : len(self.selected_country_ids),