izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

country name indo version

Browse Source
This commit is contained in:
Debby
2026-06-07 08:09:14 +07:00
parent 512c08bff3
commit ca1e0d3949
3 changed files with 344 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bigquery_aggraget_fact_selected_layer.py
+129 -48
View File
@@ -5,9 +5,12 @@ Tabel 2: agg_narrative_indicator -> fs_asean_gold
=============================================================================
PERUBAHAN:
- Ditambahkan kolom country_name_id : nama negara dalam Bahasa Indonesia [BARU]
- Ditambahkan kolom indicator_name_id : nama indikator dalam Bahasa Indonesia
- Ditambahkan kolom pillar_name_id : nama pilar dalam Bahasa Indonesia
- Kedua kolom ikut tersimpan di BigQuery (schema + DataFrame output)
- Ketiga kolom ikut tersimpan di BigQuery (schema + DataFrame output)
- Narrative versi Indonesia menggunakan nama negara & pilar dalam Bahasa Indonesia
- FIXED: "Access" -> "Akses" (konsisten di semua mapping pilar)
=============================================================================
agg_indicator_norm
@@ -35,7 +38,7 @@ Performance Label Logic:
- performance : "Good" jika norm_score_1_100 >= 60, "Bad" jika < 60, null jika null
Output Schema (agg_indicator_norm):
year, country_id, country_name,
year, country_id, country_name, country_name_id,
indicator_id, indicator_name, indicator_name_id,
unit, direction,
pillar_id, pillar_name, pillar_name_id,
@@ -54,6 +57,8 @@ Tujuan:
Menghasilkan narasi otomatis per indikator (granularity: indicator_id).
Narasi membaca kondisi nyata dari data: tren, gap, anomali, konsistensi.
Tersedia dalam dua bahasa: Inggris (narrative_en) dan Indonesia (narrative_id).
- narrative_en : menggunakan nama negara & pilar dalam Bahasa Inggris
- narrative_id : menggunakan nama negara & pilar dalam Bahasa Indonesia
Tanpa markdown bold (**) agar aman ditampilkan di Looker Studio.
Granularity:
@@ -71,6 +76,7 @@ Output Schema (agg_narrative_indicator):
n_yoy_total, n_yoy_positive,
best_yoy_from, best_yoy_to,
country_worst, country_best,
country_worst_id, country_best_id,
narrative_en,
narrative_id
"""
@@ -96,7 +102,25 @@ from google.cloud import bigquery
# MAPPING BAHASA INDONESIA
# =============================================================================
# Mapping nama negara (Inggris -> Indonesia)
COUNTRY_NAME_ID_MAP: dict = {
"Brunei Darussalam" : "Brunei Darussalam",
"Cambodia" : "Kamboja",
"Indonesia" : "Indonesia",
"Lao People's Democratic Republic" : "Laos",
"Lao PDR" : "Laos",
"Malaysia" : "Malaysia",
"Myanmar" : "Myanmar",
"Philippines" : "Filipina",
"Singapore" : "Singapura",
"Thailand" : "Thailand",
"Timor-Leste" : "Timor-Leste",
"Viet Nam" : "Vietnam",
"Vietnam" : "Vietnam",
}
# Mapping nama pilar (Inggris -> Indonesia)
# FIXED: "Access" -> "Akses" (bukan "Keterjangkauan")
PILLAR_NAME_ID_MAP: dict = {
"Availability" : "Ketersediaan",
"Access" : "Akses",
@@ -189,8 +213,6 @@ INDICATOR_NAME_ID_MAP: dict = {
"Stabilitas politik dan ketiadaan kekerasan/terorisme",
"domestic food price volatility index":
"Indeks volatilitas harga pangan domestik",
"per capita food supply variability (kcal/capita/day)":
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
"cereal import dependency ratio (percent) (3-year average)":
"Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
"value of food imports in total merchandise exports (percent) (3-year average)":
@@ -200,11 +222,19 @@ INDICATOR_NAME_ID_MAP: dict = {
}
def get_country_name_id(country_name: str) -> str:
"""Kembalikan terjemahan Bahasa Indonesia untuk nama negara."""
return COUNTRY_NAME_ID_MAP.get(
str(country_name).strip(),
str(country_name), # fallback: kembalikan nama asli
)
def get_indicator_name_id(indicator_name: str) -> str:
"""Kembalikan terjemahan Bahasa Indonesia untuk nama indikator."""
return INDICATOR_NAME_ID_MAP.get(
str(indicator_name).lower().strip(),
str(indicator_name), # fallback: kembalikan nama asli jika tidak ada mapping
str(indicator_name), # fallback: kembalikan nama asli
)
@@ -212,7 +242,7 @@ def get_pillar_name_id(pillar_name: str) -> str:
"""Kembalikan terjemahan Bahasa Indonesia untuk nama pilar."""
return PILLAR_NAME_ID_MAP.get(
str(pillar_name).strip(),
str(pillar_name), # fallback: kembalikan nama asli jika tidak ada mapping
str(pillar_name), # fallback: kembalikan nama asli
)
@@ -406,6 +436,11 @@ def _detect_anomaly_year(scores_by_year: pd.Series) -> tuple:
def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple:
"""
Mengembalikan (best_country_en, worst_country_en, is_consistent).
Nama negara dikembalikan dalam Bahasa Inggris; penerjemahan dilakukan
di layer narrative builder.
"""
country_avg = (
df_ind.groupby("country_name")["value"]
.mean()
@@ -446,34 +481,42 @@ def _detect_consistency(df_ind: pd.DataFrame, lower_better: bool) -> tuple:
# =============================================================================
# NARRATIVE BUILDER — plain text, no markdown, bilingual
# FIXED: narrative_id menggunakan nama negara & pilar dalam Bahasa Indonesia
# =============================================================================
def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tuple:
ind_id = int(row["indicator_id"])
ind_name = str(row["indicator_name"]).strip()
unit = str(row["unit"]).strip() if row["unit"] else ""
direction = str(row["direction"]).strip()
pillar = str(row["pillar_name"]).strip()
framework = str(row["framework"]).strip()
year_min = int(row["year_min"])
year_max = int(row["year_max"])
ind_id = int(row["indicator_id"])
ind_name_en = str(row["indicator_name"]).strip()
ind_name_id = str(row.get("indicator_name_id", ind_name_en)).strip()
unit = str(row["unit"]).strip() if row["unit"] else ""
direction = str(row["direction"]).strip()
pillar_en = str(row["pillar_name"]).strip()
pillar_id_ = get_pillar_name_id(pillar_en) # nama pilar dalam Bahasa Indonesia
framework = str(row["framework"]).strip()
year_min = int(row["year_min"])
year_max = int(row["year_max"])
lower_better = _is_lower_better(direction)
df_ind = df_full[df_full["indicator_id"] == ind_id].copy()
if df_ind.empty:
na_en = f"{ind_name} ({framework}, {pillar}): Insufficient data for analysis."
na_id = f"{ind_name} ({framework}, {pillar}): Data tidak cukup untuk dianalisis."
na_en = f"{ind_name_en} ({framework}, {pillar_en}): Insufficient data for analysis."
na_id = f"{ind_name_id} ({framework}, {pillar_id_}): Data tidak cukup untuk dianalisis."
return na_en, na_id
asean_avg_by_year = (
df_ind.groupby("year")["value"].mean().dropna()
)
trend_label = _detect_trend(asean_avg_by_year, lower_better)
gap_label = _detect_gap_trend(df_ind, lower_better)
trend_label = _detect_trend(asean_avg_by_year, lower_better)
gap_label = _detect_gap_trend(df_ind, lower_better)
anomaly_year, anomaly_dir = _detect_anomaly_year(asean_avg_by_year)
best_country, worst_country, is_consistent = _detect_consistency(df_ind, lower_better)
# best_country & worst_country -> nama dalam Bahasa Inggris (dari data)
best_country_en, worst_country_en, is_consistent = _detect_consistency(df_ind, lower_better)
# Terjemahan nama negara ke Bahasa Indonesia
best_country_id = get_country_name_id(best_country_en) if best_country_en else None
worst_country_id = get_country_name_id(worst_country_en) if worst_country_en else None
avg_first = row.get("avg_value_first", np.nan)
avg_last = row.get("avg_value_last", np.nan)
@@ -488,8 +531,9 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
sentences_en = []
sentences_id = []
s1_en = f"{ind_name} ({framework}, {pillar}, {year_min}-{year_max}):"
s1_id = f"{ind_name} ({framework}, {pillar}, {year_min}-{year_max}):"
# Header: EN menggunakan nama Inggris, ID menggunakan nama Indonesia
s1_en = f"{ind_name_en} ({framework}, {pillar_en}, {year_min}-{year_max}):"
s1_id = f"{ind_name_id} ({framework}, {pillar_id_}, {year_min}-{year_max}):"
sentences_en.append(s1_en)
sentences_id.append(s1_id)
@@ -528,24 +572,25 @@ def _build_narrative_per_indicator(row: pd.Series, df_full: pd.DataFrame) -> tup
sentences_en.append(f"A sharp improvement was observed in {anomaly_year}, standing out from the overall pattern.")
sentences_id.append(f"Peningkatan tajam tercatat pada tahun {anomaly_year}, yang menyimpang dari pola keseluruhan.")
if best_country and worst_country:
# Kalimat tentang negara: EN pakai nama Inggris, ID pakai nama Indonesia
if best_country_en and worst_country_en:
if is_consistent:
sentences_en.append(
f"{best_country} consistently performed above the regional average, "
f"while {worst_country} consistently lagged behind."
f"{best_country_en} consistently performed above the regional average, "
f"while {worst_country_en} consistently lagged behind."
)
sentences_id.append(
f"{best_country} secara konsisten berada di atas rata-rata regional, "
f"sementara {worst_country} secara konsisten tertinggal."
f"{best_country_id} secara konsisten berada di atas rata-rata regional, "
f"sementara {worst_country_id} secara konsisten tertinggal."
)
else:
sentences_en.append(
f"Overall, {best_country} showed the best performance, "
f"while {worst_country} had the weakest results across the period."
f"Overall, {best_country_en} showed the best performance, "
f"while {worst_country_en} had the weakest results across the period."
)
sentences_id.append(
f"Secara keseluruhan, {best_country} menunjukkan performa terbaik, "
f"sementara {worst_country} memiliki hasil terlemah sepanjang periode."
f"Secara keseluruhan, {best_country_id} menunjukkan performa terbaik, "
f"sementara {worst_country_id} memiliki hasil terlemah sepanjang periode."
)
narrative_en = " ".join(s for s in sentences_en if s)
@@ -689,23 +734,54 @@ class IndicatorNormAggregator:
self.logger.info("STEP 3b: ADD BAHASA INDONESIA NAME COLUMNS")
self.logger.info("=" * 80)
# Nama negara
self.df["country_name_id"] = (
self.df["country_name"]
.apply(get_country_name_id)
.astype(str)
)
# Nama indikator
self.df["indicator_name_id"] = (
self.df["indicator_name"]
.apply(get_indicator_name_id)
.astype(str)
)
# Nama pilar
self.df["pillar_name_id"] = (
self.df["pillar_name"]
.apply(get_pillar_name_id)
.astype(str)
)
n_country_mapped = (self.df["country_name_id"] != self.df["country_name"]).sum()
n_indicator_mapped = (self.df["indicator_name_id"] != self.df["indicator_name"]).sum()
n_pillar_mapped = (self.df["pillar_name_id"] != self.df["pillar_name"]).sum()
self.logger.info(f" country_name_id mapped rows : {n_country_mapped:,}")
self.logger.info(f" indicator_name_id mapped rows : {n_indicator_mapped:,}")
self.logger.info(f" pillar_name_id mapped rows : {n_pillar_mapped:,}")
# Log sample mapping
# Log sample negara
sample_ctr = (
self.df[["country_name", "country_name_id"]]
.drop_duplicates()
.sort_values("country_name")
)
self.logger.info("\n Terjemahan nama negara (EN -> ID):")
for _, r in sample_ctr.iterrows():
self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}")
# Log sample pilar
sample_pil = (
self.df[["pillar_name", "pillar_name_id"]]
.drop_duplicates()
)
self.logger.info("\n Pillar mapping (EN -> ID):")
for _, r in sample_pil.iterrows():
self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}")
# Log sample indikator
sample_ind = (
self.df[["indicator_name", "indicator_name_id"]]
.drop_duplicates()
@@ -716,14 +792,6 @@ class IndicatorNormAggregator:
self.logger.info(f" EN: {r['indicator_name'][:55]}")
self.logger.info(f" ID: {r['indicator_name_id'][:55]}")
sample_pil = (
self.df[["pillar_name", "pillar_name_id"]]
.drop_duplicates()
)
self.logger.info("\n Pillar mapping (EN -> ID):")
for _, r in sample_pil.iterrows():
self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}")
# =========================================================================
# STEP 4: Deteksi sdgs_start_year
# =========================================================================
@@ -925,7 +993,7 @@ class IndicatorNormAggregator:
self.logger.info("=" * 80)
out = df[[
"year", "country_id", "country_name",
"year", "country_id", "country_name", "country_name_id",
"indicator_id", "indicator_name", "indicator_name_id",
"unit", "direction",
"pillar_id", "pillar_name", "pillar_name_id",
@@ -941,6 +1009,7 @@ class IndicatorNormAggregator:
out["year"] = out["year"].astype(int)
out["country_id"] = out["country_id"].astype(int)
out["country_name"] = out["country_name"].astype(str)
out["country_name_id"] = out["country_name_id"].astype(str)
out["indicator_id"] = out["indicator_id"].astype(int)
out["indicator_name"] = out["indicator_name"].astype(str)
out["indicator_name_id"] = out["indicator_name_id"].astype(str)
@@ -966,6 +1035,7 @@ class IndicatorNormAggregator:
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("indicator_name_id", "STRING", mode="NULLABLE"),
@@ -1009,7 +1079,7 @@ class IndicatorNormAggregator:
"yoy_columns" : ["yoy_value", "yoy_norm_value"],
"performance_threshold": _PERFORMANCE_THRESHOLD,
"unit_source" : "dim_indicator",
"added_columns" : ["indicator_name_id", "pillar_name_id"],
"added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id"],
}),
"validation_metrics" : json.dumps({
"total_rows" : rows_loaded,
@@ -1062,6 +1132,7 @@ class IndicatorNormAggregator:
self.logger.info("STEP 12-17: agg_narrative_indicator")
self.logger.info(" Granularity: per indicator_id (all years + all ASEAN countries)")
self.logger.info(" Narrative : interpretatif, plain text, bilingual EN/ID")
self.logger.info(" FIXED : narrative_id pakai nama negara & pilar Bahasa Indonesia")
self.logger.info("=" * 80)
df = df_final.copy()
@@ -1150,7 +1221,7 @@ class IndicatorNormAggregator:
})
df_yoy_stats = pd.DataFrame(yoy_stats)
# Country best/worst
# Country best/worst (nama asli Bahasa Inggris)
df_country_avg = (
df.groupby(["indicator_id", "country_id", "country_name"])
.agg(country_avg_value=("value", "mean"))
@@ -1166,9 +1237,12 @@ class IndicatorNormAggregator:
worst_row = grp.loc[grp["country_avg_value"].idxmin()]
best_row = grp.loc[grp["country_avg_value"].idxmax()]
country_stats.append({
"indicator_id" : ind_id,
"country_worst": worst_row["country_name"],
"country_best" : best_row["country_name"],
"indicator_id" : ind_id,
"country_worst" : worst_row["country_name"], # nama Inggris
"country_best" : best_row["country_name"], # nama Inggris
# Tambahan: nama Indonesia untuk kedua negara
"country_worst_id": get_country_name_id(worst_row["country_name"]),
"country_best_id" : get_country_name_id(best_row["country_name"]),
})
df_country_stats = pd.DataFrame(country_stats)
@@ -1229,6 +1303,7 @@ class IndicatorNormAggregator:
"n_yoy_total", "n_yoy_positive",
"best_yoy_from", "best_yoy_to",
"country_worst", "country_best",
"country_worst_id", "country_best_id",
"narrative_en", "narrative_id",
]].copy()
@@ -1255,6 +1330,8 @@ class IndicatorNormAggregator:
out["best_yoy_to"] = pd.to_numeric(out["best_yoy_to"], errors="coerce").astype("Int64")
out["country_worst"] = out["country_worst"].astype(str).replace("nan", pd.NA).astype("string")
out["country_best"] = out["country_best"].astype(str).replace("nan", pd.NA).astype("string")
out["country_worst_id"] = out["country_worst_id"].astype(str).replace("nan", pd.NA).astype("string")
out["country_best_id"] = out["country_best_id"].astype(str).replace("nan", pd.NA).astype("string")
out["narrative_en"] = out["narrative_en"].astype(str)
out["narrative_id"] = out["narrative_id"].astype(str)
@@ -1280,6 +1357,8 @@ class IndicatorNormAggregator:
bigquery.SchemaField("best_yoy_to", "INTEGER", mode="NULLABLE"),
bigquery.SchemaField("country_worst", "STRING", mode="NULLABLE"),
bigquery.SchemaField("country_best", "STRING", mode="NULLABLE"),
bigquery.SchemaField("country_worst_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("country_best_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("narrative_en", "STRING", mode="NULLABLE"),
bigquery.SchemaField("narrative_id", "STRING", mode="NULLABLE"),
]
@@ -1310,7 +1389,8 @@ class IndicatorNormAggregator:
"narrative_dimensions" : ["trend", "gap_trend", "anomaly", "country_consistency"],
"performance_threshold": _PERFORMANCE_THRESHOLD,
"layer" : "gold",
"added_columns" : ["indicator_name_id", "pillar_name_id"],
"added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id",
"country_worst_id", "country_best_id"],
}),
"validation_metrics" : json.dumps({
"total_rows" : rows_loaded,
@@ -1334,13 +1414,14 @@ class IndicatorNormAggregator:
self.logger.info(" Dim : dim_indicator (unit)")
self.logger.info(" Output : agg_indicator_norm -> fs_asean_gold")
self.logger.info(" agg_narrative_indicator -> fs_asean_gold")
self.logger.info(" Added : indicator_name_id, pillar_name_id (Bahasa Indonesia)")
self.logger.info(" Added : country_name_id, indicator_name_id, pillar_name_id (Bahasa Indonesia)")
self.logger.info(" FIXED : 'Access' -> 'Akses', narrative_id pakai nama ID")
self.logger.info("=" * 80)
self.load_data()
self.load_units()
self._merge_unit()
self._add_indonesia_name_columns() # <-- BARU
self._add_indonesia_name_columns()
self.sdgs_start_year = self._detect_sdgs_start_year()
self._assign_framework()
df_normed = self._compute_norm_values()
scripts/bigquery_aggregate_layer.py
+143 -55
View File
@@ -13,13 +13,16 @@ Narrative style:
- Plain text, tanpa markdown bold (**)
- Interpretatif: membaca tren, gap, anomali, konsistensi dari data nyata
- Bilingual: narrative_en (Inggris) + narrative_id (Indonesia)
* narrative_en : nama negara & pilar dalam Bahasa Inggris
* narrative_id : nama negara & pilar dalam Bahasa Indonesia [FIXED]
- Granularity: per tahun (Overview & Pillar)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
- agg_pillar_composite : + pillar_name_id
- agg_pillar_by_country : + pillar_name_id
- agg_framework_by_country : (framework tidak diterjemahkan, sudah singkat)
- agg_narrative_pillar : + pillar_name_id
FIXED & ADDED:
- "Access" -> "Akses" di semua mapping pilar (bukan "Keterjangkauan")
- Tambah COUNTRY_NAME_ID_MAP untuk terjemahan nama 10 negara ASEAN
- narrative_id menggunakan nama negara & pilar dalam Bahasa Indonesia
- Tambah kolom country_name_id di tabel yang menyimpan country_name
- Kolom pillar_name_id di semua tabel pilar
"""
import pandas as pd
@@ -92,23 +95,40 @@ _FIES_DETECTION_LOWER: frozenset = frozenset([
# TRANSLATION DICTIONARIES
# =============================================================================
# Nama negara ASEAN -> Bahasa Indonesia [BARU]
COUNTRY_NAME_ID_MAP: dict = {
"Brunei Darussalam" : "Brunei Darussalam",
"Cambodia" : "Kamboja",
"Indonesia" : "Indonesia",
"Lao People's Democratic Republic" : "Laos",
"Lao PDR" : "Laos",
"Malaysia" : "Malaysia",
"Myanmar" : "Myanmar",
"Philippines" : "Filipina",
"Singapore" : "Singapura",
"Thailand" : "Thailand",
"Timor-Leste" : "Timor-Leste",
"Viet Nam" : "Vietnam",
"Vietnam" : "Vietnam",
}
# Nama pilar -> Bahasa Indonesia
# FIXED: "Access" -> "Akses" (bukan "Keterjangkauan")
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability" : "Keberlanjutan",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability" : "Keberlanjutan",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Availability" : "Ketersediaan",
"Access" : "Akses",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability" : "Keberlanjutan",
"availability" : "Ketersediaan",
"access" : "Akses",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability" : "Keberlanjutan",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Akses Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Food Sustainability": "Keberlanjutan Pangan",
}
@@ -247,6 +267,13 @@ INDICATOR_TRANSLATION_ID: dict = {
}
def translate_country(name: str) -> str:
"""Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return COUNTRY_NAME_ID_MAP.get(name.strip(), name)
def translate_indicator(name: str) -> str:
"""Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
@@ -437,6 +464,7 @@ def _find_anomaly_year(values_by_year: dict) -> tuple:
# =============================================================================
# NARRATIVE BUILDER — OVERVIEW (per tahun)
# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia
# =============================================================================
def _build_overview_narrative(
@@ -533,23 +561,34 @@ def _build_overview_narrative(
sentences_en.append(s4_en)
sentences_id.append(s4_id)
# Ranking: EN pakai nama Inggris, ID pakai nama Indonesia
if ranking_list and len(ranking_list) >= 2:
top = ranking_list[0]
bottom = ranking_list[-1]
s5_en = (
f"In {year}, {top['country_name']} led the region with a score of "
f"{_fmt_score(top['score'])}, while {bottom['country_name']} ranked last "
top_name_en = top['country_name']
top_name_id = translate_country(top_name_en)
bottom_name_en = bottom['country_name']
bottom_name_id = translate_country(bottom_name_en)
s5_en = (
f"In {year}, {top_name_en} led the region with a score of "
f"{_fmt_score(top['score'])}, while {bottom_name_en} ranked last "
f"at {_fmt_score(bottom['score'])}."
)
s5_id = (
f"Pada tahun {year}, {top['country_name']} memimpin kawasan dengan skor "
f"{_fmt_score(top['score'])}, sementara {bottom['country_name']} berada di "
f"Pada tahun {year}, {top_name_id} memimpin kawasan dengan skor "
f"{_fmt_score(top['score'])}, sementara {bottom_name_id} berada di "
f"posisi terbawah dengan skor {_fmt_score(bottom['score'])}."
)
sentences_en.append(s5_en)
sentences_id.append(s5_id)
# Peningkatan/penurunan: EN pakai nama Inggris, ID pakai nama Indonesia
if most_improved_country and most_declined_country:
improved_name_id = translate_country(most_improved_country)
declined_name_id = translate_country(most_declined_country)
if most_improved_country != most_declined_country:
s6_en = (
f"{most_improved_country} showed the biggest improvement "
@@ -558,9 +597,9 @@ def _build_overview_narrative(
f"({_fmt_delta(most_declined_delta)} pts)."
)
s6_id = (
f"{most_improved_country} mencatat peningkatan terbesar "
f"{improved_name_id} mencatat peningkatan terbesar "
f"({_fmt_delta(most_improved_delta)} poin), "
f"sementara {most_declined_country} mengalami penurunan terbesar "
f"sementara {declined_name_id} mengalami penurunan terbesar "
f"({_fmt_delta(most_declined_delta)} poin)."
)
sentences_en.append(s6_en)
@@ -573,6 +612,7 @@ def _build_overview_narrative(
# =============================================================================
# NARRATIVE BUILDER — PILLAR (per tahun per pilar)
# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia
# =============================================================================
def _build_pillar_narrative(
@@ -593,6 +633,9 @@ def _build_pillar_narrative(
sentences_en = []
sentences_id = []
# Terjemahan nama pilar
pillar_name_id = translate_pillar(pillar_name)
rank_suffix = {1: "st", 2: "nd", 3: "rd"}.get(rank_in_year, "th")
perf_word_en = "good" if pillar_score >= PERFORMANCE_THRESHOLD else "below target"
perf_word_id = "baik" if pillar_score >= PERFORMANCE_THRESHOLD else "di bawah target"
@@ -602,7 +645,7 @@ def _build_pillar_narrative(
f"{n_pillars} pillars with a score of {_fmt_score(pillar_score)} ({perf_word_en})."
)
s1_id = (
f"Pada tahun {year}, pilar {pillar_name} menempati peringkat {rank_in_year} dari "
f"Pada tahun {year}, pilar {pillar_name_id} menempati peringkat {rank_in_year} dari "
f"{n_pillars} pilar dengan skor {_fmt_score(pillar_score)} ({perf_word_id})."
)
sentences_en.append(s1_en)
@@ -632,16 +675,16 @@ def _build_pillar_narrative(
trend = _detect_series_trend(hist_scores)
if trend == "improving_consistent":
s3_en = f"This pillar has shown consistent improvement since {hist_years[0]}."
s3_id = f"Pilar ini menunjukkan perbaikan yang konsisten sejak {hist_years[0]}."
s3_id = f"Pilar {pillar_name_id} menunjukkan perbaikan yang konsisten sejak {hist_years[0]}."
elif trend == "improving_slowing":
s3_en = f"While the pillar improved since {hist_years[0]}, the pace has slowed in recent years."
s3_id = f"Meskipun pilar ini membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir."
s3_id = f"Meskipun pilar {pillar_name_id} membaik sejak {hist_years[0]}, lajunya melambat dalam beberapa tahun terakhir."
elif trend == "deteriorating":
s3_en = f"This pillar has shown a declining trend since {hist_years[0]}, requiring targeted intervention."
s3_id = f"Pilar ini menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah."
s3_id = f"Pilar {pillar_name_id} menunjukkan tren penurunan sejak {hist_years[0]}, memerlukan intervensi yang terarah."
elif trend == "fluctuating":
s3_en = f"Performance in this pillar has been inconsistent since {hist_years[0]}, with no clear trend."
s3_id = f"Performa pilar ini tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas."
s3_id = f"Performa pilar {pillar_name_id} tidak konsisten sejak {hist_years[0]}, tanpa tren yang jelas."
else:
s3_en = ""
s3_id = ""
@@ -657,10 +700,10 @@ def _build_pillar_narrative(
)
if gap_trend == "widening":
s4_en = "Country disparities within this pillar have widened over time."
s4_id = "Kesenjangan antar negara dalam pilar ini semakin melebar seiring waktu."
s4_id = f"Kesenjangan antar negara dalam pilar {pillar_name_id} semakin melebar seiring waktu."
elif gap_trend == "narrowing":
s4_en = "Country disparities within this pillar have narrowed, indicating more balanced progress."
s4_id = "Kesenjangan antar negara dalam pilar ini menyempit, mengindikasikan kemajuan yang lebih merata."
s4_id = f"Kesenjangan antar negara dalam pilar {pillar_name_id} menyempit, mengindikasikan kemajuan yang lebih merata."
else:
s4_en = ""
s4_id = ""
@@ -669,32 +712,41 @@ def _build_pillar_narrative(
sentences_en.append(s4_en)
sentences_id.append(s4_id)
# Negara terbaik/terburuk: EN pakai nama Inggris, ID pakai nama Indonesia
if top_country and bot_country and top_country != bot_country:
top_country_id = translate_country(top_country)
bot_country_id = translate_country(bot_country)
s5_en = (
f"{top_country} performed best in this pillar ({_fmt_score(top_country_score)}), "
f"while {bot_country} had the lowest score ({_fmt_score(bot_country_score)})."
)
s5_id = (
f"{top_country} memiliki performa terbaik dalam pilar ini ({_fmt_score(top_country_score)}), "
f"sementara {bot_country} memiliki skor terendah ({_fmt_score(bot_country_score)})."
f"{top_country_id} memiliki performa terbaik dalam pilar {pillar_name_id} "
f"({_fmt_score(top_country_score)}), "
f"sementara {bot_country_id} memiliki skor terendah ({_fmt_score(bot_country_score)})."
)
sentences_en.append(s5_en)
sentences_id.append(s5_id)
# Perbandingan antar pilar: EN pakai nama Inggris, ID pakai nama Indonesia
if not all_pillar_scores_year.empty and len(all_pillar_scores_year) > 1:
sorted_pillars = all_pillar_scores_year.sort_values("pillar_score_1_100", ascending=False)
strongest = sorted_pillars.iloc[0]
weakest = sorted_pillars.iloc[-1]
if strongest["pillar_name"] != pillar_name and weakest["pillar_name"] != pillar_name:
strongest_id = translate_pillar(strongest["pillar_name"])
weakest_id = translate_pillar(weakest["pillar_name"])
s6_en = (
f"Across all pillars in {year}, {strongest['pillar_name']} scored highest "
f"({_fmt_score(strongest['pillar_score_1_100'])}) and {weakest['pillar_name']} "
f"scored lowest ({_fmt_score(weakest['pillar_score_1_100'])})."
)
s6_id = (
f"Di antara semua pilar pada tahun {year}, {strongest['pillar_name']} mendapat skor "
f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest['pillar_name']} "
f"Di antara semua pilar pada tahun {year}, {strongest_id} mendapat skor "
f"tertinggi ({_fmt_score(strongest['pillar_score_1_100'])}) dan {weakest_id} "
f"mendapat skor terendah ({_fmt_score(weakest['pillar_score_1_100'])})."
)
sentences_en.append(s6_en)
@@ -758,7 +810,10 @@ class FoodSecurityAggregator:
self.logger.warning(f" [DIRECTION] {n_null_dir} rows NULL -> diisi 'positive'")
self.df["direction"] = self.df["direction"].fillna("positive")
# Pastikan kolom terjemahan Indonesia tersedia (bisa dari fact atau dibuat ulang)
# Pastikan kolom terjemahan Indonesia tersedia
if "country_name_id" not in self.df.columns:
self.df["country_name_id"] = self.df["country_name"].apply(translate_country)
self.logger.info(" [TRANSLATION] Kolom country_name_id dibuat dari mapping.")
if "indicator_name_id" not in self.df.columns:
self.df["indicator_name_id"] = self.df["indicator_name"].apply(translate_indicator)
self.logger.info(" [TRANSLATION] Kolom indicator_name_id dibuat dari mapping.")
@@ -766,7 +821,23 @@ class FoodSecurityAggregator:
self.df["pillar_name_id"] = self.df["pillar_name"].apply(translate_pillar)
self.logger.info(" [TRANSLATION] Kolom pillar_name_id dibuat dari mapping.")
self.logger.info(f" Rows : {len(self.df):,}")
# Log terjemahan negara
sample_ctr = (
self.df[["country_name", "country_name_id"]]
.drop_duplicates()
.sort_values("country_name")
)
self.logger.info("\n Terjemahan nama negara (EN -> ID):")
for _, r in sample_ctr.iterrows():
self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}")
# Log terjemahan pilar
sample_pil = self.df[["pillar_name", "pillar_name_id"]].drop_duplicates()
self.logger.info("\n Terjemahan nama pilar (EN -> ID):")
for _, r in sample_pil.iterrows():
self.logger.info(f" {r['pillar_name']:<20} -> {r['pillar_name_id']}")
self.logger.info(f"\n Rows : {len(self.df):,}")
self.logger.info(f" Countries : {self.df['country_id'].nunique()}")
self.logger.info(f" Indicators: {self.df['indicator_id'].nunique()}")
self.logger.info(
@@ -942,7 +1013,7 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["pillar_id"], "pillar_score_1_100")
# Kolom terjemahan Indonesia
# Kolom terjemahan Indonesia — FIXED: "Access" -> "Akses"
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["pillar_id"] = df["pillar_id"].astype(int)
@@ -979,7 +1050,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 3: agg_pillar_by_country
# Kolom tambahan: pillar_name_id
# Kolom tambahan: pillar_name_id, country_name_id
# =========================================================================
def calc_pillar_by_country(self) -> pd.DataFrame:
@@ -1007,8 +1078,9 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["country_id", "pillar_id"], "pillar_country_score_1_100")
# Kolom terjemahan Indonesia
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
# Kolom terjemahan Indonesia — FIXED
df["pillar_name_id"] = df["pillar_name"].apply(translate_pillar)
df["country_name_id"] = df["country_name"].apply(translate_country)
df["country_id"] = df["country_id"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
@@ -1017,10 +1089,12 @@ class FoodSecurityAggregator:
df["pillar_country_norm"] = df["pillar_country_norm"].astype(float)
df["pillar_country_score_1_100"] = df["pillar_country_score_1_100"].astype(float)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["country_name_id"] = df["country_name_id"].astype(str)
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pillar_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("pillar_name_id", "STRING", mode="REQUIRED"),
@@ -1043,7 +1117,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 4: agg_framework_by_country
# Tidak ada kolom pillar/indicator di tabel ini (sudah di level framework)
# Tambah kolom: country_name_id
# =========================================================================
def _calc_country_composite_inmemory(self) -> pd.DataFrame:
@@ -1175,18 +1249,23 @@ class FoodSecurityAggregator:
)
df = add_yoy(df, ["country_id", "framework"], "framework_score_1_100")
# Tambah kolom nama negara Indonesia
df["country_name_id"] = df["country_name"].apply(translate_country)
df["country_id"] = df["country_id"].astype(int)
df["year"] = df["year"].astype(int)
df["n_indicators"] = safe_int(df["n_indicators"], col_name="n_indicators", logger=self.logger)
df["rank_in_framework_year"] = safe_int(df["rank_in_framework_year"], col_name="rank_in_framework_year", logger=self.logger)
df["framework_norm"] = df["framework_norm"].astype(float)
df["framework_score_1_100"] = df["framework_score_1_100"].astype(float)
df["country_name_id"] = df["country_name_id"].astype(str)
self._validate_mdgs_equals_total(df, level="country")
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("framework", "STRING", mode="REQUIRED"),
bigquery.SchemaField("n_indicators", "INTEGER", mode="REQUIRED"),
@@ -1208,7 +1287,6 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 5: agg_framework_asean
# Tidak ada kolom pillar/indicator langsung di tabel ini
# =========================================================================
def calc_framework_asean(self) -> pd.DataFrame:
@@ -1371,7 +1449,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 6: agg_narrative_overview
# Tidak ada kolom pillar/indicator di tabel ini
# FIXED: narrative_id pakai nama negara Indonesia
# =========================================================================
def calc_narrative_overview(
@@ -1384,6 +1462,7 @@ class FoodSecurityAggregator:
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 6: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID")
self.logger.info(" FIXED : narrative_id pakai nama negara Bahasa Indonesia")
self.logger.info("=" * 70)
try:
@@ -1535,6 +1614,7 @@ class FoodSecurityAggregator:
# =========================================================================
# STEP 7: agg_narrative_pillar
# Kolom tambahan: pillar_name_id
# FIXED: narrative_id pakai nama negara & pilar Bahasa Indonesia
# =========================================================================
def calc_narrative_pillar(
@@ -1547,6 +1627,7 @@ class FoodSecurityAggregator:
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 7: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Narrative: interpretatif, plain text, bilingual EN/ID")
self.logger.info(" FIXED : narrative_id pakai nama negara & pilar Bahasa Indonesia")
self.logger.info("=" * 70)
try:
@@ -1576,7 +1657,7 @@ class FoodSecurityAggregator:
p_yoy = prow["year_over_year_change"]
p_yoy_val = float(p_yoy) if pd.notna(p_yoy) else None
# Terjemahan Indonesia nama pillar
# Terjemahan Indonesia nama pillar — FIXED
p_name_id = translate_pillar(p_name)
p_country = (
@@ -1626,20 +1707,24 @@ class FoodSecurityAggregator:
"rank_in_year": p_rank,
"yoy_change": p_yoy_val,
"top_country": top_country,
"top_country_id": translate_country(top_country) if top_country else None,
"top_country_score": top_country_score,
"bottom_country": bot_country,
"bottom_country_id": translate_country(bot_country) if bot_country else None,
"bottom_country_score": bot_country_score,
"narrative_en": narrative_en,
"narrative_id": narrative_id,
})
df = pd.DataFrame(records)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
df["year"] = df["year"].astype(int)
df["pillar_id"] = df["pillar_id"].astype(int)
df["rank_in_year"] = df["rank_in_year"].astype(int)
df["pillar_name_id"] = df["pillar_name_id"].astype(str)
df["narrative_en"] = df["narrative_en"].astype(str)
df["narrative_id"] = df["narrative_id"].astype(str)
df["top_country_id"] = df["top_country_id"].astype(str).replace("None", pd.NA).astype("string")
df["bottom_country_id"] = df["bottom_country_id"].astype(str).replace("None", pd.NA).astype("string")
for col in ["pillar_score", "yoy_change", "top_country_score", "bottom_country_score"]:
df[col] = pd.to_numeric(df[col], errors="coerce").astype(float)
@@ -1657,8 +1742,10 @@ class FoodSecurityAggregator:
bigquery.SchemaField("rank_in_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("yoy_change", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("top_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("top_country_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("top_country_score", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("bottom_country", "STRING", mode="NULLABLE"),
bigquery.SchemaField("bottom_country_id", "STRING", mode="NULLABLE"),
bigquery.SchemaField("bottom_country_score", "FLOAT", mode="NULLABLE"),
bigquery.SchemaField("narrative_en", "STRING", mode="REQUIRED"),
bigquery.SchemaField("narrative_id", "STRING", mode="REQUIRED"),
@@ -1750,6 +1837,7 @@ class FoodSecurityAggregator:
self.logger.info("FOOD SECURITY AGGREGATION — 6 TABLES -> fs_asean_gold")
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}")
self.logger.info(f" Narrative style : interpretive, plain text, bilingual EN/ID")
self.logger.info(f" FIXED : 'Access' -> 'Akses', nama negara & pilar ID")
self.logger.info("=" * 70)
self.load_data()
scripts/bigquery_analytical_layer.py
+72 -15
View File
@@ -10,7 +10,13 @@ Filtering Order:
5. Filter indicators with consistent presence across FIXED countries
6. Save analytical table (dengan nama/label lengkap untuk Looker Studio)
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
ADDED:
- Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
- Kolom country_name_id (terjemahan Bahasa Indonesia nama negara)
FIXED:
- Nama pilar "Access" -> "Akses" (konsisten di semua mapping)
- Nama negara ASEAN dalam Bahasa Indonesia
"""
import pandas as pd
@@ -40,21 +46,39 @@ from google.cloud import bigquery
# TRANSLATION DICTIONARIES
# =============================================================================
COUNTRY_NAME_ID_MAP: dict = {
# Nama resmi -> Bahasa Indonesia
"Brunei Darussalam" : "Brunei Darussalam",
"Cambodia" : "Kamboja",
"Indonesia" : "Indonesia",
"Lao People's Democratic Republic" : "Laos",
"Lao PDR" : "Laos",
"Malaysia" : "Malaysia",
"Myanmar" : "Myanmar",
"Philippines" : "Filipina",
"Singapore" : "Singapura",
"Thailand" : "Thailand",
"Timor-Leste" : "Timor-Leste",
"Viet Nam" : "Vietnam",
"Vietnam" : "Vietnam",
}
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability": "Keberlanjutan",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability": "Keberlanjutan",
# 4 pilar utama Food Security — "Access" -> "Akses" (FIXED, bukan "Keterjangkauan")
"Availability" : "Ketersediaan",
"Access" : "Akses",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability" : "Keberlanjutan",
# Variasi penulisan huruf kecil
"availability" : "Ketersediaan",
"access" : "Akses",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability" : "Keberlanjutan",
# Variasi dengan prefix "Food"
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Access" : "Akses Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Food Sustainability": "Keberlanjutan Pangan",
@@ -195,6 +219,13 @@ INDICATOR_TRANSLATION_ID: dict = {
}
def translate_country(name: str) -> str:
"""Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return COUNTRY_NAME_ID_MAP.get(name.strip(), name)
def translate_indicator(name: str) -> str:
"""Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
@@ -226,6 +257,7 @@ class AnalyticalLayerLoader:
Output: fact_asean_food_security_selected -> DW layer (Gold) -> fs_asean_gold
Kolom tambahan:
- country_name_id : terjemahan Bahasa Indonesia dari country_name
- indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
- pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name
"""
@@ -625,12 +657,24 @@ class AnalyticalLayerLoader:
# ------------------------------------------------------------------
# TAMBAHAN: kolom terjemahan Bahasa Indonesia
# country_name_id : terjemahan Bahasa Indonesia dari country_name [BARU]
# indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
# pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name
# ------------------------------------------------------------------
analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country)
analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator)
analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar)
# Log negara yang belum punya terjemahan
no_trans_ctr = analytical_df[
analytical_df['country_name_id'] == analytical_df['country_name']
]['country_name'].unique()
if len(no_trans_ctr) > 0:
self.logger.warning(
f" [TRANSLATION] {len(no_trans_ctr)} country/countries tidak ada di kamus "
f"(menggunakan nama asli): {list(no_trans_ctr)}"
)
# Log indikator yang belum punya terjemahan (fallback ke nama asli)
no_trans_ind = analytical_df[
analytical_df['indicator_name_id'] == analytical_df['indicator_name']
@@ -657,6 +701,7 @@ class AnalyticalLayerLoader:
# Pastikan tipe data konsisten
analytical_df['country_id'] = analytical_df['country_id'].astype(int)
analytical_df['country_name'] = analytical_df['country_name'].astype(str)
analytical_df['country_name_id'] = analytical_df['country_name_id'].astype(str)
analytical_df['indicator_id'] = analytical_df['indicator_id'].astype(int)
analytical_df['indicator_name'] = analytical_df['indicator_name'].astype(str)
analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].astype(str)
@@ -671,10 +716,21 @@ class AnalyticalLayerLoader:
self.logger.info(f" Kolom yang disimpan: {list(analytical_df.columns)}")
self.logger.info(f" Total rows: {len(analytical_df):,}")
# Log sample terjemahan negara
sample_ctr = (
analytical_df[['country_name', 'country_name_id']]
.drop_duplicates()
.sort_values('country_name')
)
self.logger.info("\n Terjemahan nama negara (EN -> ID):")
for _, r in sample_ctr.iterrows():
self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}")
# Schema BigQuery
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("indicator_name_id", "STRING", mode="REQUIRED"),
@@ -710,7 +766,8 @@ class AnalyticalLayerLoader:
'fixed_countries': len(self.selected_country_ids),
'no_gaps' : True,
'layer' : 'gold',
'columns' : 'id + name + name_id (Looker Studio ready)'
'columns' : 'id + name + name_id (Looker Studio ready)',
'added_columns' : ['country_name_id', 'indicator_name_id', 'pillar_name_id'],
}),
'validation_metrics' : json.dumps({
'fixed_countries' : len(self.selected_country_ids),