delete translator

requirements.txt

@@ -9,4 +9,3 @@ numpy
 wbgapi
 pytz
 db-dtypes
-deep-translator

scripts/bigquery_analytical_layer.py

@@ -10,10 +10,7 @@ Filtering Order:
 5. Filter indicators with consistent presence across FIXED countries
 6. Save analytical table (dengan nama/label lengkap untuk Looker Studio)
 
-ADDED:
+ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
-- Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
-- Kolom country_name_id (terjemahan Bahasa Indonesia nama negara)
-- Terjemahan indikator via Google Translate (deep-translator), tanpa dict statis
 """
 
 import pandas as pd
@@ -37,53 +34,163 @@ from scripts.bigquery_helpers import (
     save_etl_metadata,
 )
 from google.cloud import bigquery
-from deep_translator import GoogleTranslator
 
 
 # =============================================================================
-# TRANSLATION DICTIONARIES (country & pillar only, indikator via Google Translate)
+# TRANSLATION DICTIONARIES
 # =============================================================================
 
-COUNTRY_NAME_ID_MAP: dict = {
-    "Brunei Darussalam"                : "Brunei Darussalam",
-    "Cambodia"                         : "Kamboja",
-    "Indonesia"                        : "Indonesia",
-    "Lao People's Democratic Republic" : "Laos",
-    "Lao PDR"                          : "Laos",
-    "Malaysia"                         : "Malaysia",
-    "Myanmar"                          : "Myanmar",
-    "Philippines"                      : "Filipina",
-    "Singapore"                        : "Singapura",
-    "Thailand"                         : "Thailand",
-    "Timor-Leste"                       : "Timor-Leste",
-    "Viet Nam"                         : "Vietnam",
-    "Vietnam"                          : "Vietnam",
-}
-
 PILLAR_TRANSLATION_ID: dict = {
+    # 4 pilar utama Food Security
     "Availability"  : "Ketersediaan",
-    "Access"             : "Akses",
+    "Access"        : "Keterjangkauan",
     "Utilization"   : "Pemanfaatan",
     "Stability"     : "Stabilitas",
     "Sustainability": "Keberlanjutan",
+    # Variasi penulisan yang mungkin muncul
     "availability"  : "Ketersediaan",
-    "access"             : "Akses",
+    "access"        : "Keterjangkauan",
     "utilization"   : "Pemanfaatan",
     "stability"     : "Stabilitas",
     "sustainability": "Keberlanjutan",
     "Food Availability"  : "Ketersediaan Pangan",
-    "Food Access"        : "Akses Pangan",
+    "Food Access"        : "Keterjangkauan Pangan",
     "Food Utilization"   : "Pemanfaatan Pangan",
     "Food Stability"     : "Stabilitas Pangan",
     "Food Sustainability": "Keberlanjutan Pangan",
 }
 
 
-def translate_country(name: str) -> str:
+INDICATOR_TRANSLATION_ID: dict = {
-    """Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli."""
+    # -------------------------------------------------------------------------
+    # DIETARY ENERGY SUPPLY
+    # -------------------------------------------------------------------------
+    "Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)":
+        "Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari)",
+    "Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)":
+        "Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari) (rata-rata 3 tahun)",
+
+    # -------------------------------------------------------------------------
+    # WATER & SANITATION
+    # -------------------------------------------------------------------------
+    "Percentage of population using at least basic drinking water services (percent)":
+        "Persentase penduduk yang menggunakan layanan air minum dasar (persen)",
+    "Percentage of population using at least basic sanitation services (percent)":
+        "Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)",
+    "Percentage of population using safely managed drinking water services (percent)":
+        "Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)",
+    "Percentage of population using safely managed sanitation services (percent)":
+        "Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)",
+
+    # -------------------------------------------------------------------------
+    # INFRASTRUCTURE
+    # -------------------------------------------------------------------------
+    "Rail lines density (total route in km per 100 square km of land area)":
+        "Kepadatan jalur kereta api (total rute dalam km per 100 km² lahan)",
+
+    # -------------------------------------------------------------------------
+    # AVAILABILITY
+    # -------------------------------------------------------------------------
+    "Average dietary energy requirement (kcal/cap/day)":
+        "Rata-rata kebutuhan energi makanan (kkal/kapita/hari)",
+    "Average dietary energy supply adequacy (percent) (3-year average)":
+        "Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)",
+    "Average fat supply (g/cap/day) (3-year average)":
+        "Rata-rata pasokan lemak (g/kapita/hari) (rata-rata 3 tahun)",
+    "Average protein supply (g/cap/day) (3-year average)":
+        "Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)",
+    "Average supply of protein of animal origin (g/cap/day) (3-year average)":
+        "Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)",
+    "Percent of arable land equipped for irrigation (percent) (3-year average)":
+        "Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)",
+    "Cereal import dependency ratio (percent) (3-year average)":
+        "Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
+    "Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
+        "Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)",
+    "Per capita food supply variability (kcal/cap/day)":
+        "Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
+    "Value of food imports in total merchandise exports (percent) (3-year average)":
+        "Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)",
+
+    # -------------------------------------------------------------------------
+    # ACCESS
+    # -------------------------------------------------------------------------
+    "Gross domestic product per capita, PPP, (constant 2021 international $)":
+        "Produk domestik bruto per kapita, PPP (internasional konstan 2021 US$)",
+    "Political stability and absence of violence/terrorism (index)":
+        "Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)",
+    "Prevalence of undernourishment (percent) (3-year average)":
+        "Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)",
+    "Number of people undernourished (million) (3-year average)":
+        "Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)",
+    "Minimum dietary energy requirement (kcal/cap/day)":
+        "Kebutuhan energi makanan minimum (kkal/kapita/hari)",
+
+    # -------------------------------------------------------------------------
+    # UTILIZATION
+    # -------------------------------------------------------------------------
+    "Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)":
+        "Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)",
+    "Number of children under 5 years affected by wasting (million)":
+        "Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)",
+    "Number of moderately or severely food insecure female adults (million) (3-year average)":
+        "Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
+    "Number of moderately or severely food insecure male adults (million) (3-year average)":
+        "Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
+    "Number of moderately or severely food insecure people (million) (3-year average)":
+        "Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
+    "Number of severely food insecure female adults (million) (3-year average)":
+        "Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
+    "Number of severely food insecure male adults (million) (3-year average)":
+        "Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
+    "Number of severely food insecure people (million) (3-year average)":
+        "Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
+    "Number of women of reproductive age (15-49 years) affected by anemia (million)":
+        "Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)",
+    "Percentage of children under 5 years affected by wasting (percent)":
+        "Persentase anak di bawah 5 tahun yang mengalami wasting (persen)",
+    "Prevalence of anemia among women of reproductive age (15-49 years) (percent)":
+        "Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)",
+    "Coefficient of variation of habitual caloric consumption distribution (real number)":
+        "Koefisien variasi distribusi konsumsi kalori kebiasaan (bilangan riil)",
+    "Incidence of caloric losses at retail distribution level (percent)":
+        "Insidensi kehilangan kalori pada tingkat distribusi ritel (persen)",
+    "Number of children under 5 years of age who are overweight (modeled estimates) (million)":
+        "Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)",
+    "Number of children under 5 years of age who are stunted (modeled estimates) (million)":
+        "Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)",
+    "Number of newborns with low birthweight (million)":
+        "Jumlah bayi baru lahir dengan berat badan lahir rendah (juta jiwa)",
+    "Number of obese adults (18 years and older) (million)":
+        "Jumlah orang dewasa yang mengalami obesitas (18 tahun ke atas) (juta jiwa)",
+    "Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
+        "Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)",
+    "Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
+        "Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)",
+    "Prevalence of low birthweight (percent)":
+        "Prevalensi berat badan lahir rendah (persen)",
+    "Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
+        "Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
+    "Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
+        "Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
+    "Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
+        "Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)",
+    "Prevalence of obesity in the adult population (18 years and older) (percent)":
+        "Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)",
+    "Prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
+        "Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
+    "Prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
+        "Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
+    "Prevalence of severe food insecurity in the total population (percent) (3-year average)":
+        "Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)",
+}
+
+
+def translate_indicator(name: str) -> str:
+    """Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
     if not name:
         return name
-    return COUNTRY_NAME_ID_MAP.get(name.strip(), name)
+    return INDICATOR_TRANSLATION_ID.get(name, name)
 
 
 def translate_pillar(name: str) -> str:
@@ -93,27 +200,6 @@ def translate_pillar(name: str) -> str:
     return PILLAR_TRANSLATION_ID.get(name, name)
 
 
-def translate_all_indicators(indicator_names: list[str], logger: logging.Logger) -> dict[str, str]:
-    """
-    Terjemahkan SEMUA nama indikator ke Bahasa Indonesia via Google Translate.
-    Setiap indikator diterjemahkan satu per satu (no batch).
-    Fallback ke nama asli jika terjemahan gagal.
-    """
-    translator = GoogleTranslator(source='en', target='id')
-    results    = {}
-
-    for name in indicator_names:
-        try:
-            translated    = translator.translate(name)
-            results[name] = translated
-            logger.info(f"    [OK] {name[:60]:<62} -> {translated}")
-        except Exception as e:
-            results[name] = name
-            logger.warning(f"    [FAIL] {name[:60]:<62} -> fallback (error: {e})")
-
-    return results
-
-
 # =============================================================================
 # ANALYTICAL LAYER CLASS
 # =============================================================================
@@ -131,8 +217,7 @@ class AnalyticalLayerLoader:
     Output: fact_asean_food_security_selected -> DW layer (Gold) -> fs_asean_gold
 
     Kolom tambahan:
-    - country_name_id   : terjemahan Bahasa Indonesia dari country_name
+    - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
-    - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name (via Google Translate)
     - pillar_name_id    : terjemahan Bahasa Indonesia dari pillar_name
     """
 
@@ -315,10 +400,8 @@ class AnalyticalLayerLoader:
 
         df_valid       = pd.DataFrame(valid_combinations)
         df_valid['key'] = df_valid['country_id'].astype(str) + '_' + df_valid['indicator_id'].astype(str)
-        self.df_clean['key'] = (
+        self.df_clean['key'] = (self.df_clean['country_id'].astype(str) + '_' +
-            self.df_clean['country_id'].astype(str) + '_' +
+                                self.df_clean['indicator_id'].astype(str))
-            self.df_clean['indicator_id'].astype(str)
-        )
 
         original_count = len(self.df_clean)
         self.df_clean  = self.df_clean[self.df_clean['key'].isin(df_valid['key'])].copy()
@@ -532,40 +615,21 @@ class AnalyticalLayerLoader:
             ]].copy()
 
             # ------------------------------------------------------------------
-            # Terjemahkan negara & pillar via dict statis
+            # TAMBAHAN: kolom terjemahan Bahasa Indonesia
+            # indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
+            # pillar_name_id    : terjemahan Bahasa Indonesia dari pillar_name
             # ------------------------------------------------------------------
-            analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country)
+            analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator)
             analytical_df['pillar_name_id']    = analytical_df['pillar_name'].apply(translate_pillar)
 
-            # ------------------------------------------------------------------
+            # Log indikator yang belum punya terjemahan (fallback ke nama asli)
-            # Terjemahkan SEMUA indikator unik via Google Translate (deep-translator)
+            no_trans_ind = analytical_df[
-            # ------------------------------------------------------------------
+                analytical_df['indicator_name_id'] == analytical_df['indicator_name']
-            all_indicators = analytical_df['indicator_name'].unique().tolist()
+            ]['indicator_name'].unique()
-            self.logger.info(
+            if len(no_trans_ind) > 0:
-                f"\n  [TRANSLATE] Menerjemahkan {len(all_indicators)} indikator "
-                f"via Google Translate..."
-            )
-
-            indicator_translation_map      = translate_all_indicators(all_indicators, self.logger)
-            analytical_df['indicator_name_id'] = analytical_df['indicator_name'].map(
-                indicator_translation_map
-            )
-
-            # Fallback ke nama asli jika map menghasilkan NaN
-            analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].fillna(
-                analytical_df['indicator_name']
-            )
-
-            # ------------------------------------------------------------------
-            # Log warning negara/pillar yang tidak punya terjemahan
-            # ------------------------------------------------------------------
-            no_trans_ctr = analytical_df[
-                analytical_df['country_name_id'] == analytical_df['country_name']
-            ]['country_name'].unique()
-            if len(no_trans_ctr) > 0:
                 self.logger.warning(
-                    f"  [TRANSLATION] {len(no_trans_ctr)} negara tidak ada di kamus "
+                    f"  [TRANSLATION] {len(no_trans_ind)} indicator(s) tidak ada di kamus "
-                    f"(menggunakan nama asli): {list(no_trans_ctr)}"
+                    f"(menggunakan nama asli): {list(no_trans_ind)[:5]}"
                 )
 
             no_trans_pil = analytical_df[
@@ -573,20 +637,17 @@ class AnalyticalLayerLoader:
             ]['pillar_name'].unique()
             if len(no_trans_pil) > 0:
                 self.logger.warning(
-                    f"  [TRANSLATION] {len(no_trans_pil)} pillar tidak ada di kamus "
+                    f"  [TRANSLATION] {len(no_trans_pil)} pillar(s) tidak ada di kamus "
                     f"(menggunakan nama asli): {list(no_trans_pil)}"
                 )
 
-            # ------------------------------------------------------------------
-            # Sort & pastikan tipe data konsisten
-            # ------------------------------------------------------------------
             analytical_df = analytical_df.sort_values(
                 ['year', 'country_name', 'pillar_name', 'indicator_name']
             ).reset_index(drop=True)
 
+            # Pastikan tipe data konsisten
             analytical_df['country_id']        = analytical_df['country_id'].astype(int)
             analytical_df['country_name']      = analytical_df['country_name'].astype(str)
-            analytical_df['country_name_id']   = analytical_df['country_name_id'].astype(str)
             analytical_df['indicator_id']      = analytical_df['indicator_id'].astype(int)
             analytical_df['indicator_name']    = analytical_df['indicator_name'].astype(str)
             analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].astype(str)
@@ -598,26 +659,13 @@ class AnalyticalLayerLoader:
             analytical_df['year']              = analytical_df['year'].astype(int)
             analytical_df['value']             = analytical_df['value'].astype(float)
 
-            self.logger.info(f"\n  Kolom yang disimpan: {list(analytical_df.columns)}")
+            self.logger.info(f"  Kolom yang disimpan: {list(analytical_df.columns)}")
             self.logger.info(f"  Total rows: {len(analytical_df):,}")
 
-            # Log sample terjemahan negara
-            sample_ctr = (
-                analytical_df[['country_name', 'country_name_id']]
-                .drop_duplicates()
-                .sort_values('country_name')
-            )
-            self.logger.info("\n  Terjemahan nama negara (EN -> ID):")
-            for _, r in sample_ctr.iterrows():
-                self.logger.info(f"    {r['country_name']:<35} -> {r['country_name_id']}")
-
-            # ------------------------------------------------------------------
             # Schema BigQuery
-            # ------------------------------------------------------------------
             schema = [
                 bigquery.SchemaField("country_id",        "INTEGER", mode="REQUIRED"),
                 bigquery.SchemaField("country_name",      "STRING",  mode="REQUIRED"),
-                bigquery.SchemaField("country_name_id",   "STRING",  mode="REQUIRED"),
                 bigquery.SchemaField("indicator_id",      "INTEGER", mode="REQUIRED"),
                 bigquery.SchemaField("indicator_name",    "STRING",  mode="REQUIRED"),
                 bigquery.SchemaField("indicator_name_id", "STRING",  mode="REQUIRED"),
@@ -653,9 +701,7 @@ class AnalyticalLayerLoader:
                     'fixed_countries': len(self.selected_country_ids),
                     'no_gaps'        : True,
                     'layer'          : 'gold',
-                    'columns'        : 'id + name + name_id (Looker Studio ready)',
+                    'columns'        : 'id + name + name_id (Looker Studio ready)'
-                    'added_columns'  : ['country_name_id', 'indicator_name_id', 'pillar_name_id'],
-                    'indicator_translation': 'Google Translate via deep-translator',
                 }),
                 'validation_metrics' : json.dumps({
                     'fixed_countries' : len(self.selected_country_ids),