izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

delete translator

Browse Source
This commit is contained in:
Debby
2026-06-07 09:36:49 +07:00
parent 355f31b096
commit 43f0e36233
2 changed files with 164 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
-1
View File
@@ -9,4 +9,3 @@ numpy
wbgapi
pytz
db-dtypes
deep-translator
scripts/bigquery_analytical_layer.py
+155 -109
View File
@@ -10,10 +10,7 @@ Filtering Order:
5. Filter indicators with consistent presence across FIXED countries
6. Save analytical table (dengan nama/label lengkap untuk Looker Studio)
ADDED:
- Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
- Kolom country_name_id (terjemahan Bahasa Indonesia nama negara)
- Terjemahan indikator via Google Translate (deep-translator), tanpa dict statis
ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia)
"""
import pandas as pd
@@ -37,53 +34,163 @@ from scripts.bigquery_helpers import (
save_etl_metadata,
)
from google.cloud import bigquery
from deep_translator import GoogleTranslator
# =============================================================================
# TRANSLATION DICTIONARIES (country & pillar only, indikator via Google Translate)
# TRANSLATION DICTIONARIES
# =============================================================================
COUNTRY_NAME_ID_MAP: dict = {
"Brunei Darussalam" : "Brunei Darussalam",
"Cambodia" : "Kamboja",
"Indonesia" : "Indonesia",
"Lao People's Democratic Republic" : "Laos",
"Lao PDR" : "Laos",
"Malaysia" : "Malaysia",
"Myanmar" : "Myanmar",
"Philippines" : "Filipina",
"Singapore" : "Singapura",
"Thailand" : "Thailand",
"Timor-Leste" : "Timor-Leste",
"Viet Nam" : "Vietnam",
"Vietnam" : "Vietnam",
}
PILLAR_TRANSLATION_ID: dict = {
# 4 pilar utama Food Security
"Availability" : "Ketersediaan",
"Access" : "Akses",
"Access" : "Keterjangkauan",
"Utilization" : "Pemanfaatan",
"Stability" : "Stabilitas",
"Sustainability" : "Keberlanjutan",
"Sustainability": "Keberlanjutan",
# Variasi penulisan yang mungkin muncul
"availability" : "Ketersediaan",
"access" : "Akses",
"access" : "Keterjangkauan",
"utilization" : "Pemanfaatan",
"stability" : "Stabilitas",
"sustainability" : "Keberlanjutan",
"sustainability": "Keberlanjutan",
"Food Availability" : "Ketersediaan Pangan",
"Food Access" : "Akses Pangan",
"Food Access" : "Keterjangkauan Pangan",
"Food Utilization" : "Pemanfaatan Pangan",
"Food Stability" : "Stabilitas Pangan",
"Food Sustainability": "Keberlanjutan Pangan",
}
def translate_country(name: str) -> str:
"""Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli."""
INDICATOR_TRANSLATION_ID: dict = {
# -------------------------------------------------------------------------
# DIETARY ENERGY SUPPLY
# -------------------------------------------------------------------------
"Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)":
"Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari)",
"Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)":
"Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari) (rata-rata 3 tahun)",
# -------------------------------------------------------------------------
# WATER & SANITATION
# -------------------------------------------------------------------------
"Percentage of population using at least basic drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum dasar (persen)",
"Percentage of population using at least basic sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)",
"Percentage of population using safely managed drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)",
"Percentage of population using safely managed sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)",
# -------------------------------------------------------------------------
# INFRASTRUCTURE
# -------------------------------------------------------------------------
"Rail lines density (total route in km per 100 square km of land area)":
"Kepadatan jalur kereta api (total rute dalam km per 100 km² lahan)",
# -------------------------------------------------------------------------
# AVAILABILITY
# -------------------------------------------------------------------------
"Average dietary energy requirement (kcal/cap/day)":
"Rata-rata kebutuhan energi makanan (kkal/kapita/hari)",
"Average dietary energy supply adequacy (percent) (3-year average)":
"Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)",
"Average fat supply (g/cap/day) (3-year average)":
"Rata-rata pasokan lemak (g/kapita/hari) (rata-rata 3 tahun)",
"Average protein supply (g/cap/day) (3-year average)":
"Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)",
"Average supply of protein of animal origin (g/cap/day) (3-year average)":
"Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)",
"Percent of arable land equipped for irrigation (percent) (3-year average)":
"Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)",
"Cereal import dependency ratio (percent) (3-year average)":
"Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
"Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
"Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)",
"Per capita food supply variability (kcal/cap/day)":
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
"Value of food imports in total merchandise exports (percent) (3-year average)":
"Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)",
# -------------------------------------------------------------------------
# ACCESS
# -------------------------------------------------------------------------
"Gross domestic product per capita, PPP, (constant 2021 international $)":
"Produk domestik bruto per kapita, PPP (internasional konstan 2021 US$)",
"Political stability and absence of violence/terrorism (index)":
"Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)",
"Prevalence of undernourishment (percent) (3-year average)":
"Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)",
"Number of people undernourished (million) (3-year average)":
"Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)",
"Minimum dietary energy requirement (kcal/cap/day)":
"Kebutuhan energi makanan minimum (kkal/kapita/hari)",
# -------------------------------------------------------------------------
# UTILIZATION
# -------------------------------------------------------------------------
"Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)":
"Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)",
"Number of children under 5 years affected by wasting (million)":
"Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)",
"Number of moderately or severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of women of reproductive age (15-49 years) affected by anemia (million)":
"Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)",
"Percentage of children under 5 years affected by wasting (percent)":
"Persentase anak di bawah 5 tahun yang mengalami wasting (persen)",
"Prevalence of anemia among women of reproductive age (15-49 years) (percent)":
"Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)",
"Coefficient of variation of habitual caloric consumption distribution (real number)":
"Koefisien variasi distribusi konsumsi kalori kebiasaan (bilangan riil)",
"Incidence of caloric losses at retail distribution level (percent)":
"Insidensi kehilangan kalori pada tingkat distribusi ritel (persen)",
"Number of children under 5 years of age who are overweight (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)",
"Number of children under 5 years of age who are stunted (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)",
"Number of newborns with low birthweight (million)":
"Jumlah bayi baru lahir dengan berat badan lahir rendah (juta jiwa)",
"Number of obese adults (18 years and older) (million)":
"Jumlah orang dewasa yang mengalami obesitas (18 tahun ke atas) (juta jiwa)",
"Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)",
"Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)",
"Prevalence of low birthweight (percent)":
"Prevalensi berat badan lahir rendah (persen)",
"Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)",
"Prevalence of obesity in the adult population (18 years and older) (percent)":
"Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)",
"Prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)",
}
def translate_indicator(name: str) -> str:
"""Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli."""
if not name:
return name
return COUNTRY_NAME_ID_MAP.get(name.strip(), name)
return INDICATOR_TRANSLATION_ID.get(name, name)
def translate_pillar(name: str) -> str:
@@ -93,27 +200,6 @@ def translate_pillar(name: str) -> str:
return PILLAR_TRANSLATION_ID.get(name, name)
def translate_all_indicators(indicator_names: list[str], logger: logging.Logger) -> dict[str, str]:
"""
Terjemahkan SEMUA nama indikator ke Bahasa Indonesia via Google Translate.
Setiap indikator diterjemahkan satu per satu (no batch).
Fallback ke nama asli jika terjemahan gagal.
"""
translator = GoogleTranslator(source='en', target='id')
results = {}
for name in indicator_names:
try:
translated = translator.translate(name)
results[name] = translated
logger.info(f" [OK] {name[:60]:<62} -> {translated}")
except Exception as e:
results[name] = name
logger.warning(f" [FAIL] {name[:60]:<62} -> fallback (error: {e})")
return results
# =============================================================================
# ANALYTICAL LAYER CLASS
# =============================================================================
@@ -131,8 +217,7 @@ class AnalyticalLayerLoader:
Output: fact_asean_food_security_selected -> DW layer (Gold) -> fs_asean_gold
Kolom tambahan:
- country_name_id : terjemahan Bahasa Indonesia dari country_name
- indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name (via Google Translate)
- indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
- pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name
"""
@@ -315,10 +400,8 @@ class AnalyticalLayerLoader:
df_valid = pd.DataFrame(valid_combinations)
df_valid['key'] = df_valid['country_id'].astype(str) + '_' + df_valid['indicator_id'].astype(str)
self.df_clean['key'] = (
self.df_clean['country_id'].astype(str) + '_' +
self.df_clean['indicator_id'].astype(str)
)
self.df_clean['key'] = (self.df_clean['country_id'].astype(str) + '_' +
self.df_clean['indicator_id'].astype(str))
original_count = len(self.df_clean)
self.df_clean = self.df_clean[self.df_clean['key'].isin(df_valid['key'])].copy()
@@ -532,40 +615,21 @@ class AnalyticalLayerLoader:
]].copy()
# ------------------------------------------------------------------
# Terjemahkan negara & pillar via dict statis
# TAMBAHAN: kolom terjemahan Bahasa Indonesia
# indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name
# pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name
# ------------------------------------------------------------------
analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country)
analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator)
analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar)
# ------------------------------------------------------------------
# Terjemahkan SEMUA indikator unik via Google Translate (deep-translator)
# ------------------------------------------------------------------
all_indicators = analytical_df['indicator_name'].unique().tolist()
self.logger.info(
f"\n [TRANSLATE] Menerjemahkan {len(all_indicators)} indikator "
f"via Google Translate..."
)
indicator_translation_map = translate_all_indicators(all_indicators, self.logger)
analytical_df['indicator_name_id'] = analytical_df['indicator_name'].map(
indicator_translation_map
)
# Fallback ke nama asli jika map menghasilkan NaN
analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].fillna(
analytical_df['indicator_name']
)
# ------------------------------------------------------------------
# Log warning negara/pillar yang tidak punya terjemahan
# ------------------------------------------------------------------
no_trans_ctr = analytical_df[
analytical_df['country_name_id'] == analytical_df['country_name']
]['country_name'].unique()
if len(no_trans_ctr) > 0:
# Log indikator yang belum punya terjemahan (fallback ke nama asli)
no_trans_ind = analytical_df[
analytical_df['indicator_name_id'] == analytical_df['indicator_name']
]['indicator_name'].unique()
if len(no_trans_ind) > 0:
self.logger.warning(
f" [TRANSLATION] {len(no_trans_ctr)} negara tidak ada di kamus "
f"(menggunakan nama asli): {list(no_trans_ctr)}"
f" [TRANSLATION] {len(no_trans_ind)} indicator(s) tidak ada di kamus "
f"(menggunakan nama asli): {list(no_trans_ind)[:5]}"
)
no_trans_pil = analytical_df[
@@ -573,20 +637,17 @@ class AnalyticalLayerLoader:
]['pillar_name'].unique()
if len(no_trans_pil) > 0:
self.logger.warning(
f" [TRANSLATION] {len(no_trans_pil)} pillar tidak ada di kamus "
f" [TRANSLATION] {len(no_trans_pil)} pillar(s) tidak ada di kamus "
f"(menggunakan nama asli): {list(no_trans_pil)}"
)
# ------------------------------------------------------------------
# Sort & pastikan tipe data konsisten
# ------------------------------------------------------------------
analytical_df = analytical_df.sort_values(
['year', 'country_name', 'pillar_name', 'indicator_name']
).reset_index(drop=True)
# Pastikan tipe data konsisten
analytical_df['country_id'] = analytical_df['country_id'].astype(int)
analytical_df['country_name'] = analytical_df['country_name'].astype(str)
analytical_df['country_name_id'] = analytical_df['country_name_id'].astype(str)
analytical_df['indicator_id'] = analytical_df['indicator_id'].astype(int)
analytical_df['indicator_name'] = analytical_df['indicator_name'].astype(str)
analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].astype(str)
@@ -598,26 +659,13 @@ class AnalyticalLayerLoader:
analytical_df['year'] = analytical_df['year'].astype(int)
analytical_df['value'] = analytical_df['value'].astype(float)
self.logger.info(f"\n Kolom yang disimpan: {list(analytical_df.columns)}")
self.logger.info(f" Kolom yang disimpan: {list(analytical_df.columns)}")
self.logger.info(f" Total rows: {len(analytical_df):,}")
# Log sample terjemahan negara
sample_ctr = (
analytical_df[['country_name', 'country_name_id']]
.drop_duplicates()
.sort_values('country_name')
)
self.logger.info("\n Terjemahan nama negara (EN -> ID):")
for _, r in sample_ctr.iterrows():
self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}")
# ------------------------------------------------------------------
# Schema BigQuery
# ------------------------------------------------------------------
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("indicator_name_id", "STRING", mode="REQUIRED"),
@@ -653,9 +701,7 @@ class AnalyticalLayerLoader:
'fixed_countries': len(self.selected_country_ids),
'no_gaps' : True,
'layer' : 'gold',
'columns' : 'id + name + name_id (Looker Studio ready)',
'added_columns' : ['country_name_id', 'indicator_name_id', 'pillar_name_id'],
'indicator_translation': 'Google Translate via deep-translator',
'columns' : 'id + name + name_id (Looker Studio ready)'
}),
'validation_metrics' : json.dumps({
'fixed_countries' : len(self.selected_country_ids),