diff --git a/scripts/bigquery_analytical_layer.py b/scripts/bigquery_analytical_layer.py index 98955fc..8035b2a 100644 --- a/scripts/bigquery_analytical_layer.py +++ b/scripts/bigquery_analytical_layer.py @@ -13,10 +13,7 @@ Filtering Order: ADDED: - Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) - Kolom country_name_id (terjemahan Bahasa Indonesia nama negara) - -FIXED: -- Nama pilar "Access" -> "Akses" (konsisten di semua mapping) -- Nama negara ASEAN dalam Bahasa Indonesia +- Terjemahan indikator via Google Translate (deep-translator), tanpa dict statis """ import pandas as pd @@ -40,43 +37,40 @@ from scripts.bigquery_helpers import ( save_etl_metadata, ) from google.cloud import bigquery +from deep_translator import GoogleTranslator # ============================================================================= -# TRANSLATION DICTIONARIES +# TRANSLATION DICTIONARIES (country & pillar only, indikator via Google Translate) # ============================================================================= COUNTRY_NAME_ID_MAP: dict = { - # Nama resmi -> Bahasa Indonesia - "Brunei Darussalam" : "Brunei Darussalam", - "Cambodia" : "Kamboja", - "Indonesia" : "Indonesia", - "Lao People's Democratic Republic" : "Laos", - "Lao PDR" : "Laos", - "Malaysia" : "Malaysia", - "Myanmar" : "Myanmar", - "Philippines" : "Filipina", - "Singapore" : "Singapura", - "Thailand" : "Thailand", - "Timor-Leste" : "Timor-Leste", - "Viet Nam" : "Vietnam", - "Vietnam" : "Vietnam", + "Brunei Darussalam" : "Brunei Darussalam", + "Cambodia" : "Kamboja", + "Indonesia" : "Indonesia", + "Lao People's Democratic Republic" : "Laos", + "Lao PDR" : "Laos", + "Malaysia" : "Malaysia", + "Myanmar" : "Myanmar", + "Philippines" : "Filipina", + "Singapore" : "Singapura", + "Thailand" : "Thailand", + "Timor-Leste" : "Timor-Leste", + "Viet Nam" : "Vietnam", + "Vietnam" : "Vietnam", } PILLAR_TRANSLATION_ID: dict = { - # 4 pilar utama Food Security — "Access" -> "Akses" (FIXED, bukan "Keterjangkauan") "Availability" : "Ketersediaan", "Access" : "Akses", "Utilization" : "Pemanfaatan", "Stability" : "Stabilitas", "Sustainability" : "Keberlanjutan", - # Variasi penulisan huruf kecil "availability" : "Ketersediaan", "access" : "Akses", "utilization" : "Pemanfaatan", "stability" : "Stabilitas", "sustainability" : "Keberlanjutan", - # Variasi dengan prefix "Food" "Food Availability" : "Ketersediaan Pangan", "Food Access" : "Akses Pangan", "Food Utilization" : "Pemanfaatan Pangan", @@ -84,140 +78,6 @@ PILLAR_TRANSLATION_ID: dict = { "Food Sustainability": "Keberlanjutan Pangan", } -INDICATOR_TRANSLATION_ID: dict = { - # ------------------------------------------------------------------------- - # AVAILABILITY - # ------------------------------------------------------------------------- - "Average dietary energy supply adequacy (percent) (3-year average)": - "Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)", - "Average value of food production (constant 2014-2016 thousand US$) (3-year average)": - "Nilai rata-rata produksi pangan (ribu US$ konstan 2014-2016) (rata-rata 3 tahun)", - "Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)": - "Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)", - "Average protein supply (g/cap/day) (3-year average)": - "Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)", - "Average supply of protein of animal origin (g/cap/day) (3-year average)": - "Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)", - "Cereal import dependency ratio (percent) (3-year average)": - "Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)", - "Percent of arable land equipped for irrigation (percent) (3-year average)": - "Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)", - "Crop production index (2014-2016 = 100)": - "Indeks produksi tanaman pangan (2014-2016 = 100)", - "Livestock production index (2014-2016 = 100)": - "Indeks produksi peternakan (2014-2016 = 100)", - "Value of food imports over total merchandise exports (percent) (3-year average)": - "Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)", - "Food production variability (constant 2014-2016 thousand US$ per capita)": - "Variabilitas produksi pangan (ribu US$ konstan 2014-2016 per kapita)", - "Food supply variability (kcal/cap/day)": - "Variabilitas pasokan pangan (kkal/kapita/hari)", - - # ------------------------------------------------------------------------- - # ACCESS - # ------------------------------------------------------------------------- - "Gross domestic product per capita, PPP (constant 2017 international $)": - "Produk domestik bruto per kapita, PPP (internasional konstan 2017 US$)", - "Domestic food price level index (2015 = 1.00)": - "Indeks tingkat harga pangan domestik (2015 = 1,00)", - "Domestic food price volatility index": - "Indeks volatilitas harga pangan domestik", - "Prevalence of undernourishment (percent) (3-year average)": - "Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)", - "Number of people undernourished (million) (3-year average)": - "Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)", - "Depth of the food deficit (kcal/capita/day) (3-year average)": - "Kedalaman defisit pangan (kkal/kapita/hari) (rata-rata 3 tahun)", - "Percentage of population using at least basic drinking water services (percent)": - "Persentase penduduk yang menggunakan layanan air minum dasar (persen)", - "Percentage of population using safely managed drinking water services (percent)": - "Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)", - "Percentage of population using at least basic sanitation services (percent)": - "Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)", - "Percentage of population using safely managed sanitation services (percent)": - "Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)", - "Access to electricity (percent of rural population)": - "Akses listrik (persen penduduk pedesaan)", - "Proportion of population with access to electricity (percent)": - "Proporsi penduduk dengan akses listrik (persen)", - "Road infrastructure index": - "Indeks infrastruktur jalan", - "Rail lines density (total route-km per 100 square km of land area)": - "Kepadatan jalur kereta api (total rute-km per 100 km2 lahan)", - "Gross national income per capita (Atlas method, current US$)": - "Pendapatan nasional bruto per kapita (metode Atlas, US$ terkini)", - "Food Insecurity Experience Scale (FIES)": - "Skala Pengalaman Ketidakamanan Pangan (FIES)", - - # ------------------------------------------------------------------------- - # UTILIZATION - # ------------------------------------------------------------------------- - "Prevalence of severe food insecurity in the total population (percent) (3-year average)": - "Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)", - "Prevalence of severe food insecurity in the male adult population (percent) (3-year average)": - "Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of severe food insecurity in the female adult population (percent) (3-year average)": - "Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)": - "Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)", - "Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)": - "Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)": - "Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)", - "Number of severely food insecure people (million) (3-year average)": - "Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", - "Number of severely food insecure male adults (million) (3-year average)": - "Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", - "Number of severely food insecure female adults (million) (3-year average)": - "Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", - "Number of moderately or severely food insecure people (million) (3-year average)": - "Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", - "Number of moderately or severely food insecure male adults (million) (3-year average)": - "Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", - "Number of moderately or severely food insecure female adults (million) (3-year average)": - "Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", - "Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)": - "Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)", - "Number of children under 5 years of age who are stunted (modeled estimates) (million)": - "Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)", - "Percentage of children under 5 years affected by wasting (percent)": - "Persentase anak di bawah 5 tahun yang mengalami wasting (persen)", - "Number of children under 5 years affected by wasting (million)": - "Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)", - "Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)": - "Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)", - "Number of children under 5 years of age who are overweight (modeled estimates) (million)": - "Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)", - "Prevalence of anemia among women of reproductive age (15-49 years) (percent)": - "Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)", - "Number of women of reproductive age (15-49 years) affected by anemia (million)": - "Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)", - "Prevalence of obesity in the adult population (18 years and older) (percent)": - "Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)", - "Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)": - "Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)", - "Minimum dietary diversity for women (MDD-W) (percent)": - "Keragaman pola makan minimum untuk perempuan (MDD-W) (persen)", - - # ------------------------------------------------------------------------- - # STABILITY - # ------------------------------------------------------------------------- - "Cereal import dependency ratio (percent)": - "Rasio ketergantungan impor sereal (persen)", - "Political stability and absence of violence/terrorism (index)": - "Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)", - "Domestic food price volatility": - "Volatilitas harga pangan domestik", - "Per capita food supply variability (kcal/cap/day)": - "Variabilitas pasokan pangan per kapita (kkal/kapita/hari)", - "Percentage of arable land equipped for irrigation (percent)": - "Persentase lahan pertanian yang dilengkapi irigasi (persen)", - "GDP per capita growth (annual %)": - "Pertumbuhan PDB per kapita (% tahunan)", - "GDP growth (annual %)": - "Pertumbuhan PDB (% tahunan)", -} - def translate_country(name: str) -> str: """Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli.""" @@ -226,13 +86,6 @@ def translate_country(name: str) -> str: return COUNTRY_NAME_ID_MAP.get(name.strip(), name) -def translate_indicator(name: str) -> str: - """Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli.""" - if not name: - return name - return INDICATOR_TRANSLATION_ID.get(name, name) - - def translate_pillar(name: str) -> str: """Terjemahkan nama pillar ke Bahasa Indonesia. Fallback ke nama asli.""" if not name: @@ -240,6 +93,27 @@ def translate_pillar(name: str) -> str: return PILLAR_TRANSLATION_ID.get(name, name) +def translate_all_indicators(indicator_names: list[str], logger: logging.Logger) -> dict[str, str]: + """ + Terjemahkan SEMUA nama indikator ke Bahasa Indonesia via Google Translate. + Setiap indikator diterjemahkan satu per satu (no batch). + Fallback ke nama asli jika terjemahan gagal. + """ + translator = GoogleTranslator(source='en', target='id') + results = {} + + for name in indicator_names: + try: + translated = translator.translate(name) + results[name] = translated + logger.info(f" [OK] {name[:60]:<62} -> {translated}") + except Exception as e: + results[name] = name + logger.warning(f" [FAIL] {name[:60]:<62} -> fallback (error: {e})") + + return results + + # ============================================================================= # ANALYTICAL LAYER CLASS # ============================================================================= @@ -258,7 +132,7 @@ class AnalyticalLayerLoader: Kolom tambahan: - country_name_id : terjemahan Bahasa Indonesia dari country_name - - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name + - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name (via Google Translate) - pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name """ @@ -439,10 +313,12 @@ class AnalyticalLayerLoader: self.logger.info(f"\n [+] Valid: {len(valid_combinations):,}") self.logger.info(f" [-] Removed: {len(removed_combinations):,}") - df_valid = pd.DataFrame(valid_combinations) + df_valid = pd.DataFrame(valid_combinations) df_valid['key'] = df_valid['country_id'].astype(str) + '_' + df_valid['indicator_id'].astype(str) - self.df_clean['key'] = (self.df_clean['country_id'].astype(str) + '_' + - self.df_clean['indicator_id'].astype(str)) + self.df_clean['key'] = ( + self.df_clean['country_id'].astype(str) + '_' + + self.df_clean['indicator_id'].astype(str) + ) original_count = len(self.df_clean) self.df_clean = self.df_clean[self.df_clean['key'].isin(df_valid['key'])].copy() @@ -656,49 +532,58 @@ class AnalyticalLayerLoader: ]].copy() # ------------------------------------------------------------------ - # TAMBAHAN: kolom terjemahan Bahasa Indonesia - # country_name_id : terjemahan Bahasa Indonesia dari country_name [BARU] - # indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name - # pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name + # Terjemahkan negara & pillar via dict statis # ------------------------------------------------------------------ - analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country) - analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator) - analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar) + analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country) + analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar) - # Log negara yang belum punya terjemahan + # ------------------------------------------------------------------ + # Terjemahkan SEMUA indikator unik via Google Translate (deep-translator) + # ------------------------------------------------------------------ + all_indicators = analytical_df['indicator_name'].unique().tolist() + self.logger.info( + f"\n [TRANSLATE] Menerjemahkan {len(all_indicators)} indikator " + f"via Google Translate..." + ) + + indicator_translation_map = translate_all_indicators(all_indicators, self.logger) + analytical_df['indicator_name_id'] = analytical_df['indicator_name'].map( + indicator_translation_map + ) + + # Fallback ke nama asli jika map menghasilkan NaN + analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].fillna( + analytical_df['indicator_name'] + ) + + # ------------------------------------------------------------------ + # Log warning negara/pillar yang tidak punya terjemahan + # ------------------------------------------------------------------ no_trans_ctr = analytical_df[ analytical_df['country_name_id'] == analytical_df['country_name'] ]['country_name'].unique() if len(no_trans_ctr) > 0: self.logger.warning( - f" [TRANSLATION] {len(no_trans_ctr)} country/countries tidak ada di kamus " + f" [TRANSLATION] {len(no_trans_ctr)} negara tidak ada di kamus " f"(menggunakan nama asli): {list(no_trans_ctr)}" ) - # Log indikator yang belum punya terjemahan (fallback ke nama asli) - no_trans_ind = analytical_df[ - analytical_df['indicator_name_id'] == analytical_df['indicator_name'] - ]['indicator_name'].unique() - if len(no_trans_ind) > 0: - self.logger.warning( - f" [TRANSLATION] {len(no_trans_ind)} indicator(s) tidak ada di kamus " - f"(menggunakan nama asli): {list(no_trans_ind)[:5]}" - ) - no_trans_pil = analytical_df[ analytical_df['pillar_name_id'] == analytical_df['pillar_name'] ]['pillar_name'].unique() if len(no_trans_pil) > 0: self.logger.warning( - f" [TRANSLATION] {len(no_trans_pil)} pillar(s) tidak ada di kamus " + f" [TRANSLATION] {len(no_trans_pil)} pillar tidak ada di kamus " f"(menggunakan nama asli): {list(no_trans_pil)}" ) + # ------------------------------------------------------------------ + # Sort & pastikan tipe data konsisten + # ------------------------------------------------------------------ analytical_df = analytical_df.sort_values( ['year', 'country_name', 'pillar_name', 'indicator_name'] ).reset_index(drop=True) - # Pastikan tipe data konsisten analytical_df['country_id'] = analytical_df['country_id'].astype(int) analytical_df['country_name'] = analytical_df['country_name'].astype(str) analytical_df['country_name_id'] = analytical_df['country_name_id'].astype(str) @@ -713,7 +598,7 @@ class AnalyticalLayerLoader: analytical_df['year'] = analytical_df['year'].astype(int) analytical_df['value'] = analytical_df['value'].astype(float) - self.logger.info(f" Kolom yang disimpan: {list(analytical_df.columns)}") + self.logger.info(f"\n Kolom yang disimpan: {list(analytical_df.columns)}") self.logger.info(f" Total rows: {len(analytical_df):,}") # Log sample terjemahan negara @@ -726,7 +611,9 @@ class AnalyticalLayerLoader: for _, r in sample_ctr.iterrows(): self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}") + # ------------------------------------------------------------------ # Schema BigQuery + # ------------------------------------------------------------------ schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), @@ -768,6 +655,7 @@ class AnalyticalLayerLoader: 'layer' : 'gold', 'columns' : 'id + name + name_id (Looker Studio ready)', 'added_columns' : ['country_name_id', 'indicator_name_id', 'pillar_name_id'], + 'indicator_translation': 'Google Translate via deep-translator', }), 'validation_metrics' : json.dumps({ 'fixed_countries' : len(self.selected_country_ids),