diff --git a/requirements.txt b/requirements.txt index e2547f5..f0fb1a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,4 @@ pandas numpy wbgapi pytz -db-dtypes -deep-translator \ No newline at end of file +db-dtypes \ No newline at end of file diff --git a/scripts/bigquery_analytical_layer.py b/scripts/bigquery_analytical_layer.py index 8035b2a..50e3e41 100644 --- a/scripts/bigquery_analytical_layer.py +++ b/scripts/bigquery_analytical_layer.py @@ -10,10 +10,7 @@ Filtering Order: 5. Filter indicators with consistent presence across FIXED countries 6. Save analytical table (dengan nama/label lengkap untuk Looker Studio) -ADDED: -- Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) -- Kolom country_name_id (terjemahan Bahasa Indonesia nama negara) -- Terjemahan indikator via Google Translate (deep-translator), tanpa dict statis +ADDED: Kolom indicator_name_id dan pillar_name_id (terjemahan Bahasa Indonesia) """ import pandas as pd @@ -37,53 +34,163 @@ from scripts.bigquery_helpers import ( save_etl_metadata, ) from google.cloud import bigquery -from deep_translator import GoogleTranslator # ============================================================================= -# TRANSLATION DICTIONARIES (country & pillar only, indikator via Google Translate) +# TRANSLATION DICTIONARIES # ============================================================================= -COUNTRY_NAME_ID_MAP: dict = { - "Brunei Darussalam" : "Brunei Darussalam", - "Cambodia" : "Kamboja", - "Indonesia" : "Indonesia", - "Lao People's Democratic Republic" : "Laos", - "Lao PDR" : "Laos", - "Malaysia" : "Malaysia", - "Myanmar" : "Myanmar", - "Philippines" : "Filipina", - "Singapore" : "Singapura", - "Thailand" : "Thailand", - "Timor-Leste" : "Timor-Leste", - "Viet Nam" : "Vietnam", - "Vietnam" : "Vietnam", -} - PILLAR_TRANSLATION_ID: dict = { - "Availability" : "Ketersediaan", - "Access" : "Akses", - "Utilization" : "Pemanfaatan", - "Stability" : "Stabilitas", - "Sustainability" : "Keberlanjutan", - "availability" : "Ketersediaan", - "access" : "Akses", - "utilization" : "Pemanfaatan", - "stability" : "Stabilitas", - "sustainability" : "Keberlanjutan", + # 4 pilar utama Food Security + "Availability" : "Ketersediaan", + "Access" : "Keterjangkauan", + "Utilization" : "Pemanfaatan", + "Stability" : "Stabilitas", + "Sustainability": "Keberlanjutan", + # Variasi penulisan yang mungkin muncul + "availability" : "Ketersediaan", + "access" : "Keterjangkauan", + "utilization" : "Pemanfaatan", + "stability" : "Stabilitas", + "sustainability": "Keberlanjutan", "Food Availability" : "Ketersediaan Pangan", - "Food Access" : "Akses Pangan", + "Food Access" : "Keterjangkauan Pangan", "Food Utilization" : "Pemanfaatan Pangan", "Food Stability" : "Stabilitas Pangan", "Food Sustainability": "Keberlanjutan Pangan", } -def translate_country(name: str) -> str: - """Terjemahkan nama negara ke Bahasa Indonesia. Fallback ke nama asli.""" +INDICATOR_TRANSLATION_ID: dict = { + # ------------------------------------------------------------------------- + # DIETARY ENERGY SUPPLY + # ------------------------------------------------------------------------- + "Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)": + "Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari)", + "Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)": + "Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari) (rata-rata 3 tahun)", + + # ------------------------------------------------------------------------- + # WATER & SANITATION + # ------------------------------------------------------------------------- + "Percentage of population using at least basic drinking water services (percent)": + "Persentase penduduk yang menggunakan layanan air minum dasar (persen)", + "Percentage of population using at least basic sanitation services (percent)": + "Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)", + "Percentage of population using safely managed drinking water services (percent)": + "Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)", + "Percentage of population using safely managed sanitation services (percent)": + "Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)", + + # ------------------------------------------------------------------------- + # INFRASTRUCTURE + # ------------------------------------------------------------------------- + "Rail lines density (total route in km per 100 square km of land area)": + "Kepadatan jalur kereta api (total rute dalam km per 100 km² lahan)", + + # ------------------------------------------------------------------------- + # AVAILABILITY + # ------------------------------------------------------------------------- + "Average dietary energy requirement (kcal/cap/day)": + "Rata-rata kebutuhan energi makanan (kkal/kapita/hari)", + "Average dietary energy supply adequacy (percent) (3-year average)": + "Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)", + "Average fat supply (g/cap/day) (3-year average)": + "Rata-rata pasokan lemak (g/kapita/hari) (rata-rata 3 tahun)", + "Average protein supply (g/cap/day) (3-year average)": + "Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)", + "Average supply of protein of animal origin (g/cap/day) (3-year average)": + "Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)", + "Percent of arable land equipped for irrigation (percent) (3-year average)": + "Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)", + "Cereal import dependency ratio (percent) (3-year average)": + "Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)", + "Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)": + "Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)", + "Per capita food supply variability (kcal/cap/day)": + "Variabilitas pasokan pangan per kapita (kkal/kapita/hari)", + "Value of food imports in total merchandise exports (percent) (3-year average)": + "Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)", + + # ------------------------------------------------------------------------- + # ACCESS + # ------------------------------------------------------------------------- + "Gross domestic product per capita, PPP, (constant 2021 international $)": + "Produk domestik bruto per kapita, PPP (internasional konstan 2021 US$)", + "Political stability and absence of violence/terrorism (index)": + "Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)", + "Prevalence of undernourishment (percent) (3-year average)": + "Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)", + "Number of people undernourished (million) (3-year average)": + "Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)", + "Minimum dietary energy requirement (kcal/cap/day)": + "Kebutuhan energi makanan minimum (kkal/kapita/hari)", + + # ------------------------------------------------------------------------- + # UTILIZATION + # ------------------------------------------------------------------------- + "Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)": + "Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)", + "Number of children under 5 years affected by wasting (million)": + "Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)", + "Number of moderately or severely food insecure female adults (million) (3-year average)": + "Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", + "Number of moderately or severely food insecure male adults (million) (3-year average)": + "Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", + "Number of moderately or severely food insecure people (million) (3-year average)": + "Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", + "Number of severely food insecure female adults (million) (3-year average)": + "Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", + "Number of severely food insecure male adults (million) (3-year average)": + "Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", + "Number of severely food insecure people (million) (3-year average)": + "Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", + "Number of women of reproductive age (15-49 years) affected by anemia (million)": + "Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)", + "Percentage of children under 5 years affected by wasting (percent)": + "Persentase anak di bawah 5 tahun yang mengalami wasting (persen)", + "Prevalence of anemia among women of reproductive age (15-49 years) (percent)": + "Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)", + "Coefficient of variation of habitual caloric consumption distribution (real number)": + "Koefisien variasi distribusi konsumsi kalori kebiasaan (bilangan riil)", + "Incidence of caloric losses at retail distribution level (percent)": + "Insidensi kehilangan kalori pada tingkat distribusi ritel (persen)", + "Number of children under 5 years of age who are overweight (modeled estimates) (million)": + "Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)", + "Number of children under 5 years of age who are stunted (modeled estimates) (million)": + "Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)", + "Number of newborns with low birthweight (million)": + "Jumlah bayi baru lahir dengan berat badan lahir rendah (juta jiwa)", + "Number of obese adults (18 years and older) (million)": + "Jumlah orang dewasa yang mengalami obesitas (18 tahun ke atas) (juta jiwa)", + "Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)": + "Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)", + "Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)": + "Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)", + "Prevalence of low birthweight (percent)": + "Prevalensi berat badan lahir rendah (persen)", + "Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)": + "Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)", + "Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)": + "Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)", + "Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)": + "Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)", + "Prevalence of obesity in the adult population (18 years and older) (percent)": + "Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)", + "Prevalence of severe food insecurity in the female adult population (percent) (3-year average)": + "Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)", + "Prevalence of severe food insecurity in the male adult population (percent) (3-year average)": + "Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)", + "Prevalence of severe food insecurity in the total population (percent) (3-year average)": + "Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)", +} + + +def translate_indicator(name: str) -> str: + """Terjemahkan nama indikator ke Bahasa Indonesia. Fallback ke nama asli.""" if not name: return name - return COUNTRY_NAME_ID_MAP.get(name.strip(), name) + return INDICATOR_TRANSLATION_ID.get(name, name) def translate_pillar(name: str) -> str: @@ -93,27 +200,6 @@ def translate_pillar(name: str) -> str: return PILLAR_TRANSLATION_ID.get(name, name) -def translate_all_indicators(indicator_names: list[str], logger: logging.Logger) -> dict[str, str]: - """ - Terjemahkan SEMUA nama indikator ke Bahasa Indonesia via Google Translate. - Setiap indikator diterjemahkan satu per satu (no batch). - Fallback ke nama asli jika terjemahan gagal. - """ - translator = GoogleTranslator(source='en', target='id') - results = {} - - for name in indicator_names: - try: - translated = translator.translate(name) - results[name] = translated - logger.info(f" [OK] {name[:60]:<62} -> {translated}") - except Exception as e: - results[name] = name - logger.warning(f" [FAIL] {name[:60]:<62} -> fallback (error: {e})") - - return results - - # ============================================================================= # ANALYTICAL LAYER CLASS # ============================================================================= @@ -131,8 +217,7 @@ class AnalyticalLayerLoader: Output: fact_asean_food_security_selected -> DW layer (Gold) -> fs_asean_gold Kolom tambahan: - - country_name_id : terjemahan Bahasa Indonesia dari country_name - - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name (via Google Translate) + - indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name - pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name """ @@ -313,12 +398,10 @@ class AnalyticalLayerLoader: self.logger.info(f"\n [+] Valid: {len(valid_combinations):,}") self.logger.info(f" [-] Removed: {len(removed_combinations):,}") - df_valid = pd.DataFrame(valid_combinations) + df_valid = pd.DataFrame(valid_combinations) df_valid['key'] = df_valid['country_id'].astype(str) + '_' + df_valid['indicator_id'].astype(str) - self.df_clean['key'] = ( - self.df_clean['country_id'].astype(str) + '_' + - self.df_clean['indicator_id'].astype(str) - ) + self.df_clean['key'] = (self.df_clean['country_id'].astype(str) + '_' + + self.df_clean['indicator_id'].astype(str)) original_count = len(self.df_clean) self.df_clean = self.df_clean[self.df_clean['key'].isin(df_valid['key'])].copy() @@ -532,40 +615,21 @@ class AnalyticalLayerLoader: ]].copy() # ------------------------------------------------------------------ - # Terjemahkan negara & pillar via dict statis + # TAMBAHAN: kolom terjemahan Bahasa Indonesia + # indicator_name_id : terjemahan Bahasa Indonesia dari indicator_name + # pillar_name_id : terjemahan Bahasa Indonesia dari pillar_name # ------------------------------------------------------------------ - analytical_df['country_name_id'] = analytical_df['country_name'].apply(translate_country) - analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar) + analytical_df['indicator_name_id'] = analytical_df['indicator_name'].apply(translate_indicator) + analytical_df['pillar_name_id'] = analytical_df['pillar_name'].apply(translate_pillar) - # ------------------------------------------------------------------ - # Terjemahkan SEMUA indikator unik via Google Translate (deep-translator) - # ------------------------------------------------------------------ - all_indicators = analytical_df['indicator_name'].unique().tolist() - self.logger.info( - f"\n [TRANSLATE] Menerjemahkan {len(all_indicators)} indikator " - f"via Google Translate..." - ) - - indicator_translation_map = translate_all_indicators(all_indicators, self.logger) - analytical_df['indicator_name_id'] = analytical_df['indicator_name'].map( - indicator_translation_map - ) - - # Fallback ke nama asli jika map menghasilkan NaN - analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].fillna( - analytical_df['indicator_name'] - ) - - # ------------------------------------------------------------------ - # Log warning negara/pillar yang tidak punya terjemahan - # ------------------------------------------------------------------ - no_trans_ctr = analytical_df[ - analytical_df['country_name_id'] == analytical_df['country_name'] - ]['country_name'].unique() - if len(no_trans_ctr) > 0: + # Log indikator yang belum punya terjemahan (fallback ke nama asli) + no_trans_ind = analytical_df[ + analytical_df['indicator_name_id'] == analytical_df['indicator_name'] + ]['indicator_name'].unique() + if len(no_trans_ind) > 0: self.logger.warning( - f" [TRANSLATION] {len(no_trans_ctr)} negara tidak ada di kamus " - f"(menggunakan nama asli): {list(no_trans_ctr)}" + f" [TRANSLATION] {len(no_trans_ind)} indicator(s) tidak ada di kamus " + f"(menggunakan nama asli): {list(no_trans_ind)[:5]}" ) no_trans_pil = analytical_df[ @@ -573,20 +637,17 @@ class AnalyticalLayerLoader: ]['pillar_name'].unique() if len(no_trans_pil) > 0: self.logger.warning( - f" [TRANSLATION] {len(no_trans_pil)} pillar tidak ada di kamus " + f" [TRANSLATION] {len(no_trans_pil)} pillar(s) tidak ada di kamus " f"(menggunakan nama asli): {list(no_trans_pil)}" ) - # ------------------------------------------------------------------ - # Sort & pastikan tipe data konsisten - # ------------------------------------------------------------------ analytical_df = analytical_df.sort_values( ['year', 'country_name', 'pillar_name', 'indicator_name'] ).reset_index(drop=True) + # Pastikan tipe data konsisten analytical_df['country_id'] = analytical_df['country_id'].astype(int) analytical_df['country_name'] = analytical_df['country_name'].astype(str) - analytical_df['country_name_id'] = analytical_df['country_name_id'].astype(str) analytical_df['indicator_id'] = analytical_df['indicator_id'].astype(int) analytical_df['indicator_name'] = analytical_df['indicator_name'].astype(str) analytical_df['indicator_name_id'] = analytical_df['indicator_name_id'].astype(str) @@ -598,26 +659,13 @@ class AnalyticalLayerLoader: analytical_df['year'] = analytical_df['year'].astype(int) analytical_df['value'] = analytical_df['value'].astype(float) - self.logger.info(f"\n Kolom yang disimpan: {list(analytical_df.columns)}") + self.logger.info(f" Kolom yang disimpan: {list(analytical_df.columns)}") self.logger.info(f" Total rows: {len(analytical_df):,}") - # Log sample terjemahan negara - sample_ctr = ( - analytical_df[['country_name', 'country_name_id']] - .drop_duplicates() - .sort_values('country_name') - ) - self.logger.info("\n Terjemahan nama negara (EN -> ID):") - for _, r in sample_ctr.iterrows(): - self.logger.info(f" {r['country_name']:<35} -> {r['country_name_id']}") - - # ------------------------------------------------------------------ # Schema BigQuery - # ------------------------------------------------------------------ schema = [ bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("country_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("country_name_id", "STRING", mode="REQUIRED"), bigquery.SchemaField("indicator_id", "INTEGER", mode="REQUIRED"), bigquery.SchemaField("indicator_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("indicator_name_id", "STRING", mode="REQUIRED"), @@ -653,9 +701,7 @@ class AnalyticalLayerLoader: 'fixed_countries': len(self.selected_country_ids), 'no_gaps' : True, 'layer' : 'gold', - 'columns' : 'id + name + name_id (Looker Studio ready)', - 'added_columns' : ['country_name_id', 'indicator_name_id', 'pillar_name_id'], - 'indicator_translation': 'Google Translate via deep-translator', + 'columns' : 'id + name + name_id (Looker Studio ready)' }), 'validation_metrics' : json.dumps({ 'fixed_countries' : len(self.selected_country_ids),