diff --git a/scripts/bigquery_aggraget_fact_selected_layer.py b/scripts/bigquery_aggraget_fact_selected_layer.py index 2f41387..c258a48 100644 --- a/scripts/bigquery_aggraget_fact_selected_layer.py +++ b/scripts/bigquery_aggraget_fact_selected_layer.py @@ -3,6 +3,11 @@ BIGQUERY ANALYSIS LAYER - INDICATOR NORM AGGREGATION Tabel 1: agg_indicator_norm -> fs_asean_gold Tabel 2: agg_narrative_indicator -> fs_asean_gold +============================================================================= +BUGFIX: + - INDICATOR_NAME_ID_MAP: semua key diubah ke lowercase agar cocok dengan + lookup menggunakan .lower().strip(). Sebelumnya key Title Case tidak pernah + match karena fungsi get_indicator_name_id() melakukan .lower() sebelum lookup. ============================================================================= PERUBAHAN: - Ditambahkan kolom country_name_id : nama negara dalam Bahasa Indonesia [BARU] @@ -134,128 +139,132 @@ PILLAR_NAME_ID_MAP: dict = { "sustainability": "Keberlanjutan", } -# Kunci: indicator_name lowercase stripped +# ============================================================================= +# BUGFIX: Semua key INDICATOR_NAME_ID_MAP ditulis lowercase +# agar cocok dengan lookup: str(indicator_name).lower().strip() +# Sebelumnya key Title Case tidak pernah match -> kolom selalu fallback ke EN. +# ============================================================================= INDICATOR_NAME_ID_MAP: dict = { # ------------------------------------------------------------------------- # DIETARY ENERGY SUPPLY # ------------------------------------------------------------------------- - "Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)": + "dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)": "Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari)", - "Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)": + "dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)": "Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari) (rata-rata 3 tahun)", # ------------------------------------------------------------------------- # WATER & SANITATION # ------------------------------------------------------------------------- - "Percentage of population using at least basic drinking water services (percent)": + "percentage of population using at least basic drinking water services (percent)": "Persentase penduduk yang menggunakan layanan air minum dasar (persen)", - "Percentage of population using at least basic sanitation services (percent)": + "percentage of population using at least basic sanitation services (percent)": "Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)", - "Percentage of population using safely managed drinking water services (percent)": + "percentage of population using safely managed drinking water services (percent)": "Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)", - "Percentage of population using safely managed sanitation services (percent)": + "percentage of population using safely managed sanitation services (percent)": "Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)", # ------------------------------------------------------------------------- # INFRASTRUCTURE # ------------------------------------------------------------------------- - "Rail lines density (total route in km per 100 square km of land area)": + "rail lines density (total route in km per 100 square km of land area)": "Kepadatan jalur kereta api (total rute dalam km per 100 km² lahan)", # ------------------------------------------------------------------------- # AVAILABILITY # ------------------------------------------------------------------------- - "Average dietary energy requirement (kcal/cap/day)": + "average dietary energy requirement (kcal/cap/day)": "Rata-rata kebutuhan energi makanan (kkal/kapita/hari)", - "Average dietary energy supply adequacy (percent) (3-year average)": + "average dietary energy supply adequacy (percent) (3-year average)": "Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)", - "Average fat supply (g/cap/day) (3-year average)": + "average fat supply (g/cap/day) (3-year average)": "Rata-rata pasokan lemak (g/kapita/hari) (rata-rata 3 tahun)", - "Average protein supply (g/cap/day) (3-year average)": + "average protein supply (g/cap/day) (3-year average)": "Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)", - "Average supply of protein of animal origin (g/cap/day) (3-year average)": + "average supply of protein of animal origin (g/cap/day) (3-year average)": "Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)", - "Percent of arable land equipped for irrigation (percent) (3-year average)": + "percent of arable land equipped for irrigation (percent) (3-year average)": "Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)", - "Cereal import dependency ratio (percent) (3-year average)": + "cereal import dependency ratio (percent) (3-year average)": "Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)", - "Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)": + "share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)": "Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)", - "Per capita food supply variability (kcal/cap/day)": + "per capita food supply variability (kcal/cap/day)": "Variabilitas pasokan pangan per kapita (kkal/kapita/hari)", - "Value of food imports in total merchandise exports (percent) (3-year average)": + "value of food imports in total merchandise exports (percent) (3-year average)": "Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)", # ------------------------------------------------------------------------- # ACCESS # ------------------------------------------------------------------------- - "Gross domestic product per capita, PPP, (constant 2021 international $)": + "gross domestic product per capita, ppp, (constant 2021 international $)": "Produk domestik bruto per kapita, PPP (internasional konstan 2021 US$)", - "Political stability and absence of violence/terrorism (index)": + "political stability and absence of violence/terrorism (index)": "Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)", - "Prevalence of undernourishment (percent) (3-year average)": + "prevalence of undernourishment (percent) (3-year average)": "Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)", - "Number of people undernourished (million) (3-year average)": + "number of people undernourished (million) (3-year average)": "Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)", - "Minimum dietary energy requirement (kcal/cap/day)": + "minimum dietary energy requirement (kcal/cap/day)": "Kebutuhan energi makanan minimum (kkal/kapita/hari)", # ------------------------------------------------------------------------- # UTILIZATION # ------------------------------------------------------------------------- - "Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)": + "prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)": "Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)", - "Number of children under 5 years affected by wasting (million)": + "number of children under 5 years affected by wasting (million)": "Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)", - "Number of moderately or severely food insecure female adults (million) (3-year average)": + "number of moderately or severely food insecure female adults (million) (3-year average)": "Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", - "Number of moderately or severely food insecure male adults (million) (3-year average)": + "number of moderately or severely food insecure male adults (million) (3-year average)": "Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", - "Number of moderately or severely food insecure people (million) (3-year average)": + "number of moderately or severely food insecure people (million) (3-year average)": "Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)", - "Number of severely food insecure female adults (million) (3-year average)": + "number of severely food insecure female adults (million) (3-year average)": "Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", - "Number of severely food insecure male adults (million) (3-year average)": + "number of severely food insecure male adults (million) (3-year average)": "Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", - "Number of severely food insecure people (million) (3-year average)": + "number of severely food insecure people (million) (3-year average)": "Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)", - "Number of women of reproductive age (15-49 years) affected by anemia (million)": + "number of women of reproductive age (15-49 years) affected by anemia (million)": "Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)", - "Percentage of children under 5 years affected by wasting (percent)": + "percentage of children under 5 years affected by wasting (percent)": "Persentase anak di bawah 5 tahun yang mengalami wasting (persen)", - "Prevalence of anemia among women of reproductive age (15-49 years) (percent)": + "prevalence of anemia among women of reproductive age (15-49 years) (percent)": "Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)", - "Coefficient of variation of habitual caloric consumption distribution (real number)": + "coefficient of variation of habitual caloric consumption distribution (real number)": "Koefisien variasi distribusi konsumsi kalori kebiasaan (bilangan riil)", - "Incidence of caloric losses at retail distribution level (percent)": + "incidence of caloric losses at retail distribution level (percent)": "Insidensi kehilangan kalori pada tingkat distribusi ritel (persen)", - "Number of children under 5 years of age who are overweight (modeled estimates) (million)": + "number of children under 5 years of age who are overweight (modeled estimates) (million)": "Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)", - "Number of children under 5 years of age who are stunted (modeled estimates) (million)": + "number of children under 5 years of age who are stunted (modeled estimates) (million)": "Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)", - "Number of newborns with low birthweight (million)": + "number of newborns with low birthweight (million)": "Jumlah bayi baru lahir dengan berat badan lahir rendah (juta jiwa)", - "Number of obese adults (18 years and older) (million)": + "number of obese adults (18 years and older) (million)": "Jumlah orang dewasa yang mengalami obesitas (18 tahun ke atas) (juta jiwa)", - "Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)": + "percentage of children under 5 years of age who are overweight (modelled estimates) (percent)": "Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)", - "Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)": + "percentage of children under 5 years of age who are stunted (modelled estimates) (percent)": "Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)", - "Prevalence of low birthweight (percent)": + "prevalence of low birthweight (percent)": "Prevalensi berat badan lahir rendah (persen)", - "Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)": + "prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)": "Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)": + "prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)": "Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)": + "prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)": "Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)", - "Prevalence of obesity in the adult population (18 years and older) (percent)": + "prevalence of obesity in the adult population (18 years and older) (percent)": "Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)", - "Prevalence of severe food insecurity in the female adult population (percent) (3-year average)": + "prevalence of severe food insecurity in the female adult population (percent) (3-year average)": "Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of severe food insecurity in the male adult population (percent) (3-year average)": + "prevalence of severe food insecurity in the male adult population (percent) (3-year average)": "Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)", - "Prevalence of severe food insecurity in the total population (percent) (3-year average)": + "prevalence of severe food insecurity in the total population (percent) (3-year average)": "Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)", } @@ -269,7 +278,11 @@ def get_country_name_id(country_name: str) -> str: def get_indicator_name_id(indicator_name: str) -> str: - """Kembalikan terjemahan Bahasa Indonesia untuk nama indikator.""" + """ + Kembalikan terjemahan Bahasa Indonesia untuk nama indikator. + BUGFIX: key di INDICATOR_NAME_ID_MAP sudah lowercase sehingga + .lower().strip() lookup akan selalu match dengan benar. + """ return INDICATOR_NAME_ID_MAP.get( str(indicator_name).lower().strip(), str(indicator_name), # fallback: kembalikan nama asli @@ -779,7 +792,7 @@ class IndicatorNormAggregator: .astype(str) ) - # Nama indikator + # Nama indikator — BUGFIX: key di map sudah lowercase, lookup .lower() match dengan benar self.df["indicator_name_id"] = ( self.df["indicator_name"] .apply(get_indicator_name_id) @@ -800,6 +813,23 @@ class IndicatorNormAggregator: self.logger.info(f" indicator_name_id mapped rows : {n_indicator_mapped:,}") self.logger.info(f" pillar_name_id mapped rows : {n_pillar_mapped:,}") + # Validasi: peringatkan jika ada indikator yang tidak ter-mapping + unmapped_indicators = ( + self.df[self.df["indicator_name_id"] == self.df["indicator_name"]] + ["indicator_name"] + .drop_duplicates() + .tolist() + ) + if unmapped_indicators: + self.logger.warning( + f" [WARNING] {len(unmapped_indicators)} indicator_name tidak ditemukan " + f"di INDICATOR_NAME_ID_MAP (fallback ke nama Inggris):" + ) + for name in unmapped_indicators[:10]: + self.logger.warning(f" - {name}") + if len(unmapped_indicators) > 10: + self.logger.warning(f" ... dan {len(unmapped_indicators) - 10} lainnya") + # Log sample negara sample_ctr = ( self.df[["country_name", "country_name_id"]] @@ -821,14 +851,15 @@ class IndicatorNormAggregator: # Log sample indikator sample_ind = ( - self.df[["indicator_name", "indicator_name_id"]] + self.df[self.df["indicator_name_id"] != self.df["indicator_name"]] + [["indicator_name", "indicator_name_id"]] .drop_duplicates() .head(5) ) - self.logger.info("\n Sample indicator mapping (EN -> ID):") + self.logger.info("\n Sample indicator mapping (EN -> ID) [ter-mapping]:") for _, r in sample_ind.iterrows(): - self.logger.info(f" EN: {r['indicator_name'][:55]}") - self.logger.info(f" ID: {r['indicator_name_id'][:55]}") + self.logger.info(f" EN: {r['indicator_name'][:75]}") + self.logger.info(f" ID: {r['indicator_name_id'][:75]}") # ========================================================================= # STEP 4: Deteksi sdgs_start_year @@ -1118,6 +1149,7 @@ class IndicatorNormAggregator: "performance_threshold": _PERFORMANCE_THRESHOLD, "unit_source" : "dim_indicator", "added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id"], + "bugfix" : "INDICATOR_NAME_ID_MAP keys lowercased to match .lower() lookup", }), "validation_metrics" : json.dumps({ "total_rows" : rows_loaded, @@ -1275,10 +1307,9 @@ class IndicatorNormAggregator: worst_row = grp.loc[grp["country_avg_value"].idxmin()] best_row = grp.loc[grp["country_avg_value"].idxmax()] country_stats.append({ - "indicator_id" : ind_id, - "country_worst" : worst_row["country_name"], # nama Inggris - "country_best" : best_row["country_name"], # nama Inggris - # Tambahan: nama Indonesia untuk kedua negara + "indicator_id" : ind_id, + "country_worst" : worst_row["country_name"], + "country_best" : best_row["country_name"], "country_worst_id": get_country_name_id(worst_row["country_name"]), "country_best_id" : get_country_name_id(best_row["country_name"]), }) @@ -1429,6 +1460,7 @@ class IndicatorNormAggregator: "layer" : "gold", "added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id", "country_worst_id", "country_best_id"], + "bugfix" : "INDICATOR_NAME_ID_MAP keys lowercased to match .lower() lookup", }), "validation_metrics" : json.dumps({ "total_rows" : rows_loaded, @@ -1453,7 +1485,7 @@ class IndicatorNormAggregator: self.logger.info(" Output : agg_indicator_norm -> fs_asean_gold") self.logger.info(" agg_narrative_indicator -> fs_asean_gold") self.logger.info(" Added : country_name_id, indicator_name_id, pillar_name_id (Bahasa Indonesia)") - self.logger.info(" FIXED : 'Access' -> 'Akses', narrative_id pakai nama ID") + self.logger.info(" BUGFIX : INDICATOR_NAME_ID_MAP keys -> lowercase (cocok dengan .lower() lookup)") self.logger.info("=" * 80) self.load_data()