izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lowercase key

Browse Source
This commit is contained in:
Debby
2026-06-07 12:11:13 +07:00
parent 1007e30764
commit 17cc7833ad
1 changed files with 93 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bigquery_aggraget_fact_selected_layer.py
+92 -60
View File
@@ -3,6 +3,11 @@ BIGQUERY ANALYSIS LAYER - INDICATOR NORM AGGREGATION
Tabel 1: agg_indicator_norm -> fs_asean_gold
Tabel 2: agg_narrative_indicator -> fs_asean_gold
=============================================================================
BUGFIX:
- INDICATOR_NAME_ID_MAP: semua key diubah ke lowercase agar cocok dengan
lookup menggunakan .lower().strip(). Sebelumnya key Title Case tidak pernah
match karena fungsi get_indicator_name_id() melakukan .lower() sebelum lookup.
=============================================================================
PERUBAHAN:
- Ditambahkan kolom country_name_id : nama negara dalam Bahasa Indonesia [BARU]
@@ -134,128 +139,132 @@ PILLAR_NAME_ID_MAP: dict = {
"sustainability": "Keberlanjutan",
}
# Kunci: indicator_name lowercase stripped
# =============================================================================
# BUGFIX: Semua key INDICATOR_NAME_ID_MAP ditulis lowercase
# agar cocok dengan lookup: str(indicator_name).lower().strip()
# Sebelumnya key Title Case tidak pernah match -> kolom selalu fallback ke EN.
# =============================================================================
INDICATOR_NAME_ID_MAP: dict = {
# -------------------------------------------------------------------------
# DIETARY ENERGY SUPPLY
# -------------------------------------------------------------------------
"Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)":
"dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day)":
"Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari)",
"Dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)":
"dietary energy supply used in the estimation of the prevalence of undernourishment (kcal/cap/day) (3-year average)":
"Pasokan energi makanan yang digunakan dalam estimasi prevalensi kekurangan gizi (kkal/kapita/hari) (rata-rata 3 tahun)",
# -------------------------------------------------------------------------
# WATER & SANITATION
# -------------------------------------------------------------------------
"Percentage of population using at least basic drinking water services (percent)":
"percentage of population using at least basic drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum dasar (persen)",
"Percentage of population using at least basic sanitation services (percent)":
"percentage of population using at least basic sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi dasar (persen)",
"Percentage of population using safely managed drinking water services (percent)":
"percentage of population using safely managed drinking water services (percent)":
"Persentase penduduk yang menggunakan layanan air minum yang dikelola dengan aman (persen)",
"Percentage of population using safely managed sanitation services (percent)":
"percentage of population using safely managed sanitation services (percent)":
"Persentase penduduk yang menggunakan layanan sanitasi yang dikelola dengan aman (persen)",
# -------------------------------------------------------------------------
# INFRASTRUCTURE
# -------------------------------------------------------------------------
"Rail lines density (total route in km per 100 square km of land area)":
"rail lines density (total route in km per 100 square km of land area)":
"Kepadatan jalur kereta api (total rute dalam km per 100 km² lahan)",
# -------------------------------------------------------------------------
# AVAILABILITY
# -------------------------------------------------------------------------
"Average dietary energy requirement (kcal/cap/day)":
"average dietary energy requirement (kcal/cap/day)":
"Rata-rata kebutuhan energi makanan (kkal/kapita/hari)",
"Average dietary energy supply adequacy (percent) (3-year average)":
"average dietary energy supply adequacy (percent) (3-year average)":
"Kecukupan rata-rata pasokan energi makanan (persen) (rata-rata 3 tahun)",
"Average fat supply (g/cap/day) (3-year average)":
"average fat supply (g/cap/day) (3-year average)":
"Rata-rata pasokan lemak (g/kapita/hari) (rata-rata 3 tahun)",
"Average protein supply (g/cap/day) (3-year average)":
"average protein supply (g/cap/day) (3-year average)":
"Rata-rata pasokan protein (g/kapita/hari) (rata-rata 3 tahun)",
"Average supply of protein of animal origin (g/cap/day) (3-year average)":
"average supply of protein of animal origin (g/cap/day) (3-year average)":
"Rata-rata pasokan protein hewani (g/kapita/hari) (rata-rata 3 tahun)",
"Percent of arable land equipped for irrigation (percent) (3-year average)":
"percent of arable land equipped for irrigation (percent) (3-year average)":
"Persentase lahan pertanian yang dilengkapi irigasi (persen) (rata-rata 3 tahun)",
"Cereal import dependency ratio (percent) (3-year average)":
"cereal import dependency ratio (percent) (3-year average)":
"Rasio ketergantungan impor sereal (persen) (rata-rata 3 tahun)",
"Share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
"share of dietary energy supply derived from cereals, roots and tubers (percent) (3-year average)":
"Proporsi pasokan energi makanan dari serealia, akar, dan umbi-umbian (persen) (rata-rata 3 tahun)",
"Per capita food supply variability (kcal/cap/day)":
"per capita food supply variability (kcal/cap/day)":
"Variabilitas pasokan pangan per kapita (kkal/kapita/hari)",
"Value of food imports in total merchandise exports (percent) (3-year average)":
"value of food imports in total merchandise exports (percent) (3-year average)":
"Nilai impor pangan terhadap total ekspor barang (persen) (rata-rata 3 tahun)",
# -------------------------------------------------------------------------
# ACCESS
# -------------------------------------------------------------------------
"Gross domestic product per capita, PPP, (constant 2021 international $)":
"gross domestic product per capita, ppp, (constant 2021 international $)":
"Produk domestik bruto per kapita, PPP (internasional konstan 2021 US$)",
"Political stability and absence of violence/terrorism (index)":
"political stability and absence of violence/terrorism (index)":
"Stabilitas politik dan tidak adanya kekerasan/terorisme (indeks)",
"Prevalence of undernourishment (percent) (3-year average)":
"prevalence of undernourishment (percent) (3-year average)":
"Prevalensi kekurangan gizi (persen) (rata-rata 3 tahun)",
"Number of people undernourished (million) (3-year average)":
"number of people undernourished (million) (3-year average)":
"Jumlah penduduk kekurangan gizi (juta jiwa) (rata-rata 3 tahun)",
"Minimum dietary energy requirement (kcal/cap/day)":
"minimum dietary energy requirement (kcal/cap/day)":
"Kebutuhan energi makanan minimum (kkal/kapita/hari)",
# -------------------------------------------------------------------------
# UTILIZATION
# -------------------------------------------------------------------------
"Prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)":
"prevalence of exclusive breastfeeding among infants 0-5 months of age (percent)":
"Prevalensi pemberian ASI eksklusif pada bayi usia 0-5 bulan (persen)",
"Number of children under 5 years affected by wasting (million)":
"number of children under 5 years affected by wasting (million)":
"Jumlah anak di bawah 5 tahun yang mengalami wasting (juta jiwa)",
"Number of moderately or severely food insecure female adults (million) (3-year average)":
"number of moderately or severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure male adults (million) (3-year average)":
"number of moderately or severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of moderately or severely food insecure people (million) (3-year average)":
"number of moderately or severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan sedang atau berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure female adults (million) (3-year average)":
"number of severely food insecure female adults (million) (3-year average)":
"Jumlah perempuan dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure male adults (million) (3-year average)":
"number of severely food insecure male adults (million) (3-year average)":
"Jumlah laki-laki dewasa yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of severely food insecure people (million) (3-year average)":
"number of severely food insecure people (million) (3-year average)":
"Jumlah penduduk yang mengalami kerawanan pangan berat (juta jiwa) (rata-rata 3 tahun)",
"Number of women of reproductive age (15-49 years) affected by anemia (million)":
"number of women of reproductive age (15-49 years) affected by anemia (million)":
"Jumlah perempuan usia reproduksi (15-49 tahun) yang menderita anemia (juta jiwa)",
"Percentage of children under 5 years affected by wasting (percent)":
"percentage of children under 5 years affected by wasting (percent)":
"Persentase anak di bawah 5 tahun yang mengalami wasting (persen)",
"Prevalence of anemia among women of reproductive age (15-49 years) (percent)":
"prevalence of anemia among women of reproductive age (15-49 years) (percent)":
"Prevalensi anemia pada perempuan usia reproduksi (15-49 tahun) (persen)",
"Coefficient of variation of habitual caloric consumption distribution (real number)":
"coefficient of variation of habitual caloric consumption distribution (real number)":
"Koefisien variasi distribusi konsumsi kalori kebiasaan (bilangan riil)",
"Incidence of caloric losses at retail distribution level (percent)":
"incidence of caloric losses at retail distribution level (percent)":
"Insidensi kehilangan kalori pada tingkat distribusi ritel (persen)",
"Number of children under 5 years of age who are overweight (modeled estimates) (million)":
"number of children under 5 years of age who are overweight (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (juta jiwa)",
"Number of children under 5 years of age who are stunted (modeled estimates) (million)":
"number of children under 5 years of age who are stunted (modeled estimates) (million)":
"Jumlah anak di bawah 5 tahun yang mengalami stunting (estimasi model) (juta jiwa)",
"Number of newborns with low birthweight (million)":
"number of newborns with low birthweight (million)":
"Jumlah bayi baru lahir dengan berat badan lahir rendah (juta jiwa)",
"Number of obese adults (18 years and older) (million)":
"number of obese adults (18 years and older) (million)":
"Jumlah orang dewasa yang mengalami obesitas (18 tahun ke atas) (juta jiwa)",
"Percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
"percentage of children under 5 years of age who are overweight (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami kelebihan berat badan (estimasi model) (persen)",
"Percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
"percentage of children under 5 years of age who are stunted (modelled estimates) (percent)":
"Persentase anak di bawah 5 tahun yang mengalami stunting (estimasi model) (persen)",
"Prevalence of low birthweight (percent)":
"prevalence of low birthweight (percent)":
"Prevalensi berat badan lahir rendah (persen)",
"Prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
"prevalence of moderate or severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
"prevalence of moderate or severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
"prevalence of moderate or severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan sedang atau berat pada total penduduk (persen) (rata-rata 3 tahun)",
"Prevalence of obesity in the adult population (18 years and older) (percent)":
"prevalence of obesity in the adult population (18 years and older) (percent)":
"Prevalensi obesitas pada penduduk dewasa (18 tahun ke atas) (persen)",
"Prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
"prevalence of severe food insecurity in the female adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk perempuan dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
"prevalence of severe food insecurity in the male adult population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada penduduk laki-laki dewasa (persen) (rata-rata 3 tahun)",
"Prevalence of severe food insecurity in the total population (percent) (3-year average)":
"prevalence of severe food insecurity in the total population (percent) (3-year average)":
"Prevalensi kerawanan pangan berat pada total penduduk (persen) (rata-rata 3 tahun)",
}
@@ -269,7 +278,11 @@ def get_country_name_id(country_name: str) -> str:
def get_indicator_name_id(indicator_name: str) -> str:
"""Kembalikan terjemahan Bahasa Indonesia untuk nama indikator."""
"""
Kembalikan terjemahan Bahasa Indonesia untuk nama indikator.
BUGFIX: key di INDICATOR_NAME_ID_MAP sudah lowercase sehingga
.lower().strip() lookup akan selalu match dengan benar.
"""
return INDICATOR_NAME_ID_MAP.get(
str(indicator_name).lower().strip(),
str(indicator_name), # fallback: kembalikan nama asli
@@ -779,7 +792,7 @@ class IndicatorNormAggregator:
.astype(str)
)
# Nama indikator
# Nama indikator — BUGFIX: key di map sudah lowercase, lookup .lower() match dengan benar
self.df["indicator_name_id"] = (
self.df["indicator_name"]
.apply(get_indicator_name_id)
@@ -800,6 +813,23 @@ class IndicatorNormAggregator:
self.logger.info(f" indicator_name_id mapped rows : {n_indicator_mapped:,}")
self.logger.info(f" pillar_name_id mapped rows : {n_pillar_mapped:,}")
# Validasi: peringatkan jika ada indikator yang tidak ter-mapping
unmapped_indicators = (
self.df[self.df["indicator_name_id"] == self.df["indicator_name"]]
["indicator_name"]
.drop_duplicates()
.tolist()
)
if unmapped_indicators:
self.logger.warning(
f" [WARNING] {len(unmapped_indicators)} indicator_name tidak ditemukan "
f"di INDICATOR_NAME_ID_MAP (fallback ke nama Inggris):"
)
for name in unmapped_indicators[:10]:
self.logger.warning(f" - {name}")
if len(unmapped_indicators) > 10:
self.logger.warning(f" ... dan {len(unmapped_indicators) - 10} lainnya")
# Log sample negara
sample_ctr = (
self.df[["country_name", "country_name_id"]]
@@ -821,14 +851,15 @@ class IndicatorNormAggregator:
# Log sample indikator
sample_ind = (
self.df[["indicator_name", "indicator_name_id"]]
self.df[self.df["indicator_name_id"] != self.df["indicator_name"]]
[["indicator_name", "indicator_name_id"]]
.drop_duplicates()
.head(5)
)
self.logger.info("\n Sample indicator mapping (EN -> ID):")
self.logger.info("\n Sample indicator mapping (EN -> ID) [ter-mapping]:")
for _, r in sample_ind.iterrows():
self.logger.info(f" EN: {r['indicator_name'][:55]}")
self.logger.info(f" ID: {r['indicator_name_id'][:55]}")
self.logger.info(f" EN: {r['indicator_name'][:75]}")
self.logger.info(f" ID: {r['indicator_name_id'][:75]}")
# =========================================================================
# STEP 4: Deteksi sdgs_start_year
@@ -1118,6 +1149,7 @@ class IndicatorNormAggregator:
"performance_threshold": _PERFORMANCE_THRESHOLD,
"unit_source" : "dim_indicator",
"added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id"],
"bugfix" : "INDICATOR_NAME_ID_MAP keys lowercased to match .lower() lookup",
}),
"validation_metrics" : json.dumps({
"total_rows" : rows_loaded,
@@ -1276,9 +1308,8 @@ class IndicatorNormAggregator:
best_row = grp.loc[grp["country_avg_value"].idxmax()]
country_stats.append({
"indicator_id" : ind_id,
"country_worst" : worst_row["country_name"], # nama Inggris
"country_best" : best_row["country_name"], # nama Inggris
# Tambahan: nama Indonesia untuk kedua negara
"country_worst" : worst_row["country_name"],
"country_best" : best_row["country_name"],
"country_worst_id": get_country_name_id(worst_row["country_name"]),
"country_best_id" : get_country_name_id(best_row["country_name"]),
})
@@ -1429,6 +1460,7 @@ class IndicatorNormAggregator:
"layer" : "gold",
"added_columns" : ["country_name_id", "indicator_name_id", "pillar_name_id",
"country_worst_id", "country_best_id"],
"bugfix" : "INDICATOR_NAME_ID_MAP keys lowercased to match .lower() lookup",
}),
"validation_metrics" : json.dumps({
"total_rows" : rows_loaded,
@@ -1453,7 +1485,7 @@ class IndicatorNormAggregator:
self.logger.info(" Output : agg_indicator_norm -> fs_asean_gold")
self.logger.info(" agg_narrative_indicator -> fs_asean_gold")
self.logger.info(" Added : country_name_id, indicator_name_id, pillar_name_id (Bahasa Indonesia)")
self.logger.info(" FIXED : 'Access' -> 'Akses', narrative_id pakai nama ID")
self.logger.info(" BUGFIX : INDICATOR_NAME_ID_MAP keys -> lowercase (cocok dengan .lower() lookup)")
self.logger.info("=" * 80)
self.load_data()