izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add condition at framework and edit narrative indicator

Browse Source
This commit is contained in:
Debby
2026-07-16 09:57:16 +07:00
parent 989d3445a0
commit d21fac69aa
2 changed files with 239 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bigquery_aggregate_layer.py
+56 -3
View File
@@ -25,6 +25,11 @@ KONDISI PILAR (pillar_condition_en / pillar_condition_id):
Kolom tambahan di agg_pillar_by_country untuk mendeskripsikan kondisi
tiap pilar per negara per tahun secara kontekstual dan kuantitatif.
KONDISI FRAMEWORK (framework_condition_en / framework_condition_id):
Kolom tambahan di agg_framework_by_country untuk mendeskripsikan kondisi
tiap framework (MDGs/SDGs/Total) per negara per tahun, mengikuti tier
dan referensi yang sama dengan kondisi pilar.
Landasan teori:
1. FAO & CFS (1996 World Food Summit; CFS Reform Document 2009):
Definisi 4 pilar ketahanan pangan dan makna substantif masing-masing.
@@ -997,7 +1002,10 @@ class FoodSecurityAggregator:
"asean_country_id" : ASEAN_COUNTRY_ID,
"pillar_change" : "Sustainability renamed to Food Other, all pillars prefixed with Food",
"architecture" : "ASEAN merged into country tables (country_id=0)",
"condition_column" : "pillar_condition_en/id added to agg_pillar_by_country",
"condition_column" : (
"pillar_condition_en/id added to agg_pillar_by_country; "
"framework_condition_en/id added to agg_framework_by_country"
),
"condition_reference" : (
"GFSI 2022 (Economist Impact) score tiers >= 75/60/40/20; "
"IPC Technical Manual 2019; FAO/CFS 4-pillar framework 1996/2009; "
@@ -1187,7 +1195,7 @@ class FoodSecurityAggregator:
return df
# =========================================================================
# STEP 3: agg_framework_by_country (termasuk ASEAN)
# STEP 3: agg_framework_by_country (termasuk ASEAN + kolom kondisi)
# =========================================================================
def calc_framework_by_country(self) -> pd.DataFrame:
@@ -1196,6 +1204,7 @@ class FoodSecurityAggregator:
self.logger.info("\n" + "=" * 70)
self.logger.info(f"STEP 3: {table_name} -> [Gold] fs_asean_gold")
self.logger.info(" Termasuk baris ASEAN (country_id=0)")
self.logger.info(" Kolom baru: framework_condition_en, framework_condition_id")
self.logger.info("=" * 70)
try:
@@ -1313,6 +1322,31 @@ class FoodSecurityAggregator:
df = check_and_dedup(df, ["country_id", "framework", "year"], context=table_name, logger=self.logger)
# -----------------------------------------------------------------
# TAMBAHAN: kolom kondisi framework (tier + deskripsi dipisah)
# Dibangkitkan SETELAH framework_score_1_100 tersedia dan SETELAH
# dedup, mengikuti pola yang sama dengan pillar_condition di
# agg_pillar_by_country. Referensi tier: GFSI 2022 (Economist
# Impact); IPC 2019; FAO/CFS 1996/2009; FAO SOFI 2024.
#
# Kolom yang dihasilkan:
# framework_condition_en — label tier EN, e.g. "At Risk"
# framework_condition_id — label tier ID, e.g. "Berisiko"
# framework_condition_desc_en — deskripsi kontekstual EN
# framework_condition_desc_id — deskripsi kontekstual ID
# -----------------------------------------------------------------
fw_conditions = df.apply(
lambda row: get_framework_condition(
row["framework"],
row["framework_score_1_100"]
),
axis=1
)
df["framework_condition_en"] = fw_conditions.apply(lambda x: x[0])
df["framework_condition_id"] = fw_conditions.apply(lambda x: x[1])
df["framework_condition_desc_en"] = fw_conditions.apply(lambda x: x[2])
df["framework_condition_desc_id"] = fw_conditions.apply(lambda x: x[3])
country_mask = df["country_id"] != ASEAN_COUNTRY_ID
df.loc[country_mask, "rank_in_framework_year"] = (
df[country_mask]
@@ -1331,6 +1365,18 @@ class FoodSecurityAggregator:
df["framework_norm"] = df["framework_norm"].astype(float)
df["framework_score_1_100"] = df["framework_score_1_100"].astype(float)
df["country_name_id"] = df["country_name_id"].astype(str)
df["framework_condition_en"] = df["framework_condition_en"].astype(str)
df["framework_condition_id"] = df["framework_condition_id"].astype(str)
df["framework_condition_desc_en"] = df["framework_condition_desc_en"].astype(str)
df["framework_condition_desc_id"] = df["framework_condition_desc_id"].astype(str)
self.logger.info(f"\n Total rows: {len(df):,}")
# Log distribusi kondisi untuk QA
self.logger.info("\n Distribusi framework_condition_en (sample):")
fw_cond_dist = df["framework_condition_en"].value_counts().head(10)
for cond, cnt in fw_cond_dist.items():
self.logger.info(f" {cnt:>6,} {cond}")
schema = [
bigquery.SchemaField("country_id", "INTEGER", mode="REQUIRED"),
@@ -1343,6 +1389,13 @@ class FoodSecurityAggregator:
bigquery.SchemaField("framework_score_1_100", "FLOAT", mode="REQUIRED"),
bigquery.SchemaField("rank_in_framework_year", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("year_over_year_change", "FLOAT", mode="NULLABLE"),
# --- KOLOM KONDISI (sama pola dengan agg_pillar_by_country) ---
# Tier label (GFSI 2022): Secure / Adequate / Moderate / At Risk / Critical
bigquery.SchemaField("framework_condition_en", "STRING", mode="REQUIRED"),
bigquery.SchemaField("framework_condition_id", "STRING", mode="REQUIRED"),
# Deskripsi kontekstual per framework (MDGs/SDGs/Total)
bigquery.SchemaField("framework_condition_desc_en", "STRING", mode="REQUIRED"),
bigquery.SchemaField("framework_condition_desc_id", "STRING", mode="REQUIRED"),
]
rows = load_to_bigquery(
self.client, df, table_name, layer='gold',
@@ -1600,7 +1653,7 @@ class FoodSecurityAggregator:
self.logger.info("\n" + "=" * 70)
self.logger.info("FOOD SECURITY AGGREGATION — 3 TABLES -> fs_asean_gold")
self.logger.info(" ASEAN aggregate DIGABUNG ke tabel yang sama (country_id=0)")
self.logger.info(" Kolom baru : pillar_condition_en, pillar_condition_id")
self.logger.info(" Kolom baru : pillar_condition_en/id, framework_condition_en/id")
self.logger.info(f" Performance threshold: {PERFORMANCE_THRESHOLD}")
self.logger.info(f" Condition tiers (GFSI 2022): >=75 Secure | >=60 Adequate |")
self.logger.info(f" >=40 Moderate | >=20 At Risk | <20 Critical")