From a357e0622e5ba9612eba2947cd07d4e1e4c1f3f6 Mon Sep 17 00:00:00 2001 From: Power BI Dev Date: Mon, 27 Jul 2026 22:33:04 +0700 Subject: [PATCH] throttle polling outside business hours Single DAG, single 30-min schedule instead of 5-min or two separate DAGs. run_sync() itself decides whether to do real work: always during business hours (default 07:00-17:00 WIB), otherwise only once per EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120) tracked via a last_full_sync timestamp in the sqlite state. Off-hours skips return immediately without touching DB/Calendar/Gmail at all. --- dags/edoxid_calendar_sync.py | 20 ++++++++++-- docker-compose.yaml | 4 +++ scripts/edoxid_calendar_config.py | 14 ++++++++ scripts/edoxid_calendar_state_store.py | 17 ++++++++++ scripts/edoxid_calendar_sync.py | 44 ++++++++++++++++++++++---- 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/dags/edoxid_calendar_sync.py b/dags/edoxid_calendar_sync.py index f88c56b..09abf64 100644 --- a/dags/edoxid_calendar_sync.py +++ b/dags/edoxid_calendar_sync.py @@ -1,7 +1,19 @@ """ DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email. -Baca DB doxid2022 (read-only, user calendar_sync_ro) tiap 5 menit, lalu: +DAG ini trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule — bukan 2 DAG +terpisah untuk jam-kerja/luar-jam-kerja, biar sederhana), tapi `run_sync()` +sendiri yang memutuskan kerja BENERAN atau skip: + - Jam kerja (default 07:00-17:00 WIB, EDOXID_BUSINESS_HOURS_START/END): tiap + trigger (~30 menit) langsung kerja penuh. + - Di luar jam kerja: cuma kerja penuh kalau sudah >= + EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120 menit) sejak sync terakhir; + kalau belum, skip (tidak buka koneksi DB/Calendar/Gmail sama sekali, murah). + Alasan: 30 menit sepanjang hari dianggap kurang efisien (bukan soal Google + menganggap ini "bot" — ini OAuth API resmi dengan izin, bukan scraping — + tapi soal efisiensi jumlah run), tapi tetap ingin responsif di jam kerja. + +Kerja penuh artinya: 1. create/update/delete event Google Calendar sesuai jadwal sidang yang baru/berubah/dibatalkan. 2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan @@ -25,7 +37,9 @@ Environment yang wajib di-set di Coolify (service airflow-scheduler): (refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar) Opsional (lihat scripts/edoxid_calendar_config.py untuk default): EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME, - EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE + EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE, + EDOXID_BUSINESS_HOURS_START (default 7), EDOXID_BUSINESS_HOURS_END (default 17), + EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120) """ from datetime import datetime, timedelta @@ -39,7 +53,7 @@ with DAG( dag_id="edoxid_calendar_sync", description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'", start_date=datetime(2026, 7, 27), - schedule_interval="*/5 * * * *", + schedule_interval="*/30 * * * *", catchup=False, max_active_runs=1, default_args={ diff --git a/docker-compose.yaml b/docker-compose.yaml index 71bdd61..f090051 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -52,6 +52,10 @@ services: - EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID} - EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET} - EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN} + # jam kerja WIB — di luar rentang ini sync beneran cuma jalan tiap N menit, bukan tiap trigger DAG + - EDOXID_BUSINESS_HOURS_START=${EDOXID_BUSINESS_HOURS_START:-7} + - EDOXID_BUSINESS_HOURS_END=${EDOXID_BUSINESS_HOURS_END:-17} + - EDOXID_OFF_HOURS_INTERVAL_MINUTES=${EDOXID_OFF_HOURS_INTERVAL_MINUTES:-120} extra_hosts: - "host.docker.internal:host-gateway" volumes: diff --git a/scripts/edoxid_calendar_config.py b/scripts/edoxid_calendar_config.py index 9b6de87..23a3f20 100644 --- a/scripts/edoxid_calendar_config.py +++ b/scripts/edoxid_calendar_config.py @@ -56,3 +56,17 @@ def get_sync_min_date() -> str: yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja. """ return os.environ.get("EDOXID_SYNC_MIN_DATE", "today") + + +def get_business_hours() -> tuple[int, int]: + """(jam_mulai, jam_selesai) WIB — di luar rentang ini, sync sungguhan cuma + jalan tiap `get_off_hours_interval_minutes()`, bukan tiap trigger DAG. + Tidak perlu 2 DAG terpisah — DAG tetap 1 jadwal (tiap 30 menit), tapi + run_sync() sendiri yang skip kalau lagi di luar jam kerja & belum waktunya.""" + start = int(os.environ.get("EDOXID_BUSINESS_HOURS_START", "7")) + end = int(os.environ.get("EDOXID_BUSINESS_HOURS_END", "17")) + return start, end + + +def get_off_hours_interval_minutes() -> int: + return int(os.environ.get("EDOXID_OFF_HOURS_INTERVAL_MINUTES", "120")) diff --git a/scripts/edoxid_calendar_state_store.py b/scripts/edoxid_calendar_state_store.py index 2161d03..dff1f5b 100644 --- a/scripts/edoxid_calendar_state_store.py +++ b/scripts/edoxid_calendar_state_store.py @@ -25,6 +25,23 @@ class StateStore: ) """ ) + self._conn.execute( + "CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + self._conn.commit() + + def get_last_full_sync(self) -> datetime | None: + row = self._conn.execute( + "SELECT value FROM sync_meta WHERE key = 'last_full_sync'" + ).fetchone() + return datetime.fromisoformat(row[0]) if row else None + + def set_last_full_sync(self, when: datetime) -> None: + self._conn.execute( + "INSERT INTO sync_meta (key, value) VALUES ('last_full_sync', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (when.isoformat(),), + ) self._conn.commit() def close(self) -> None: diff --git a/scripts/edoxid_calendar_sync.py b/scripts/edoxid_calendar_sync.py index 2c49242..d6de93d 100644 --- a/scripts/edoxid_calendar_sync.py +++ b/scripts/edoxid_calendar_sync.py @@ -6,8 +6,10 @@ from datetime import datetime, timedelta import pytz from scripts.edoxid_calendar_config import ( + get_business_hours, get_db_config, get_google_config, + get_off_hours_interval_minutes, get_state_sqlite_path, get_sync_min_date, ) @@ -27,15 +29,32 @@ logger = logging.getLogger(__name__) JAKARTA_TZ = pytz.timezone("Asia/Jakarta") -# Toleransi jendela waktu untuk cek reminder — harus >= interval polling DAG (5 menit) -# supaya tidak ada window yang "kelewat" di antara 2 run. -REMINDER_WINDOW = timedelta(minutes=10) + +def _should_run_now(now: datetime, state: StateStore) -> bool: + """DAG tetap trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule — + lebih sederhana dari 2 DAG terpisah), tapi kerja BENERAN (baca DB, panggil + Calendar/Gmail API) cuma jalan penuh saat jam kerja, atau di luar jam kerja + kalau sudah >= EDOXID_OFF_HOURS_INTERVAL_MINUTES sejak sync terakhir.""" + start, end = get_business_hours() + if start <= now.hour < end: + return True + + last = state.get_last_full_sync() + if last is None: + return True + if last.tzinfo is None: + last = JAKARTA_TZ.localize(last) + + return now - last >= timedelta(minutes=get_off_hours_interval_minutes()) def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int: """Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee — terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee).""" + # Toleransi jendela harus >= jarak terbesar antar-run BENERAN (di luar jam + # kerja), supaya H-1/1-jam tidak "kelewat" gara-gara run di antaranya di-skip. + reminder_window = timedelta(minutes=get_off_hours_interval_minutes() + 5) now = datetime.now(JAKARTA_TZ) sent = 0 @@ -47,7 +66,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: if not emails: continue - if h1_time <= now < h1_time + REMINDER_WINDOW and not state.is_notified(submission["id"], "h1"): + if h1_time <= now < h1_time + reminder_window and not state.is_notified(submission["id"], "h1"): subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)} — {submission['student_name']}" gmail.send(emails, subject, build_description(submission)) state.mark_notified(submission["id"], "h1") @@ -57,7 +76,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: submission["id"], format_submission_summary(submission), ", ".join(emails), ) - if one_hour_time <= now < one_hour_time + REMINDER_WINDOW and not state.is_notified( + if one_hour_time <= now < one_hour_time + reminder_window and not state.is_notified( submission["id"], "1h" ): subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)} — {submission['student_name']}" @@ -73,8 +92,21 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: def run_sync() -> dict: - db = Database(get_db_config()) state = StateStore(get_state_sqlite_path()) + now = datetime.now(JAKARTA_TZ) + + if not _should_run_now(now, state): + start, end = get_business_hours() + logger.info( + "Di luar jam kerja (%02d:00-%02d:00 WIB) dan belum %s menit sejak sync terakhir — skip run ini.", + start, end, get_off_hours_interval_minutes(), + ) + state.close() + return {"skipped_off_hours": True} + + state.set_last_full_sync(now) + + db = Database(get_db_config()) google_config = get_google_config() calendar = GoogleCalendarService(google_config) gmail = GmailService(google_config)