throttle polling outside business hours

Single DAG, single 30-min schedule instead of 5-min or two separate
DAGs. run_sync() itself decides whether to do real work: always during
business hours (default 07:00-17:00 WIB), otherwise only once per
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120) tracked via a
last_full_sync timestamp in the sqlite state. Off-hours skips return
immediately without touching DB/Calendar/Gmail at all.
This commit is contained in:
Power BI Dev
2026-07-27 22:33:04 +07:00
parent 7a6dc7d9ae
commit a357e0622e
5 changed files with 90 additions and 9 deletions
+17 -3
View File
@@ -1,7 +1,19 @@
""" """
DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email. DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email.
Baca DB doxid2022 (read-only, user calendar_sync_ro) tiap 5 menit, lalu: DAG ini trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule — bukan 2 DAG
terpisah untuk jam-kerja/luar-jam-kerja, biar sederhana), tapi `run_sync()`
sendiri yang memutuskan kerja BENERAN atau skip:
- Jam kerja (default 07:00-17:00 WIB, EDOXID_BUSINESS_HOURS_START/END): tiap
trigger (~30 menit) langsung kerja penuh.
- Di luar jam kerja: cuma kerja penuh kalau sudah >=
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120 menit) sejak sync terakhir;
kalau belum, skip (tidak buka koneksi DB/Calendar/Gmail sama sekali, murah).
Alasan: 30 menit sepanjang hari dianggap kurang efisien (bukan soal Google
menganggap ini "bot" — ini OAuth API resmi dengan izin, bukan scraping —
tapi soal efisiensi jumlah run), tapi tetap ingin responsif di jam kerja.
Kerja penuh artinya:
1. create/update/delete event Google Calendar sesuai jadwal sidang yang 1. create/update/delete event Google Calendar sesuai jadwal sidang yang
baru/berubah/dibatalkan. baru/berubah/dibatalkan.
2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan 2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan
@@ -25,7 +37,9 @@ Environment yang wajib di-set di Coolify (service airflow-scheduler):
(refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar) (refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar)
Opsional (lihat scripts/edoxid_calendar_config.py untuk default): Opsional (lihat scripts/edoxid_calendar_config.py untuk default):
EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME, EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME,
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE,
EDOXID_BUSINESS_HOURS_START (default 7), EDOXID_BUSINESS_HOURS_END (default 17),
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120)
""" """
from datetime import datetime, timedelta from datetime import datetime, timedelta
@@ -39,7 +53,7 @@ with DAG(
dag_id="edoxid_calendar_sync", dag_id="edoxid_calendar_sync",
description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'", description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'",
start_date=datetime(2026, 7, 27), start_date=datetime(2026, 7, 27),
schedule_interval="*/5 * * * *", schedule_interval="*/30 * * * *",
catchup=False, catchup=False,
max_active_runs=1, max_active_runs=1,
default_args={ default_args={
+4
View File
@@ -52,6 +52,10 @@ services:
- EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID} - EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID}
- EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET} - EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET}
- EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN} - EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN}
# jam kerja WIB — di luar rentang ini sync beneran cuma jalan tiap N menit, bukan tiap trigger DAG
- EDOXID_BUSINESS_HOURS_START=${EDOXID_BUSINESS_HOURS_START:-7}
- EDOXID_BUSINESS_HOURS_END=${EDOXID_BUSINESS_HOURS_END:-17}
- EDOXID_OFF_HOURS_INTERVAL_MINUTES=${EDOXID_OFF_HOURS_INTERVAL_MINUTES:-120}
extra_hosts: extra_hosts:
- "host.docker.internal:host-gateway" - "host.docker.internal:host-gateway"
volumes: volumes:
+14
View File
@@ -56,3 +56,17 @@ def get_sync_min_date() -> str:
yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja. yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja.
""" """
return os.environ.get("EDOXID_SYNC_MIN_DATE", "today") return os.environ.get("EDOXID_SYNC_MIN_DATE", "today")
def get_business_hours() -> tuple[int, int]:
"""(jam_mulai, jam_selesai) WIB — di luar rentang ini, sync sungguhan cuma
jalan tiap `get_off_hours_interval_minutes()`, bukan tiap trigger DAG.
Tidak perlu 2 DAG terpisah — DAG tetap 1 jadwal (tiap 30 menit), tapi
run_sync() sendiri yang skip kalau lagi di luar jam kerja & belum waktunya."""
start = int(os.environ.get("EDOXID_BUSINESS_HOURS_START", "7"))
end = int(os.environ.get("EDOXID_BUSINESS_HOURS_END", "17"))
return start, end
def get_off_hours_interval_minutes() -> int:
return int(os.environ.get("EDOXID_OFF_HOURS_INTERVAL_MINUTES", "120"))
+17
View File
@@ -25,6 +25,23 @@ class StateStore:
) )
""" """
) )
self._conn.execute(
"CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
)
self._conn.commit()
def get_last_full_sync(self) -> datetime | None:
row = self._conn.execute(
"SELECT value FROM sync_meta WHERE key = 'last_full_sync'"
).fetchone()
return datetime.fromisoformat(row[0]) if row else None
def set_last_full_sync(self, when: datetime) -> None:
self._conn.execute(
"INSERT INTO sync_meta (key, value) VALUES ('last_full_sync', ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(when.isoformat(),),
)
self._conn.commit() self._conn.commit()
def close(self) -> None: def close(self) -> None:
+38 -6
View File
@@ -6,8 +6,10 @@ from datetime import datetime, timedelta
import pytz import pytz
from scripts.edoxid_calendar_config import ( from scripts.edoxid_calendar_config import (
get_business_hours,
get_db_config, get_db_config,
get_google_config, get_google_config,
get_off_hours_interval_minutes,
get_state_sqlite_path, get_state_sqlite_path,
get_sync_min_date, get_sync_min_date,
) )
@@ -27,15 +29,32 @@ logger = logging.getLogger(__name__)
JAKARTA_TZ = pytz.timezone("Asia/Jakarta") JAKARTA_TZ = pytz.timezone("Asia/Jakarta")
# Toleransi jendela waktu untuk cek reminder — harus >= interval polling DAG (5 menit)
# supaya tidak ada window yang "kelewat" di antara 2 run. def _should_run_now(now: datetime, state: StateStore) -> bool:
REMINDER_WINDOW = timedelta(minutes=10) """DAG tetap trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule —
lebih sederhana dari 2 DAG terpisah), tapi kerja BENERAN (baca DB, panggil
Calendar/Gmail API) cuma jalan penuh saat jam kerja, atau di luar jam kerja
kalau sudah >= EDOXID_OFF_HOURS_INTERVAL_MINUTES sejak sync terakhir."""
start, end = get_business_hours()
if start <= now.hour < end:
return True
last = state.get_last_full_sync()
if last is None:
return True
if last.tzinfo is None:
last = JAKARTA_TZ.localize(last)
return now - last >= timedelta(minutes=get_off_hours_interval_minutes())
def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int: def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int:
"""Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee — """Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee —
terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk
calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee).""" calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee)."""
# Toleransi jendela harus >= jarak terbesar antar-run BENERAN (di luar jam
# kerja), supaya H-1/1-jam tidak "kelewat" gara-gara run di antaranya di-skip.
reminder_window = timedelta(minutes=get_off_hours_interval_minutes() + 5)
now = datetime.now(JAKARTA_TZ) now = datetime.now(JAKARTA_TZ)
sent = 0 sent = 0
@@ -47,7 +66,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
if not emails: if not emails:
continue continue
if h1_time <= now < h1_time + REMINDER_WINDOW and not state.is_notified(submission["id"], "h1"): if h1_time <= now < h1_time + reminder_window and not state.is_notified(submission["id"], "h1"):
subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)}{submission['student_name']}" subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)}{submission['student_name']}"
gmail.send(emails, subject, build_description(submission)) gmail.send(emails, subject, build_description(submission))
state.mark_notified(submission["id"], "h1") state.mark_notified(submission["id"], "h1")
@@ -57,7 +76,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
submission["id"], format_submission_summary(submission), ", ".join(emails), submission["id"], format_submission_summary(submission), ", ".join(emails),
) )
if one_hour_time <= now < one_hour_time + REMINDER_WINDOW and not state.is_notified( if one_hour_time <= now < one_hour_time + reminder_window and not state.is_notified(
submission["id"], "1h" submission["id"], "1h"
): ):
subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)}{submission['student_name']}" subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)}{submission['student_name']}"
@@ -73,8 +92,21 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
def run_sync() -> dict: def run_sync() -> dict:
db = Database(get_db_config())
state = StateStore(get_state_sqlite_path()) state = StateStore(get_state_sqlite_path())
now = datetime.now(JAKARTA_TZ)
if not _should_run_now(now, state):
start, end = get_business_hours()
logger.info(
"Di luar jam kerja (%02d:00-%02d:00 WIB) dan belum %s menit sejak sync terakhir — skip run ini.",
start, end, get_off_hours_interval_minutes(),
)
state.close()
return {"skipped_off_hours": True}
state.set_last_full_sync(now)
db = Database(get_db_config())
google_config = get_google_config() google_config = get_google_config()
calendar = GoogleCalendarService(google_config) calendar = GoogleCalendarService(google_config)
gmail = GmailService(google_config) gmail = GmailService(google_config)