throttle polling outside business hours

Single DAG, single 30-min schedule instead of 5-min or two separate
DAGs. run_sync() itself decides whether to do real work: always during
business hours (default 07:00-17:00 WIB), otherwise only once per
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120) tracked via a
last_full_sync timestamp in the sqlite state. Off-hours skips return
immediately without touching DB/Calendar/Gmail at all.
This commit is contained in:
Power BI Dev
2026-07-27 22:33:04 +07:00
parent 7a6dc7d9ae
commit a357e0622e
5 changed files with 90 additions and 9 deletions
+17 -3
View File
@@ -1,7 +1,19 @@
"""
DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email.
Baca DB doxid2022 (read-only, user calendar_sync_ro) tiap 5 menit, lalu:
DAG ini trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule — bukan 2 DAG
terpisah untuk jam-kerja/luar-jam-kerja, biar sederhana), tapi `run_sync()`
sendiri yang memutuskan kerja BENERAN atau skip:
- Jam kerja (default 07:00-17:00 WIB, EDOXID_BUSINESS_HOURS_START/END): tiap
trigger (~30 menit) langsung kerja penuh.
- Di luar jam kerja: cuma kerja penuh kalau sudah >=
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120 menit) sejak sync terakhir;
kalau belum, skip (tidak buka koneksi DB/Calendar/Gmail sama sekali, murah).
Alasan: 30 menit sepanjang hari dianggap kurang efisien (bukan soal Google
menganggap ini "bot" — ini OAuth API resmi dengan izin, bukan scraping —
tapi soal efisiensi jumlah run), tapi tetap ingin responsif di jam kerja.
Kerja penuh artinya:
1. create/update/delete event Google Calendar sesuai jadwal sidang yang
baru/berubah/dibatalkan.
2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan
@@ -25,7 +37,9 @@ Environment yang wajib di-set di Coolify (service airflow-scheduler):
(refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar)
Opsional (lihat scripts/edoxid_calendar_config.py untuk default):
EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME,
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE,
EDOXID_BUSINESS_HOURS_START (default 7), EDOXID_BUSINESS_HOURS_END (default 17),
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120)
"""
from datetime import datetime, timedelta
@@ -39,7 +53,7 @@ with DAG(
dag_id="edoxid_calendar_sync",
description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'",
start_date=datetime(2026, 7, 27),
schedule_interval="*/5 * * * *",
schedule_interval="*/30 * * * *",
catchup=False,
max_active_runs=1,
default_args={
+4
View File
@@ -52,6 +52,10 @@ services:
- EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID}
- EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET}
- EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN}
# jam kerja WIB — di luar rentang ini sync beneran cuma jalan tiap N menit, bukan tiap trigger DAG
- EDOXID_BUSINESS_HOURS_START=${EDOXID_BUSINESS_HOURS_START:-7}
- EDOXID_BUSINESS_HOURS_END=${EDOXID_BUSINESS_HOURS_END:-17}
- EDOXID_OFF_HOURS_INTERVAL_MINUTES=${EDOXID_OFF_HOURS_INTERVAL_MINUTES:-120}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
+14
View File
@@ -56,3 +56,17 @@ def get_sync_min_date() -> str:
yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja.
"""
return os.environ.get("EDOXID_SYNC_MIN_DATE", "today")
def get_business_hours() -> tuple[int, int]:
"""(jam_mulai, jam_selesai) WIB — di luar rentang ini, sync sungguhan cuma
jalan tiap `get_off_hours_interval_minutes()`, bukan tiap trigger DAG.
Tidak perlu 2 DAG terpisah — DAG tetap 1 jadwal (tiap 30 menit), tapi
run_sync() sendiri yang skip kalau lagi di luar jam kerja & belum waktunya."""
start = int(os.environ.get("EDOXID_BUSINESS_HOURS_START", "7"))
end = int(os.environ.get("EDOXID_BUSINESS_HOURS_END", "17"))
return start, end
def get_off_hours_interval_minutes() -> int:
return int(os.environ.get("EDOXID_OFF_HOURS_INTERVAL_MINUTES", "120"))
+17
View File
@@ -25,6 +25,23 @@ class StateStore:
)
"""
)
self._conn.execute(
"CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
)
self._conn.commit()
def get_last_full_sync(self) -> datetime | None:
row = self._conn.execute(
"SELECT value FROM sync_meta WHERE key = 'last_full_sync'"
).fetchone()
return datetime.fromisoformat(row[0]) if row else None
def set_last_full_sync(self, when: datetime) -> None:
self._conn.execute(
"INSERT INTO sync_meta (key, value) VALUES ('last_full_sync', ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(when.isoformat(),),
)
self._conn.commit()
def close(self) -> None:
+38 -6
View File
@@ -6,8 +6,10 @@ from datetime import datetime, timedelta
import pytz
from scripts.edoxid_calendar_config import (
get_business_hours,
get_db_config,
get_google_config,
get_off_hours_interval_minutes,
get_state_sqlite_path,
get_sync_min_date,
)
@@ -27,15 +29,32 @@ logger = logging.getLogger(__name__)
JAKARTA_TZ = pytz.timezone("Asia/Jakarta")
# Toleransi jendela waktu untuk cek reminder — harus >= interval polling DAG (5 menit)
# supaya tidak ada window yang "kelewat" di antara 2 run.
REMINDER_WINDOW = timedelta(minutes=10)
def _should_run_now(now: datetime, state: StateStore) -> bool:
"""DAG tetap trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule —
lebih sederhana dari 2 DAG terpisah), tapi kerja BENERAN (baca DB, panggil
Calendar/Gmail API) cuma jalan penuh saat jam kerja, atau di luar jam kerja
kalau sudah >= EDOXID_OFF_HOURS_INTERVAL_MINUTES sejak sync terakhir."""
start, end = get_business_hours()
if start <= now.hour < end:
return True
last = state.get_last_full_sync()
if last is None:
return True
if last.tzinfo is None:
last = JAKARTA_TZ.localize(last)
return now - last >= timedelta(minutes=get_off_hours_interval_minutes())
def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int:
"""Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee —
terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk
calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee)."""
# Toleransi jendela harus >= jarak terbesar antar-run BENERAN (di luar jam
# kerja), supaya H-1/1-jam tidak "kelewat" gara-gara run di antaranya di-skip.
reminder_window = timedelta(minutes=get_off_hours_interval_minutes() + 5)
now = datetime.now(JAKARTA_TZ)
sent = 0
@@ -47,7 +66,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
if not emails:
continue
if h1_time <= now < h1_time + REMINDER_WINDOW and not state.is_notified(submission["id"], "h1"):
if h1_time <= now < h1_time + reminder_window and not state.is_notified(submission["id"], "h1"):
subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)}{submission['student_name']}"
gmail.send(emails, subject, build_description(submission))
state.mark_notified(submission["id"], "h1")
@@ -57,7 +76,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
submission["id"], format_submission_summary(submission), ", ".join(emails),
)
if one_hour_time <= now < one_hour_time + REMINDER_WINDOW and not state.is_notified(
if one_hour_time <= now < one_hour_time + reminder_window and not state.is_notified(
submission["id"], "1h"
):
subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)}{submission['student_name']}"
@@ -73,8 +92,21 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
def run_sync() -> dict:
db = Database(get_db_config())
state = StateStore(get_state_sqlite_path())
now = datetime.now(JAKARTA_TZ)
if not _should_run_now(now, state):
start, end = get_business_hours()
logger.info(
"Di luar jam kerja (%02d:00-%02d:00 WIB) dan belum %s menit sejak sync terakhir — skip run ini.",
start, end, get_off_hours_interval_minutes(),
)
state.close()
return {"skipped_off_hours": True}
state.set_last_full_sync(now)
db = Database(get_db_config())
google_config = get_google_config()
calendar = GoogleCalendarService(google_config)
gmail = GmailService(google_config)