throttle polling outside business hours
Single DAG, single 30-min schedule instead of 5-min or two separate DAGs. run_sync() itself decides whether to do real work: always during business hours (default 07:00-17:00 WIB), otherwise only once per EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120) tracked via a last_full_sync timestamp in the sqlite state. Off-hours skips return immediately without touching DB/Calendar/Gmail at all.
This commit is contained in:
@@ -1,7 +1,19 @@
|
||||
"""
|
||||
DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email.
|
||||
|
||||
Baca DB doxid2022 (read-only, user calendar_sync_ro) tiap 5 menit, lalu:
|
||||
DAG ini trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule — bukan 2 DAG
|
||||
terpisah untuk jam-kerja/luar-jam-kerja, biar sederhana), tapi `run_sync()`
|
||||
sendiri yang memutuskan kerja BENERAN atau skip:
|
||||
- Jam kerja (default 07:00-17:00 WIB, EDOXID_BUSINESS_HOURS_START/END): tiap
|
||||
trigger (~30 menit) langsung kerja penuh.
|
||||
- Di luar jam kerja: cuma kerja penuh kalau sudah >=
|
||||
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120 menit) sejak sync terakhir;
|
||||
kalau belum, skip (tidak buka koneksi DB/Calendar/Gmail sama sekali, murah).
|
||||
Alasan: 30 menit sepanjang hari dianggap kurang efisien (bukan soal Google
|
||||
menganggap ini "bot" — ini OAuth API resmi dengan izin, bukan scraping —
|
||||
tapi soal efisiensi jumlah run), tapi tetap ingin responsif di jam kerja.
|
||||
|
||||
Kerja penuh artinya:
|
||||
1. create/update/delete event Google Calendar sesuai jadwal sidang yang
|
||||
baru/berubah/dibatalkan.
|
||||
2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan
|
||||
@@ -25,7 +37,9 @@ Environment yang wajib di-set di Coolify (service airflow-scheduler):
|
||||
(refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar)
|
||||
Opsional (lihat scripts/edoxid_calendar_config.py untuk default):
|
||||
EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME,
|
||||
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE
|
||||
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE,
|
||||
EDOXID_BUSINESS_HOURS_START (default 7), EDOXID_BUSINESS_HOURS_END (default 17),
|
||||
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120)
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
@@ -39,7 +53,7 @@ with DAG(
|
||||
dag_id="edoxid_calendar_sync",
|
||||
description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'",
|
||||
start_date=datetime(2026, 7, 27),
|
||||
schedule_interval="*/5 * * * *",
|
||||
schedule_interval="*/30 * * * *",
|
||||
catchup=False,
|
||||
max_active_runs=1,
|
||||
default_args={
|
||||
|
||||
@@ -52,6 +52,10 @@ services:
|
||||
- EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID}
|
||||
- EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET}
|
||||
- EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN}
|
||||
# jam kerja WIB — di luar rentang ini sync beneran cuma jalan tiap N menit, bukan tiap trigger DAG
|
||||
- EDOXID_BUSINESS_HOURS_START=${EDOXID_BUSINESS_HOURS_START:-7}
|
||||
- EDOXID_BUSINESS_HOURS_END=${EDOXID_BUSINESS_HOURS_END:-17}
|
||||
- EDOXID_OFF_HOURS_INTERVAL_MINUTES=${EDOXID_OFF_HOURS_INTERVAL_MINUTES:-120}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
|
||||
@@ -56,3 +56,17 @@ def get_sync_min_date() -> str:
|
||||
yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja.
|
||||
"""
|
||||
return os.environ.get("EDOXID_SYNC_MIN_DATE", "today")
|
||||
|
||||
|
||||
def get_business_hours() -> tuple[int, int]:
|
||||
"""(jam_mulai, jam_selesai) WIB — di luar rentang ini, sync sungguhan cuma
|
||||
jalan tiap `get_off_hours_interval_minutes()`, bukan tiap trigger DAG.
|
||||
Tidak perlu 2 DAG terpisah — DAG tetap 1 jadwal (tiap 30 menit), tapi
|
||||
run_sync() sendiri yang skip kalau lagi di luar jam kerja & belum waktunya."""
|
||||
start = int(os.environ.get("EDOXID_BUSINESS_HOURS_START", "7"))
|
||||
end = int(os.environ.get("EDOXID_BUSINESS_HOURS_END", "17"))
|
||||
return start, end
|
||||
|
||||
|
||||
def get_off_hours_interval_minutes() -> int:
|
||||
return int(os.environ.get("EDOXID_OFF_HOURS_INTERVAL_MINUTES", "120"))
|
||||
|
||||
@@ -25,6 +25,23 @@ class StateStore:
|
||||
)
|
||||
"""
|
||||
)
|
||||
self._conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def get_last_full_sync(self) -> datetime | None:
|
||||
row = self._conn.execute(
|
||||
"SELECT value FROM sync_meta WHERE key = 'last_full_sync'"
|
||||
).fetchone()
|
||||
return datetime.fromisoformat(row[0]) if row else None
|
||||
|
||||
def set_last_full_sync(self, when: datetime) -> None:
|
||||
self._conn.execute(
|
||||
"INSERT INTO sync_meta (key, value) VALUES ('last_full_sync', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(when.isoformat(),),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def close(self) -> None:
|
||||
|
||||
@@ -6,8 +6,10 @@ from datetime import datetime, timedelta
|
||||
import pytz
|
||||
|
||||
from scripts.edoxid_calendar_config import (
|
||||
get_business_hours,
|
||||
get_db_config,
|
||||
get_google_config,
|
||||
get_off_hours_interval_minutes,
|
||||
get_state_sqlite_path,
|
||||
get_sync_min_date,
|
||||
)
|
||||
@@ -27,15 +29,32 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
JAKARTA_TZ = pytz.timezone("Asia/Jakarta")
|
||||
|
||||
# Toleransi jendela waktu untuk cek reminder — harus >= interval polling DAG (5 menit)
|
||||
# supaya tidak ada window yang "kelewat" di antara 2 run.
|
||||
REMINDER_WINDOW = timedelta(minutes=10)
|
||||
|
||||
def _should_run_now(now: datetime, state: StateStore) -> bool:
|
||||
"""DAG tetap trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule —
|
||||
lebih sederhana dari 2 DAG terpisah), tapi kerja BENERAN (baca DB, panggil
|
||||
Calendar/Gmail API) cuma jalan penuh saat jam kerja, atau di luar jam kerja
|
||||
kalau sudah >= EDOXID_OFF_HOURS_INTERVAL_MINUTES sejak sync terakhir."""
|
||||
start, end = get_business_hours()
|
||||
if start <= now.hour < end:
|
||||
return True
|
||||
|
||||
last = state.get_last_full_sync()
|
||||
if last is None:
|
||||
return True
|
||||
if last.tzinfo is None:
|
||||
last = JAKARTA_TZ.localize(last)
|
||||
|
||||
return now - last >= timedelta(minutes=get_off_hours_interval_minutes())
|
||||
|
||||
|
||||
def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int:
|
||||
"""Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee —
|
||||
terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk
|
||||
calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee)."""
|
||||
# Toleransi jendela harus >= jarak terbesar antar-run BENERAN (di luar jam
|
||||
# kerja), supaya H-1/1-jam tidak "kelewat" gara-gara run di antaranya di-skip.
|
||||
reminder_window = timedelta(minutes=get_off_hours_interval_minutes() + 5)
|
||||
now = datetime.now(JAKARTA_TZ)
|
||||
sent = 0
|
||||
|
||||
@@ -47,7 +66,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
|
||||
if not emails:
|
||||
continue
|
||||
|
||||
if h1_time <= now < h1_time + REMINDER_WINDOW and not state.is_notified(submission["id"], "h1"):
|
||||
if h1_time <= now < h1_time + reminder_window and not state.is_notified(submission["id"], "h1"):
|
||||
subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)} — {submission['student_name']}"
|
||||
gmail.send(emails, subject, build_description(submission))
|
||||
state.mark_notified(submission["id"], "h1")
|
||||
@@ -57,7 +76,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
|
||||
submission["id"], format_submission_summary(submission), ", ".join(emails),
|
||||
)
|
||||
|
||||
if one_hour_time <= now < one_hour_time + REMINDER_WINDOW and not state.is_notified(
|
||||
if one_hour_time <= now < one_hour_time + reminder_window and not state.is_notified(
|
||||
submission["id"], "1h"
|
||||
):
|
||||
subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)} — {submission['student_name']}"
|
||||
@@ -73,8 +92,21 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
|
||||
|
||||
|
||||
def run_sync() -> dict:
|
||||
db = Database(get_db_config())
|
||||
state = StateStore(get_state_sqlite_path())
|
||||
now = datetime.now(JAKARTA_TZ)
|
||||
|
||||
if not _should_run_now(now, state):
|
||||
start, end = get_business_hours()
|
||||
logger.info(
|
||||
"Di luar jam kerja (%02d:00-%02d:00 WIB) dan belum %s menit sejak sync terakhir — skip run ini.",
|
||||
start, end, get_off_hours_interval_minutes(),
|
||||
)
|
||||
state.close()
|
||||
return {"skipped_off_hours": True}
|
||||
|
||||
state.set_last_full_sync(now)
|
||||
|
||||
db = Database(get_db_config())
|
||||
google_config = get_google_config()
|
||||
calendar = GoogleCalendarService(google_config)
|
||||
gmail = GmailService(google_config)
|
||||
|
||||
Reference in New Issue
Block a user