throttle polling outside business hours
Single DAG, single 30-min schedule instead of 5-min or two separate DAGs. run_sync() itself decides whether to do real work: always during business hours (default 07:00-17:00 WIB), otherwise only once per EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120) tracked via a last_full_sync timestamp in the sqlite state. Off-hours skips return immediately without touching DB/Calendar/Gmail at all.
This commit is contained in:
@@ -1,7 +1,19 @@
|
|||||||
"""
|
"""
|
||||||
DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email.
|
DAG: sync jadwal sidang eDOXID -> Google Calendar "Jadwal Sidang" + reminder email.
|
||||||
|
|
||||||
Baca DB doxid2022 (read-only, user calendar_sync_ro) tiap 5 menit, lalu:
|
DAG ini trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule — bukan 2 DAG
|
||||||
|
terpisah untuk jam-kerja/luar-jam-kerja, biar sederhana), tapi `run_sync()`
|
||||||
|
sendiri yang memutuskan kerja BENERAN atau skip:
|
||||||
|
- Jam kerja (default 07:00-17:00 WIB, EDOXID_BUSINESS_HOURS_START/END): tiap
|
||||||
|
trigger (~30 menit) langsung kerja penuh.
|
||||||
|
- Di luar jam kerja: cuma kerja penuh kalau sudah >=
|
||||||
|
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120 menit) sejak sync terakhir;
|
||||||
|
kalau belum, skip (tidak buka koneksi DB/Calendar/Gmail sama sekali, murah).
|
||||||
|
Alasan: 30 menit sepanjang hari dianggap kurang efisien (bukan soal Google
|
||||||
|
menganggap ini "bot" — ini OAuth API resmi dengan izin, bukan scraping —
|
||||||
|
tapi soal efisiensi jumlah run), tapi tetap ingin responsif di jam kerja.
|
||||||
|
|
||||||
|
Kerja penuh artinya:
|
||||||
1. create/update/delete event Google Calendar sesuai jadwal sidang yang
|
1. create/update/delete event Google Calendar sesuai jadwal sidang yang
|
||||||
baru/berubah/dibatalkan.
|
baru/berubah/dibatalkan.
|
||||||
2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan
|
2. Kirim email reminder terpisah (subjek "[PENGINGAT SEBELUM H-1]" dan
|
||||||
@@ -25,7 +37,9 @@ Environment yang wajib di-set di Coolify (service airflow-scheduler):
|
|||||||
(refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar)
|
(refresh token WAJIB scope "calendar" + "gmail.send" sekaligus, bukan cuma calendar)
|
||||||
Opsional (lihat scripts/edoxid_calendar_config.py untuk default):
|
Opsional (lihat scripts/edoxid_calendar_config.py untuk default):
|
||||||
EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME,
|
EDOXID_DB_HOST, EDOXID_DB_PORT, EDOXID_DB_DATABASE, EDOXID_DB_USERNAME,
|
||||||
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE
|
EDOXID_GOOGLE_CALENDAR_ID, EDOXID_STATE_SQLITE_PATH, EDOXID_SYNC_MIN_DATE,
|
||||||
|
EDOXID_BUSINESS_HOURS_START (default 7), EDOXID_BUSINESS_HOURS_END (default 17),
|
||||||
|
EDOXID_OFF_HOURS_INTERVAL_MINUTES (default 120)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
@@ -39,7 +53,7 @@ with DAG(
|
|||||||
dag_id="edoxid_calendar_sync",
|
dag_id="edoxid_calendar_sync",
|
||||||
description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'",
|
description="Sync jadwal sidang eDOXID ke Google Calendar 'Jadwal Sidang'",
|
||||||
start_date=datetime(2026, 7, 27),
|
start_date=datetime(2026, 7, 27),
|
||||||
schedule_interval="*/5 * * * *",
|
schedule_interval="*/30 * * * *",
|
||||||
catchup=False,
|
catchup=False,
|
||||||
max_active_runs=1,
|
max_active_runs=1,
|
||||||
default_args={
|
default_args={
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ services:
|
|||||||
- EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID}
|
- EDOXID_GOOGLE_CLIENT_ID=${EDOXID_GOOGLE_CLIENT_ID}
|
||||||
- EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET}
|
- EDOXID_GOOGLE_CLIENT_SECRET=${EDOXID_GOOGLE_CLIENT_SECRET}
|
||||||
- EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN}
|
- EDOXID_GOOGLE_REFRESH_TOKEN=${EDOXID_GOOGLE_REFRESH_TOKEN}
|
||||||
|
# jam kerja WIB — di luar rentang ini sync beneran cuma jalan tiap N menit, bukan tiap trigger DAG
|
||||||
|
- EDOXID_BUSINESS_HOURS_START=${EDOXID_BUSINESS_HOURS_START:-7}
|
||||||
|
- EDOXID_BUSINESS_HOURS_END=${EDOXID_BUSINESS_HOURS_END:-17}
|
||||||
|
- EDOXID_OFF_HOURS_INTERVAL_MINUTES=${EDOXID_OFF_HOURS_INTERVAL_MINUTES:-120}
|
||||||
extra_hosts:
|
extra_hosts:
|
||||||
- "host.docker.internal:host-gateway"
|
- "host.docker.internal:host-gateway"
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -56,3 +56,17 @@ def get_sync_min_date() -> str:
|
|||||||
yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja.
|
yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja.
|
||||||
"""
|
"""
|
||||||
return os.environ.get("EDOXID_SYNC_MIN_DATE", "today")
|
return os.environ.get("EDOXID_SYNC_MIN_DATE", "today")
|
||||||
|
|
||||||
|
|
||||||
|
def get_business_hours() -> tuple[int, int]:
|
||||||
|
"""(jam_mulai, jam_selesai) WIB — di luar rentang ini, sync sungguhan cuma
|
||||||
|
jalan tiap `get_off_hours_interval_minutes()`, bukan tiap trigger DAG.
|
||||||
|
Tidak perlu 2 DAG terpisah — DAG tetap 1 jadwal (tiap 30 menit), tapi
|
||||||
|
run_sync() sendiri yang skip kalau lagi di luar jam kerja & belum waktunya."""
|
||||||
|
start = int(os.environ.get("EDOXID_BUSINESS_HOURS_START", "7"))
|
||||||
|
end = int(os.environ.get("EDOXID_BUSINESS_HOURS_END", "17"))
|
||||||
|
return start, end
|
||||||
|
|
||||||
|
|
||||||
|
def get_off_hours_interval_minutes() -> int:
|
||||||
|
return int(os.environ.get("EDOXID_OFF_HOURS_INTERVAL_MINUTES", "120"))
|
||||||
|
|||||||
@@ -25,6 +25,23 @@ class StateStore:
|
|||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
self._conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def get_last_full_sync(self) -> datetime | None:
|
||||||
|
row = self._conn.execute(
|
||||||
|
"SELECT value FROM sync_meta WHERE key = 'last_full_sync'"
|
||||||
|
).fetchone()
|
||||||
|
return datetime.fromisoformat(row[0]) if row else None
|
||||||
|
|
||||||
|
def set_last_full_sync(self, when: datetime) -> None:
|
||||||
|
self._conn.execute(
|
||||||
|
"INSERT INTO sync_meta (key, value) VALUES ('last_full_sync', ?) "
|
||||||
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||||
|
(when.isoformat(),),
|
||||||
|
)
|
||||||
self._conn.commit()
|
self._conn.commit()
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ from datetime import datetime, timedelta
|
|||||||
import pytz
|
import pytz
|
||||||
|
|
||||||
from scripts.edoxid_calendar_config import (
|
from scripts.edoxid_calendar_config import (
|
||||||
|
get_business_hours,
|
||||||
get_db_config,
|
get_db_config,
|
||||||
get_google_config,
|
get_google_config,
|
||||||
|
get_off_hours_interval_minutes,
|
||||||
get_state_sqlite_path,
|
get_state_sqlite_path,
|
||||||
get_sync_min_date,
|
get_sync_min_date,
|
||||||
)
|
)
|
||||||
@@ -27,15 +29,32 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
JAKARTA_TZ = pytz.timezone("Asia/Jakarta")
|
JAKARTA_TZ = pytz.timezone("Asia/Jakarta")
|
||||||
|
|
||||||
# Toleransi jendela waktu untuk cek reminder — harus >= interval polling DAG (5 menit)
|
|
||||||
# supaya tidak ada window yang "kelewat" di antara 2 run.
|
def _should_run_now(now: datetime, state: StateStore) -> bool:
|
||||||
REMINDER_WINDOW = timedelta(minutes=10)
|
"""DAG tetap trigger tiap 30 menit sepanjang hari (1 DAG, 1 schedule —
|
||||||
|
lebih sederhana dari 2 DAG terpisah), tapi kerja BENERAN (baca DB, panggil
|
||||||
|
Calendar/Gmail API) cuma jalan penuh saat jam kerja, atau di luar jam kerja
|
||||||
|
kalau sudah >= EDOXID_OFF_HOURS_INTERVAL_MINUTES sejak sync terakhir."""
|
||||||
|
start, end = get_business_hours()
|
||||||
|
if start <= now.hour < end:
|
||||||
|
return True
|
||||||
|
|
||||||
|
last = state.get_last_full_sync()
|
||||||
|
if last is None:
|
||||||
|
return True
|
||||||
|
if last.tzinfo is None:
|
||||||
|
last = JAKARTA_TZ.localize(last)
|
||||||
|
|
||||||
|
return now - last >= timedelta(minutes=get_off_hours_interval_minutes())
|
||||||
|
|
||||||
|
|
||||||
def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int:
|
def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int:
|
||||||
"""Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee —
|
"""Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee —
|
||||||
terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk
|
terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk
|
||||||
calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee)."""
|
calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee)."""
|
||||||
|
# Toleransi jendela harus >= jarak terbesar antar-run BENERAN (di luar jam
|
||||||
|
# kerja), supaya H-1/1-jam tidak "kelewat" gara-gara run di antaranya di-skip.
|
||||||
|
reminder_window = timedelta(minutes=get_off_hours_interval_minutes() + 5)
|
||||||
now = datetime.now(JAKARTA_TZ)
|
now = datetime.now(JAKARTA_TZ)
|
||||||
sent = 0
|
sent = 0
|
||||||
|
|
||||||
@@ -47,7 +66,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
|
|||||||
if not emails:
|
if not emails:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if h1_time <= now < h1_time + REMINDER_WINDOW and not state.is_notified(submission["id"], "h1"):
|
if h1_time <= now < h1_time + reminder_window and not state.is_notified(submission["id"], "h1"):
|
||||||
subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)} — {submission['student_name']}"
|
subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)} — {submission['student_name']}"
|
||||||
gmail.send(emails, subject, build_description(submission))
|
gmail.send(emails, subject, build_description(submission))
|
||||||
state.mark_notified(submission["id"], "h1")
|
state.mark_notified(submission["id"], "h1")
|
||||||
@@ -57,7 +76,7 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
|
|||||||
submission["id"], format_submission_summary(submission), ", ".join(emails),
|
submission["id"], format_submission_summary(submission), ", ".join(emails),
|
||||||
)
|
)
|
||||||
|
|
||||||
if one_hour_time <= now < one_hour_time + REMINDER_WINDOW and not state.is_notified(
|
if one_hour_time <= now < one_hour_time + reminder_window and not state.is_notified(
|
||||||
submission["id"], "1h"
|
submission["id"], "1h"
|
||||||
):
|
):
|
||||||
subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)} — {submission['student_name']}"
|
subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)} — {submission['student_name']}"
|
||||||
@@ -73,8 +92,21 @@ def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail:
|
|||||||
|
|
||||||
|
|
||||||
def run_sync() -> dict:
|
def run_sync() -> dict:
|
||||||
db = Database(get_db_config())
|
|
||||||
state = StateStore(get_state_sqlite_path())
|
state = StateStore(get_state_sqlite_path())
|
||||||
|
now = datetime.now(JAKARTA_TZ)
|
||||||
|
|
||||||
|
if not _should_run_now(now, state):
|
||||||
|
start, end = get_business_hours()
|
||||||
|
logger.info(
|
||||||
|
"Di luar jam kerja (%02d:00-%02d:00 WIB) dan belum %s menit sejak sync terakhir — skip run ini.",
|
||||||
|
start, end, get_off_hours_interval_minutes(),
|
||||||
|
)
|
||||||
|
state.close()
|
||||||
|
return {"skipped_off_hours": True}
|
||||||
|
|
||||||
|
state.set_last_full_sync(now)
|
||||||
|
|
||||||
|
db = Database(get_db_config())
|
||||||
google_config = get_google_config()
|
google_config = get_google_config()
|
||||||
calendar = GoogleCalendarService(google_config)
|
calendar = GoogleCalendarService(google_config)
|
||||||
gmail = GmailService(google_config)
|
gmail = GmailService(google_config)
|
||||||
|
|||||||
Reference in New Issue
Block a user