izu/airflow-coolify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add edoxid calendar sync dag

Browse Source 
Sync jadwal sidang eDOXID ke Google Calendar "Jadwal Sidang" tiap 5 menit
(read-only ke doxid2022), plus reminder email H-1/1-jam via Gmail API
karena reminder Calendar bawaan tidak ikut ke kalender pribadi attendee.
This commit is contained in:
Power BI Dev
2026-07-27 21:59:27 +07:00
parent c6edee7aeb
commit 33f4666bad
9 changed files with 658 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/edoxid_calendar_config.py
+56
View File
@@ -0,0 +1,56 @@
import os
from dataclasses import dataclass
@dataclass(frozen=True)
class DbConfig:
host: str
port: int
database: str
username: str
password: str
@dataclass(frozen=True)
class GoogleConfig:
client_id: str
client_secret: str
refresh_token: str
calendar_id: str
def get_db_config() -> DbConfig:
return DbConfig(
host=os.environ.get("EDOXID_DB_HOST", "203.24.50.52"),
port=int(os.environ.get("EDOXID_DB_PORT", "3306")),
database=os.environ.get("EDOXID_DB_DATABASE", "doxid2022"),
username=os.environ.get("EDOXID_DB_USERNAME", "calendar_sync_ro"),
password=os.environ["EDOXID_DB_PASSWORD"],
)
def get_google_config() -> GoogleConfig:
return GoogleConfig(
client_id=os.environ["EDOXID_GOOGLE_CLIENT_ID"],
client_secret=os.environ["EDOXID_GOOGLE_CLIENT_SECRET"],
refresh_token=os.environ["EDOXID_GOOGLE_REFRESH_TOKEN"],
# Calendar "Jadwal Sidang" milik tu@informatika.untan.ac.id
calendar_id=os.environ.get(
"EDOXID_GOOGLE_CALENDAR_ID",
"c_1a4ilomn51i1r0mccnjqe9cumg@group.calendar.google.com",
),
)
def get_state_sqlite_path() -> str:
return os.environ.get("EDOXID_STATE_SQLITE_PATH", "/opt/airflow/scripts/edoxid_calendar_state.sqlite")
def get_sync_min_date() -> str:
"""Batas bawah tanggal sidang yang disinkron, format 'YYYY-MM-DD'.
Default: hari ini — supaya run pertama TIDAK langsung mengirim invite
Calendar ke semua submission lama (2019-2026, ~1156 baris per cek 2026-07-27)
yang sudah punya schedule. Ubah lewat env var kalau memang mau backfill sengaja.
"""
return os.environ.get("EDOXID_SYNC_MIN_DATE", "today")
scripts/edoxid_calendar_database.py
+93
View File
@@ -0,0 +1,93 @@
from datetime import date, datetime
import pymysql
import pymysql.cursors
from scripts.edoxid_calendar_config import DbConfig
class Database:
"""Koneksi READ-ONLY ke doxid2022 (DB aplikasi eDOXID) pakai user
`calendar_sync_ro` yang cuma punya grant SELECT ke submissions/verificators/users.
Tidak pernah menulis apa pun ke DB ini."""
def __init__(self, config: DbConfig):
self._conn = pymysql.connect(
host=config.host,
port=config.port,
user=config.username,
password=config.password,
database=config.database,
cursorclass=pymysql.cursors.DictCursor,
autocommit=True,
)
def close(self) -> None:
self._conn.close()
def get_scheduled_submissions(self, min_date: str = "today") -> list[dict]:
"""Submission yang sudah punya jadwal sidang (schedule IS NOT NULL),
dibatasi >= min_date supaya tidak menyapu semua histori lama.
min_date: 'today' atau string 'YYYY-MM-DD'.
"""
cutoff = date.today() if min_date == "today" else datetime.strptime(min_date, "%Y-%m-%d").date()
with self._conn.cursor() as cur:
cur.execute(
"""
SELECT s.id, s.schedule, s.room, s.title, s.type_id,
u.name AS student_name, u.email AS student_email, u.unique_id AS student_nim
FROM submissions s
JOIN users u ON u.id = s.student_id
WHERE s.schedule IS NOT NULL
AND s.schedule >= %s
""",
(cutoff,),
)
submissions = cur.fetchall()
if not submissions:
return []
ids = [s["id"] for s in submissions]
placeholders = ",".join(["%s"] * len(ids))
# Diurutkan v.id supaya "Pembimbing 1/2" & "Penguji 1/2" konsisten
# dengan urutan input aslinya.
with self._conn.cursor() as cur:
cur.execute(
f"""
SELECT v.submission_id, v.type, u.name, u.email
FROM verificators v
JOIN users u ON u.id = v.user_id
WHERE v.submission_id IN ({placeholders})
AND v.type IN ('Pembimbing', 'Penguji')
AND u.email IS NOT NULL AND u.email != ''
ORDER BY v.id
""",
ids,
)
verificator_rows = cur.fetchall()
verificators_by_submission: dict[int, list[dict]] = {}
for row in verificator_rows:
verificators_by_submission.setdefault(row["submission_id"], []).append(
{"type": row["type"], "name": row["name"], "email": row["email"]}
)
for submission in submissions:
submission["verificators"] = verificators_by_submission.get(submission["id"], [])
return submissions
def get_ids_with_active_schedule(self) -> set[int]:
"""ID semua submission yang MASIH punya schedule (tanpa filter tanggal).
Dipakai KHUSUS untuk deteksi orphan/cancelled — supaya sidang yang
tanggalnya sudah lewat (tapi memang belum dibatalkan) tidak salah
dianggap "cancelled" dan event Calendar-nya ikut terhapus, gara-gara
filter min_date di get_scheduled_submissions() menyisihkannya.
"""
with self._conn.cursor() as cur:
cur.execute("SELECT id FROM submissions WHERE schedule IS NOT NULL")
return {row["id"] for row in cur.fetchall()}
scripts/edoxid_calendar_gmail.py
+35
View File
@@ -0,0 +1,35 @@
import base64
from email.mime.text import MIMEText
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
from scripts.edoxid_calendar_config import GoogleConfig
class GmailService:
"""Kirim email reminder H-1/1-jam sebagai tu@informatika.untan.ac.id
langsung (scope gmail.send, akun OAuth yang sama dengan Calendar)."""
def __init__(self, config: GoogleConfig):
creds = Credentials(
token=None,
refresh_token=config.refresh_token,
token_uri="https://oauth2.googleapis.com/token",
client_id=config.client_id,
client_secret=config.client_secret,
scopes=["https://www.googleapis.com/auth/gmail.send"],
)
creds.refresh(Request())
self._service = build("gmail", "v1", credentials=creds, cache_discovery=False)
def send(self, to: list[str], subject: str, body_text: str) -> str:
message = MIMEText(body_text, "plain", "utf-8")
message["to"] = ", ".join(to)
message["subject"] = subject
raw = base64.urlsafe_b64encode(message.as_bytes()).decode("utf-8")
result = self._service.users().messages().send(userId="me", body={"raw": raw}).execute()
return result["id"]
scripts/edoxid_calendar_google.py
+184
View File
@@ -0,0 +1,184 @@
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from scripts.edoxid_calendar_config import GoogleConfig
JAKARTA_TZ = ZoneInfo("Asia/Jakarta")
TYPE_LABELS = {
1: "Seminar Proposal",
2: "Seminar Hasil",
3: "Sidang Akhir",
}
DAY_NAMES = {
"Monday": "Senin", "Tuesday": "Selasa", "Wednesday": "Rabu", "Thursday": "Kamis",
"Friday": "Jumat", "Saturday": "Sabtu", "Sunday": "Minggu",
}
MONTH_NAMES = {
"January": "Januari", "February": "Februari", "March": "Maret", "April": "April",
"May": "Mei", "June": "Juni", "July": "Juli", "August": "Agustus",
"September": "September", "October": "Oktober", "November": "November", "December": "Desember",
}
# --- Fungsi format, dipakai bareng oleh event Calendar & email reminder (edoxid_calendar_gmail_reminder.py) ---
def get_schedule_datetime(submission: dict) -> datetime:
schedule = submission["schedule"]
if isinstance(schedule, str):
schedule = datetime.fromisoformat(schedule)
return schedule.replace(tzinfo=JAKARTA_TZ)
def get_type_label(submission: dict) -> str:
return TYPE_LABELS.get(submission["type_id"], "Sidang")
def format_indonesian_date(date: datetime) -> str:
day = DAY_NAMES.get(date.strftime("%A"), date.strftime("%A"))
month = MONTH_NAMES.get(date.strftime("%B"), date.strftime("%B"))
return f"{day}, {date.strftime('%d')} {month} {date.strftime('%Y')}"
def build_description(submission: dict) -> str:
schedule = get_schedule_datetime(submission)
type_label = get_type_label(submission)
pembimbing = [v for v in submission["verificators"] if v["type"] == "Pembimbing"]
penguji = [v for v in submission["verificators"] if v["type"] == "Penguji"]
lines = [
f"UNDANGAN {type_label.upper()} SECARA LURING",
"JURUSAN INFORMATIKA",
"",
"Mengundang Bapak dan Ibu dosen tim pembimbing dan penguji untuk hadir pada "
f"{type_label.lower()} mahasiswa yang dilaksanakan secara Luring.",
"",
f"👨‍🎓 {submission['student_name']} - {submission['student_nim']}",
"",
f"📖 {submission['title']}",
"",
f"📅 {format_indonesian_date(schedule)}",
f"🕑 {schedule.strftime('%H:%M')} WIB",
"",
]
for i, v in enumerate(pembimbing, start=1):
lines.append(f"📕 Pembimbing {i} : {v['name']}")
for i, v in enumerate(penguji, start=1):
lines.append(f"📒 Penguji {i} : {v['name']}")
lines += [
"",
"Tempat:",
submission.get("room") or "-",
"",
"Terimakasih.",
"ttd",
"Kajur Informatika",
]
return "\n".join(lines)
def format_submission_summary(submission: dict) -> str:
"""Ringkasan 1-baris untuk log Airflow — supaya kelihatan jelas di task log
siapa yang dapat undangan/reminder tanpa perlu buka Calendar/email manual."""
schedule = get_schedule_datetime(submission)
pembimbing = [v["name"] for v in submission["verificators"] if v["type"] == "Pembimbing"]
penguji = [v["name"] for v in submission["verificators"] if v["type"] == "Penguji"]
return (
f"{submission['student_name']} ({submission['student_nim']}) — "
f"{get_type_label(submission)} — \"{submission['title']}\" — "
f"{format_indonesian_date(schedule)} {schedule.strftime('%H:%M')} WIB "
f"@ {submission.get('room') or '-'} | "
f"Pembimbing: {', '.join(pembimbing) or '-'} | "
f"Penguji: {', '.join(penguji) or '-'}"
)
def build_attendee_emails(submission: dict) -> list[str]:
emails = [submission["student_email"]] + [v["email"] for v in submission["verificators"]]
seen = set()
deduped = []
for email in emails:
if email and email not in seen:
seen.add(email)
deduped.append(email)
return deduped
class GoogleCalendarService:
"""Auth via OAuth refresh token sebagai tu@informatika.untan.ac.id LANGSUNG
(bukan service account) — service account tidak bisa invite attendee tanpa
Domain-Wide Delegation of Authority (dicoba & ditolak Google 2026-07-27)."""
def __init__(self, config: GoogleConfig):
self._calendar_id = config.calendar_id
creds = Credentials(
token=None,
refresh_token=config.refresh_token,
token_uri="https://oauth2.googleapis.com/token",
client_id=config.client_id,
client_secret=config.client_secret,
scopes=["https://www.googleapis.com/auth/calendar"],
)
creds.refresh(Request())
self._service = build("calendar", "v3", credentials=creds, cache_discovery=False)
def create_event(self, submission: dict) -> str:
event = self._build_event(submission)
result = (
self._service.events()
.insert(calendarId=self._calendar_id, body=event, sendUpdates="all")
.execute()
)
return result["id"]
def update_event(self, event_id: str, submission: dict) -> None:
event = self._build_event(submission)
self._service.events().update(
calendarId=self._calendar_id, eventId=event_id, body=event, sendUpdates="all"
).execute()
def delete_event(self, event_id: str) -> None:
try:
self._service.events().delete(
calendarId=self._calendar_id, eventId=event_id, sendUpdates="all"
).execute()
except HttpError as e:
# 404/410 berarti event sudah tidak ada (mis. dihapus manual) — aman diabaikan
if e.resp.status not in (404, 410):
raise
def _build_event(self, submission: dict) -> dict:
schedule = get_schedule_datetime(submission)
end = schedule + timedelta(hours=1)
return {
"summary": f"{get_type_label(submission)} — {submission['student_name']}",
"description": build_description(submission),
"location": submission.get("room") or "",
"start": {"dateTime": schedule.isoformat(), "timeZone": "Asia/Jakarta"},
"end": {"dateTime": end.isoformat(), "timeZone": "Asia/Jakarta"},
"attendees": [{"email": email} for email in build_attendee_emails(submission)],
"reminders": {"useDefault": False, "overrides": self._build_reminders(schedule)},
}
@staticmethod
def _build_reminders(schedule: datetime) -> list[dict]:
h1 = schedule.replace(hour=7, minute=0, second=0, microsecond=0) - timedelta(days=1)
minutes_h1 = max(0, int((schedule - h1).total_seconds() // 60))
return [
{"method": "popup", "minutes": minutes_h1},
{"method": "popup", "minutes": 60},
]
scripts/edoxid_calendar_state_store.py
+98
View File
@@ -0,0 +1,98 @@
import hashlib
import json
import sqlite3
from datetime import datetime
class StateStore:
"""State lokal (SQLite) tracking submission_id -> event_id + content_hash.
Sengaja TIDAK di doxid2022 — kalau file ini hilang/reset, paling buruk
event lama ke-recreate di Calendar, tidak merusak apa pun di DB eDOXID."""
def __init__(self, path: str):
self._conn = sqlite3.connect(path)
self._conn.execute(
"""
CREATE TABLE IF NOT EXISTS sync_state (
submission_id INTEGER PRIMARY KEY,
event_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
updated_at TEXT NOT NULL,
notified_h1 INTEGER NOT NULL DEFAULT 0,
notified_1h INTEGER NOT NULL DEFAULT 0
)
"""
)
self._conn.commit()
def close(self) -> None:
self._conn.close()
@staticmethod
def compute_hash(submission: dict) -> str:
payload = json.dumps(
[
str(submission["schedule"]),
submission["room"],
submission["title"],
submission["type_id"],
submission["student_email"],
submission["student_nim"],
submission["verificators"],
],
sort_keys=True,
default=str,
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def find(self, submission_id: int) -> dict | None:
row = self._conn.execute(
"SELECT event_id, content_hash FROM sync_state WHERE submission_id = ?",
(submission_id,),
).fetchone()
if row is None:
return None
return {"event_id": row[0], "content_hash": row[1]}
def save(self, submission_id: int, event_id: str, content_hash: str) -> None:
self._conn.execute(
"""
INSERT INTO sync_state (submission_id, event_id, content_hash, updated_at)
VALUES (?, ?, ?, ?)
""",
(submission_id, event_id, content_hash, datetime.utcnow().isoformat()),
)
self._conn.commit()
def update_hash(self, submission_id: int, content_hash: str) -> None:
self._conn.execute(
"UPDATE sync_state SET content_hash = ?, updated_at = ? WHERE submission_id = ?",
(content_hash, datetime.utcnow().isoformat(), submission_id),
)
self._conn.commit()
def remove(self, submission_id: int) -> None:
self._conn.execute("DELETE FROM sync_state WHERE submission_id = ?", (submission_id,))
self._conn.commit()
def is_notified(self, submission_id: int, which: str) -> bool:
column = "notified_h1" if which == "h1" else "notified_1h"
row = self._conn.execute(
f"SELECT {column} FROM sync_state WHERE submission_id = ?", (submission_id,)
).fetchone()
return bool(row and row[0])
def mark_notified(self, submission_id: int, which: str) -> None:
column = "notified_h1" if which == "h1" else "notified_1h"
self._conn.execute(
f"UPDATE sync_state SET {column} = 1 WHERE submission_id = ?", (submission_id,)
)
self._conn.commit()
def find_orphaned(self, active_submission_ids: set[int]) -> dict[int, str]:
"""Submission yang tercatat di state tapi schedule-nya sudah di-null-kan
(dibatalkan) atau baris submission-nya sudah dihapus — bukan sekadar
tanggalnya lewat. `active_submission_ids` harus dari query TANPA filter
tanggal (lihat Database.get_ids_with_active_schedule)."""
rows = self._conn.execute("SELECT submission_id, event_id FROM sync_state").fetchall()
return {sid: eid for sid, eid in rows if sid not in active_submission_ids}
scripts/edoxid_calendar_sync.py
+127
View File
@@ -0,0 +1,127 @@
import logging
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
from scripts.edoxid_calendar_config import (
get_db_config,
get_google_config,
get_state_sqlite_path,
get_sync_min_date,
)
from scripts.edoxid_calendar_database import Database
from scripts.edoxid_calendar_gmail import GmailService
from scripts.edoxid_calendar_google import (
GoogleCalendarService,
build_attendee_emails,
build_description,
format_submission_summary,
get_schedule_datetime,
get_type_label,
)
from scripts.edoxid_calendar_state_store import StateStore
logger = logging.getLogger(__name__)
JAKARTA_TZ = ZoneInfo("Asia/Jakarta")
# Toleransi jendela waktu untuk cek reminder — harus >= interval polling DAG (5 menit)
# supaya tidak ada window yang "kelewat" di antara 2 run.
REMINDER_WINDOW = timedelta(minutes=10)
def _check_and_send_reminders(submissions: list[dict], state: StateStore, gmail: GmailService) -> int:
"""Kirim email reminder H-1 (07:00) & 1 jam sebelum ke semua attendee —
terpisah dari reminders.overrides Calendar (yang cuma berlaku untuk
calendar TU sendiri, tidak ikut ke kalender pribadi tiap attendee)."""
now = datetime.now(JAKARTA_TZ)
sent = 0
for submission in submissions:
schedule = get_schedule_datetime(submission)
h1_time = schedule.replace(hour=7, minute=0, second=0, microsecond=0) - timedelta(days=1)
one_hour_time = schedule - timedelta(hours=1)
emails = build_attendee_emails(submission)
if not emails:
continue
if h1_time <= now < h1_time + REMINDER_WINDOW and not state.is_notified(submission["id"], "h1"):
subject = f"[PENGINGAT SEBELUM H-1] {get_type_label(submission)} — {submission['student_name']}"
gmail.send(emails, subject, build_description(submission))
state.mark_notified(submission["id"], "h1")
sent += 1
logger.info(
"REMINDER H-1 terkirim | submission #%s | %s | penerima: %s",
submission["id"], format_submission_summary(submission), ", ".join(emails),
)
if one_hour_time <= now < one_hour_time + REMINDER_WINDOW and not state.is_notified(
submission["id"], "1h"
):
subject = f"[PENGINGAT 1 JAM SEBELUM] {get_type_label(submission)} — {submission['student_name']}"
gmail.send(emails, subject, build_description(submission))
state.mark_notified(submission["id"], "1h")
sent += 1
logger.info(
"REMINDER 1-JAM terkirim | submission #%s | %s | penerima: %s",
submission["id"], format_submission_summary(submission), ", ".join(emails),
)
return sent
def run_sync() -> dict:
db = Database(get_db_config())
state = StateStore(get_state_sqlite_path())
google_config = get_google_config()
calendar = GoogleCalendarService(google_config)
gmail = GmailService(google_config)
created = updated = deleted = skipped = 0
try:
submissions = db.get_scheduled_submissions(get_sync_min_date())
for submission in submissions:
content_hash = state.compute_hash(submission)
existing = state.find(submission["id"])
if existing is None:
event_id = calendar.create_event(submission)
state.save(submission["id"], event_id, content_hash)
created += 1
logger.info(
"CREATE event %s | submission #%s | %s",
event_id, submission["id"], format_submission_summary(submission),
)
elif existing["content_hash"] != content_hash:
calendar.update_event(existing["event_id"], submission)
state.update_hash(submission["id"], content_hash)
updated += 1
logger.info(
"UPDATE event %s | submission #%s | %s",
existing["event_id"], submission["id"], format_submission_summary(submission),
)
else:
skipped += 1
active_ids = db.get_ids_with_active_schedule()
for submission_id, event_id in state.find_orphaned(active_ids).items():
calendar.delete_event(event_id)
state.remove(submission_id)
deleted += 1
logger.info("DELETE submission #%s -> event %s", submission_id, event_id)
reminders_sent = _check_and_send_reminders(submissions, state, gmail)
finally:
db.close()
state.close()
summary = {
"created": created,
"updated": updated,
"deleted": deleted,
"skipped": skipped,
"reminders_sent": reminders_sent,
}
logger.info("Sync selesai: %s", summary)
return summary