diff --git a/dags/scholar_penelitian_monthly.py b/dags/scholar_penelitian_monthly.py new file mode 100644 index 0000000..fff5c55 --- /dev/null +++ b/dags/scholar_penelitian_monthly.py @@ -0,0 +1,50 @@ +""" +DAG bulanan: sync data penelitian dosen Informatika ke DB dashboard akreditasi. + +Task 1 tarik_publikasi_serpapi : semua publikasi tiap dosen dari Google + Scholar via SerpAPI (berdasar dosen.scholar; + dosen.scholar/sinta yang kosong diisi + otomatis dari halaman departemen SINTA). +Task 2 update_kuartil_scopus_sinta : label kuartil Q1-Q4 dari view Scopus di + profil SINTA -> kolom penelitian.scopus. + +Environment yang wajib di-set di Coolify (service airflow-scheduler): + DB_HOST, DB_PORT, DB_USER, DB_PASS, DB_NAME -> MySQL dashboard (Coolify) + SERPAPI_KEY -> kunci SerpAPI +Opsional: SERPAPI_MAX_REQUESTS (default 90), SINTA_USERNAME/SINTA_PASSWORD. +""" + +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.python import PythonOperator + +# Import fungsi dari folder scripts +from scripts.scholar_api_scraper import run_publications, run_scopus + +with DAG( + dag_id="scholar_penelitian_monthly", + description="Tarik publikasi dosen via SerpAPI + kuartil Scopus dari SINTA", + start_date=datetime(2026, 7, 1), + schedule_interval="0 2 1 * *", + catchup=False, + max_active_runs=1, + default_args={ + "owner": "dashboard-informatika", + "retries": 1, + "retry_delay": timedelta(minutes=30), + }, + tags=["scholar", "sinta", "penelitian", "dashboard"], +) as dag: + + task_publikasi = PythonOperator( + task_id="tarik_publikasi_serpapi", + python_callable=run_publications, + ) + + task_scopus = PythonOperator( + task_id="update_kuartil_scopus_sinta", + python_callable=run_scopus, + ) + + task_publikasi >> task_scopus diff --git a/docker-compose.yaml b/docker-compose.yaml index 7bc554e..d1e3105 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,6 +35,16 @@ services: - PYTHONPATH=/opt/airflow - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow - AIRFLOW__CORE__EXECUTOR=LocalExecutor + # koneksi DB dashboard akreditasi + SerpAPI (nilai di-set di Coolify UI) + - DB_HOST=${DB_HOST:-host.docker.internal} + - DB_PORT=${DB_PORT:-7891} + - DB_USER=${DB_USER:-root} + - DB_PASS=${DB_PASS} + - DB_NAME=${DB_NAME:-informatika} + - SERPAPI_KEY=${SERPAPI_KEY} + - SERPAPI_MAX_REQUESTS=${SERPAPI_MAX_REQUESTS:-90} + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - airflow_logs:/opt/airflow/logs - airflow_plugins:/opt/airflow/plugins diff --git a/requirements.txt b/requirements.txt index f0fb1a1..2c30da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ beautifulsoup4 requests +PyMySQL pandas-gbq apache-airflow-providers-google google-cloud-bigquery diff --git a/scripts/scholar_api_scraper.py b/scripts/scholar_api_scraper.py new file mode 100644 index 0000000..16a4108 --- /dev/null +++ b/scripts/scholar_api_scraper.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +""" +Penarik data penelitian dosen via SerpAPI (Google Scholar) + kuartil Scopus dari SINTA. + +Dua langkah: + 1. publications : tarik SEMUA publikasi tiap dosen dari Google Scholar lewat + SerpAPI (engine google_scholar_author, 100 artikel/request) berdasarkan + kolom dosen.scholar, lalu insert baris baru ke tabel `penelitian`. + Kolom `link` diisi citation_id (format sama dgn data lama). + 2. scopus : tarik daftar publikasi Scopus tiap dosen dari SINTA + (view=scopus, ada label kuartil), cocokkan judul, lalu UPDATE kolom + `penelitian.scopus` (Q1..Q4). + +Konfigurasi environment: + DB_HOST, DB_USER, DB_PASS, DB_NAME + SERPAPI_KEY (fallback: file apikey.json di root project) + SERPAPI_MAX_REQUESTS (default 90 - pengaman kuota free tier 100/bulan) + +CLI: + python scholar_api_scraper.py --dry-run # tanpa tulis DB + python scholar_api_scraper.py # publications + scopus + python scholar_api_scraper.py --only publications + python scholar_api_scraper.py --only scopus # tidak pakai kuota SerpAPI +""" + +import argparse +import logging +import os +import re + +import requests + +try: # di image Airflow, modul berada di /opt/airflow/scripts (PYTHONPATH=/opt/airflow) + from scripts.sinta_scraper import ( + db_connect, + ensure_schema, + get_author_publications, + get_department_authors, + make_session, + match_dosen, + norm_title, + DEFAULT_DEPT_URL, + ) +except ImportError: + from sinta_scraper import ( + db_connect, + ensure_schema, + get_author_publications, + get_department_authors, + make_session, + match_dosen, + norm_title, + DEFAULT_DEPT_URL, + ) + +log = logging.getLogger("scholar_api") +SERPAPI_URL = "https://serpapi.com/search.json" + + +def get_api_key(): + key = os.getenv("SERPAPI_KEY") + if key: + return key.strip() + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "apikey.json") + if os.path.exists(path): + return open(path, encoding="utf-8").read().strip().strip('"') + raise RuntimeError("SERPAPI_KEY tidak di-set dan apikey.json tidak ditemukan") + + +def load_dosen(): + """Daftar dosen dari tabel dosen. Fallback ke halaman departemen SINTA + (ID Scholar dari URL foto) bila DB tidak bisa diakses (mis. saat dry-run).""" + try: + conn = db_connect() + with conn.cursor() as cur: + cur.execute("SELECT id_dosen, nama, scholar, sinta FROM dosen") + rows = [ + { + "id_dosen": r[0], + "nama": re.sub(r"\s+", " ", (r[1] or "")).strip(), + "scholar": (r[2] or "").strip(), + "sinta": (r[3] or "").strip() if r[3] else "", + } + for r in cur.fetchall() + ] + conn.close() + return rows, True + except Exception as e: + log.warning("DB tidak bisa diakses (%s) - fallback ke daftar SINTA", e) + authors = get_department_authors(make_session(), DEFAULT_DEPT_URL) + return [ + {"id_dosen": None, "nama": a["nama"], "scholar": a["gs_id"] or "", "sinta": a["sinta_id"]} + for a in authors + ], False + + +# ------------------------------------------------- langkah 1: SerpAPI Scholar + +def fetch_author_articles(key, author_id, budget): + """Semua artikel satu author dari SerpAPI. Return (nama_profil_gs, list artikel).""" + articles, start, profile_name = [], 0, "" + while True: + if budget["used"] >= budget["max"]: + log.warning("Budget SerpAPI (%d request) habis - stop di %s", budget["max"], author_id) + break + resp = requests.get( + SERPAPI_URL, + params={ + "engine": "google_scholar_author", + "author_id": author_id, + "num": 100, + "start": start, + "api_key": key, + "hl": "id", + }, + timeout=60, + ) + budget["used"] += 1 + data = resp.json() + if "error" in data: + raise RuntimeError(data["error"]) + profile_name = data.get("author", {}).get("name", profile_name) + batch = data.get("articles", []) + articles.extend(batch) + if len(batch) < 100 or "next" not in data.get("serpapi_pagination", {}): + break + start += 100 + return profile_name, articles + + +def _apply_filter(rows, dosen_filter): + if not dosen_filter: + return rows + f = dosen_filter.lower() + return [r for r in rows if f in r["nama"].lower() or f == r["scholar"].lower()] + + +def enrich_dosen(session, conn, dosen_rows, dry_run=False): + """Isi dosen.sinta dan dosen.scholar yang masih kosong dari halaman + departemen SINTA (ID Scholar diambil dari URL foto profil).""" + if not any(not d["sinta"] or not d["scholar"] for d in dosen_rows): + return + for a in get_department_authors(session, DEFAULT_DEPT_URL): + d = match_dosen(a["nama"], dosen_rows) + if d is None: + continue + updates = {} + if not d["sinta"]: + d["sinta"] = updates["sinta"] = a["sinta_id"] + if not d["scholar"] and a["gs_id"]: + d["scholar"] = updates["scholar"] = a["gs_id"] + if updates and conn and d["id_dosen"] is not None and not dry_run: + with conn.cursor() as cur: + for col, val in updates.items(): + cur.execute(f"UPDATE dosen SET {col}=%s WHERE id_dosen=%s", (val, d["id_dosen"])) + log.info("dosen '%s' dilengkapi: %s", d["nama"], updates) + if conn and not dry_run: + conn.commit() + + +def run_publications(dry_run=False, dosen_filter=None): + key = get_api_key() + budget = {"used": 0, "max": int(os.getenv("SERPAPI_MAX_REQUESTS", "90"))} + dosen_rows, db_ok = load_dosen() + conn = db_connect() if (not dry_run and db_ok) else None + if conn: + ensure_schema(conn) + enrich_dosen(make_session(), conn, dosen_rows, dry_run) + targets = _apply_filter([d for d in dosen_rows if d["scholar"]], dosen_filter) + log.info("%d dosen punya ID Scholar (dari %d)", len(targets), len(dosen_rows)) + + stats = {"authors": 0, "articles": 0, "inserted": 0, "skipped": 0, "errors": 0} + try: + existing = set() + if conn: + with conn.cursor() as cur: + cur.execute("SELECT id_author, judul FROM penelitian") + existing = {(r[0], norm_title(r[1] or "")) for r in cur.fetchall()} + + for d in targets: + try: + gs_name, articles = fetch_author_articles(key, d["scholar"], budget) + except Exception as e: + log.error("Gagal tarik %s (%s): %s", d["nama"], d["scholar"], e) + stats["errors"] += 1 + continue + stats["authors"] += 1 + stats["articles"] += len(articles) + author_name = gs_name or d["nama"] + log.info("%s: %d artikel (request terpakai: %d)", author_name, len(articles), budget["used"]) + if not conn: + continue + with conn.cursor() as cur: + for art in articles: + judul = (art.get("title") or "").strip() + if not judul: + continue + key_row = (d["scholar"], norm_title(judul)) + if key_row in existing: + stats["skipped"] += 1 + continue + ym = re.search(r"\d{4}", str(art.get("year") or "")) + cur.execute( + "INSERT INTO penelitian " + "(judul, author, link, tahun, author_full, id_author, biaya) " + "VALUES (%s, %s, %s, %s, %s, %s, '')", + ( + judul[:400], + author_name[:100], + (art.get("citation_id") or "")[:300], + ym.group(0) if ym else "", + (art.get("authors") or "")[:255], + d["scholar"], + ), + ) + existing.add(key_row) + stats["inserted"] += 1 + if conn: + conn.commit() + except Exception: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + stats["serpapi_requests"] = budget["used"] + log.info("Publications selesai: %s", stats) + return stats + + +# --------------------------------------------- langkah 2: kuartil dari SINTA + +def run_scopus(dry_run=False, dosen_filter=None): + session = make_session() + dosen_rows, db_ok = load_dosen() + conn = db_connect() if db_ok else None + if conn: + ensure_schema(conn) + + enrich_dosen(session, conn, dosen_rows, dry_run) + + targets = _apply_filter([d for d in dosen_rows if d["sinta"]], dosen_filter) + log.info("%d dosen punya SINTA ID", len(targets)) + + stats = {"authors": 0, "scopus_items": 0, "updated": 0, "unmatched": 0} + pubs_by_author = {} + if conn: + with conn.cursor() as cur: + cur.execute("SELECT id_artikel, id_author, judul, scopus FROM penelitian") + for r in cur.fetchall(): + pubs_by_author.setdefault(r[1], []).append( + {"id_artikel": r[0], "nt": norm_title(r[2] or ""), "scopus": r[3]} + ) + try: + for d in targets: + items = get_author_publications(session, d["sinta"], view="scopus") + stats["authors"] += 1 + stats["scopus_items"] += len(items) + if not conn: + continue + rows = pubs_by_author.get(d["scholar"], []) + with conn.cursor() as cur: + for it in items: + q = it["quartile"] or "Scopus" + match = next((r for r in rows if r["nt"] == norm_title(it["judul"])), None) + if not match: + stats["unmatched"] += 1 + continue + if (match["scopus"] or "") != q and not dry_run: + cur.execute( + "UPDATE penelitian SET scopus=%s WHERE id_artikel=%s", + (q, match["id_artikel"]), + ) + match["scopus"] = q + stats["updated"] += 1 + if conn and not dry_run: + conn.commit() + except Exception: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + log.info("Scopus selesai: %s", stats) + return stats + + +def run(dry_run=False): + """Entry point gabungan - dipakai CLI; Airflow memanggil per-langkah.""" + return { + "publications": run_publications(dry_run=dry_run), + "scopus": run_scopus(dry_run=dry_run), + } + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + parser = argparse.ArgumentParser(description="Tarik penelitian via SerpAPI + kuartil SINTA") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--only", choices=["publications", "scopus"]) + parser.add_argument("--dosen", help="filter satu dosen: potongan nama atau ID Scholar") + args = parser.parse_args() + if args.only == "publications": + print(run_publications(dry_run=args.dry_run, dosen_filter=args.dosen)) + elif args.only == "scopus": + print(run_scopus(dry_run=args.dry_run, dosen_filter=args.dosen)) + else: + print(run_publications(dry_run=args.dry_run, dosen_filter=args.dosen)) + print(run_scopus(dry_run=args.dry_run, dosen_filter=args.dosen)) diff --git a/scripts/sinta_scraper.py b/scripts/sinta_scraper.py new file mode 100644 index 0000000..bcab245 --- /dev/null +++ b/scripts/sinta_scraper.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +Scraper data penelitian (publikasi) dosen dari SINTA. + +Sumber : halaman authors departemen SINTA -> profil tiap dosen (view googlescholar). +Target : tabel `penelitian` pada database dashboard (skema sama dengan crawler_scholar.php lama). + +Konfigurasi lewat environment variable (default mengikuti config.php dashboard): + DB_HOST, DB_USER, DB_PASS, DB_NAME + SINTA_USERNAME, SINTA_PASSWORD (opsional; tanpa login SINTA hanya memberi + 10 publikasi terbaru per dosen) + SINTA_DEPT_URL (opsional; default = departemen Informatika UNTAN) + +Bisa dijalankan standalone: + python sinta_scraper.py --dry-run # scrape saja, tampilkan hasil, tanpa tulis DB + python sinta_scraper.py # scrape + sync ke database +""" + +import argparse +import logging +import os +import re +import time + +import requests +from bs4 import BeautifulSoup + +log = logging.getLogger("sinta_scraper") + +BASE_URL = "https://sinta.kemdiktisaintek.go.id" +DEFAULT_DEPT_URL = ( + BASE_URL + + "/departments/authors/475/EF2A55B7-1CB8-40FC-8D37-B110440E425E/1AD9D155-C081-4913-8B9E-63464F5B0F93" +) +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0 Safari/537.36" +) +REQUEST_DELAY = 0.8 # jeda antar request (detik), jangan terlalu agresif ke SINTA +MAX_PAGES = 100 # pengaman agar loop pagination tidak lari tanpa batas + + +# ---------------------------------------------------------------- HTTP / login + +def make_session(): + s = requests.Session() + s.headers["User-Agent"] = USER_AGENT + return s + + +def login(session): + """Login ke SINTA bila kredensial tersedia. Return True bila berhasil.""" + user = os.getenv("SINTA_USERNAME") + pwd = os.getenv("SINTA_PASSWORD") + if not user or not pwd: + log.warning( + "SINTA_USERNAME/SINTA_PASSWORD tidak di-set - tanpa login SINTA hanya " + "menampilkan 10 publikasi terbaru per dosen." + ) + return False + + session.get(f"{BASE_URL}/logins", timeout=30) # ambil cookie sesi + resp = session.post( + f"{BASE_URL}/logins/do_login", + data={"username": user, "password": pwd}, + timeout=30, + ) + ok = "logout" in resp.text.lower() or "dashboard" in resp.url + if ok: + log.info("Login SINTA berhasil sebagai %s", user) + else: + log.warning("Login SINTA GAGAL - lanjut tanpa login (10 publikasi/dosen).") + return ok + + +def fetch(session, url): + time.sleep(REQUEST_DELAY) + resp = session.get(url, timeout=30) + resp.raise_for_status() + return BeautifulSoup(resp.text, "html.parser") + + +# ------------------------------------------------------------------- scraping + +def get_department_authors(session, dept_url): + """Ambil daftar dosen dari halaman authors departemen (semua halaman). + + Return list of dict: {sinta_id, nama, gs_id} - gs_id diambil dari URL foto + profil (foto SINTA di-serve dari Google Scholar, mengandung parameter user=). + """ + authors, seen = [], set() + for page in range(1, MAX_PAGES): + soup = fetch(session, f"{dept_url}?page={page}") + found_new = False + for name_el in soup.select(".profile-name a[href*='authors/profile/']"): + m = re.search(r"authors/profile/(\d+)", name_el["href"]) + if not m or m.group(1) in seen: + continue + seen.add(m.group(1)) + found_new = True + + gs_id = None + row = name_el.find_parent("div", class_="col-lg") + row = row.parent if row else None + if row: + img = row.select_one("img") + gm = re.search(r"[?&]user=([\w-]+)", img["src"]) if img else None + gs_id = gm.group(1) if gm else None + + authors.append( + { + "sinta_id": m.group(1), + "nama": name_el.get_text(strip=True).title(), + "gs_id": gs_id, + } + ) + if not found_new: + break + log.info("Ditemukan %d dosen di halaman departemen", len(authors)) + return authors + + +def parse_publication_items(soup): + pubs = [] + for item in soup.select(".ar-list-item"): + title_el = item.select_one(".ar-title a") + if not title_el: + continue + authors_full = "" + for a in item.select(".ar-meta a"): + txt = a.get_text(strip=True) + if txt.startswith("Authors"): + authors_full = re.sub(r"^Authors\s*:\s*", "", txt) + break + year_el = item.select_one(".ar-year") + ym = re.search(r"\d{4}", year_el.get_text()) if year_el else None + cited_el = item.select_one(".ar-cited") + cm = re.search(r"\d+", cited_el.get_text()) if cited_el else None + quart_el = item.select_one(".ar-quartile") # hanya ada di view=scopus + qm = re.search(r"Q\d", quart_el.get_text()) if quart_el else None + pubs.append( + { + "judul": title_el.get_text(strip=True), + "link": title_el.get("href", ""), + "author_full": authors_full, + "tahun": ym.group(0) if ym else "", + "cited": int(cm.group(0)) if cm else 0, + "quartile": qm.group(0) if qm else None, + } + ) + return pubs + + +def get_author_publications(session, sinta_id, view="googlescholar"): + """Ambil semua publikasi seorang dosen. + + Pagination SINTA tanpa login tidak konsisten - kadang halaman berikutnya + berisi halaman sebelumnya lagi. Karena itu judul di-dedup, dan loop baru + berhenti setelah beberapa halaman berturut-turut tidak memberi judul baru. + """ + all_pubs, seen, stale_pages = [], set(), 0 + for page in range(1, MAX_PAGES): + new = [] + for _ in range(3): # retry: kadang SINTA menyajikan ulang halaman lama + soup = fetch( + session, + f"{BASE_URL}/authors/profile/{sinta_id}?view={view}&page={page}", + ) + pubs = parse_publication_items(soup) + new = [p for p in pubs if p["judul"] not in seen] + if new or not pubs: + break + if new: + stale_pages = 0 + all_pubs.extend(new) + seen.update(p["judul"] for p in new) + else: + stale_pages += 1 + if stale_pages >= 2: + break + return all_pubs + + +# ------------------------------------------------------------------- database + +def db_connect(): + import pymysql + + return pymysql.connect( + host=os.getenv("DB_HOST", "localhost"), + port=int(os.getenv("DB_PORT", "3306")), + user=os.getenv("DB_USER", "root"), + password=os.getenv("DB_PASS", "root"), + database=os.getenv("DB_NAME", "informatika"), + charset="utf8mb4", + autocommit=False, + ) + + +def ensure_schema(conn): + """Tambah kolom penelitian.updated_at bila belum ada (dikelola MySQL: + terisi otomatis saat INSERT dan ter-update saat UPDATE). Kolom ditaruh + paling akhir agar indeks kolom SELECT * di dashboard PHP tidak bergeser.""" + with conn.cursor() as cur: + cur.execute( + "SELECT COUNT(*) FROM information_schema.columns " + "WHERE table_schema=DATABASE() AND table_name='penelitian' " + "AND column_name='updated_at'" + ) + if cur.fetchone()[0] == 0: + log.info("Menambahkan kolom penelitian.updated_at") + cur.execute( + "ALTER TABLE penelitian ADD COLUMN updated_at TIMESTAMP NOT NULL " + "DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP" + ) + conn.commit() + + +def norm_title(t): + return re.sub(r"[^a-z0-9]+", "", t.lower()) + + +def norm_name_tokens(name): + """Token nama tanpa gelar: buang kata bertitik (Dr., S.Kom., M.T., dst).""" + name = name.split(",")[0] + tokens = { + w.lower() + for w in re.findall(r"[A-Za-z]+", name) + if len(w) > 1 and w.lower() not in {"dr", "ir", "prof", "st", "mt", "kom"} + } + return tokens + + +def _norm_name_concat(name): + """Nama tanpa gelar sebagai satu string huruf kecil, utk kasus token pecah + beda ('Tsana'Uddin' vs 'Tsanauddin').""" + return "".join(re.findall(r"[A-Za-z]+", name.split(",")[0])).lower() + + +def match_dosen(nama_sinta, dosen_rows): + """Cari baris dosen yang namanya paling cocok (jaccard token >= 0.5, + atau string nama gabungan identik).""" + target = norm_name_tokens(nama_sinta) + target_concat = _norm_name_concat(nama_sinta) + best, best_score = None, 0.0 + for row in dosen_rows: + if target_concat and target_concat == _norm_name_concat(row["nama"] or ""): + return row + tokens = norm_name_tokens(row["nama"] or "") + if not tokens or not target: + continue + score = len(target & tokens) / len(target | tokens) + if score > best_score: + best, best_score = row, score + return best if best_score >= 0.5 else None + + +def sync_to_db(authors, pubs_per_author): + """Upsert hasil scrape ke tabel penelitian + isi kolom dosen.sinta.""" + conn = db_connect() + ensure_schema(conn) + stats = {"inserted": 0, "skipped": 0, "dosen_updated": 0} + try: + with conn.cursor() as cur: + cur.execute("SELECT id_dosen, nama, scholar, sinta FROM dosen") + dosen_rows = [ + {"id_dosen": r[0], "nama": r[1], "scholar": (r[2] or "").strip(), "sinta": r[3]} + for r in cur.fetchall() + ] + cur.execute("SELECT id_author, judul FROM penelitian") + existing = {(r[0], norm_title(r[1] or "")) for r in cur.fetchall()} + + for author in authors: + dosen = match_dosen(author["nama"], dosen_rows) + # id_author konsisten dgn data lama: utamakan kolom dosen.scholar + # agar publikasi lama & baru menyatu, baru fallback ke ID dari + # foto profil SINTA, terakhir ID SINTA itu sendiri + id_author = ( + (dosen["scholar"] if dosen and dosen["scholar"] else None) + or author["gs_id"] + or author["sinta_id"] + ) + if dosen and dosen["sinta"] != author["sinta_id"]: + cur.execute( + "UPDATE dosen SET sinta=%s WHERE id_dosen=%s", + (author["sinta_id"], dosen["id_dosen"]), + ) + stats["dosen_updated"] += 1 + if dosen and not dosen["scholar"] and author["gs_id"]: + cur.execute( + "UPDATE dosen SET scholar=%s WHERE id_dosen=%s", + (author["gs_id"], dosen["id_dosen"]), + ) + dosen["scholar"] = author["gs_id"] + + for pub in pubs_per_author.get(author["sinta_id"], []): + key = (id_author, norm_title(pub["judul"])) + if key in existing: + stats["skipped"] += 1 + continue + cur.execute( + "INSERT INTO penelitian " + "(judul, author, link, tahun, author_full, id_author, biaya) " + "VALUES (%s, %s, %s, %s, %s, %s, '')", + ( + pub["judul"][:400], + author["nama"][:100], + pub["link"][:300], + pub["tahun"][:4], + pub["author_full"][:255], + id_author, + ), + ) + existing.add(key) + stats["inserted"] += 1 + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + return stats + + +# ------------------------------------------------------------------ orchestra + +def run(dry_run=False): + """Entry point - dipakai oleh Airflow PythonOperator maupun CLI.""" + session = make_session() + logged_in = login(session) + + dept_url = os.getenv("SINTA_DEPT_URL", DEFAULT_DEPT_URL) + authors = get_department_authors(session, dept_url) + if not authors: + raise RuntimeError( + "Tidak ada dosen terbaca dari halaman departemen SINTA - " + "kemungkinan struktur HTML berubah atau situs tidak bisa diakses." + ) + + pubs_per_author, total = {}, 0 + for i, author in enumerate(authors, 1): + pubs = get_author_publications(session, author["sinta_id"]) + pubs_per_author[author["sinta_id"]] = pubs + total += len(pubs) + log.info("[%d/%d] %s: %d publikasi", i, len(authors), author["nama"], len(pubs)) + + log.info("Total %d publikasi dari %d dosen (login=%s)", total, len(authors), logged_in) + + if dry_run: + return {"authors": len(authors), "publications": total, "logged_in": logged_in} + + stats = sync_to_db(authors, pubs_per_author) + log.info( + "DB sync: %d baris baru, %d sudah ada, %d dosen di-update kolom sinta", + stats["inserted"], stats["skipped"], stats["dosen_updated"], + ) + return {"authors": len(authors), "publications": total, "logged_in": logged_in, **stats} + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + parser = argparse.ArgumentParser(description="Scrape penelitian dosen dari SINTA") + parser.add_argument("--dry-run", action="store_true", help="scrape tanpa menulis ke DB") + args = parser.parse_args() + print(run(dry_run=args.dry_run))