penelitian scheduler
This commit is contained in:
Power BI Dev
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Penarik data penelitian dosen via SerpAPI (Google Scholar) + kuartil Scopus dari SINTA.
|
||||
|
||||
Dua langkah:
|
||||
1. publications : tarik SEMUA publikasi tiap dosen dari Google Scholar lewat
|
||||
SerpAPI (engine google_scholar_author, 100 artikel/request) berdasarkan
|
||||
kolom dosen.scholar, lalu insert baris baru ke tabel `penelitian`.
|
||||
Kolom `link` diisi citation_id (format sama dgn data lama).
|
||||
2. scopus : tarik daftar publikasi Scopus tiap dosen dari SINTA
|
||||
(view=scopus, ada label kuartil), cocokkan judul, lalu UPDATE kolom
|
||||
`penelitian.scopus` (Q1..Q4).
|
||||
|
||||
Konfigurasi environment:
|
||||
DB_HOST, DB_USER, DB_PASS, DB_NAME
|
||||
SERPAPI_KEY (fallback: file apikey.json di root project)
|
||||
SERPAPI_MAX_REQUESTS (default 90 - pengaman kuota free tier 100/bulan)
|
||||
|
||||
CLI:
|
||||
python scholar_api_scraper.py --dry-run # tanpa tulis DB
|
||||
python scholar_api_scraper.py # publications + scopus
|
||||
python scholar_api_scraper.py --only publications
|
||||
python scholar_api_scraper.py --only scopus # tidak pakai kuota SerpAPI
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
import requests
|
||||
|
||||
try: # di image Airflow, modul berada di /opt/airflow/scripts (PYTHONPATH=/opt/airflow)
|
||||
from scripts.sinta_scraper import (
|
||||
db_connect,
|
||||
ensure_schema,
|
||||
get_author_publications,
|
||||
get_department_authors,
|
||||
make_session,
|
||||
match_dosen,
|
||||
norm_title,
|
||||
DEFAULT_DEPT_URL,
|
||||
)
|
||||
except ImportError:
|
||||
from sinta_scraper import (
|
||||
db_connect,
|
||||
ensure_schema,
|
||||
get_author_publications,
|
||||
get_department_authors,
|
||||
make_session,
|
||||
match_dosen,
|
||||
norm_title,
|
||||
DEFAULT_DEPT_URL,
|
||||
)
|
||||
|
||||
log = logging.getLogger("scholar_api")
|
||||
SERPAPI_URL = "https://serpapi.com/search.json"
|
||||
|
||||
|
||||
def get_api_key():
|
||||
key = os.getenv("SERPAPI_KEY")
|
||||
if key:
|
||||
return key.strip()
|
||||
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "apikey.json")
|
||||
if os.path.exists(path):
|
||||
return open(path, encoding="utf-8").read().strip().strip('"')
|
||||
raise RuntimeError("SERPAPI_KEY tidak di-set dan apikey.json tidak ditemukan")
|
||||
|
||||
|
||||
def load_dosen():
|
||||
"""Daftar dosen dari tabel dosen. Fallback ke halaman departemen SINTA
|
||||
(ID Scholar dari URL foto) bila DB tidak bisa diakses (mis. saat dry-run)."""
|
||||
try:
|
||||
conn = db_connect()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id_dosen, nama, scholar, sinta FROM dosen")
|
||||
rows = [
|
||||
{
|
||||
"id_dosen": r[0],
|
||||
"nama": re.sub(r"\s+", " ", (r[1] or "")).strip(),
|
||||
"scholar": (r[2] or "").strip(),
|
||||
"sinta": (r[3] or "").strip() if r[3] else "",
|
||||
}
|
||||
for r in cur.fetchall()
|
||||
]
|
||||
conn.close()
|
||||
return rows, True
|
||||
except Exception as e:
|
||||
log.warning("DB tidak bisa diakses (%s) - fallback ke daftar SINTA", e)
|
||||
authors = get_department_authors(make_session(), DEFAULT_DEPT_URL)
|
||||
return [
|
||||
{"id_dosen": None, "nama": a["nama"], "scholar": a["gs_id"] or "", "sinta": a["sinta_id"]}
|
||||
for a in authors
|
||||
], False
|
||||
|
||||
|
||||
# ------------------------------------------------- langkah 1: SerpAPI Scholar
|
||||
|
||||
def fetch_author_articles(key, author_id, budget):
|
||||
"""Semua artikel satu author dari SerpAPI. Return (nama_profil_gs, list artikel)."""
|
||||
articles, start, profile_name = [], 0, ""
|
||||
while True:
|
||||
if budget["used"] >= budget["max"]:
|
||||
log.warning("Budget SerpAPI (%d request) habis - stop di %s", budget["max"], author_id)
|
||||
break
|
||||
resp = requests.get(
|
||||
SERPAPI_URL,
|
||||
params={
|
||||
"engine": "google_scholar_author",
|
||||
"author_id": author_id,
|
||||
"num": 100,
|
||||
"start": start,
|
||||
"api_key": key,
|
||||
"hl": "id",
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
budget["used"] += 1
|
||||
data = resp.json()
|
||||
if "error" in data:
|
||||
raise RuntimeError(data["error"])
|
||||
profile_name = data.get("author", {}).get("name", profile_name)
|
||||
batch = data.get("articles", [])
|
||||
articles.extend(batch)
|
||||
if len(batch) < 100 or "next" not in data.get("serpapi_pagination", {}):
|
||||
break
|
||||
start += 100
|
||||
return profile_name, articles
|
||||
|
||||
|
||||
def _apply_filter(rows, dosen_filter):
|
||||
if not dosen_filter:
|
||||
return rows
|
||||
f = dosen_filter.lower()
|
||||
return [r for r in rows if f in r["nama"].lower() or f == r["scholar"].lower()]
|
||||
|
||||
|
||||
def enrich_dosen(session, conn, dosen_rows, dry_run=False):
|
||||
"""Isi dosen.sinta dan dosen.scholar yang masih kosong dari halaman
|
||||
departemen SINTA (ID Scholar diambil dari URL foto profil)."""
|
||||
if not any(not d["sinta"] or not d["scholar"] for d in dosen_rows):
|
||||
return
|
||||
for a in get_department_authors(session, DEFAULT_DEPT_URL):
|
||||
d = match_dosen(a["nama"], dosen_rows)
|
||||
if d is None:
|
||||
continue
|
||||
updates = {}
|
||||
if not d["sinta"]:
|
||||
d["sinta"] = updates["sinta"] = a["sinta_id"]
|
||||
if not d["scholar"] and a["gs_id"]:
|
||||
d["scholar"] = updates["scholar"] = a["gs_id"]
|
||||
if updates and conn and d["id_dosen"] is not None and not dry_run:
|
||||
with conn.cursor() as cur:
|
||||
for col, val in updates.items():
|
||||
cur.execute(f"UPDATE dosen SET {col}=%s WHERE id_dosen=%s", (val, d["id_dosen"]))
|
||||
log.info("dosen '%s' dilengkapi: %s", d["nama"], updates)
|
||||
if conn and not dry_run:
|
||||
conn.commit()
|
||||
|
||||
|
||||
def run_publications(dry_run=False, dosen_filter=None):
|
||||
key = get_api_key()
|
||||
budget = {"used": 0, "max": int(os.getenv("SERPAPI_MAX_REQUESTS", "90"))}
|
||||
dosen_rows, db_ok = load_dosen()
|
||||
conn = db_connect() if (not dry_run and db_ok) else None
|
||||
if conn:
|
||||
ensure_schema(conn)
|
||||
enrich_dosen(make_session(), conn, dosen_rows, dry_run)
|
||||
targets = _apply_filter([d for d in dosen_rows if d["scholar"]], dosen_filter)
|
||||
log.info("%d dosen punya ID Scholar (dari %d)", len(targets), len(dosen_rows))
|
||||
|
||||
stats = {"authors": 0, "articles": 0, "inserted": 0, "skipped": 0, "errors": 0}
|
||||
try:
|
||||
existing = set()
|
||||
if conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id_author, judul FROM penelitian")
|
||||
existing = {(r[0], norm_title(r[1] or "")) for r in cur.fetchall()}
|
||||
|
||||
for d in targets:
|
||||
try:
|
||||
gs_name, articles = fetch_author_articles(key, d["scholar"], budget)
|
||||
except Exception as e:
|
||||
log.error("Gagal tarik %s (%s): %s", d["nama"], d["scholar"], e)
|
||||
stats["errors"] += 1
|
||||
continue
|
||||
stats["authors"] += 1
|
||||
stats["articles"] += len(articles)
|
||||
author_name = gs_name or d["nama"]
|
||||
log.info("%s: %d artikel (request terpakai: %d)", author_name, len(articles), budget["used"])
|
||||
if not conn:
|
||||
continue
|
||||
with conn.cursor() as cur:
|
||||
for art in articles:
|
||||
judul = (art.get("title") or "").strip()
|
||||
if not judul:
|
||||
continue
|
||||
key_row = (d["scholar"], norm_title(judul))
|
||||
if key_row in existing:
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
ym = re.search(r"\d{4}", str(art.get("year") or ""))
|
||||
cur.execute(
|
||||
"INSERT INTO penelitian "
|
||||
"(judul, author, link, tahun, author_full, id_author, biaya) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, '')",
|
||||
(
|
||||
judul[:400],
|
||||
author_name[:100],
|
||||
(art.get("citation_id") or "")[:300],
|
||||
ym.group(0) if ym else "",
|
||||
(art.get("authors") or "")[:255],
|
||||
d["scholar"],
|
||||
),
|
||||
)
|
||||
existing.add(key_row)
|
||||
stats["inserted"] += 1
|
||||
if conn:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
if conn:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
stats["serpapi_requests"] = budget["used"]
|
||||
log.info("Publications selesai: %s", stats)
|
||||
return stats
|
||||
|
||||
|
||||
# --------------------------------------------- langkah 2: kuartil dari SINTA
|
||||
|
||||
def run_scopus(dry_run=False, dosen_filter=None):
|
||||
session = make_session()
|
||||
dosen_rows, db_ok = load_dosen()
|
||||
conn = db_connect() if db_ok else None
|
||||
if conn:
|
||||
ensure_schema(conn)
|
||||
|
||||
enrich_dosen(session, conn, dosen_rows, dry_run)
|
||||
|
||||
targets = _apply_filter([d for d in dosen_rows if d["sinta"]], dosen_filter)
|
||||
log.info("%d dosen punya SINTA ID", len(targets))
|
||||
|
||||
stats = {"authors": 0, "scopus_items": 0, "updated": 0, "unmatched": 0}
|
||||
pubs_by_author = {}
|
||||
if conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id_artikel, id_author, judul, scopus FROM penelitian")
|
||||
for r in cur.fetchall():
|
||||
pubs_by_author.setdefault(r[1], []).append(
|
||||
{"id_artikel": r[0], "nt": norm_title(r[2] or ""), "scopus": r[3]}
|
||||
)
|
||||
try:
|
||||
for d in targets:
|
||||
items = get_author_publications(session, d["sinta"], view="scopus")
|
||||
stats["authors"] += 1
|
||||
stats["scopus_items"] += len(items)
|
||||
if not conn:
|
||||
continue
|
||||
rows = pubs_by_author.get(d["scholar"], [])
|
||||
with conn.cursor() as cur:
|
||||
for it in items:
|
||||
q = it["quartile"] or "Scopus"
|
||||
match = next((r for r in rows if r["nt"] == norm_title(it["judul"])), None)
|
||||
if not match:
|
||||
stats["unmatched"] += 1
|
||||
continue
|
||||
if (match["scopus"] or "") != q and not dry_run:
|
||||
cur.execute(
|
||||
"UPDATE penelitian SET scopus=%s WHERE id_artikel=%s",
|
||||
(q, match["id_artikel"]),
|
||||
)
|
||||
match["scopus"] = q
|
||||
stats["updated"] += 1
|
||||
if conn and not dry_run:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
if conn:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
log.info("Scopus selesai: %s", stats)
|
||||
return stats
|
||||
|
||||
|
||||
def run(dry_run=False):
|
||||
"""Entry point gabungan - dipakai CLI; Airflow memanggil per-langkah."""
|
||||
return {
|
||||
"publications": run_publications(dry_run=dry_run),
|
||||
"scopus": run_scopus(dry_run=dry_run),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
parser = argparse.ArgumentParser(description="Tarik penelitian via SerpAPI + kuartil SINTA")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--only", choices=["publications", "scopus"])
|
||||
parser.add_argument("--dosen", help="filter satu dosen: potongan nama atau ID Scholar")
|
||||
args = parser.parse_args()
|
||||
if args.only == "publications":
|
||||
print(run_publications(dry_run=args.dry_run, dosen_filter=args.dosen))
|
||||
elif args.only == "scopus":
|
||||
print(run_scopus(dry_run=args.dry_run, dosen_filter=args.dosen))
|
||||
else:
|
||||
print(run_publications(dry_run=args.dry_run, dosen_filter=args.dosen))
|
||||
print(run_scopus(dry_run=args.dry_run, dosen_filter=args.dosen))
|
||||
Reference in New Issue
Block a user