13 lines
426 B
Python
13 lines
426 B
Python
import requests
|
|
import pandas as pd
|
|
|
|
def run_scraping_logic():
|
|
# Contoh scraping
|
|
url = "http://universities.hipolabs.com/search?country=Indonesia"
|
|
res = requests.get(url)
|
|
df = pd.DataFrame(res.json())
|
|
|
|
# Logika cleaning atau upload ke BigQuery ditaruh di sini
|
|
output_path = "/opt/airflow/logs/data_akademik.csv"
|
|
df.to_csv(output_path, index=False)
|
|
return f"Berhasil menarik {len(df)} data." |