init new dags
This commit is contained in:
0
scripts/.gitkeep
Normal file
0
scripts/.gitkeep
Normal file
13
scripts/scraper_pddikti.py
Normal file
13
scripts/scraper_pddikti.py
Normal file
@@ -0,0 +1,13 @@
|
||||
import requests
|
||||
import pandas as pd
|
||||
|
||||
def run_scraping_logic():
|
||||
# Contoh scraping
|
||||
url = "http://universities.hipolabs.com/search?country=Indonesia"
|
||||
res = requests.get(url)
|
||||
df = pd.DataFrame(res.json())
|
||||
|
||||
# Logika cleaning atau upload ke BigQuery ditaruh di sini
|
||||
output_path = "/opt/airflow/logs/data_akademik.csv"
|
||||
df.to_csv(output_path, index=False)
|
||||
return f"Berhasil menarik {len(df)} data."
|
||||
Reference in New Issue
Block a user