29 lines
624 B
Python
29 lines
624 B
Python
from datetime import datetime
|
|
|
|
import pandas as pd
|
|
from parser import pipeline
|
|
|
|
from . import models
|
|
from .database import get_db
|
|
|
|
|
|
def job():
|
|
fetch_start = datetime.now()
|
|
print("Starting refetch job: " + fetch_start.isoformat())
|
|
|
|
db = next(get_db())
|
|
|
|
parser = pipeline()
|
|
|
|
db.query(models.Record).delete()
|
|
db.commit()
|
|
|
|
print("Rewriting db: " + datetime.now().isoformat())
|
|
|
|
for i, row in parser.df.iterrows():
|
|
row = row.where((pd.notnull(row)), None)
|
|
db.add(models.Record(**row.to_dict()))
|
|
db.commit()
|
|
|
|
print(f"Fetched in {datetime.now() - fetch_start}\n{parser}")
|