from rosseti_parser import pipeline, preprocess_read_df
import pandas as pd
import numpy as np
from datetime import datetime
import logging

from .database import get_db
from . import models

from io import StringIO


def job():
    fetch_start = datetime.now()
    print("Starting refetch job: " + fetch_start.isoformat())

    db = next(get_db())

    parser = pipeline()

    db.query(models.Record).delete()
    db.commit()

    print("Rewriting db: " + datetime.now().isoformat())

    for i, row in parser.df.iterrows():
        row = row.where((pd.notnull(row)), None)
        db.add(models.Record(**row.to_dict()))
        db.commit()

    print(f"Fetched in {datetime.now() - fetch_start}\n{parser}")