42 lines
906 B
Python
42 lines
906 B
Python
from datetime import datetime
|
|
from parser import pipeline
|
|
|
|
import pandas as pd
|
|
import psycopg
|
|
|
|
from .database import db_credentials
|
|
|
|
sql_statement = """COPY records (
|
|
index,
|
|
region,
|
|
area,
|
|
town,
|
|
street,
|
|
branch,
|
|
res,
|
|
comment,
|
|
building_id,
|
|
lat,
|
|
lng,
|
|
start,
|
|
finish
|
|
) FROM STDIN"""
|
|
|
|
|
|
def job():
|
|
fetch_start = datetime.now()
|
|
print("Starting refetch job: " + fetch_start.isoformat())
|
|
|
|
parser = pipeline()
|
|
|
|
print("Rewriting db: " + datetime.now().isoformat())
|
|
|
|
with psycopg.connect(**db_credentials) as connection:
|
|
with connection.cursor() as cursor:
|
|
with cursor.copy(sql_statement) as copy:
|
|
for _, row in parser.df.iterrows():
|
|
row = row.where((pd.notnull(row)), None)
|
|
copy.write_row(row.to_list())
|
|
|
|
print(f"Fetched in {datetime.now() - fetch_start}\n{parser}")
|