FInished work

(Too lazy to split by commits)

parser_api/config.py

@@ -0,0 +1,9 @@
+import os
+
+REFETCH_PERIOD_H = int(os.environ.get('REFETCH_PERIOD_H', '4'))
+
+POSTGRES_USER = os.environ.get('POSTGRES_USER', 'rosseti')
+POSTGRES_PASSWORD = os.environ.get('POSTGRES_PASSWORD', 'rosseti')
+POSTGRES_DB = os.environ.get('POSTGRES_DB', 'rosseti')
+POSTGRES_HOST = os.environ.get('POSTGRES_HOST', 'localhost')
+POSTGRES_PORT = int(os.environ.get('POSTGRES_PORT', '5432'))

parser_api/controller.py

@@ -0,0 +1,111 @@
+from typing import List, Optional
+from functools import reduce
+import datetime
+
+from fastapi import HTTPException
+from sqlalchemy import func, True_
+from sqlalchemy.orm import Session
+from sqlalchemy.sql import operators
+from sqlalchemy.sql.expression import BinaryExpression
+
+from . import models, schemas
+
+
+def create_record(db: Session, record: schemas.Record):
+    db_record = models.Record(
+        region=record.region,
+        area=record.area,
+        town=record.town,
+        street=record.street,
+        start=record.start,
+        finish=record.finish,
+        branch=record.branch,
+        res=record.res,
+        comment=record.comment,
+        building_id=record.building_id,
+        lat=record.lat,
+        lng=record.lng,
+    )
+    db.add(db_record)
+    db.commit()
+    db.refresh(db_record)
+
+    return db_record
+
+
+def contains_lower(name, val):
+    if type(val) == str:
+        return getattr(models.Record, name).icontains(val)
+    else:
+        return getattr(models.Record, name) == val
+
+
+def and_if_can(a: BinaryExpression, b: Optional[BinaryExpression]):
+    if b is not None:
+        return a & b
+    else:
+        return a
+
+
+def search_each(db: Session, filters: schemas.RecordRequest) -> List[schemas.Record]:
+    query = None
+
+    if filters.start:
+        query = (models.Record.start <= filters.start)
+    if filters.finish:
+        query = and_if_can(models.Record.finish >= filters.finish, query)
+
+    filters = list(
+        filter(lambda x: x[1] is not None and x[0] not in ('start, finish'), filters))
+
+    query = reduce(lambda acc, ftr: and_if_can(
+        contains_lower(*ftr), acc), filters, query)
+
+    if query is None:
+        res = db.query(models.Record).all()
+
+    res = db.query(models.Record).filter(query).all()
+
+    return res
+
+
+def search_all(db: Session, prompt: str) -> List[schemas.Record]:
+    prompt = prompt.strip()
+
+    query = reduce(lambda acc, name: acc | contains_lower(name, prompt), (
+        'region',
+        'area',
+        'town',
+        'street',
+        'branch',
+        'res'
+    ), contains_lower('comment', prompt))
+
+    res = db.query(models.Record).filter(query).all()
+
+    return res
+
+
+def check_outage(db: Session, building_id: int) -> schemas.CheckResponse:
+    building_query = db.query(models.Record).filter(
+        (models.Record.building_id == building_id))
+
+    if building_query.count() == 0:
+        raise HTTPException(404, 'No such building')
+
+    now = datetime.datetime.now()
+
+    res = building_query.filter(
+        (models.Record.start <= now) &
+        (now <= models.Record.finish)
+    ).first()
+
+    if res is None:
+        return {
+            'is_outage': False
+        }
+
+    return {
+        'is_outage': True,
+        'when_finish': res.finish
+    }

parser_api/database.py

@@ -0,0 +1,31 @@
+from typing import Generator
+import os
+
+from sqlalchemy import create_engine, URL
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, Session
+
+from .config import POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB
+
+engine = create_engine(
+    URL.create(
+        "postgresql+psycopg",
+        username=POSTGRES_USER,
+        password=POSTGRES_PASSWORD,
+        host=POSTGRES_HOST,
+        port=POSTGRES_PORT,
+        database=POSTGRES_DB,
+    ),
+    client_encoding='utf8',
+)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+# Dependency
+def get_db() -> Generator[Session, None, None]:
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

parser_api/job.py

@@ -0,0 +1,31 @@
+from rosseti_parser import pipeline, preprocess_read_df
+import pandas as pd
+import numpy as np
+from datetime import datetime
+import logging
+
+from .database import get_db
+from . import models
+
+from io import StringIO
+
+
+def job():
+    fetch_start = datetime.now()
+    print("Starting refetch job: " + fetch_start.isoformat())
+
+    db = next(get_db())
+
+    parser = pipeline()
+
+    db.query(models.Record).delete()
+    db.commit()
+
+    print("Rewriting db: " + datetime.now().isoformat())
+
+    for i, row in parser.df.iterrows():
+        row = row.where((pd.notnull(row)), None)
+        db.add(models.Record(**row.to_dict()))
+        db.commit()
+
+    print(f"Fetched in {datetime.now() - fetch_start}\n{parser}")

parser_api/main.py

@@ -0,0 +1,37 @@
+from contextlib import asynccontextmanager
+import datetime
+
+from fastapi import FastAPI
+import schedule
+
+from . import models, router
+from .database import engine
+from .scheduler import run_continuously, run_threaded
+from .job import job
+from .config import REFETCH_PERIOD_H
+
+models.Base.metadata.create_all(bind=engine)
+
+start_stamp = datetime.datetime.now()
+
+
+async def lifespan(app: FastAPI):
+    schedule.every(REFETCH_PERIOD_H).hours.do(job)
+    stop_run_continuously = run_continuously()
+
+    run_threaded(job)
+
+    yield
+
+    stop_run_continuously()
+
+app = FastAPI(lifespan=lifespan)
+
+app.include_router(router.router)
+
+
+@app.get('/')
+def root():
+    return {
+        "up_since": start_stamp
+    }

parser_api/models.py

@@ -0,0 +1,23 @@
+from sqlalchemy import Boolean, Column, Integer, String, DateTime, Float
+from sqlalchemy.orm import relationship
+
+from .database import Base
+
+
+class Record(Base):
+    __tablename__ = 'records'
+
+    id = Column(Integer, primary_key=True, index=True)
+    index = Column(Integer)
+    region = Column(String, nullable=True)
+    area = Column(String, nullable=True)
+    town = Column(String, nullable=True)
+    street = Column(String, nullable=True)
+    start = Column(DateTime)
+    finish = Column(DateTime)
+    branch = Column(String, nullable=True)
+    res = Column(String, nullable=True)
+    comment = Column(String, nullable=True)
+    building_id = Column(Integer, nullable=True)
+    lat = Column(Float, nullable=True)
+    lng = Column(Float, nullable=True)

parser_api/router.py

@@ -0,0 +1,33 @@
+from fastapi import HTTPException, Depends
+from sqlalchemy.orm import Session
+from typing import List, Annotated
+
+from fastapi import APIRouter
+
+from . import models, schemas, controller
+from .database import SessionLocal, get_db
+
+router = APIRouter(prefix='/api')
+
+
+@router.get('/list', response_model=List[schemas.Record])
+def list_rows(
+    filters: Annotated[schemas.RecordRequest, Depends()],
+    db: Session = Depends(get_db)
+):
+    return controller.search_each(db, filters)
+
+
+@router.get('/search', response_model=List[schemas.Record])
+def search_rows(query: str, db: Session = Depends(get_db)):
+    return controller.search_all(db, query)
+
+
+@router.get('/check', response_model=schemas.CheckResponse)
+def check(building_id: int, db: Session = Depends(get_db)):
+    return controller.check_outage(db, building_id)
+
+
+@router.put('/create', response_model=schemas.Record)
+def create_record(record: schemas.RecordCreate, db: Session = Depends(get_db)):
+    return controller.create_record(db, record)

parser_api/scheduler.py

@@ -0,0 +1,25 @@
+import threading
+import time
+
+import schedule
+
+
+def run_continuously(interval=1):
+    cease_continuous_run = threading.Event()
+
+    class ScheduleThread(threading.Thread):
+        @classmethod
+        def run(cls):
+            while not cease_continuous_run.is_set():
+                schedule.run_pending()
+                time.sleep(interval)
+
+    continuous_thread = ScheduleThread()
+    continuous_thread.start()
+
+    return cease_continuous_run.set
+
+
+def run_threaded(job):
+    job_thread = threading.Thread(target=job)
+    job_thread.start()

parser_api/schemas.py

@@ -0,0 +1,39 @@
+import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class BaseRecord(BaseModel):
+    index: Optional[int] = None
+    region: Optional[str] = None
+    area: Optional[str] = None
+    town: Optional[str] = None
+    street: Optional[str] = None
+    branch: Optional[str] = None
+    res: Optional[str] = None
+    comment: Optional[str] = None
+    building_id: Optional[int] = None
+    lat: Optional[float] = None
+    lng: Optional[float] = None
+
+
+class Record(BaseRecord):
+    id: int
+    start: datetime.datetime
+    finish: datetime.datetime
+
+
+class RecordRequest(BaseRecord):
+    start: Optional[datetime.datetime] = None
+    finish: Optional[datetime.datetime] = None
+
+
+class RecordCreate(BaseRecord):
+    start: datetime.datetime
+    finish: datetime.datetime
+
+
+class CheckResponse(BaseModel):
+    is_outage: bool
+    when_finish: Optional[datetime.datetime] = None