Format code, renamed modules

runner/README.md

@@ -0,0 +1,27 @@
+# Parser API
+
+## Описание
+
+FastAPI REST API, предоставляющий доступ к данным с сайта [Россети Ленэнерго](https://rosseti-lenenergo.ru/planned_work/).
+
+## Доступные методы
+
+- GET `/api/list` - Поиск по каждому полю в отдельности
+- GET `/api/search` - Поиск по всем полям сразу
+- GET `/api/check` - Проверка, является ли отключение в вашем доме сейчас официальным и если да, то когда сеть снова включат.
+- PUT `/api/create` - Отладочное поле для добавления записей в БД
+- GET `/` - Healthcheck
+
+Подробнее: [Swagger UI](http://localhost:8000/docs) когда запущенно приложение
+
+## Инструкция по запуску
+
+В корневой папке проекта:
+
+```bash
+python -m venv .venv
+
+pip install -r requirements.txt
+
+python -m uvicorn parser_api.main:app
+```

runner/config.py

@@ -0,0 +1,9 @@
+import os
+
+REFETCH_PERIOD_H = int(os.environ.get("REFETCH_PERIOD_H", "4"))
+
+POSTGRES_USER = os.environ.get("POSTGRES_USER", "rosseti")
+POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "rosseti")
+POSTGRES_DB = os.environ.get("POSTGRES_DB", "rosseti")
+POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost")
+POSTGRES_PORT = int(os.environ.get("POSTGRES_PORT", "5432"))

runner/controller.py

@@ -0,0 +1,109 @@
+import datetime
+from functools import reduce
+from typing import List, Optional
+
+from fastapi import HTTPException
+from parser import get_building_id
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.expression import BinaryExpression
+
+from . import models, schemas
+
+
+def create_record(db: Session, record: schemas.Record):
+    db_record = models.Record(
+        region=record.region,
+        area=record.area,
+        town=record.town,
+        street=record.street,
+        start=record.start,
+        finish=record.finish,
+        branch=record.branch,
+        res=record.res,
+        comment=record.comment,
+        building_id=record.building_id,
+        lat=record.lat,
+        lng=record.lng,
+    )
+    db.add(db_record)
+    db.commit()
+    db.refresh(db_record)
+
+    return db_record
+
+
+def contains_lower(name, val):
+    if isinstance(val, str):
+        return getattr(models.Record, name).icontains(val)
+    else:
+        return getattr(models.Record, name) == val
+
+
+def and_if_can(a: BinaryExpression, b: Optional[BinaryExpression]) -> BinaryExpression:
+    if b is not None:
+        return a & b
+    else:
+        return a
+
+
+def search_each(db: Session, filters: schemas.RecordRequest) -> List[schemas.Record]:
+    query = None
+
+    if filters.start:
+        query = models.Record.start <= filters.start
+    if filters.finish:
+        query = and_if_can(models.Record.finish >= filters.finish, query)
+
+    filters = list(
+        filter(lambda x: x[1] is not None and x[0] not in ("start, finish"), filters)
+    )
+
+    query = reduce(
+        lambda acc, ftr: and_if_can(contains_lower(*ftr), acc), filters, query
+    )
+
+    if query is None:
+        res = db.query(models.Record).all()
+    else:
+        res = db.query(models.Record).filter(query).all()
+
+    return res
+
+
+def search_all(db: Session, prompt: str) -> List[schemas.Record]:
+    prompt = prompt.strip()
+
+    query = reduce(
+        lambda acc, name: acc | contains_lower(name, prompt),
+        ("region", "area", "town", "street", "branch", "res"),
+        contains_lower("comment", prompt),
+    )
+
+    building_id, *_ = get_building_id(prompt)
+
+    if building_id is not None:
+        query |= models.Record.building_id == building_id
+
+    res = db.query(models.Record).filter(query).all()
+
+    return res
+
+
+def check_outage(db: Session, building_id: int) -> schemas.CheckResponse:
+    building_query = db.query(models.Record).filter(
+        (models.Record.building_id == building_id)
+    )
+
+    if building_query.count() == 0:
+        raise HTTPException(404, "No such building")
+
+    now = datetime.datetime.now()
+
+    res = building_query.filter(
+        (models.Record.start <= now) & (now <= models.Record.finish)
+    ).first()
+
+    if res is None:
+        return {"is_outage": False}
+
+    return {"is_outage": True, "when_finish": res.finish}

runner/database.py

@@ -0,0 +1,37 @@
+from typing import Generator
+
+from sqlalchemy import URL, create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import Session, sessionmaker
+
+from .config import (
+    POSTGRES_DB,
+    POSTGRES_HOST,
+    POSTGRES_PASSWORD,
+    POSTGRES_PORT,
+    POSTGRES_USER,
+)
+
+engine = create_engine(
+    URL.create(
+        "postgresql+psycopg",
+        username=POSTGRES_USER,
+        password=POSTGRES_PASSWORD,
+        host=POSTGRES_HOST,
+        port=POSTGRES_PORT,
+        database=POSTGRES_DB,
+    ),
+    client_encoding="utf8",
+)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+
+# Dependency
+def get_db() -> Generator[Session, None, None]:
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

runner/job.py

@@ -0,0 +1,28 @@
+from datetime import datetime
+
+import pandas as pd
+from parser import pipeline
+
+from . import models
+from .database import get_db
+
+
+def job():
+    fetch_start = datetime.now()
+    print("Starting refetch job: " + fetch_start.isoformat())
+
+    db = next(get_db())
+
+    parser = pipeline()
+
+    db.query(models.Record).delete()
+    db.commit()
+
+    print("Rewriting db: " + datetime.now().isoformat())
+
+    for i, row in parser.df.iterrows():
+        row = row.where((pd.notnull(row)), None)
+        db.add(models.Record(**row.to_dict()))
+        db.commit()
+
+    print(f"Fetched in {datetime.now() - fetch_start}\n{parser}")

runner/main.py

@@ -0,0 +1,35 @@
+import datetime
+
+import schedule
+from fastapi import FastAPI
+
+from . import models, router, schemas
+from .config import REFETCH_PERIOD_H
+from .database import engine
+from .job import job
+from .scheduler import run_continuously, run_threaded
+
+models.Base.metadata.create_all(bind=engine)
+
+start_stamp = datetime.datetime.now()
+
+
+async def lifespan(app: FastAPI):
+    schedule.every(REFETCH_PERIOD_H).hours.do(job)
+    stop_run_continuously = run_continuously()
+
+    run_threaded(job)
+
+    yield
+
+    stop_run_continuously()
+
+
+app = FastAPI(lifespan=lifespan)
+
+app.include_router(router.router)
+
+
+@app.get("/", response_model=schemas.Healthcheck)
+def Healthcheck():
+    return {"up_since": start_stamp}

runner/models.py

@@ -0,0 +1,22 @@
+from sqlalchemy import Column, DateTime, Float, Integer, String
+
+from .database import Base
+
+
+class Record(Base):
+    __tablename__ = "records"
+
+    id = Column(Integer, primary_key=True, index=True)
+    index = Column(Integer)
+    region = Column(String, nullable=True)
+    area = Column(String, nullable=True)
+    town = Column(String, nullable=True)
+    street = Column(String, nullable=True)
+    start = Column(DateTime)
+    finish = Column(DateTime)
+    branch = Column(String, nullable=True)
+    res = Column(String, nullable=True)
+    comment = Column(String, nullable=True)
+    building_id = Column(Integer, nullable=True)
+    lat = Column(Float, nullable=True)
+    lng = Column(Float, nullable=True)

runner/router.py

@@ -0,0 +1,74 @@
+from typing import Annotated, List
+
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+
+from . import controller, schemas
+from .database import get_db
+
+router = APIRouter(prefix="/api")
+
+
+@router.get("/list", response_model=List[schemas.Record], summary="Search by filters")
+def list_rows(
+    filters: Annotated[schemas.RecordRequest, Depends()], db: Session = Depends(get_db)
+):
+    """
+    Searches rows with specified filters.
+
+    Case insensitive contains:
+    - **region**
+    - **area**
+    - **town**
+    - **street**
+    - **start**
+    - **finish**
+    - **branch**
+    - **res**
+    - **comment**
+    Exact match:
+    - **index**
+    - **building_id**
+    - **lat**
+    - **lng**
+    Later or earlier than respectively:
+    - **start**
+    - **finish**
+    """
+    return controller.search_each(db, filters)
+
+
+@router.get("/search", response_model=List[schemas.Record], summary="Search by query")
+def search_rows(query: str, db: Session = Depends(get_db)):
+    """
+    Selects rows with cells containing case insensitive prompt as its part.
+    In addition, geocoding is being applied to prompt and if building_id found, corresponding row is being returned.
+
+    Rows to be searched:
+    - **region**
+    - **area**
+    - **town**
+    - **street**
+    - **branch**
+    - **res**
+    - **comment**
+    """
+    return controller.search_all(db, query)
+
+
+@router.get(
+    "/check", response_model=schemas.CheckResponse, summary="Check when outage ends"
+)
+def check(building_id: int, db: Session = Depends(get_db)):
+    """
+    Checks if there is an active outage for building_id and if there is, also returns when will it end
+    """
+    return controller.check_outage(db, building_id)
+
+
+@router.put("/create", response_model=schemas.Record)
+def create_record(record: schemas.RecordCreate, db: Session = Depends(get_db)):
+    """
+    Not for public usage
+    """
+    return controller.create_record(db, record)

runner/scheduler.py

@@ -0,0 +1,25 @@
+import threading
+import time
+
+import schedule
+
+
+def run_continuously(interval=1):
+    cease_continuous_run = threading.Event()
+
+    class ScheduleThread(threading.Thread):
+        @classmethod
+        def run(cls):
+            while not cease_continuous_run.is_set():
+                schedule.run_pending()
+                time.sleep(interval)
+
+    continuous_thread = ScheduleThread()
+    continuous_thread.start()
+
+    return cease_continuous_run.set
+
+
+def run_threaded(job):
+    job_thread = threading.Thread(target=job)
+    job_thread.start()

runner/schemas.py

@@ -0,0 +1,43 @@
+import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class BaseRecord(BaseModel):
+    index: Optional[int] = None
+    region: Optional[str] = None
+    area: Optional[str] = None
+    town: Optional[str] = None
+    street: Optional[str] = None
+    branch: Optional[str] = None
+    res: Optional[str] = None
+    comment: Optional[str] = None
+    building_id: Optional[int] = None
+    lat: Optional[float] = None
+    lng: Optional[float] = None
+
+
+class Record(BaseRecord):
+    id: int
+    start: datetime.datetime
+    finish: datetime.datetime
+
+
+class RecordRequest(BaseRecord):
+    start: Optional[datetime.datetime] = None
+    finish: Optional[datetime.datetime] = None
+
+
+class RecordCreate(BaseRecord):
+    start: datetime.datetime
+    finish: datetime.datetime
+
+
+class CheckResponse(BaseModel):
+    is_outage: bool
+    when_finish: Optional[datetime.datetime] = None
+
+
+class Healthcheck(BaseModel):
+    up_since: datetime.datetime