dm1sh/lenengro_parser
1
0
Fork 0
Code Pull Requests Activity

classification(full address)

Browse Source
This commit is contained in:
AnastasiaOnimovma 2023-10-16 22:02:12 +03:00
parent cb422b9a2f
commit d822c5012b
2 changed files with 92 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
parser/address.py
View File

@ -7,16 +7,17 @@ import pandas as pd
T = TypeVar("T") T = TypeVar("T")
CLASSES = ("s", "h", "b", "l", "?") CLASSES = ("d", "c", "t", "s", "h", "b", "l", "r", "w")
DISTRICTS_PREFIXES = ("мо ", "р-н")
SETTLEMENTS_PREFIXES = ( COUNTRYSIDE_PREFIXES = (
"г", "мо", "р-н", "п", "д", "гп", "c", "хутор", "массив", "тер", "СНТ", "СТ", "ДСК", "ДНП", "ДПК", "НП", " г", " п", " д", " гп", " рп", " кп", " пгт", " c", "хутор", " урочище"
"садоводство") "г.", "п.", "д.", "гп.", "рп.", "кп.", "пгт.", "c.")
TERRITORY_PREFIXES =("тер.", " тер", "снт ", "ст ", "дск ", "днп ", "дпк ", "нп ", "пдк ", "т/б ", "садоводство", "массив", "хоз","сад-во","с-во")
STREET_PREFIXES = ( STREET_PREFIXES = (
" ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе","линия","аллея", "мост", "парк", "кольцо","проезд", " ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе","линия","аллея", "мост", " парк", "кольцо","проезд", "съезд",
"ул.", "бул.", "пр.", "ш.", "пер.", "дор.", "маг.", "наб.", "пл.", "просп.", "туп.") "ул.", "бул.", "пр.", "ш.", "пер.", "дор.", "маг.", "наб.", "пл.", "просп.", "туп.")
HOUSES_PREFIXES = ("д.", "уч.", "участок") HOUSES_PREFIXES = ("д.", "уч.", "участок","мкд","тп")
BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение") BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение","корпус")
LETTER = ("лит.", "литера"," л.") LETTER = ("лит.", "литера"," л.")
@ -58,46 +59,67 @@ def any_of_in(substrings: Iterable[str], string: str) -> bool:
def flatten(arr: Iterable[List[T]]) -> List[T]: def flatten(arr: Iterable[List[T]]) -> List[T]:
return sum(arr, []) return sum(arr, [])
def find_room(token: pd.Series, pre_token: pd.Series) -> str:
if re.search(r"пом\.?", token['obj']):
return "r"
return ""
def find_litera(token: pd.Series, pre_token: pd.Series) -> str: def find_litera(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(LETTER, token['obj']) \ if any_of_in(LETTER, token['obj'].lower()) \
or re.search(r"\d{1,3}[А-Яа-я]( |$)", token['obj']): or re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", token['obj']):
return "l" return "l"
# не работает if (re.search(r"\b([А-Я]|[а-я]){1}$", token['obj']) \
if (re.search(r"\b[А-Яа-я]{1}\b", token['obj']) and "l" in pre_token['class']): and ("l" in pre_token['class'] or "h" in pre_token['class'])) \
and not (" ш" in token["obj"]) \
and not find_countryside(token,pre_token):
return "l" return "l"
return "" return ""
def find_building(token: pd.Series, pre_token: pd.Series) -> str: def find_building(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(BUILDING_PREFIXES, token['obj']) \ if re.search(r"\d", token['obj']):
or (re.search(r"\d", token['obj']) and "b" in pre_token['class']) \ if any_of_in(BUILDING_PREFIXES, token['obj'].lower()) \
or re.search(r"к\.*\d", token['obj']) \ or "b" in pre_token['class'] and not ("h" in token['class'])\
or re.search(r"\d", token['obj']) and "b" in pre_token['class']: or re.search(r"к\.* ?\d", token['obj']):
return "b" return "b"
return "" return ""
def find_house(token: pd.Series, pre_token: pd.Series) -> str: def find_house(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(HOUSES_PREFIXES, token['obj']): if re.search(r"\d{1,4}", token['obj']):
return "h" if any_of_in(HOUSES_PREFIXES, token['obj'].lower()):
if re.search(r"(д|д\.) ?\d{1,3} ?\/*\d* ?", token['obj']) and not ("-я" in token['obj']): return "h"
if "h" in pre_token['class'] \ if re.search(r"(д|д\.) ?\d{1,4} ?\/*\d* ?", token['obj']):
or "s" in pre_token['class'] \ return "h"
or "s" in token['class']: if ("s" in pre_token['class'] or "h" in pre_token['class'] or "s" in token['class']) \
and not any_of_in(("-я", "-й", "-Я"), token['obj'])\
and not find_building(token,pre_token):
return "h" return "h"
# не работает
if re.search(r"\d{1,3}", token['obj']) and ("s" in pre_token['class'] or "h" in pre_token['class']):
return "h"
return "" return ""
def find_street(token: pd.Series, pre_token: pd.Series) -> str: def find_street(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(STREET_PREFIXES, token['obj']) \ if any_of_in(STREET_PREFIXES, token['obj'].lower()) \
or (re.search(r"[А-Я]{1}[а-я]+", token['obj']) and "s" in pre_token['class']): or re.search(r"[А-Я]{1}[а-я]+ая", token['obj']):
return "s" return "s"
return "" return ""
def find_territory(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(TERRITORY_PREFIXES, token['obj'].lower()):
return "t"
return ""
def find_countryside(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(COUNTRYSIDE_PREFIXES, token['obj'].lower()) \
and not find_house(token,pre_token) \
and not find_street(token,pre_token):
return "c"
return ""
def find_district(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(DISTRICTS_PREFIXES, token['obj'].lower()):
return "d"
return ""
# TODO: переработать систему из if в нормальный вид и классификация чисел/букв # TODO: переработать систему из if в нормальный вид и классификация чисел/букв
def split_address(address: str) -> List[str]: def split_address(address: str) -> List[str]:
@ -125,43 +147,68 @@ def split_address(address: str) -> List[str]:
pre_token = pd.Series(data=["", ""], index=['obj', 'class']) pre_token = pd.Series(data=["", ""], index=['obj', 'class'])
else: else:
pre_token = tokens.iloc[i - 1] pre_token = tokens.iloc[i - 1]
obj_class = find_district(cur_tk, pre_token)
if obj_class:
cur_tk["class"] += obj_class
if "d" in pre_token['class']:
res.append(accumulator)
accumulator = ""
accumulator += cur_tk["obj"]
obj_class = find_countryside(cur_tk, pre_token)
if obj_class:
cur_tk["class"] += obj_class
if "c" in pre_token['class']:
res.append(accumulator)
accumulator = ""
accumulator += cur_tk["obj"]
obj_class = find_territory(cur_tk, pre_token)
if obj_class:
cur_tk["class"] += obj_class
if "t" in pre_token['class']:
res.append(accumulator)
accumulator = ""
accumulator +=cur_tk["obj"]
obj_class = find_street(cur_tk, pre_token) obj_class = find_street(cur_tk, pre_token)
if obj_class: if obj_class:
cur_tk["class"] += obj_class cur_tk["class"] += obj_class
if "s" in tokens['class'].iloc[i - 1]: if "s" in pre_token['class']:
res.append(accumulator) res.append(accumulator)
accumulator = "" accumulator = ""
accumulator += tokens["obj"].iloc[i] accumulator += cur_tk["obj"]
obj_class = find_house(cur_tk, pre_token) obj_class = find_house(cur_tk, pre_token)
if obj_class: if obj_class:
cur_tk["class"] += obj_class cur_tk["class"] += obj_class
if "h" in tokens['class'].iloc[i - 1]: if "h" in pre_token["class"]:
res.append(accumulator) res.append(accumulator)
num = re.findall("\d{,3}", tokens['obj'].iloc[i])[-1] num = re.findall("\d{1,4}", cur_tk['obj'])[-1]
accumulator = re.sub(r"\d{,3} ?\/*\d* ?", num,accumulator) accumulator = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator)
else: else:
accumulator += tokens["obj"].iloc[i] accumulator += cur_tk["obj"]
obj_class = find_building(cur_tk, pre_token) obj_class = find_building(cur_tk, pre_token)
if obj_class: if obj_class:
cur_tk["class"] += obj_class cur_tk["class"] += obj_class
if "b" in tokens['class'].iloc[i - 1]: if "b" in pre_token["class"]:
res.append(accumulator) res.append(accumulator)
num = re.findall("\d", tokens['obj'].iloc[i])[-1] num = re.findall("\d", tokens['obj'].iloc[i])[-1]
accumulator = re.sub(r"\d$", num, accumulator) accumulator = re.sub(r"\d$", num, accumulator)
else: else:
accumulator += tokens["obj"].iloc[i] accumulator += pre_token["obj"]
obj_class = find_litera(cur_tk, pre_token) obj_class = find_litera(cur_tk, pre_token)
if obj_class: if obj_class:
cur_tk["class"] += obj_class cur_tk["class"] += obj_class
if "l" in tokens['class'].iloc[i - 1]: if "l" in pre_token["class"]:
res.append(accumulator) res.append(accumulator)
num = re.findall("[А-яа-я]", tokens['obj'].iloc[i].strip())[-1] num = re.findall("[А-яа-я]", cur_tk["obj"].strip())[-1]
accumulator = re.sub(r"[А-яа-я]$", num, accumulator) accumulator = re.sub(r"[А-яа-я]$", num, accumulator)
else: else:
accumulator += tokens["obj"].iloc[i] accumulator += cur_tk["obj"]
if cur_tk['class'] == "": if cur_tk['class'] == "":
cur_tk['class'] = "w" cur_tk['class'] = "w"
print(cur_tk) tokens.iloc[i] = cur_tk
print(tokens.iloc[i])
# print(cur_tk)
return res return res
return [address] return [address]

6
parser/util.py
View File

@ -10,12 +10,16 @@ from . import (
def pipeline(parser: Optional[LenenergoParser] = None) -> LenenergoParser: def pipeline(parser: Optional[LenenergoParser] = None) -> LenenergoParser:
if parser is None: if parser is None:
parser = LenenergoParser() parser = LenenergoParser(ndays=15)
print(parser) print(parser)
parser.df = split_addresses(parser.df) parser.df = split_addresses(parser.df)
for i in range(len(parser.df)):
print(parser.df['Улица'].iloc[i])
parser.df = concurrent_fetch_builing_ids(parser.df) parser.df = concurrent_fetch_builing_ids(parser.df)
parser.df = preprocess_df(parser.df) parser.df = preprocess_df(parser.df)