dm1sh/lenengro_parser
1
0
Fork 0
Code Pull Requests Activity

Street recognition

Browse Source
This commit is contained in:
AnastasiaOnimovma 2023-10-23 00:42:27 +03:00
parent 1fd7a123f9
commit 259c71b17b
1 changed files with 57 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
parser/address.py
View File

@ -7,8 +7,7 @@ import pandas as pd
T = TypeVar("T")
CLASSES = ("w", "d", "c", "t", "s", "h", "b", "l", "r")
CLASSES = ("w", "d", "c", "t", "s", "h", "b","e", "l", "r")
DISTRICTS_PREFIXES = ("мо ", "р-н","городское","лесхоз")
COUNTRYSIDE_PREFIXES = (
"г", "п", "д", "гп", "рп", "кп", "пгт", "c", "хутор", " урочище")
@ -18,9 +17,11 @@ STREET_PREFIXES = (
" ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе", "лини", "аллея",
"мост", " парк", "кольцо", "проезд", "съезд","переулок",
"ул.", "бул.", "пр.", "ш.", "пер.", "дор.", "маг.", "наб.", "пл.", "просп.", "туп.")
HOUSES_PREFIXES = ("д.", "уч.", "участок", "мкд", "тп","дом")
BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение", "корпус")
HOUSES_PREFIXES = ("д.", "уч.", "участок", "мкд", "тп","дом","дома")
BUILDING_PREFIXES = ("к.", "к ","корп", "корпус")
EDIFICE_PREFIXES=("стр.", "строение")
LETTER = ("лит.", "литера", " л.")
PREFIXES = (DISTRICTS_PREFIXES, COUNTRYSIDE_PREFIXES, TERRITORY_PREFIXES, STREET_PREFIXES, HOUSES_PREFIXES, BUILDING_PREFIXES, EDIFICE_PREFIXES,LETTER)
def unfold_house_ranges(token: str) -> List[str]:
@ -47,12 +48,14 @@ def flatten(arr: Iterable[List[T]]) -> List[T]:
def find_room(token: pd.Series, pre_token: pd.Series) -> str:
if re.search(r"пом\.?", token['obj']):
if re.search(r"\bпом\.?", token['obj']):
return "r"
return ""
def find_litera(token: pd.Series, pre_token: pd.Series) -> str:
if find_room(token, pre_token):
return ""
if any_of_in(LETTER, token['obj'].lower()) \
or re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", token['obj']):
return "l"
@ -62,33 +65,47 @@ def find_litera(token: pd.Series, pre_token: pd.Series) -> str:
and not find_countryside(token, pre_token):
return "l"
return ""
def find_edifice(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(EDIFICE_PREFIXES, token['obj'].lower()):
return "e"
return ""
def find_building(token: pd.Series, pre_token: pd.Series) -> str:
if re.search(r"\d", token['obj']):
if re.search(r"\d", token['obj']) and not find_room(token,pre_token):
if any_of_in(BUILDING_PREFIXES, token['obj'].lower()) \
or "b" in pre_token['class'] and not ("h" in token['class']) \
or "b" in pre_token['class'] and not ("h" in token['class']) and not find_edifice(token,pre_token)\
or re.search(r"к\.* ?\d", token['obj']):
return "b"
return ""
def find_house(token: pd.Series, pre_token: pd.Series) -> str:
if re.search(r"\d{1,4}", token['obj']):
if re.search(r"\d{1,4}", token['obj']) and not find_room(token,pre_token):
if any_of_in(HOUSES_PREFIXES, token['obj'].lower()):
return "h"
if re.search(r"(д|д\.) ?\d{1,4} ?\/*\d* ?", token['obj']):
return "h"
if ("s" in pre_token['class'] or "h" in pre_token['class'] or "s" in token['class']) \
and not any_of_in(("-я", "-й", "-Я"), token['obj']) \
and not find_building(token, pre_token):
and not find_building(token, pre_token)\
and not find_edifice(token,pre_token):
return "h"
if find_building(token, pre_token) \
and not any_of_in(("-я", "-й", "-Я"), token['obj']) \
and True:
if len(re.findall(r"\d{1,4}", token['obj'])) > 1:
return "h"
if int(re.search(r"\d{1,4}", token['obj']).group()) // 10 >0:
return "h"
return ""
def find_street(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(STREET_PREFIXES, token['obj'].lower()) \
or re.search(r"[а-я]+ая", token['obj']):
if any_of_in(STREET_PREFIXES, token['obj'].lower()):
return "s"
if re.search(r"\b[А-Яа-я]{4,}\b", token['obj']) \
and not any([el in token["obj"].lower() for pr in PREFIXES for el in pr if len(el)>2]) \
and not ("d" in token["class"] or "t" in token["class"] or "c" in token["class"]):
return "s"
return ""
@ -103,7 +120,7 @@ def find_countryside(token: pd.Series, pre_token: pd.Series) -> str:
if any_of_in(COUNTRYSIDE_PREFIXES, token['obj'].lower()) \
and re.search(r"\b[гпдрпктc]{1,3}(\b|\. )", token['obj']) \
and not find_house(token, pre_token) \
and not find_street(token, pre_token):
and not any_of_in(STREET_PREFIXES, token['obj'].lower()):
return "c"
return ""
@ -123,7 +140,9 @@ def address_classification(token: pd.Series, pre_token: pd.Series) -> pd.Series:
token["class"] += find_street(token, pre_token)
token["class"] += find_house(token, pre_token)
token["class"] += find_building(token, pre_token)
token["class"] += find_edifice(token, pre_token)
token["class"] += find_litera(token, pre_token)
token["class"] += find_room(token, pre_token)
if token['class'] == "":
token['class'] = "w"
if brackets:
@ -138,6 +157,8 @@ def cut_address(ad: pd.Series, cl: str) -> pd.Series:
elif ad["class"][-1] == "b":
num = re.findall("к{0,1}\.? ?\d", ad["address"])[-1]
ad["address"] = re.sub(num, "", ad["address"])
elif ad["class"][-1] == "e":
ad["address"] = re.sub(r"cтр\.? ?\d", "", ad["address"])
elif ad["class"][-1] == "l":
ad["address"] = re.sub(r"[литера]*\.? ?[А-Яа-я]{1}$", "", ad["address"])
elif ad["class"][-1] == "r":
@ -157,7 +178,8 @@ def split_address(address: str) -> List[str]:
tokens = pd.DataFrame()
tokens['obj'] = t
tokens = tokens[tokens["obj"] != ""]
for el in ("", "уг.", "д."):
tokens = tokens[tokens["obj"] != el]
tokens.insert(len(tokens.columns), "class", "")
res = []
accumulator = pd.Series(data={"address": "", "class": ""})
@ -195,8 +217,14 @@ def split_address(address: str) -> List[str]:
accumulator["address"] = cur_tk["obj"]
if cur_tk["class"][0] == "h":
num = re.findall("\d{1,4} ?\/?\d* ?", cur_tk['obj'])[0]
accumulator["address"] = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator["address"])
num = re.findall("\d{1,4} ?[\/\-]?\d* ?", cur_tk['obj'])[0]
if any_of_in(("-я", "-й", "-Я"), accumulator["address"]):
idx = 1
else:
idx = 0
num_ac = re.findall("\d{1,4} ?[\/\-]?\d* ?", accumulator["address"])
if num_ac:
accumulator["address"] = re.sub(num_ac[idx], num, accumulator["address"])
cur_tk["class"] =cur_tk["class"][1:]
if cur_tk["class"] and cur_tk["class"][0] == "b":
@ -208,6 +236,13 @@ def split_address(address: str) -> List[str]:
accumulator["address"] = re.sub(r"\d$", num, accumulator["address"])
cur_tk["class"] = cur_tk["class"][1:]
if cur_tk["class"] and cur_tk["class"][0] == "e":
num = re.findall("стр\.? ?\d", cur_tk["obj"].strip())[-1]
accumulator["address"] = re.sub(r"cтр\. ?\d", num, accumulator["address"].strip())
if num and not "e" in accumulator["class"]:
accumulator["class"] += "e"
cur_tk["class"] = cur_tk["class"][1:]
if cur_tk["class"] and cur_tk["class"][0] == "l":
num = re.findall("[А-Яа-я]", cur_tk["obj"].strip())[-1]
accumulator["address"] = re.sub(r"[А-Яа-я]$", "", accumulator["address"].strip())
@ -217,6 +252,12 @@ def split_address(address: str) -> List[str]:
else:
if re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", accumulator["address"]):
accumulator["address"] = re.sub(r"[А-Яа-я]$", "", accumulator["address"].strip())
if cur_tk["class"] and cur_tk["class"][0] == "r":
num = re.findall("пом\. ?\-?\d*\w?", cur_tk["obj"].strip())[-1]
accumulator["address"] = re.sub(r"пом\. ?\d\-?\d*\w?", num, accumulator["address"].strip())
if num and not "r" in accumulator["class"]:
accumulator["class"] += "r"
cur_tk["class"] = cur_tk["class"][1:]
res.extend(unfold_house_ranges(accumulator["address"]))
print(res)
return res