diff --git a/parser/address.py b/parser/address.py index a7ca5dd..b9d18c9 100644 --- a/parser/address.py +++ b/parser/address.py @@ -7,16 +7,17 @@ import pandas as pd T = TypeVar("T") -CLASSES = ("s", "h", "b", "l", "?") - -SETTLEMENTS_PREFIXES = ( - "г", "мо", "р-н", "п", "д", "гп", "c", "хутор", "массив", "тер", "СНТ", "СТ", "ДСК", "ДНП", "ДПК", "НП", - "садоводство") +CLASSES = ("d", "c", "t", "s", "h", "b", "l", "r", "w") +DISTRICTS_PREFIXES = ("мо ", "р-н") +COUNTRYSIDE_PREFIXES = ( + " г", " п", " д", " гп", " рп", " кп", " пгт", " c", "хутор", " урочище" + "г.", "п.", "д.", "гп.", "рп.", "кп.", "пгт.", "c.") +TERRITORY_PREFIXES =("тер.", " тер", "снт ", "ст ", "дск ", "днп ", "дпк ", "нп ", "пдк ", "т/б ", "садоводство", "массив", "хоз","сад-во","с-во") STREET_PREFIXES = ( - " ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе","линия","аллея", "мост", "парк", "кольцо","проезд", + " ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе","линия","аллея", "мост", " парк", "кольцо","проезд", "съезд", "ул.", "бул.", "пр.", "ш.", "пер.", "дор.", "маг.", "наб.", "пл.", "просп.", "туп.") -HOUSES_PREFIXES = ("д.", "уч.", "участок") -BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение") +HOUSES_PREFIXES = ("д.", "уч.", "участок","мкд","тп") +BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение","корпус") LETTER = ("лит.", "литера"," л.") @@ -58,46 +59,67 @@ def any_of_in(substrings: Iterable[str], string: str) -> bool: def flatten(arr: Iterable[List[T]]) -> List[T]: return sum(arr, []) +def find_room(token: pd.Series, pre_token: pd.Series) -> str: + if re.search(r"пом\.?", token['obj']): + return "r" + return "" def find_litera(token: pd.Series, pre_token: pd.Series) -> str: - if any_of_in(LETTER, token['obj']) \ - or re.search(r"\d{1,3}[А-Яа-я]( |$)", token['obj']): + if any_of_in(LETTER, token['obj'].lower()) \ + or re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", token['obj']): return "l" - # не работает - if (re.search(r"\b[А-Яа-я]{1}\b", token['obj']) and "l" in pre_token['class']): + if (re.search(r"\b([А-Я]|[а-я]){1}$", token['obj']) \ + and ("l" in pre_token['class'] or "h" in pre_token['class'])) \ + and not (" ш" in token["obj"]) \ + and not find_countryside(token,pre_token): return "l" return "" def find_building(token: pd.Series, pre_token: pd.Series) -> str: - if any_of_in(BUILDING_PREFIXES, token['obj']) \ - or (re.search(r"\d", token['obj']) and "b" in pre_token['class']) \ - or re.search(r"к\.*\d", token['obj']) \ - or re.search(r"\d", token['obj']) and "b" in pre_token['class']: - return "b" + if re.search(r"\d", token['obj']): + if any_of_in(BUILDING_PREFIXES, token['obj'].lower()) \ + or "b" in pre_token['class'] and not ("h" in token['class'])\ + or re.search(r"к\.* ?\d", token['obj']): + return "b" return "" def find_house(token: pd.Series, pre_token: pd.Series) -> str: - if any_of_in(HOUSES_PREFIXES, token['obj']): - return "h" - if re.search(r"(д|д\.) ?\d{1,3} ?\/*\d* ?", token['obj']) and not ("-я" in token['obj']): - if "h" in pre_token['class'] \ - or "s" in pre_token['class'] \ - or "s" in token['class']: + if re.search(r"\d{1,4}", token['obj']): + if any_of_in(HOUSES_PREFIXES, token['obj'].lower()): + return "h" + if re.search(r"(д|д\.) ?\d{1,4} ?\/*\d* ?", token['obj']): + return "h" + if ("s" in pre_token['class'] or "h" in pre_token['class'] or "s" in token['class']) \ + and not any_of_in(("-я", "-й", "-Я"), token['obj'])\ + and not find_building(token,pre_token): return "h" - # не работает - if re.search(r"\d{1,3}", token['obj']) and ("s" in pre_token['class'] or "h" in pre_token['class']): - return "h" return "" def find_street(token: pd.Series, pre_token: pd.Series) -> str: - if any_of_in(STREET_PREFIXES, token['obj']) \ - or (re.search(r"[А-Я]{1}[а-я]+", token['obj']) and "s" in pre_token['class']): + if any_of_in(STREET_PREFIXES, token['obj'].lower()) \ + or re.search(r"[А-Я]{1}[а-я]+ая", token['obj']): return "s" return "" +def find_territory(token: pd.Series, pre_token: pd.Series) -> str: + if any_of_in(TERRITORY_PREFIXES, token['obj'].lower()): + return "t" + return "" +def find_countryside(token: pd.Series, pre_token: pd.Series) -> str: + if any_of_in(COUNTRYSIDE_PREFIXES, token['obj'].lower()) \ + and not find_house(token,pre_token) \ + and not find_street(token,pre_token): + return "c" + return "" + +def find_district(token: pd.Series, pre_token: pd.Series) -> str: + if any_of_in(DISTRICTS_PREFIXES, token['obj'].lower()): + return "d" + return "" + # TODO: переработать систему из if в нормальный вид и классификация чисел/букв def split_address(address: str) -> List[str]: @@ -125,43 +147,68 @@ def split_address(address: str) -> List[str]: pre_token = pd.Series(data=["", ""], index=['obj', 'class']) else: pre_token = tokens.iloc[i - 1] + + obj_class = find_district(cur_tk, pre_token) + if obj_class: + cur_tk["class"] += obj_class + if "d" in pre_token['class']: + res.append(accumulator) + accumulator = "" + accumulator += cur_tk["obj"] + obj_class = find_countryside(cur_tk, pre_token) + if obj_class: + cur_tk["class"] += obj_class + if "c" in pre_token['class']: + res.append(accumulator) + accumulator = "" + accumulator += cur_tk["obj"] + obj_class = find_territory(cur_tk, pre_token) + if obj_class: + cur_tk["class"] += obj_class + if "t" in pre_token['class']: + res.append(accumulator) + accumulator = "" + accumulator +=cur_tk["obj"] obj_class = find_street(cur_tk, pre_token) if obj_class: cur_tk["class"] += obj_class - if "s" in tokens['class'].iloc[i - 1]: + if "s" in pre_token['class']: res.append(accumulator) accumulator = "" - accumulator += tokens["obj"].iloc[i] + accumulator += cur_tk["obj"] obj_class = find_house(cur_tk, pre_token) if obj_class: cur_tk["class"] += obj_class - if "h" in tokens['class'].iloc[i - 1]: + if "h" in pre_token["class"]: res.append(accumulator) - num = re.findall("\d{,3}", tokens['obj'].iloc[i])[-1] - accumulator = re.sub(r"\d{,3} ?\/*\d* ?", num,accumulator) + num = re.findall("\d{1,4}", cur_tk['obj'])[-1] + accumulator = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator) else: - accumulator += tokens["obj"].iloc[i] + accumulator += cur_tk["obj"] obj_class = find_building(cur_tk, pre_token) if obj_class: cur_tk["class"] += obj_class - if "b" in tokens['class'].iloc[i - 1]: + if "b" in pre_token["class"]: res.append(accumulator) num = re.findall("\d", tokens['obj'].iloc[i])[-1] accumulator = re.sub(r"\d$", num, accumulator) else: - accumulator += tokens["obj"].iloc[i] + accumulator += pre_token["obj"] obj_class = find_litera(cur_tk, pre_token) if obj_class: cur_tk["class"] += obj_class - if "l" in tokens['class'].iloc[i - 1]: + if "l" in pre_token["class"]: res.append(accumulator) - num = re.findall("[А-яа-я]", tokens['obj'].iloc[i].strip())[-1] + num = re.findall("[А-яа-я]", cur_tk["obj"].strip())[-1] accumulator = re.sub(r"[А-яа-я]$", num, accumulator) else: - accumulator += tokens["obj"].iloc[i] + accumulator += cur_tk["obj"] if cur_tk['class'] == "": cur_tk['class'] = "w" - print(cur_tk) + tokens.iloc[i] = cur_tk + print(tokens.iloc[i]) + + # print(cur_tk) return res return [address] diff --git a/parser/util.py b/parser/util.py index fe36f0e..6b54416 100644 --- a/parser/util.py +++ b/parser/util.py @@ -10,12 +10,16 @@ from . import ( def pipeline(parser: Optional[LenenergoParser] = None) -> LenenergoParser: if parser is None: - parser = LenenergoParser() + parser = LenenergoParser(ndays=15) print(parser) parser.df = split_addresses(parser.df) + for i in range(len(parser.df)): + print(parser.df['Улица'].iloc[i]) + + parser.df = concurrent_fetch_builing_ids(parser.df) parser.df = preprocess_df(parser.df)