From a71acc2ddf325b6af5406a70312adfde247e6d0f Mon Sep 17 00:00:00 2001 From: AnastasiaOnimovma Date: Sat, 21 Oct 2023 17:32:37 +0300 Subject: [PATCH] Working version of classification (not final) --- .gitignore | 3 +- parser/address.py | 225 ++++++++++++++++++++++++---------------------- parser/util.py | 6 +- 3 files changed, 121 insertions(+), 113 deletions(-) diff --git a/.gitignore b/.gitignore index 71444f4..fe14a37 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ __pycache__ .env data*.csv -.idea/ \ No newline at end of file +.idea/ +.ipynb_checkpoints \ No newline at end of file diff --git a/parser/address.py b/parser/address.py index b9d18c9..4c3bdca 100644 --- a/parser/address.py +++ b/parser/address.py @@ -7,50 +7,36 @@ import pandas as pd T = TypeVar("T") -CLASSES = ("d", "c", "t", "s", "h", "b", "l", "r", "w") -DISTRICTS_PREFIXES = ("мо ", "р-н") +CLASSES = ("w", "d", "c", "t", "s", "h", "b", "l", "r") + +DISTRICTS_PREFIXES = ("мо ", "р-н","городское","лесхоз") COUNTRYSIDE_PREFIXES = ( - " г", " п", " д", " гп", " рп", " кп", " пгт", " c", "хутор", " урочище" - "г.", "п.", "д.", "гп.", "рп.", "кп.", "пгт.", "c.") -TERRITORY_PREFIXES =("тер.", " тер", "снт ", "ст ", "дск ", "днп ", "дпк ", "нп ", "пдк ", "т/б ", "садоводство", "массив", "хоз","сад-во","с-во") + "г", "п", "д", "гп", "рп", "кп", "пгт", "c", "хутор", " урочище") +TERRITORY_PREFIXES = ( +"тер.", " тер", "снт ", "ст ", "дск ", "днп ", "дпк ", "нп ", "пдк ", "т/б ", "садоводство", "массив", "хозя", "сад-во") STREET_PREFIXES = ( - " ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе","линия","аллея", "мост", " парк", "кольцо","проезд", "съезд", + " ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе", "лини", "аллея", + "мост", " парк", "кольцо", "проезд", "съезд","переулок", "ул.", "бул.", "пр.", "ш.", "пер.", "дор.", "маг.", "наб.", "пл.", "просп.", "туп.") -HOUSES_PREFIXES = ("д.", "уч.", "участок","мкд","тп") -BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение","корпус") -LETTER = ("лит.", "литера"," л.") +HOUSES_PREFIXES = ("д.", "уч.", "участок", "мкд", "тп","дом") +BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение", "корпус") +LETTER = ("лит.", "литера", " л.") -def unfold_house_ranges(address: str, token: str) -> List[str]: - adresses = [] +def unfold_house_ranges(token: str) -> List[str]: + addresses = [] pairs_strings = re.findall(r"([\d]+-[\d]+)", token) for pair_string in pairs_strings: a, b = pair_string.split("-") a, b = int(a), int(b) if b > a: - token = token.replace(pair_string, "") - adresses += [address + " " + token + number for number in map(str, range(a, b + 1))] - - + addresses += [re.sub(r"([\d]+-[\d]+)", number, token) for number in map(str, range(a, b + 1))] else: token = token.replace("-", "/") - adresses += address + " " + token - if not adresses: - adresses.append(address + " " + token) - return adresses - - -def unfold_houses_list(token: str) -> List[str]: - token = unfold_house_ranges(token) - - reg = re.compile(r"(д|д\.)? ?\d+[а-яА-Я\/]*\d*(,|$| )") - - if len(re.findall(reg, token)) > 1: - tokens = token.split(",") - return [*[tokens[0] + " " + house_token for house_token in tokens[1:]]] - return [token] - + if not addresses: + addresses.append(token) + return addresses def any_of_in(substrings: Iterable[str], string: str) -> bool: return any(map(lambda substring: substring in string, substrings)) @@ -59,19 +45,21 @@ def any_of_in(substrings: Iterable[str], string: str) -> bool: def flatten(arr: Iterable[List[T]]) -> List[T]: return sum(arr, []) + def find_room(token: pd.Series, pre_token: pd.Series) -> str: - if re.search(r"пом\.?", token['obj']): + if re.search(r"пом\.?", token['obj']): return "r" return "" + def find_litera(token: pd.Series, pre_token: pd.Series) -> str: if any_of_in(LETTER, token['obj'].lower()) \ or re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", token['obj']): return "l" if (re.search(r"\b([А-Я]|[а-я]){1}$", token['obj']) \ - and ("l" in pre_token['class'] or "h" in pre_token['class'])) \ + and ("l" in pre_token['class'] or "h" in pre_token['class'])) \ and not (" ш" in token["obj"]) \ - and not find_countryside(token,pre_token): + and not find_countryside(token, pre_token): return "l" return "" @@ -79,7 +67,7 @@ def find_litera(token: pd.Series, pre_token: pd.Series) -> str: def find_building(token: pd.Series, pre_token: pd.Series) -> str: if re.search(r"\d", token['obj']): if any_of_in(BUILDING_PREFIXES, token['obj'].lower()) \ - or "b" in pre_token['class'] and not ("h" in token['class'])\ + or "b" in pre_token['class'] and not ("h" in token['class']) \ or re.search(r"к\.* ?\d", token['obj']): return "b" return "" @@ -92,36 +80,58 @@ def find_house(token: pd.Series, pre_token: pd.Series) -> str: if re.search(r"(д|д\.) ?\d{1,4} ?\/*\d* ?", token['obj']): return "h" if ("s" in pre_token['class'] or "h" in pre_token['class'] or "s" in token['class']) \ - and not any_of_in(("-я", "-й", "-Я"), token['obj'])\ - and not find_building(token,pre_token): + and not any_of_in(("-я", "-й", "-Я"), token['obj']) \ + and not find_building(token, pre_token): return "h" return "" def find_street(token: pd.Series, pre_token: pd.Series) -> str: if any_of_in(STREET_PREFIXES, token['obj'].lower()) \ - or re.search(r"[А-Я]{1}[а-я]+ая", token['obj']): + or re.search(r"[а-я]+ая", token['obj']): return "s" return "" + def find_territory(token: pd.Series, pre_token: pd.Series) -> str: if any_of_in(TERRITORY_PREFIXES, token['obj'].lower()): return "t" return "" + + def find_countryside(token: pd.Series, pre_token: pd.Series) -> str: if any_of_in(COUNTRYSIDE_PREFIXES, token['obj'].lower()) \ - and not find_house(token,pre_token) \ - and not find_street(token,pre_token): + and re.search(r"\b[гпдрпктc]{1,3}(\b|\. )", token['obj']) \ + and not find_house(token, pre_token) \ + and not find_street(token, pre_token): return "c" return "" + def find_district(token: pd.Series, pre_token: pd.Series) -> str: if any_of_in(DISTRICTS_PREFIXES, token['obj'].lower()): return "d" return "" +def address_classification(token: pd.Series, pre_token: pd.Series) -> pd.Series: + brackets = re.search(r"\(.+\)", token["obj"]) + if brackets: + token["obj"] = re.sub(r"\(.+\)", "()", token["obj"]) + token["class"] += find_district(token, pre_token) + token["class"] += find_countryside(token, pre_token) + token["class"] += find_territory(token, pre_token) + token["class"] += find_street(token, pre_token) + token["class"] += find_house(token, pre_token) + token["class"] += find_building(token, pre_token) + token["class"] += find_litera(token, pre_token) + if token['class'] == "": + token['class'] = "w" + if brackets: + token["obj"] = re.sub(r"\(\)", brackets.group(), token["obj"]) + return token -# TODO: переработать систему из if в нормальный вид и классификация чисел/букв + +# TODO: переработать систему из if в нормальный вид def split_address(address: str) -> List[str]: if ";" in address: address = address.replace(";", ",") @@ -129,18 +139,15 @@ def split_address(address: str) -> List[str]: tokens = address.split(",") t = list(map(str.strip, filter(lambda token: token != "", tokens))) - # токены в датафрэйм + tokens = pd.DataFrame() tokens['obj'] = t + tokens = tokens[tokens["obj"] != ""] tokens.insert(len(tokens.columns), "class", "") res = [] - accumulator = "" + accumulator = pd.Series(data={"address": "", "class": ""}) for i in range(len(tokens)): - - # TODO: напселённые пункты - # if any_of_in(SETTLEMENTS_PREFIXES, tokens[i].lower()) - # accumulator += tokens[i] cur_tk = tokens.iloc[i] if i == 0: @@ -148,71 +155,72 @@ def split_address(address: str) -> List[str]: else: pre_token = tokens.iloc[i - 1] - obj_class = find_district(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "d" in pre_token['class']: - res.append(accumulator) - accumulator = "" - accumulator += cur_tk["obj"] - obj_class = find_countryside(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "c" in pre_token['class']: - res.append(accumulator) - accumulator = "" - accumulator += cur_tk["obj"] - obj_class = find_territory(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "t" in pre_token['class']: - res.append(accumulator) - accumulator = "" - accumulator +=cur_tk["obj"] - obj_class = find_street(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "s" in pre_token['class']: - res.append(accumulator) - accumulator = "" - accumulator += cur_tk["obj"] - obj_class = find_house(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "h" in pre_token["class"]: - res.append(accumulator) - num = re.findall("\d{1,4}", cur_tk['obj'])[-1] - accumulator = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator) - else: - accumulator += cur_tk["obj"] - obj_class = find_building(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "b" in pre_token["class"]: - res.append(accumulator) - num = re.findall("\d", tokens['obj'].iloc[i])[-1] - accumulator = re.sub(r"\d$", num, accumulator) - else: - accumulator += pre_token["obj"] - obj_class = find_litera(cur_tk, pre_token) - if obj_class: - cur_tk["class"] += obj_class - if "l" in pre_token["class"]: - res.append(accumulator) - num = re.findall("[А-яа-я]", cur_tk["obj"].strip())[-1] - accumulator = re.sub(r"[А-яа-я]$", num, accumulator) - else: - accumulator += cur_tk["obj"] - if cur_tk['class'] == "": - cur_tk['class'] = "w" + cur_tk = address_classification(cur_tk, pre_token) tokens.iloc[i] = cur_tk print(tokens.iloc[i]) - # print(cur_tk) + if not accumulator["class"]: + accumulator["class"] = cur_tk['class'] + accumulator["address"] = cur_tk["obj"] + continue + if CLASSES.index(accumulator["class"][-1]) < CLASSES.index(cur_tk["class"][0]) and accumulator["class"]!="w": + accumulator["class"] += cur_tk['class'] + accumulator["address"] += " " + cur_tk["obj"] + else: + ad_no_ranges = unfold_house_ranges(accumulator["address"]) + accumulator["address"] = ad_no_ranges[-1] + res.extend(ad_no_ranges) + while accumulator["class"] and CLASSES.index(accumulator["class"][-1]) > CLASSES.index(cur_tk["class"][0]): + if accumulator["class"][-1] == "h": + accumulator["address"] = re.sub(r"[мкдтпучасток]*\.? ?\d{1,4} ?\/*\d* ?", "", accumulator["address"].lower()) + elif accumulator["class"][-1] == "b": + num = re.findall("к{0,1}\.? ?\d", accumulator["address"])[-1] + accumulator["address"] = re.sub(num, "", accumulator["address"]) + elif accumulator["class"][-1] == "l": + accumulator ["address"] = re.sub(r"[литера]*\.? ?[А-Яа-я]{1}$","", accumulator["address"]) + elif accumulator["class"][-1] == "r": + accumulator["address"] = re.sub(r"пом\.? ?\d+","", accumulator["address"]) + accumulator["class"] = accumulator["class"][:-1] + if not accumulator["class"] or CLASSES.index(cur_tk["class"][0]) <= CLASSES.index("s") or accumulator["class"]=="w": + accumulator["class"] = cur_tk["class"] + accumulator["address"] = cur_tk["obj"] + if cur_tk["class"][0] == "h": + num = re.findall("\d{1,4} ?\/?\d* ?", cur_tk['obj'])[0] + accumulator["address"] = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator["address"]) + cur_tk["class"] =cur_tk["class"][1:] + if cur_tk["class"] and cur_tk["class"][0] == "b": + num = re.findall("\d", cur_tk["obj"])[-1] + if num and not "b" in accumulator["class"]: + accumulator["class"] += "b" + accumulator["address"] += "к." + num + else: + accumulator["address"] = re.sub(r"\d$", num, accumulator["address"]) + cur_tk["class"] = cur_tk["class"][1:] + + if cur_tk["class"] and cur_tk["class"][0] == "l": + num = re.findall("[А-Яа-я]", cur_tk["obj"].strip())[-1] + accumulator["address"] = re.sub(r"[А-Яа-я]$", "", accumulator["address"].strip()) + accumulator["address"] += num + if num and not "l" in accumulator["class"]: + accumulator["class"] += "l" + else: + if re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", accumulator["address"]): + accumulator["address"] = re.sub(r"[А-Яа-я]$", "", accumulator["address"].strip()) + res.extend(unfold_house_ranges(accumulator["address"])) + print(res) return res return [address] +def split_pesoch_res(address: str) -> List[str]: + t = re.sub(r",", " ", address) + t = re.split(r"(Санкт-Петербург|Ленинградская обл|Л\.О)", t) + t = list(map(str.strip, filter(lambda token: token != "", t))) + tokens = [t[i] + " " + t[i+1] for i in range(0, len(t)-1, 2)] + + if tokens: + return list(set(tokens)) + return [address] def process_row(row: pd.Series[str]) -> pd.Series[str]: row = row.copy() @@ -220,7 +228,10 @@ def process_row(row: pd.Series[str]) -> pd.Series[str]: if pd.isnull(row["Улица"]): row["Улица"] = [None] else: - addresses = split_address(row["Улица"]) + if row["РЭС"] == "Песочинский РЭС": + addresses = split_pesoch_res(row["Улица"]) + else: + addresses = split_address(row["Улица"]) row["Улица"] = addresses return row @@ -229,4 +240,4 @@ def process_row(row: pd.Series[str]) -> pd.Series[str]: def split_addresses(df: pd.DataFrame) -> pd.DataFrame: merged_df = df.apply(process_row, axis=1).reset_index() - return merged_df.explode("Улица", ignore_index=True) + return merged_df.explode("Улица", ignore_index=True) \ No newline at end of file diff --git a/parser/util.py b/parser/util.py index 6b54416..983c9b8 100644 --- a/parser/util.py +++ b/parser/util.py @@ -10,16 +10,12 @@ from . import ( def pipeline(parser: Optional[LenenergoParser] = None) -> LenenergoParser: if parser is None: - parser = LenenergoParser(ndays=15) + parser = LenenergoParser(file_path = r"C:\Users\Юля\PycharmProjects\machine_learning\lenengro_parser\data_Rosseti.csv") print(parser) parser.df = split_addresses(parser.df) - for i in range(len(parser.df)): - print(parser.df['Улица'].iloc[i]) - - parser.df = concurrent_fetch_builing_ids(parser.df) parser.df = preprocess_df(parser.df)