Working version of classification (not final)
This commit is contained in:
AnastasiaOnimovma
parent
d822c5012b
commit
a71acc2ddf
3
.gitignore
vendored
3
.gitignore
vendored
@ -3,4 +3,5 @@
|
||||
__pycache__
|
||||
.env
|
||||
data*.csv
|
||||
.idea/
|
||||
.idea/
|
||||
.ipynb_checkpoints
|
||||
@ -7,50 +7,36 @@ import pandas as pd
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
CLASSES = ("d", "c", "t", "s", "h", "b", "l", "r", "w")
|
||||
DISTRICTS_PREFIXES = ("мо ", "р-н")
|
||||
CLASSES = ("w", "d", "c", "t", "s", "h", "b", "l", "r")
|
||||
|
||||
DISTRICTS_PREFIXES = ("мо ", "р-н","городское","лесхоз")
|
||||
COUNTRYSIDE_PREFIXES = (
|
||||
" г", " п", " д", " гп", " рп", " кп", " пгт", " c", "хутор", " урочище"
|
||||
"г.", "п.", "д.", "гп.", "рп.", "кп.", "пгт.", "c.")
|
||||
TERRITORY_PREFIXES =("тер.", " тер", "снт ", "ст ", "дск ", "днп ", "дпк ", "нп ", "пдк ", "т/б ", "садоводство", "массив", "хоз","сад-во","с-во")
|
||||
"г", "п", "д", "гп", "рп", "кп", "пгт", "c", "хутор", " урочище")
|
||||
TERRITORY_PREFIXES = (
|
||||
"тер.", " тер", "снт ", "ст ", "дск ", "днп ", "дпк ", "нп ", "пдк ", "т/б ", "садоводство", "массив", "хозя", "сад-во")
|
||||
STREET_PREFIXES = (
|
||||
" ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе","линия","аллея", "мост", " парк", "кольцо","проезд", "съезд",
|
||||
" ул", " бул", " пр", " ш", " пер", " дор", " маг", " наб", " пл", " просп", " туп", "шоссе", "лини", "аллея",
|
||||
"мост", " парк", "кольцо", "проезд", "съезд","переулок",
|
||||
"ул.", "бул.", "пр.", "ш.", "пер.", "дор.", "маг.", "наб.", "пл.", "просп.", "туп.")
|
||||
HOUSES_PREFIXES = ("д.", "уч.", "участок","мкд","тп")
|
||||
BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение","корпус")
|
||||
LETTER = ("лит.", "литера"," л.")
|
||||
HOUSES_PREFIXES = ("д.", "уч.", "участок", "мкд", "тп","дом")
|
||||
BUILDING_PREFIXES = ("к.", "корп", 'стр.', "строение", "корпус")
|
||||
LETTER = ("лит.", "литера", " л.")
|
||||
|
||||
|
||||
def unfold_house_ranges(address: str, token: str) -> List[str]:
|
||||
adresses = []
|
||||
def unfold_house_ranges(token: str) -> List[str]:
|
||||
addresses = []
|
||||
pairs_strings = re.findall(r"([\d]+-[\d]+)", token)
|
||||
for pair_string in pairs_strings:
|
||||
a, b = pair_string.split("-")
|
||||
a, b = int(a), int(b)
|
||||
|
||||
if b > a:
|
||||
token = token.replace(pair_string, "")
|
||||
adresses += [address + " " + token + number for number in map(str, range(a, b + 1))]
|
||||
|
||||
|
||||
addresses += [re.sub(r"([\d]+-[\d]+)", number, token) for number in map(str, range(a, b + 1))]
|
||||
else:
|
||||
token = token.replace("-", "/")
|
||||
adresses += address + " " + token
|
||||
if not adresses:
|
||||
adresses.append(address + " " + token)
|
||||
return adresses
|
||||
|
||||
|
||||
def unfold_houses_list(token: str) -> List[str]:
|
||||
token = unfold_house_ranges(token)
|
||||
|
||||
reg = re.compile(r"(д|д\.)? ?\d+[а-яА-Я\/]*\d*(,|$| )")
|
||||
|
||||
if len(re.findall(reg, token)) > 1:
|
||||
tokens = token.split(",")
|
||||
return [*[tokens[0] + " " + house_token for house_token in tokens[1:]]]
|
||||
return [token]
|
||||
|
||||
if not addresses:
|
||||
addresses.append(token)
|
||||
return addresses
|
||||
|
||||
def any_of_in(substrings: Iterable[str], string: str) -> bool:
|
||||
return any(map(lambda substring: substring in string, substrings))
|
||||
@ -59,19 +45,21 @@ def any_of_in(substrings: Iterable[str], string: str) -> bool:
|
||||
def flatten(arr: Iterable[List[T]]) -> List[T]:
|
||||
return sum(arr, [])
|
||||
|
||||
|
||||
def find_room(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if re.search(r"пом\.?", token['obj']):
|
||||
if re.search(r"пом\.?", token['obj']):
|
||||
return "r"
|
||||
return ""
|
||||
|
||||
|
||||
def find_litera(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if any_of_in(LETTER, token['obj'].lower()) \
|
||||
or re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", token['obj']):
|
||||
return "l"
|
||||
if (re.search(r"\b([А-Я]|[а-я]){1}$", token['obj']) \
|
||||
and ("l" in pre_token['class'] or "h" in pre_token['class'])) \
|
||||
and ("l" in pre_token['class'] or "h" in pre_token['class'])) \
|
||||
and not (" ш" in token["obj"]) \
|
||||
and not find_countryside(token,pre_token):
|
||||
and not find_countryside(token, pre_token):
|
||||
return "l"
|
||||
return ""
|
||||
|
||||
@ -79,7 +67,7 @@ def find_litera(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
def find_building(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if re.search(r"\d", token['obj']):
|
||||
if any_of_in(BUILDING_PREFIXES, token['obj'].lower()) \
|
||||
or "b" in pre_token['class'] and not ("h" in token['class'])\
|
||||
or "b" in pre_token['class'] and not ("h" in token['class']) \
|
||||
or re.search(r"к\.* ?\d", token['obj']):
|
||||
return "b"
|
||||
return ""
|
||||
@ -92,36 +80,58 @@ def find_house(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if re.search(r"(д|д\.) ?\d{1,4} ?\/*\d* ?", token['obj']):
|
||||
return "h"
|
||||
if ("s" in pre_token['class'] or "h" in pre_token['class'] or "s" in token['class']) \
|
||||
and not any_of_in(("-я", "-й", "-Я"), token['obj'])\
|
||||
and not find_building(token,pre_token):
|
||||
and not any_of_in(("-я", "-й", "-Я"), token['obj']) \
|
||||
and not find_building(token, pre_token):
|
||||
return "h"
|
||||
return ""
|
||||
|
||||
|
||||
def find_street(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if any_of_in(STREET_PREFIXES, token['obj'].lower()) \
|
||||
or re.search(r"[А-Я]{1}[а-я]+ая", token['obj']):
|
||||
or re.search(r"[а-я]+ая", token['obj']):
|
||||
return "s"
|
||||
return ""
|
||||
|
||||
|
||||
def find_territory(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if any_of_in(TERRITORY_PREFIXES, token['obj'].lower()):
|
||||
return "t"
|
||||
return ""
|
||||
|
||||
|
||||
def find_countryside(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if any_of_in(COUNTRYSIDE_PREFIXES, token['obj'].lower()) \
|
||||
and not find_house(token,pre_token) \
|
||||
and not find_street(token,pre_token):
|
||||
and re.search(r"\b[гпдрпктc]{1,3}(\b|\. )", token['obj']) \
|
||||
and not find_house(token, pre_token) \
|
||||
and not find_street(token, pre_token):
|
||||
return "c"
|
||||
return ""
|
||||
|
||||
|
||||
def find_district(token: pd.Series, pre_token: pd.Series) -> str:
|
||||
if any_of_in(DISTRICTS_PREFIXES, token['obj'].lower()):
|
||||
return "d"
|
||||
return ""
|
||||
|
||||
def address_classification(token: pd.Series, pre_token: pd.Series) -> pd.Series:
|
||||
brackets = re.search(r"\(.+\)", token["obj"])
|
||||
if brackets:
|
||||
token["obj"] = re.sub(r"\(.+\)", "()", token["obj"])
|
||||
token["class"] += find_district(token, pre_token)
|
||||
token["class"] += find_countryside(token, pre_token)
|
||||
token["class"] += find_territory(token, pre_token)
|
||||
token["class"] += find_street(token, pre_token)
|
||||
token["class"] += find_house(token, pre_token)
|
||||
token["class"] += find_building(token, pre_token)
|
||||
token["class"] += find_litera(token, pre_token)
|
||||
if token['class'] == "":
|
||||
token['class'] = "w"
|
||||
if brackets:
|
||||
token["obj"] = re.sub(r"\(\)", brackets.group(), token["obj"])
|
||||
return token
|
||||
|
||||
# TODO: переработать систему из if в нормальный вид и классификация чисел/букв
|
||||
|
||||
# TODO: переработать систему из if в нормальный вид
|
||||
def split_address(address: str) -> List[str]:
|
||||
if ";" in address:
|
||||
address = address.replace(";", ",")
|
||||
@ -129,18 +139,15 @@ def split_address(address: str) -> List[str]:
|
||||
tokens = address.split(",")
|
||||
|
||||
t = list(map(str.strip, filter(lambda token: token != "", tokens)))
|
||||
# токены в датафрэйм
|
||||
|
||||
tokens = pd.DataFrame()
|
||||
tokens['obj'] = t
|
||||
tokens = tokens[tokens["obj"] != ""]
|
||||
tokens.insert(len(tokens.columns), "class", "")
|
||||
res = []
|
||||
accumulator = ""
|
||||
accumulator = pd.Series(data={"address": "", "class": ""})
|
||||
|
||||
for i in range(len(tokens)):
|
||||
|
||||
# TODO: напселённые пункты
|
||||
# if any_of_in(SETTLEMENTS_PREFIXES, tokens[i].lower())
|
||||
# accumulator += tokens[i]
|
||||
cur_tk = tokens.iloc[i]
|
||||
|
||||
if i == 0:
|
||||
@ -148,71 +155,72 @@ def split_address(address: str) -> List[str]:
|
||||
else:
|
||||
pre_token = tokens.iloc[i - 1]
|
||||
|
||||
obj_class = find_district(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "d" in pre_token['class']:
|
||||
res.append(accumulator)
|
||||
accumulator = ""
|
||||
accumulator += cur_tk["obj"]
|
||||
obj_class = find_countryside(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "c" in pre_token['class']:
|
||||
res.append(accumulator)
|
||||
accumulator = ""
|
||||
accumulator += cur_tk["obj"]
|
||||
obj_class = find_territory(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "t" in pre_token['class']:
|
||||
res.append(accumulator)
|
||||
accumulator = ""
|
||||
accumulator +=cur_tk["obj"]
|
||||
obj_class = find_street(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "s" in pre_token['class']:
|
||||
res.append(accumulator)
|
||||
accumulator = ""
|
||||
accumulator += cur_tk["obj"]
|
||||
obj_class = find_house(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "h" in pre_token["class"]:
|
||||
res.append(accumulator)
|
||||
num = re.findall("\d{1,4}", cur_tk['obj'])[-1]
|
||||
accumulator = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator)
|
||||
else:
|
||||
accumulator += cur_tk["obj"]
|
||||
obj_class = find_building(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "b" in pre_token["class"]:
|
||||
res.append(accumulator)
|
||||
num = re.findall("\d", tokens['obj'].iloc[i])[-1]
|
||||
accumulator = re.sub(r"\d$", num, accumulator)
|
||||
else:
|
||||
accumulator += pre_token["obj"]
|
||||
obj_class = find_litera(cur_tk, pre_token)
|
||||
if obj_class:
|
||||
cur_tk["class"] += obj_class
|
||||
if "l" in pre_token["class"]:
|
||||
res.append(accumulator)
|
||||
num = re.findall("[А-яа-я]", cur_tk["obj"].strip())[-1]
|
||||
accumulator = re.sub(r"[А-яа-я]$", num, accumulator)
|
||||
else:
|
||||
accumulator += cur_tk["obj"]
|
||||
if cur_tk['class'] == "":
|
||||
cur_tk['class'] = "w"
|
||||
cur_tk = address_classification(cur_tk, pre_token)
|
||||
tokens.iloc[i] = cur_tk
|
||||
print(tokens.iloc[i])
|
||||
|
||||
# print(cur_tk)
|
||||
if not accumulator["class"]:
|
||||
accumulator["class"] = cur_tk['class']
|
||||
accumulator["address"] = cur_tk["obj"]
|
||||
continue
|
||||
if CLASSES.index(accumulator["class"][-1]) < CLASSES.index(cur_tk["class"][0]) and accumulator["class"]!="w":
|
||||
accumulator["class"] += cur_tk['class']
|
||||
accumulator["address"] += " " + cur_tk["obj"]
|
||||
else:
|
||||
ad_no_ranges = unfold_house_ranges(accumulator["address"])
|
||||
accumulator["address"] = ad_no_ranges[-1]
|
||||
res.extend(ad_no_ranges)
|
||||
while accumulator["class"] and CLASSES.index(accumulator["class"][-1]) > CLASSES.index(cur_tk["class"][0]):
|
||||
if accumulator["class"][-1] == "h":
|
||||
accumulator["address"] = re.sub(r"[мкдтпучасток]*\.? ?\d{1,4} ?\/*\d* ?", "", accumulator["address"].lower())
|
||||
elif accumulator["class"][-1] == "b":
|
||||
num = re.findall("к{0,1}\.? ?\d", accumulator["address"])[-1]
|
||||
accumulator["address"] = re.sub(num, "", accumulator["address"])
|
||||
elif accumulator["class"][-1] == "l":
|
||||
accumulator ["address"] = re.sub(r"[литера]*\.? ?[А-Яа-я]{1}$","", accumulator["address"])
|
||||
elif accumulator["class"][-1] == "r":
|
||||
accumulator["address"] = re.sub(r"пом\.? ?\d+","", accumulator["address"])
|
||||
accumulator["class"] = accumulator["class"][:-1]
|
||||
if not accumulator["class"] or CLASSES.index(cur_tk["class"][0]) <= CLASSES.index("s") or accumulator["class"]=="w":
|
||||
accumulator["class"] = cur_tk["class"]
|
||||
accumulator["address"] = cur_tk["obj"]
|
||||
if cur_tk["class"][0] == "h":
|
||||
num = re.findall("\d{1,4} ?\/?\d* ?", cur_tk['obj'])[0]
|
||||
accumulator["address"] = re.sub(r"\d{1,4} ?\/*\d* ?", num, accumulator["address"])
|
||||
cur_tk["class"] =cur_tk["class"][1:]
|
||||
if cur_tk["class"] and cur_tk["class"][0] == "b":
|
||||
num = re.findall("\d", cur_tk["obj"])[-1]
|
||||
if num and not "b" in accumulator["class"]:
|
||||
accumulator["class"] += "b"
|
||||
accumulator["address"] += "к." + num
|
||||
else:
|
||||
accumulator["address"] = re.sub(r"\d$", num, accumulator["address"])
|
||||
cur_tk["class"] = cur_tk["class"][1:]
|
||||
|
||||
if cur_tk["class"] and cur_tk["class"][0] == "l":
|
||||
num = re.findall("[А-Яа-я]", cur_tk["obj"].strip())[-1]
|
||||
accumulator["address"] = re.sub(r"[А-Яа-я]$", "", accumulator["address"].strip())
|
||||
accumulator["address"] += num
|
||||
if num and not "l" in accumulator["class"]:
|
||||
accumulator["class"] += "l"
|
||||
else:
|
||||
if re.search(r"\d{1,3}([А-Я]|[а-я])( |$)", accumulator["address"]):
|
||||
accumulator["address"] = re.sub(r"[А-Яа-я]$", "", accumulator["address"].strip())
|
||||
res.extend(unfold_house_ranges(accumulator["address"]))
|
||||
print(res)
|
||||
return res
|
||||
|
||||
return [address]
|
||||
|
||||
def split_pesoch_res(address: str) -> List[str]:
|
||||
t = re.sub(r",", " ", address)
|
||||
t = re.split(r"(Санкт-Петербург|Ленинградская обл|Л\.О)", t)
|
||||
t = list(map(str.strip, filter(lambda token: token != "", t)))
|
||||
tokens = [t[i] + " " + t[i+1] for i in range(0, len(t)-1, 2)]
|
||||
|
||||
if tokens:
|
||||
return list(set(tokens))
|
||||
return [address]
|
||||
|
||||
def process_row(row: pd.Series[str]) -> pd.Series[str]:
|
||||
row = row.copy()
|
||||
@ -220,7 +228,10 @@ def process_row(row: pd.Series[str]) -> pd.Series[str]:
|
||||
if pd.isnull(row["Улица"]):
|
||||
row["Улица"] = [None]
|
||||
else:
|
||||
addresses = split_address(row["Улица"])
|
||||
if row["РЭС"] == "Песочинский РЭС":
|
||||
addresses = split_pesoch_res(row["Улица"])
|
||||
else:
|
||||
addresses = split_address(row["Улица"])
|
||||
row["Улица"] = addresses
|
||||
|
||||
return row
|
||||
@ -229,4 +240,4 @@ def process_row(row: pd.Series[str]) -> pd.Series[str]:
|
||||
def split_addresses(df: pd.DataFrame) -> pd.DataFrame:
|
||||
merged_df = df.apply(process_row, axis=1).reset_index()
|
||||
|
||||
return merged_df.explode("Улица", ignore_index=True)
|
||||
return merged_df.explode("Улица", ignore_index=True)
|
||||
@ -10,16 +10,12 @@ from . import (
|
||||
|
||||
def pipeline(parser: Optional[LenenergoParser] = None) -> LenenergoParser:
|
||||
if parser is None:
|
||||
parser = LenenergoParser(ndays=15)
|
||||
parser = LenenergoParser(file_path = r"C:\Users\Юля\PycharmProjects\machine_learning\lenengro_parser\data_Rosseti.csv")
|
||||
|
||||
print(parser)
|
||||
|
||||
parser.df = split_addresses(parser.df)
|
||||
|
||||
for i in range(len(parser.df)):
|
||||
print(parser.df['Улица'].iloc[i])
|
||||
|
||||
|
||||
parser.df = concurrent_fetch_builing_ids(parser.df)
|
||||
|
||||
parser.df = preprocess_df(parser.df)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user