From 6af43f76b09ba8d8291223b75dc536142067c19f Mon Sep 17 00:00:00 2001 From: Sergey Date: Sun, 4 Oct 2020 17:24:44 +0300 Subject: [PATCH] Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Лонг-поллинг ТГ Актуальный API Протестированы: ВК → ТГ - Текст - Фото, альбомы (+ фото документом) - Документы - Стикеры (анимированные только превью) - Войсы + их текстовый транскрипт из ВК - Групповые сообщения ТГ → ВК - Текст - Фото, (+ фото документом) - Документы - Стикеры (анимированные только малюсеньким превью) - Войсы - Групповые сообщения --- .gitignore | 1 + conf.d/web.conf | 23 ---------- config.py | 24 +--------- docker-compose.yml | 17 -------- install.sh | 3 +- obtaincert.py | 35 --------------- telegram.py | 29 ++++++++----- vk_messages.py | 106 ++++++++++++++++++++++++++++++++++----------- 8 files changed, 101 insertions(+), 137 deletions(-) delete mode 100644 conf.d/web.conf delete mode 100644 obtaincert.py diff --git a/.gitignore b/.gitignore index 12fad12..b0552ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ config.py *.pem env_file openssl_config +venv/ diff --git a/conf.d/web.conf b/conf.d/web.conf deleted file mode 100644 index 3c1b43e..0000000 --- a/conf.d/web.conf +++ /dev/null @@ -1,23 +0,0 @@ -upstream tgbot { - ip_hash; - server tgbot:7777; -} - - -server { - listen 8443 ssl; - ssl_certificate /src/webhook_cert.pem; - ssl_certificate_key /src/webhook_pkey.pem; - - location / { - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; - - proxy_pass http://tgbot/; - - } - -} \ No newline at end of file diff --git a/config.py b/config.py index f7af819..00c5302 100644 --- a/config.py +++ b/config.py @@ -1,27 +1,5 @@ import os - -def get_external_host(): - import urllib.request - host = urllib.request.urlopen('https://api.ipify.org').read().decode('utf8') - return host - - -HOST = '' - -WEBHOOK_HOST = os.environ.get('WEBHOOK_HOST', HOST or get_external_host()) -WEBHOOK_PORT = os.environ.get('WEBHOOK_PORT', 8443) -WEBHOOK_URL_PATH = os.environ.get('WEBHOOK_URL_PATH', '/tgwebhook') - -WEBHOOK_URL = "https://{WEBHOOK_HOST}:{WEBHOOK_PORT}{WEBHOOK_URL_PATH}".format(WEBHOOK_HOST=WEBHOOK_HOST, - WEBHOOK_PORT=WEBHOOK_PORT, - WEBHOOK_URL_PATH=WEBHOOK_URL_PATH) - -WEBHOOK_SSL_CERT = './webhook_cert.pem' - -WEBAPP_HOST = os.environ.get('WEBAPP_HOST', '0.0.0.0') -WEBAPP_PORT = os.environ.get('WEBAPP_PORT', 7777) - DATABASE_USER = os.environ.get('POSTGRES_USER', 'postgres') DATABASE_PASSWORD = os.environ.get('POSTGRES_PASSWORD', 'postgres') DATABASE_HOST = os.environ.get('DATABASE_HOST', 'db') @@ -47,7 +25,7 @@ SETTINGS_VAR = os.environ.get('SETTINGS_VAR', 'DJANGO_TGVKBOT_SETTINGS_MODULE') MAX_FILE_SIZE = os.environ.get('MAX_FILE_SIZE', 52428800) -API_VERSION = os.environ.get('API_VERSION', '5.71') +API_VERSION = os.environ.get('API_VERSION', '5.124') AUDIO_API_VERSION = os.environ.get('API_VERSION', '5.78') # https://www.miniwebtool.com/django-secret-key-generator/ diff --git a/docker-compose.yml b/docker-compose.yml index 6c6209f..2277bf6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,27 +23,10 @@ services: restart: always networks: - db_nw - - web_nw depends_on: - db - nginx: - image: "nginx:1.13.5" - ports: - - "8443:8443" - restart: always - volumes: - - .:/src - - ./conf.d:/etc/nginx/conf.d - env_file: - - env_file - networks: - - web_nw - depends_on: - - tgbot networks: db_nw: driver: bridge - web_nw: - driver: bridge volumes: dbdata: diff --git a/install.sh b/install.sh index 2124aaa..bb17dc8 100755 --- a/install.sh +++ b/install.sh @@ -5,6 +5,5 @@ sudo usermod -aG docker $(whoami) && \ sudo curl -L https://github.com/docker/compose/releases/download/1.20.1/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose && \ sudo chmod +x /usr/local/bin/docker-compose && \ python3 set_env.py && \ -python3 obtaincert.py && \ sudo docker-compose build && \ -sudo docker-compose up -d \ No newline at end of file +sudo docker-compose up -d diff --git a/obtaincert.py b/obtaincert.py deleted file mode 100644 index 28dd32e..0000000 --- a/obtaincert.py +++ /dev/null @@ -1,35 +0,0 @@ -from subprocess import call -from config import WEBHOOK_HOST - -OPENSSL_CONFIG_TEMPLATE = """ -prompt = no -distinguished_name = req_distinguished_name -req_extensions = v3_req -[ req_distinguished_name ] -C = RU -ST = Saint-Petersburg -L = Saint-Petersburg -O = tgvkbot -OU = tgvkbot -CN = %(domain)s -emailAddress = tgvkbot@gmail.com -[ v3_req ] -# Extensions to add to a certificate request -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names -[ alt_names ] -DNS.1 = %(domain)s -DNS.2 = *.%(domain)s -""" - -call([ - 'openssl', 'genrsa', '-out', 'webhook_pkey.pem', '2048' -]) -config = open('openssl_config', 'w') -config.write(OPENSSL_CONFIG_TEMPLATE % {'domain': WEBHOOK_HOST}) -config.close() -call([ - 'openssl', 'req', '-new', '-x509', '-days', '3650', '-key', 'webhook_pkey.pem', '-out', 'webhook_cert.pem', - '-config', 'openssl_config' -]) diff --git a/telegram.py b/telegram.py index dafabff..eee5cb5 100644 --- a/telegram.py +++ b/telegram.py @@ -1,7 +1,6 @@ from aiogram import types from aiogram.bot.api import FILE_URL -from aiogram.dispatcher.webhook import get_new_configured_app -from aiohttp import web +from aiogram.utils import executor from aiohttp.client_exceptions import ContentTypeError from bot import * @@ -73,8 +72,16 @@ async def is_bot_in_iterator(msg: types.Message): return False +import secrets + + +def generate_random_id(): + return secrets.randbelow(2_147_483_647) + + async def vk_sender(token, tg_message, **kwargs): session = VkSession(access_token=token, driver=await get_driver(token)) + kwargs['random_id'] = generate_random_id() try: api = API(session) vk_msg_id = await api('messages.send', **kwargs) @@ -199,10 +206,12 @@ async def upload_attachment(msg, vk_user, file_id, peer_id, attachment_type, upl save_options = dict({'file': file_on_server['file']}) save_options['title'] = title attachment = await api(save_method, **save_options) - return f'{attachment_type}{attachment[0]["owner_id"]}_{attachment[0]["id"]}' + return f'{attachment_type}{attachment[attachment["type"]]["owner_id"]}_{attachment[attachment["type"]]["id"]}' -async def get_dialogs(token, exclude=list()): +async def get_dialogs(token, exclude=None): + if not exclude: + exclude = [] session = VkSession(access_token=token, driver=await get_driver(token)) api = API(session) dialogs = await api('messages.getDialogs', count=200) @@ -491,6 +500,7 @@ async def choose_chat(call: types.CallbackQuery): tg_id=tg_message.message_id, vk_chat=vk_chat_id ) + await bot.answer_callback_query(call.id) else: forward = Forward.objects.filter(tgchat=tgchat).first() vkchat = (await get_vk_chat(int(vk_chat_id)))[0] @@ -787,6 +797,8 @@ async def handle_documents(msg: types.Message): upload_attachment_options['upload_type'] = 'audio_message' if msg.content_type == 'sticker': + if msg.sticker.to_python()['is_animated']: + file_id = msg.sticker.thumb.file_id upload_attachment_options['upload_type'] = 'graffiti' upload_attachment_options['rewrite_name'] = True upload_attachment_options['default_name'] = 'graffiti.png' @@ -914,15 +926,8 @@ async def handle_chat_migration(msg: types.Message): forward.tgchat.save() -async def on_startup(app): - # Set new URL for webhook - await bot.set_webhook(WEBHOOK_URL, certificate=open(WEBHOOK_SSL_CERT, 'rb')) - - if __name__ == '__main__': TASKS = vk_polling_tasks() asyncio.gather(*[task['task'] for task in TASKS]) - app = get_new_configured_app(dispatcher=dp, path=WEBHOOK_URL_PATH) - app.on_startup.append(on_startup) - web.run_app(app, host=WEBAPP_HOST, port=WEBAPP_PORT) + executor.start_polling(dp) diff --git a/vk_messages.py b/vk_messages.py index 716b6df..e22c663 100644 --- a/vk_messages.py +++ b/vk_messages.py @@ -1,10 +1,10 @@ +import urllib from concurrent.futures._base import CancelledError, TimeoutError +from aiogram.utils.markdown import quote_html, hlink from aiovk.longpoll import LongPoll from bot import * -from aiogram.utils.markdown import quote_html, hlink -import urllib log = logging.getLogger('vk_messages') inline_link_re = re.compile('\[([a-zA-Z0-9_]*)\|(.*?)\]', re.MULTILINE) @@ -102,7 +102,7 @@ class MessageEventData(object): data.user_id = int(obj['user_id']) data.true_user_id = int(obj['user_id']) - data.full_text = obj['body'] + data.full_text = obj['text'] data.time = int(obj['date']) data.is_out = obj.get('out', False) data.is_forwarded = False @@ -525,7 +525,7 @@ async def process_longpoll_event(api, new_event): async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, user_id=None, forward_settings=None, vkchat=None, full_msg=None, forwarded=False, vk_msg_id=None, main_message=None, known_users=None, - force_disable_notify=None): + force_disable_notify=None, full_chat=None): token = token or msg.api._session.access_token is_multichat = is_multichat or msg.is_multichat vk_msg_id = vk_msg_id or msg.msg_id @@ -553,6 +553,10 @@ async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, u forward_setting = forward_settings or Forward.objects.filter(owner=vkuser.owner, vkchat=vkchat).first() full_msg = full_msg or await msg.api('messages.getById', message_ids=', '.join(str(x) for x in [vk_msg_id])) + + # Узнаем title чата + if is_multichat: + full_chat = await msg.api('messages.getChat', chat_id=vk_chat_id - 2000000000) if full_msg.get('items'): for vk_msg in full_msg['items']: disable_notify = force_disable_notify or bool(vk_msg.get('push_settings', False)) @@ -579,11 +583,11 @@ async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, u if forwarded or not is_multichat: header = f'{name}' + '\n' elif is_multichat: - header = f'{name} @ {quote_html(vk_msg["title"])}' + '\n' + header = f'{name} @ {quote_html(full_chat["title"])}' + '\n' to_tg_chat = vkuser.owner.uid body_parts = [] - body = quote_html(vk_msg.get('body', '')) + body = quote_html(vk_msg.get('text', '')) if body: if (len(header) + len(body)) > MAX_MESSAGE_LENGTH: @@ -602,6 +606,14 @@ async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, u body += first_text_attach['content'] attaches_scheme.remove(first_text_attach) + first_voice_attach = next( + (attach for attach in attaches_scheme if attach and attach['type'] == 'audio_message'), + None) + if first_voice_attach: + # Будем отправлять только те войсы, в которых завершен транскрипт сообщений + if first_voice_attach.get('transcript_state') != 'done': + return + if body_parts: for body_part in range(len(body_parts)): m = inline_link_re.finditer(body_parts[body_part]) @@ -687,9 +699,17 @@ async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, u attachment.get('content', '') or attachment.get('url'), reply_to_message_id=main_message, disable_notification=disable_notify) if 'content' in attachment: - attachment['content'].close() - os.remove(os.path.join(attachment['temp_path'], - attachment['file_name'] + attachment['custom_ext'])) + try: + # Иногда тут появляется url, лень проверять откуда растут ноги + attachment['content'].close() + except: + pass + try: + # Тут вообще не оч понятно, почему не удаляет + os.remove(os.path.join(attachment['temp_path'], + attachment['file_name'] + attachment['custom_ext'])) + except: + pass elif attachment['type'] == 'video': await bot.send_chat_action(to_tg_chat, ChatActions.UPLOAD_VIDEO) tg_message = await tgsend(bot.send_video, to_tg_chat, attachment['content'], @@ -714,6 +734,22 @@ async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, u title=attachment.get('title', None), reply_to_message_id=main_message, disable_notification=disable_notify, parse_mode='HTML') + elif attachment['type'] == 'audio_message': + await bot.send_chat_action(to_tg_chat, ChatActions.RECORD_AUDIO) + tg_message = await tgsend(bot.send_voice, to_tg_chat, voice=attachment['content']) + + if attachment.get('transcript'): + transcript_text = 'Войс: ' + attachment['transcript'] + transcript_message = await tgsend(bot.send_message, to_tg_chat, text=transcript_text, + reply_to_message_id=tg_message.message_id, + parse_mode=ParseMode.HTML) + Message.objects.create( + vk_chat=vk_chat_id, + vk_id=vk_msg_id, + tg_chat=transcript_message.chat.id, + tg_id=transcript_message.message_id + ) + Message.objects.create( vk_chat=vk_chat_id, vk_id=vk_msg_id, @@ -724,7 +760,7 @@ async def process_message(msg, token=None, is_multichat=None, vk_chat_id=None, u await bot.send_chat_action(to_tg_chat, ChatActions.TYPING) for fwd_message in vk_msg['fwd_messages']: await process_message(msg, token=token, is_multichat=is_multichat, vk_chat_id=vk_chat_id, - user_id=fwd_message['user_id'], + user_id=fwd_message['from_id'], forward_settings=forward_settings, vk_msg_id=vk_msg_id, vkchat=vkchat, full_msg={'items': [fwd_message]}, forwarded=True, main_message=header_message.message_id if header_message else None, @@ -749,7 +785,7 @@ async def tgsend(method, *args, **kwargs): tg_message = await method(*args, **kwargs) return tg_message except RetryAfter as e: - asyncio.sleep(e.timeout) + await asyncio.sleep(e.timeout) await tgsend(method, *args, **kwargs) except Exception: log.exception(msg='Error in message sending', exc_info=True) @@ -786,9 +822,19 @@ def form_audio_title(data: dict, delimer=' '): async def process_attachment(attachment, token=None): atype = attachment.get('type') if atype == 'photo': - photo_url = attachment[atype][await get_max_photo(attachment[atype])] + photo_url = attachment[atype]['sizes'][-1]['url'] return {'content': photo_url, 'type': 'photo'} + elif atype == 'audio_message': + voice_url = attachment[atype]['link_ogg'] + + res = {'content': voice_url, 'type': 'audio_message'} + if attachment[atype].get('transcript'): + res['transcript'] = attachment[atype]['transcript'] + if attachment[atype].get('transcript_state'): + res['transcript_state'] = attachment[atype]['transcript_state'] + return res + elif atype == 'audio': if attachment[atype].get('url') and AUDIO_PROXY_URL: try: @@ -884,15 +930,15 @@ async def process_attachment(attachment, token=None): if size > MAX_FILE_SIZE: return {'content': f'GIF', 'type': 'text'} return {'content': gif_url, 'type': 'document'} - elif 'preview' in attachment[atype] and attachment[atype]['preview'].get('graffiti'): - graffiti_url = attachment[atype]['preview']['photo']['sizes'][-1]['src'] - with aiohttp.ClientSession() as session: - img = await (await session.request('GET', graffiti_url)).read() - imgdata = Image.open(io.BytesIO(img)) - webp = io.BytesIO() - imgdata.save(webp, format='WebP') - file_bytes = webp.getvalue() - return {'content': file_bytes, 'type': 'sticker'} + # elif 'preview' in attachment[atype] and attachment[atype]['preview'].get('graffiti'): + # graffiti_url = attachment[atype]['preview']['photo']['sizes'][-1]['src'] + # with aiohttp.ClientSession() as session: + # img = await (await session.request('GET', graffiti_url)).read() + # imgdata = Image.open(io.BytesIO(img)) + # webp = io.BytesIO() + # imgdata.save(webp, format='WebP') + # file_bytes = webp.getvalue() + # return {'content': file_bytes, 'type': 'sticker'} else: size = attachment[atype]['size'] doc_url = attachment[atype]['url'] # + f'&{ext}=1' @@ -907,8 +953,18 @@ async def process_attachment(attachment, token=None): else: return {'content': f'📄 {content["docname"]}', 'type': 'text'} + elif atype == 'graffiti': + graffiti_url = attachment[atype]['url'] + with aiohttp.ClientSession() as session: + img = await (await session.request('GET', graffiti_url)).read() + imgdata = Image.open(io.BytesIO(img)) + webp = io.BytesIO() + imgdata.save(webp, format='WebP') + file_bytes = webp.getvalue() + return {'content': file_bytes, 'type': 'sticker'} + elif atype == 'sticker': - sticker_url = attachment[atype][await get_max_photo(attachment[atype])] + sticker_url = attachment[atype]['images'][-1]['url'] with aiohttp.ClientSession() as session: img = await (await session.request('GET', sticker_url)).read() imgdata = Image.open(io.BytesIO(img)) @@ -1004,7 +1060,7 @@ async def vk_polling(vkuser: VkUser): break except VkLongPollError: log.error('Longpoll error! {}'.format(vkuser.pk)) - asyncio.sleep(5) + await asyncio.sleep(5) except VkAuthError: log.error('Auth Error! {}'.format(vkuser.pk)) vkuser.is_polling = False @@ -1012,7 +1068,7 @@ async def vk_polling(vkuser: VkUser): break except TimeoutError: log.warning('Polling timeout') - asyncio.sleep(5) + await asyncio.sleep(5) except CancelledError: log.warning('Stopped polling for id: ' + str(vkuser.pk)) break @@ -1023,7 +1079,7 @@ async def vk_polling(vkuser: VkUser): pass except Exception: log.exception(msg='Error in longpolling', exc_info=True) - asyncio.sleep(5) + await asyncio.sleep(5) def vk_polling_tasks():