reduce number of requests
This commit is contained in:
parent
231f7878b2
commit
c021da3441
GPG Key ID:
E14D26A036F21656
@ -12,7 +12,6 @@ import tornado.web
|
||||
import tornado.routing
|
||||
import aiohttp
|
||||
import aiohttp_socks
|
||||
import html.parser
|
||||
import stream_providers
|
||||
|
||||
logging.basicConfig(format='[%(filename)s:%(lineno)d] %(message)s', stream=sys.stdout, level=logging.INFO)
|
||||
@ -112,26 +111,6 @@ for key in providers:
|
||||
for proxy in current:
|
||||
proxies[key].append(ProxyElem(proxy))
|
||||
|
||||
class MetaParser(html.parser.HTMLParser):
|
||||
def __init__(self):
|
||||
self.meta_data = {}
|
||||
self.accepted_attrs = []
|
||||
self.accepted_attrs.append("og:title")
|
||||
self.accepted_attrs.append("og:description")
|
||||
self.accepted_attrs.append("og:image")
|
||||
super().__init__()
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "meta":
|
||||
name = None
|
||||
for attr in (attrs + attrs):
|
||||
if len(attr) == 2:
|
||||
if isinstance(name, str):
|
||||
if attr[0] == "content":
|
||||
self.meta_data[name] = attr[1]
|
||||
return
|
||||
elif attr[0] == "property" and attr[1] in self.accepted_attrs:
|
||||
name = attr[1]
|
||||
|
||||
class UpstreamHandler():
|
||||
def __init__(self):
|
||||
self.provider = None
|
||||
@ -192,76 +171,6 @@ class UpstreamHandler():
|
||||
for session in sessions:
|
||||
await session.close()
|
||||
|
||||
async def meta(self):
|
||||
data = {}
|
||||
try:
|
||||
embed_url = f'https://noembed.com/embed?url={self.upstream_safe}'
|
||||
async with self.proxy.session() as session:
|
||||
resp_embed_future = session.get(embed_url)
|
||||
resp_upstream_future = session.get(self.upstream)
|
||||
try:
|
||||
resp_embed = await resp_embed_future
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
resp_embed = None
|
||||
try:
|
||||
resp_upstream = await resp_upstream_future
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
resp_upstream = None
|
||||
text_embed_future = resp_embed.text()
|
||||
text_upstream_future = resp_upstream.text()
|
||||
try:
|
||||
text_embed = await text_embed_future
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
text_embed = None
|
||||
try:
|
||||
text_upstream = await text_upstream_future
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
text_upstream = None
|
||||
parser = MetaParser()
|
||||
parser.feed(text_upstream)
|
||||
data_raw = json.loads(text_embed)
|
||||
if isinstance(data_raw, dict):
|
||||
data_new = {}
|
||||
data_valid = True
|
||||
data_new["og:title"] = data_raw.get("title")
|
||||
data_new["og:description"] = data_raw.get("author_name")
|
||||
data_new["og:image"] = data_raw.get("thumbnail_url")
|
||||
data_filtered = {}
|
||||
for key in data_new:
|
||||
value = data_new.get(key)
|
||||
if isinstance(value, str):
|
||||
data_filtered[key] = value
|
||||
data_filtered.update(parser.meta_data)
|
||||
data = data_filtered
|
||||
image = data.get("og:image")
|
||||
if isinstance(image, str):
|
||||
if self.provider == "youtube":
|
||||
full_image = re.sub(r'\/[a-zA-Z0-9]+\.([a-zA-Z0-9]+)$', r'/maxresdefault.\1', image)
|
||||
standard_image = re.sub(r'\/[a-zA-Z0-9]+\.([a-zA-Z0-9]+)$', r'/sddefault.\1', image)
|
||||
image_status_full_future = session.head(full_image)
|
||||
image_status_standard_future = session.head(standard_image)
|
||||
try:
|
||||
image_status_full = await image_status_full_future
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
image_status_full = None
|
||||
try:
|
||||
image_status_standard = await image_status_standard_future
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
image_status_standard = None
|
||||
if hasattr(image_status_full, "status") and (image_status_full.status < 400):
|
||||
data["og:image"] = full_image
|
||||
elif hasattr(image_status_standard, "status") and (image_status_standard.status < 400):
|
||||
data["og:image"] = standard_image
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
return data
|
||||
|
||||
if icecast_server is not None and stream_server is not None:
|
||||
try:
|
||||
with open("/app/sources.json", "r") as f:
|
||||
@ -376,32 +285,29 @@ class MainHandler(tornado.web.RequestHandler):
|
||||
|
||||
async def handle_render(self, handler):
|
||||
if script_file is not None and template_html is not None:
|
||||
meta = await handler.meta()
|
||||
meta_list = list(meta.items())
|
||||
title = meta.get("og:title")
|
||||
provider_data = await stream_providers.get_any(handler.upstream, handler.proxy, logger)
|
||||
data["script"] = script_file
|
||||
data["videojs_version"] = videojs_version
|
||||
data["chromecast_version"] = chromecast_version
|
||||
data["font_awesome_version"] = font_awesome_version
|
||||
data["custom_style"] = custom_style
|
||||
rendered_html = template_html.generate(data=data, meta=meta_list, title=title)
|
||||
rendered_html = template_html.generate(data=data, meta=provider_data.meta(), title=provider_data.title)
|
||||
self.write(rendered_html)
|
||||
else:
|
||||
self.set_status(404)
|
||||
self.write("HTML template missing.")
|
||||
|
||||
async def handle_stream(self, handler, redir):
|
||||
upstream = None
|
||||
if handler.provider == "nextcloud":
|
||||
upstream = handler.upstream + "/download"
|
||||
else:
|
||||
if not redir:
|
||||
meta = await handler.meta()
|
||||
image = meta.get("og:image")
|
||||
if isinstance(image, str):
|
||||
image = await handler.proxy.proxy_url(image, None)
|
||||
provider_data = await stream_providers.get_any(handler.upstream, handler.proxy, logger)
|
||||
upstream = provider_data.upstream()
|
||||
if isinstance(provider_data.thumbnail(), str):
|
||||
image = await handler.proxy.proxy_url(provider_data.thumbnail(), None)
|
||||
if isinstance(image, str):
|
||||
self.set_header("Custom-Poster", image)
|
||||
upstream = await stream_providers.get_any(handler.upstream, handler.proxy, logger)
|
||||
if upstream is None:
|
||||
logger.info(f'invalid upstream ({handler.provider})')
|
||||
self.set_status(404)
|
||||
|
||||
@ -1,7 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import youtube_dl
|
||||
import streamlink
|
||||
import requests
|
||||
import asyncio
|
||||
import html.parser
|
||||
import re
|
||||
|
||||
ytimg_pattern = re.compile(r'(https:\/\/[a-z0-9.]+ytimg\.com\/.+\/)[a-z0-9]+(\.[a-z0-9]+)')
|
||||
|
||||
class DummyLogger():
|
||||
def debug(self, msg):
|
||||
@ -11,71 +15,149 @@ class DummyLogger():
|
||||
def error(self, msg):
|
||||
pass
|
||||
|
||||
class MetaParser(html.parser.HTMLParser):
|
||||
def __init__(self):
|
||||
self.meta_data = {}
|
||||
self.accepted_attrs = []
|
||||
self.accepted_attrs.append("og:title")
|
||||
self.accepted_attrs.append("og:description")
|
||||
self.accepted_attrs.append("og:image")
|
||||
super().__init__()
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "meta":
|
||||
name = None
|
||||
for attr in (attrs + attrs):
|
||||
if len(attr) == 2:
|
||||
if isinstance(name, str):
|
||||
if attr[0] == "content":
|
||||
self.meta_data[name] = attr[1]
|
||||
return
|
||||
elif attr[0] == "property" and attr[1] in self.accepted_attrs:
|
||||
name = attr[1]
|
||||
|
||||
class StreamData():
|
||||
def __init__(self, upstream, thumbnail, title, description, override):
|
||||
self.values = {}
|
||||
self.values["upstream"] = upstream
|
||||
self.values["thumbnail"] = thumbnail
|
||||
self.values["title"] = title
|
||||
self.values["description"] = description
|
||||
self.override = override
|
||||
def update(self, key, value, override):
|
||||
missing = not isinstance(self.values.get(key), str)
|
||||
override = override and isinstance(value, str)
|
||||
if missing or override:
|
||||
self.values[key] = value
|
||||
def upstream(self):
|
||||
return self.values.get("upstream")
|
||||
def thumbnail(self):
|
||||
return self.values.get("thumbnail")
|
||||
def title(self):
|
||||
return self.values.get("title")
|
||||
def description(self):
|
||||
return self.values.get("description")
|
||||
def meta(self):
|
||||
data = []
|
||||
if isinstance(self.values.get("thumbnail"), str):
|
||||
data.append(("og:image", self.values.get("thumbnail")))
|
||||
if isinstance(self.values.get("title"), str):
|
||||
data.append(("og:title", self.values.get("title")))
|
||||
if isinstance(self.values.get("description"), str):
|
||||
data.append(("og:description", self.values.get("description")))
|
||||
return data
|
||||
|
||||
class StreamProvider():
|
||||
def __init__(self, upstream, proxy):
|
||||
def __init__(self, upstream, proxy, logger):
|
||||
self.upstream = upstream
|
||||
self.proxy = None
|
||||
self.logger = logger
|
||||
proxy = str(proxy)
|
||||
if len(proxy) > 5:
|
||||
self.proxy = "socks5://" + proxy
|
||||
|
||||
class StreamlinkRunner(StreamProvider):
|
||||
def stream(self):
|
||||
session = streamlink.Streamlink()
|
||||
if self.proxy is not None:
|
||||
session.set_option("https-proxy", self.proxy)
|
||||
session.set_option("http-proxy", self.proxy)
|
||||
streams = session.streams(self.upstream)
|
||||
if streams is not None:
|
||||
for key in reversed(streams):
|
||||
stream = streams.get(key)
|
||||
if hasattr(stream, "url"):
|
||||
return stream.url
|
||||
return None
|
||||
try:
|
||||
session = streamlink.Streamlink()
|
||||
if self.proxy is not None:
|
||||
session.set_option("https-proxy", self.proxy)
|
||||
session.set_option("http-proxy", self.proxy)
|
||||
streams = session.streams(self.upstream)
|
||||
if streams is not None:
|
||||
for key in reversed(streams):
|
||||
stream = streams.get(key)
|
||||
if hasattr(stream, "url"):
|
||||
return StreamData(stream.url, None, None, None, False)
|
||||
except Exception as e:
|
||||
self.logger.info(e)
|
||||
return StreamData(None, None, None, None, False)
|
||||
async def run(self):
|
||||
return await asyncio.to_thread(self.stream)
|
||||
|
||||
class YoutubeRunner(StreamProvider):
|
||||
def stream(self):
|
||||
opts = {}
|
||||
opts["logger"] = DummyLogger()
|
||||
if isinstance(self.proxy, str):
|
||||
opts["proxy"] = self.proxy
|
||||
best_url = None
|
||||
with youtube_dl.YoutubeDL(opts) as ydl:
|
||||
info = ydl.extract_info(self.upstream, download=False)
|
||||
vformats = info.get("formats")
|
||||
best_format = {}
|
||||
best_format["width"] = 10
|
||||
best_format["height"] = 10
|
||||
if isinstance(vformats, list):
|
||||
for vformat in vformats:
|
||||
acodec = vformat.get("acodec")
|
||||
vcodec = vformat.get("vcodec")
|
||||
current_width = vformat.get("height")
|
||||
current_height = vformat.get("width")
|
||||
best_width = best_format.get("width")
|
||||
best_height = best_format.get("height")
|
||||
new_url = vformat.get("url")
|
||||
if (isinstance(best_width, int) and
|
||||
isinstance(best_height, int) and
|
||||
isinstance(current_width, int) and
|
||||
isinstance(current_height, int) and
|
||||
isinstance(new_url, str) and
|
||||
current_width > best_width and
|
||||
current_height > best_height and
|
||||
acodec != "none" and
|
||||
vcodec != "none"):
|
||||
best_format = vformat
|
||||
best_url = new_url
|
||||
return best_url
|
||||
thumbnail = None
|
||||
title = None
|
||||
description = None
|
||||
try:
|
||||
opts = {}
|
||||
opts["logger"] = DummyLogger()
|
||||
if isinstance(self.proxy, str):
|
||||
opts["proxy"] = self.proxy
|
||||
with youtube_dl.YoutubeDL(opts) as ydl:
|
||||
info = ydl.extract_info(self.upstream, download=False)
|
||||
vformats = info.get("formats")
|
||||
thumbnail = info.get("thumbnail")
|
||||
description = info.get("channel")
|
||||
title = info.get("title")
|
||||
best_format = {}
|
||||
best_format["width"] = 10
|
||||
best_format["height"] = 10
|
||||
if isinstance(vformats, list):
|
||||
for vformat in vformats:
|
||||
acodec = vformat.get("acodec")
|
||||
vcodec = vformat.get("vcodec")
|
||||
current_width = vformat.get("height")
|
||||
current_height = vformat.get("width")
|
||||
best_width = best_format.get("width")
|
||||
best_height = best_format.get("height")
|
||||
new_url = vformat.get("url")
|
||||
if (isinstance(best_width, int) and
|
||||
isinstance(best_height, int) and
|
||||
isinstance(current_width, int) and
|
||||
isinstance(current_height, int) and
|
||||
isinstance(new_url, str) and
|
||||
current_width > best_width and
|
||||
current_height > best_height and
|
||||
acodec != "none" and
|
||||
vcodec != "none"):
|
||||
best_format = vformat
|
||||
best_url = new_url
|
||||
except Exception as e:
|
||||
self.logger.info(e)
|
||||
return StreamData(best_url, thumbnail, title, description, True)
|
||||
async def run(self):
|
||||
return await asyncio.to_thread(self.stream)
|
||||
|
||||
class MetaRunner(StreamProvider):
|
||||
def stream(self):
|
||||
data = {}
|
||||
try:
|
||||
resp = requests.get(self.upstream)
|
||||
parser = MetaParser()
|
||||
parser.feed(resp.text)
|
||||
data = parser.meta_data
|
||||
except Exception as e:
|
||||
self.logger.info(e)
|
||||
return StreamData(None, data.get("og:image"), data.get("og:title"), data.get("og:description"), False)
|
||||
async def run(self):
|
||||
return await asyncio.to_thread(self.stream)
|
||||
|
||||
async def get_ytdl(upstream, proxy, logger):
|
||||
result = None
|
||||
try:
|
||||
runner = YoutubeRunner(upstream, proxy)
|
||||
runner = YoutubeRunner(upstream, proxy, logger)
|
||||
result_temp = await runner.run()
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
@ -86,7 +168,18 @@ async def get_ytdl(upstream, proxy, logger):
|
||||
async def get_streamlink(upstream, proxy, logger):
|
||||
result = None
|
||||
try:
|
||||
runner = StreamlinkRunner(upstream, proxy)
|
||||
runner = StreamlinkRunner(upstream, proxy, logger)
|
||||
result_temp = await runner.run()
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
else:
|
||||
result = result_temp
|
||||
return result
|
||||
|
||||
async def get_meta(upstream, proxy, logger):
|
||||
result = None
|
||||
try:
|
||||
runner = MetaRunner(upstream, proxy, logger)
|
||||
result_temp = await runner.run()
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
@ -98,16 +191,19 @@ async def get_any(upstream, proxy, logger):
|
||||
tasks = []
|
||||
tasks.append(asyncio.create_task(get_streamlink(upstream, proxy, logger)))
|
||||
tasks.append(asyncio.create_task(get_ytdl(upstream, proxy, logger)))
|
||||
result = None
|
||||
tasks.append(asyncio.create_task(get_meta(upstream, proxy, logger)))
|
||||
result = StreamData(None, None, None, None, False)
|
||||
for task in asyncio.as_completed(tasks, timeout=5.0):
|
||||
temp_result = None
|
||||
try:
|
||||
temp_result = await task
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
if isinstance(temp_result, str):
|
||||
result = temp_result
|
||||
break
|
||||
if isinstance(temp_result, StreamData):
|
||||
result.update("upstream", temp_result.upstream(), temp_result.override)
|
||||
result.update("thumbnail", temp_result.thumbnail(), temp_result.override)
|
||||
result.update("title", temp_result.title(), temp_result.override)
|
||||
result.update("description", temp_result.description(), temp_result.override)
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user