474 lines
20 KiB
Python
474 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Manage Youtube subscriptions using youtube-dl (https://github.com/ytdl-org/youtube-dl)
|
|
# Requirements: youtube-dl module (pip install youtube_dl)
|
|
# (c) 2019-08-17 Eric Le Lay <elelay.fr:contact>
|
|
# Released under the same license terms as gPodder itself.
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
import youtube_dl
|
|
from youtube_dl.utils import DownloadError, ExtractorError, sanitize_url
|
|
|
|
import gpodder
|
|
from gpodder import download, feedcore, model, registry, youtube
|
|
from gpodder.util import mimetype_from_extension, remove_html_tags
|
|
|
|
_ = gpodder.gettext
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
__title__ = 'Youtube-dl'
|
|
__description__ = _('Manage Youtube subscriptions using youtube-dl (pip install youtube_dl)')
|
|
__only_for__ = 'gtk, cli'
|
|
__authors__ = 'Eric Le Lay <elelay.fr:contact>'
|
|
__doc__ = 'https://gpodder.github.io/docs/extensions/youtubedl.html'
|
|
|
|
want_ytdl_version = '2020.11.12'
|
|
want_ytdl_version_msg = _('Your version of youtube-dl %(have_version)s has known issues, please upgrade to %(want_version)s or newer.')
|
|
|
|
DefaultConfig = {
|
|
# youtube-dl downloads and parses each video page to get informations about it, which is very slow.
|
|
# Set to False to fall back to the fast but limited (only 15 episodes) gpodder code
|
|
'manage_channel': True,
|
|
# If for some reason youtube-dl download doesn't work for you, you can fallback to gpodder code.
|
|
# Set to False to fall back to default gpodder code (less available formats).
|
|
'manage_downloads': True,
|
|
}
|
|
|
|
|
|
# youtube feed still preprocessed by youtube.py (compat)
|
|
CHANNEL_RE = re.compile(r'''https://www.youtube.com/feeds/videos.xml\?channel_id=(.+)''')
|
|
USER_RE = re.compile(r'''https://www.youtube.com/feeds/videos.xml\?user=(.+)''')
|
|
PLAYLIST_RE = re.compile(r'''https://www.youtube.com/feeds/videos.xml\?playlist_id=(.+)''')
|
|
|
|
|
|
def youtube_parsedate(s):
|
|
"""Parse a string into a unix timestamp
|
|
|
|
Only strings provided by Youtube-dl API are
|
|
parsed with this function (20170920).
|
|
"""
|
|
if s:
|
|
return time.mktime(time.strptime(s, "%Y%m%d"))
|
|
return 0
|
|
|
|
|
|
def video_guid(video_id):
|
|
"""
|
|
generate same guid as youtube
|
|
"""
|
|
return 'yt:video:{}'.format(video_id)
|
|
|
|
|
|
class YoutubeCustomDownload(download.CustomDownload):
|
|
"""
|
|
Represents the download of a single episode using youtube-dl.
|
|
|
|
Actual youtube-dl interaction via gPodderYoutubeDL.
|
|
"""
|
|
def __init__(self, ytdl, url, episode):
|
|
self._ytdl = ytdl
|
|
self._url = url
|
|
self._reporthook = None
|
|
self._prev_dl_bytes = 0
|
|
self._episode = episode
|
|
|
|
def retrieve_resume(self, tempname, reporthook=None):
|
|
"""
|
|
called by download.DownloadTask to perform the download.
|
|
"""
|
|
self._reporthook = reporthook
|
|
# outtmpl: use given tempname by DownloadTask
|
|
# (escape % and $ because outtmpl used as a string template by youtube-dl)
|
|
outtmpl = tempname.replace('%', '%%').replace('$', '$$')
|
|
res = self._ytdl.fetch_video(self._url, outtmpl, self._my_hook)
|
|
if outtmpl != tempname:
|
|
if 'ext' in res and os.path.isfile(outtmpl + '.{}'.format(res['ext'])):
|
|
os.rename(outtmpl + '.{}'.format(res['ext']), tempname)
|
|
else:
|
|
os.rename(outtmpl, tempname)
|
|
if 'duration' in res and res['duration']:
|
|
self._episode.total_time = res['duration']
|
|
headers = {}
|
|
# youtube-dl doesn't return a content-type but an extension
|
|
if 'ext' in res:
|
|
dot_ext = '.{}'.format(res['ext'])
|
|
# See #673 when merging multiple formats, the extension is appended to the tempname
|
|
# by YoutubeDL resulting in empty .partial file + .partial.mp4 exists
|
|
# and #796 .mkv is chosen by ytdl sometimes
|
|
tempstat = os.stat(tempname)
|
|
if not tempstat.st_size:
|
|
for try_ext in (dot_ext, ".mp4", ".m4a", ".webm", ".mkv"):
|
|
tempname_with_ext = tempname + try_ext
|
|
if os.path.isfile(tempname_with_ext):
|
|
logger.debug('Youtubedl downloaded to "%s" instead of "%s", moving',
|
|
os.path.basename(tempname_with_ext),
|
|
os.path.basename(tempname))
|
|
os.remove(tempname)
|
|
os.rename(tempname_with_ext, tempname)
|
|
dot_ext = try_ext
|
|
break
|
|
ext_filetype = mimetype_from_extension(dot_ext)
|
|
if ext_filetype:
|
|
headers['content-type'] = ext_filetype
|
|
return headers, res.get('url', self._url)
|
|
|
|
def _my_hook(self, d):
|
|
if d['status'] == 'downloading':
|
|
if self._reporthook:
|
|
dl_bytes = d['downloaded_bytes']
|
|
total_bytes = d.get('total_bytes') or d.get('total_bytes_estimate') or 0
|
|
self._reporthook(self._prev_dl_bytes + dl_bytes,
|
|
1,
|
|
self._prev_dl_bytes + total_bytes)
|
|
elif d['status'] == 'finished':
|
|
dl_bytes = d['downloaded_bytes']
|
|
self._prev_dl_bytes += dl_bytes
|
|
if self._reporthook:
|
|
self._reporthook(self._prev_dl_bytes, 1, self._prev_dl_bytes)
|
|
elif d['status'] == 'error':
|
|
logger.error('download hook error: %r', d)
|
|
else:
|
|
logger.debug('unknown download hook status: %r', d)
|
|
|
|
|
|
class YoutubeFeed(model.Feed):
|
|
"""
|
|
Represents the youtube feed for model.PodcastChannel
|
|
"""
|
|
def __init__(self, url, cover_url, description, max_episodes, ie_result, downloader):
|
|
self._url = url
|
|
self._cover_url = cover_url
|
|
self._description = description
|
|
self._max_episodes = max_episodes
|
|
ie_result['entries'] = self._process_entries(ie_result.get('entries', []))
|
|
self._ie_result = ie_result
|
|
self._downloader = downloader
|
|
|
|
def _process_entries(self, entries):
|
|
filtered_entries = []
|
|
seen_guids = set()
|
|
for i, e in enumerate(entries): # consumes the generator!
|
|
if e.get('_type', 'video') in ('url', 'url_transparent') and e.get('ie_key') == 'Youtube':
|
|
guid = video_guid(e['id'])
|
|
e['guid'] = guid
|
|
if guid in seen_guids:
|
|
logger.debug('dropping already seen entry %s title="%s"', guid, e.get('title'))
|
|
else:
|
|
filtered_entries.append(e)
|
|
seen_guids.add(guid)
|
|
else:
|
|
logger.debug('dropping entry not youtube video %r', e)
|
|
if len(filtered_entries) == self._max_episodes:
|
|
# entries is a generator: stopping now prevents it to download more pages
|
|
logger.debug('stopping entry enumeration')
|
|
break
|
|
return filtered_entries
|
|
|
|
def get_title(self):
|
|
return '{} (Youtube)'.format(self._ie_result.get('title') or self._ie_result.get('id') or self._url)
|
|
|
|
def get_link(self):
|
|
return self._ie_result.get('webpage_url')
|
|
|
|
def get_description(self):
|
|
return self._description
|
|
|
|
def get_cover_url(self):
|
|
return self._cover_url
|
|
|
|
def get_http_etag(self):
|
|
""" :return str: optional -- last HTTP etag header, for conditional request next time """
|
|
# youtube-dl doesn't provide it!
|
|
return None
|
|
|
|
def get_http_last_modified(self):
|
|
""" :return str: optional -- last HTTP Last-Modified header, for conditional request next time """
|
|
# youtube-dl doesn't provide it!
|
|
return None
|
|
|
|
def get_new_episodes(self, channel, existing_guids):
|
|
# entries are already sorted by decreasing date
|
|
# trim guids to max episodes
|
|
entries = [e for i, e in enumerate(self._ie_result['entries'])
|
|
if not self._max_episodes or i < self._max_episodes]
|
|
all_seen_guids = set(e['guid'] for e in entries)
|
|
# only fetch new ones from youtube since they are so slow to get
|
|
new_entries = [e for e in entries if e['guid'] not in existing_guids]
|
|
logger.debug('%i/%i new entries', len(new_entries), len(all_seen_guids))
|
|
self._ie_result['entries'] = new_entries
|
|
self._downloader.refresh_entries(self._ie_result)
|
|
# episodes from entries
|
|
episodes = []
|
|
for en in self._ie_result['entries']:
|
|
guid = video_guid(en['id'])
|
|
description = remove_html_tags(en.get('description') or _('No description available'))
|
|
html_description = self.nice_html_description(en, description)
|
|
if en.get('ext'):
|
|
mime_type = mimetype_from_extension('.{}'.format(en['ext']))
|
|
else:
|
|
mime_type = 'application/octet-stream'
|
|
if en.get('filesize'):
|
|
filesize = int(en['filesize'] or 0)
|
|
else:
|
|
filesize = sum(int(f.get('filesize') or 0)
|
|
for f in en.get('requested_formats', []))
|
|
ep = {
|
|
'title': en.get('title', guid),
|
|
'link': en.get('webpage_url'),
|
|
'description': description,
|
|
'description_html': html_description,
|
|
'url': en.get('webpage_url'),
|
|
'file_size': filesize,
|
|
'mime_type': mime_type,
|
|
'guid': guid,
|
|
'published': youtube_parsedate(en.get('upload_date', None)),
|
|
'total_time': int(en.get('duration') or 0),
|
|
}
|
|
episode = channel.episode_factory(ep)
|
|
episode.save()
|
|
episodes.append(episode)
|
|
return episodes, all_seen_guids
|
|
|
|
def get_next_page(self, channel, max_episodes):
|
|
"""
|
|
Paginated feed support (RFC 5005).
|
|
If the feed is paged, return the next feed page.
|
|
Returned page will in turn be asked for the next page, until None is returned.
|
|
:return feedcore.Result: the next feed's page,
|
|
as a fully parsed Feed or None
|
|
"""
|
|
return None
|
|
|
|
@staticmethod
|
|
def nice_html_description(en, description):
|
|
"""
|
|
basic html formating + hyperlink highlighting + video thumbnail
|
|
"""
|
|
description = re.sub(r'''https?://[^\s]+''',
|
|
r'''<a href="\g<0>">\g<0></a>''',
|
|
description)
|
|
description = description.replace('\n', '<br>')
|
|
html = """<style type="text/css">
|
|
body > img { float: left; max-width: 30vw; margin: 0 1em 1em 0; }
|
|
</style>
|
|
"""
|
|
img = en.get('thumbnail')
|
|
if img:
|
|
html += '<img src="{}">'.format(img)
|
|
html += '<p>{}</p>'.format(description)
|
|
return html
|
|
|
|
|
|
class gPodderYoutubeDL(download.CustomDownloader):
|
|
def __init__(self, gpodder_config, my_config, force=False):
|
|
"""
|
|
:param force: force using this downloader even if config says don't manage downloads
|
|
"""
|
|
self.gpodder_config = gpodder_config
|
|
self.my_config = my_config
|
|
self.force = force
|
|
# cachedir is not much used in youtube-dl, but set it anyway
|
|
cachedir = os.path.join(gpodder.home, 'youtube-dl')
|
|
os.makedirs(cachedir, exist_ok=True)
|
|
self._ydl_opts = {
|
|
'cachedir': cachedir,
|
|
'no_color': True, # prevent escape codes in desktop notifications on errors
|
|
}
|
|
if gpodder.verbose:
|
|
self._ydl_opts['verbose'] = True
|
|
else:
|
|
self._ydl_opts['quiet'] = True
|
|
# #686 on windows without a console, sys.stdout is None, causing exceptions
|
|
# when adding podcasts.
|
|
# See https://docs.python.org/3/library/sys.html#sys.__stderr__ Note
|
|
if not sys.stdout:
|
|
logger.debug('no stdout, setting YoutubeDL logger')
|
|
self._ydl_opts['logger'] = logger
|
|
|
|
def add_format(self, gpodder_config, opts, fallback=None):
|
|
""" construct youtube-dl -f argument from configured format. """
|
|
# You can set a custom format or custom formats by editing the config for key
|
|
# `youtube.preferred_fmt_ids`
|
|
#
|
|
# It takes a list of format strings separated by comma: bestaudio, 18
|
|
# they are translated to youtube dl format bestaudio/18, meaning preferably
|
|
# the best audio quality (audio-only) and MP4 360p if it's not available.
|
|
#
|
|
# See https://github.com/ytdl-org/youtube-dl#format-selection for details
|
|
# about youtube-dl format specification.
|
|
fmt_ids = youtube.get_fmt_ids(gpodder_config.youtube, False)
|
|
opts['format'] = '/'.join(str(fmt) for fmt in fmt_ids)
|
|
if fallback:
|
|
opts['format'] += '/' + fallback
|
|
logger.debug('format=%s', opts['format'])
|
|
|
|
def fetch_video(self, url, tempname, reporthook):
|
|
opts = {
|
|
'outtmpl': tempname,
|
|
'nopart': True, # don't append .part (already .partial)
|
|
'retries': 3, # retry a few times
|
|
'progress_hooks': [reporthook] # to notify UI
|
|
}
|
|
opts.update(self._ydl_opts)
|
|
self.add_format(self.gpodder_config, opts)
|
|
with youtube_dl.YoutubeDL(opts) as ydl:
|
|
return ydl.extract_info(url, download=True)
|
|
|
|
def refresh_entries(self, ie_result):
|
|
# only interested in video metadata
|
|
opts = {
|
|
'skip_download': True, # don't download the video
|
|
'youtube_include_dash_manifest': False, # don't download the DASH manifest
|
|
}
|
|
self.add_format(self.gpodder_config, opts, fallback='18')
|
|
opts.update(self._ydl_opts)
|
|
new_entries = []
|
|
# refresh videos one by one to catch single videos blocked by youtube
|
|
for e in ie_result.get('entries', []):
|
|
tmp = {k: v for k, v in ie_result.items() if k != 'entries'}
|
|
tmp['entries'] = [e]
|
|
try:
|
|
with youtube_dl.YoutubeDL(opts) as ydl:
|
|
ydl.process_ie_result(tmp, download=False)
|
|
new_entries.extend(tmp.get('entries'))
|
|
except DownloadError as ex:
|
|
if ex.exc_info[0] == ExtractorError:
|
|
# for instance "This video contains content from xyz, who has blocked it on copyright grounds"
|
|
logger.warning('Skipping %s: %s', e.get('title', ''), ex.exc_info[1])
|
|
continue
|
|
logger.exception('Skipping %r: %s', tmp, ex.exc_info)
|
|
ie_result['entries'] = new_entries
|
|
|
|
def refresh(self, url, channel_url, max_episodes):
|
|
"""
|
|
Fetch a channel or playlist contents.
|
|
|
|
Doesn't yet fetch video entry informations, so we only get the video id and title.
|
|
"""
|
|
# Duplicate a bit of the YoutubeDL machinery here because we only
|
|
# want to parse the channel/playlist first, not to fetch video entries.
|
|
# We call YoutubeDL.extract_info(process=False), so we
|
|
# have to call extract_info again ourselves when we get a result of type 'url'.
|
|
def extract_type(ie_result):
|
|
result_type = ie_result.get('_type', 'video')
|
|
if result_type not in ('url', 'playlist', 'multi_video'):
|
|
raise Exception('Unsuported result_type: {}'.format(result_type))
|
|
has_playlist = result_type in ('playlist', 'multi_video')
|
|
return result_type, has_playlist
|
|
|
|
opts = {
|
|
'youtube_include_dash_manifest': False, # only interested in video title and id
|
|
}
|
|
opts.update(self._ydl_opts)
|
|
with youtube_dl.YoutubeDL(opts) as ydl:
|
|
ie_result = ydl.extract_info(url, download=False, process=False)
|
|
result_type, has_playlist = extract_type(ie_result)
|
|
while not has_playlist:
|
|
if result_type in ('url', 'url_transparent'):
|
|
ie_result['url'] = sanitize_url(ie_result['url'])
|
|
if result_type == 'url':
|
|
logger.debug("extract_info(%s) to get the video list", ie_result['url'])
|
|
# We have to add extra_info to the results because it may be
|
|
# contained in a playlist
|
|
ie_result = ydl.extract_info(ie_result['url'],
|
|
download=False,
|
|
process=False,
|
|
ie_key=ie_result.get('ie_key'))
|
|
result_type, has_playlist = extract_type(ie_result)
|
|
cover_url = youtube.get_cover(channel_url) # youtube-dl doesn't provide the cover url!
|
|
description = youtube.get_channel_desc(channel_url) # youtube-dl doesn't provide the description!
|
|
return feedcore.Result(feedcore.UPDATED_FEED,
|
|
YoutubeFeed(url, cover_url, description, max_episodes, ie_result, self))
|
|
|
|
def fetch_channel(self, channel, max_episodes=0):
|
|
"""
|
|
called by model.gPodderFetcher to get a custom feed.
|
|
:returns feedcore.Result: a YoutubeFeed or None if channel is not a youtube channel or playlist
|
|
"""
|
|
if not self.my_config.manage_channel:
|
|
return None
|
|
url = None
|
|
m = CHANNEL_RE.match(channel.url)
|
|
if m:
|
|
url = 'https://www.youtube.com/channel/{}/videos'.format(m.group(1))
|
|
else:
|
|
m = USER_RE.match(channel.url)
|
|
if m:
|
|
url = 'https://www.youtube.com/user/{}/videos'.format(m.group(1))
|
|
else:
|
|
m = PLAYLIST_RE.match(channel.url)
|
|
if m:
|
|
url = 'https://www.youtube.com/playlist?list={}'.format(m.group(1))
|
|
if url:
|
|
logger.info('Youtube-dl Handling %s => %s', channel.url, url)
|
|
return self.refresh(url, channel.url, max_episodes)
|
|
return None
|
|
|
|
def custom_downloader(self, unused_config, episode):
|
|
"""
|
|
called from registry.custom_downloader.resolve
|
|
"""
|
|
if not self.force and not self.my_config.manage_downloads:
|
|
return None
|
|
if re.match(r'''https://www.youtube.com/watch\?v=.+''', episode.url):
|
|
return YoutubeCustomDownload(self, episode.url, episode)
|
|
elif re.match(r'''https://www.youtube.com/watch\?v=.+''', episode.link):
|
|
return YoutubeCustomDownload(self, episode.link, episode)
|
|
return None
|
|
|
|
|
|
class gPodderExtension:
|
|
def __init__(self, container):
|
|
self.container = container
|
|
self.ytdl = None
|
|
|
|
def on_load(self):
|
|
self.ytdl = gPodderYoutubeDL(self.container.manager.core.config, self.container.config)
|
|
logger.info('Registering youtube-dl.')
|
|
registry.feed_handler.register(self.ytdl.fetch_channel)
|
|
registry.custom_downloader.register(self.ytdl.custom_downloader)
|
|
|
|
logger.debug('Youtube-DL %s' % youtube_dl.version.__version__)
|
|
|
|
if youtube_dl.utils.version_tuple(youtube_dl.version.__version__) < youtube_dl.utils.version_tuple(want_ytdl_version):
|
|
logger.error(want_ytdl_version_msg
|
|
% {'have_version': youtube_dl.version.__version__, 'want_version': want_ytdl_version})
|
|
|
|
def on_unload(self):
|
|
logger.info('Unregistering youtube-dl.')
|
|
try:
|
|
registry.feed_handler.unregister(self.ytdl.fetch_channel)
|
|
except ValueError:
|
|
pass
|
|
try:
|
|
registry.custom_downloader.unregister(self.ytdl.custom_downloader)
|
|
except ValueError:
|
|
pass
|
|
self.ytdl = None
|
|
|
|
def on_ui_object_available(self, name, ui_object):
|
|
if name == 'gpodder-gtk':
|
|
self.gpodder = ui_object
|
|
|
|
if youtube_dl.utils.version_tuple(youtube_dl.version.__version__) < youtube_dl.utils.version_tuple(want_ytdl_version):
|
|
ui_object.notification(want_ytdl_version_msg %
|
|
{'have_version': youtube_dl.version.__version__, 'want_version': want_ytdl_version},
|
|
_('Old Youtube-DL'), important=True, widget=ui_object.main_window)
|
|
|
|
def on_episodes_context_menu(self, episodes):
|
|
if not self.container.config.manage_downloads \
|
|
and not all(e.was_downloaded(and_exists=True) for e in episodes):
|
|
return [(_("Download with Youtube-DL"), self.download_episodes)]
|
|
|
|
def download_episodes(self, episodes):
|
|
# create a new gPodderYoutubeDL to force using it even if manage_downloads is False
|
|
downloader = gPodderYoutubeDL(self.container.manager.core.config, self.container.config, force=True)
|
|
self.gpodder.download_episode_list(episodes, downloader=downloader)
|