Merge remote-tracking branch 'origin/master' into requests

.appveyor.yml

@@ -15,11 +15,21 @@ build_script:
   - set
   - set PATH=C:\msys64\%MSYSTEM%\bin;C:\msys64\usr\bin;%PATH%
   - set CHERE_INVOKING=yes
-  - bash -lc "pacman --noconfirm --ask 20 -Suy"
+  # workaround new msys2 packagers
+  - curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
+  - curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig
+  - bash -lc "pacman-key --verify msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig"
+  - bash -lc "pacman --noconfirm  -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
   # remove precisely conflicting packages
   - bash -lc "pacman --noconfirm --ask 20 --remove mingw-w64-x86_64-gcc-ada mingw-w64-x86_64-gcc-objc mingw-w64-i686-gcc-ada mingw-w64-i686-gcc-objc"
-  # rerun per the "terminate MSYS2 without returning to shell and check for updates again" message
-  - bash -lc "pacman --noconfirm --ask 20 -Suy"
+  # workaround updating msys2-runtime breaks all programs until last one exited
+  - bash -lc "pacman -Syuu --noconfirm"
+  - Powershell.exe "Stop-Process -name dirmngr -Erroraction silentlycontinue; echo killing_dirmng"
+  - Powershell.exe "Stop-Process -name gpg-agent -Erroraction silentlycontinue; echo killing_gpg-agent"
+  - bash -lc "pacman -Syuu --noconfirm"
+  - Powershell.exe "Stop-Process -name dirmngr -Erroraction silentlycontinue; echo killing_dirmng"
+  - Powershell.exe "Stop-Process -name gpg-agent -Erroraction silentlycontinue; echo killing_gpg-agent"
+  # finally run the install process
   - bash -lc "bash .appveyor/msys2.sh"
 
 artifacts:

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
       # important: must be same as mac bundle's python
       - image: python:3.8
     environment:
-      - BUNDLE_TAG: base-5.2.0
+      - BUNDLE_TAG: base-5.2.2
     shell: /bin/bash --login -o pipefail
     steps:
       - checkout

bin/gpo

@@ -721,8 +721,10 @@ class gPodderCli(object):
         return True
 
     def youtube(self, url):
-        fmt_ids = youtube.get_fmt_ids(self._config.youtube)
-        yurl = youtube.get_real_download_url(url, fmt_ids)
+        fmt_ids = youtube.get_fmt_ids(self._config.youtube, False)
+        yurl, duration = youtube.get_real_download_url(url, False, fmt_ids)
+        if duration is not None:
+            episode.total_time = int(int(duration) / 1000)
         print(yurl)
 
         return True

share/gpodder/extensions/youtube-dl.py

@@ -41,6 +41,7 @@ DefaultConfig = {
 
 # youtube feed still preprocessed by youtube.py (compat)
 CHANNEL_RE = re.compile(r'''https://www.youtube.com/feeds/videos.xml\?channel_id=(.+)''')
+USER_RE = re.compile(r'''https://www.youtube.com/feeds/videos.xml\?user=(.+)''')
 PLAYLIST_RE = re.compile(r'''https://www.youtube.com/feeds/videos.xml\?playlist_id=(.+)''')
 
 
@@ -95,20 +96,24 @@ class YoutubeCustomDownload(download.CustomDownload):
         # youtube-dl doesn't return a content-type but an extension
         if 'ext' in res:
             dot_ext = '.{}'.format(res['ext'])
+            # See #673 when merging multiple formats, the extension is appended to the tempname
+            # by YoutubeDL resulting in empty .partial file + .partial.mp4 exists
+            # and #796 .mkv is chosen by ytdl sometimes
+            tempstat = os.stat(tempname)
+            if not tempstat.st_size:
+                for try_ext in (dot_ext, ".mp4", ".m4a", ".webm", ".mkv"):
+                    tempname_with_ext = tempname + try_ext
+                    if os.path.isfile(tempname_with_ext):
+                        logger.debug('Youtubedl downloaded to "%s" instead of "%s", moving',
+                                     os.path.basename(tempname_with_ext),
+                                     os.path.basename(tempname))
+                        os.remove(tempname)
+                        os.rename(tempname_with_ext, tempname)
+                        dot_ext = try_ext
+                        break
             ext_filetype = mimetype_from_extension(dot_ext)
             if ext_filetype:
                 headers['content-type'] = ext_filetype
-            # See #673 when merging multiple formats, the extension is appended to the tempname
-            # by YoutubeDL resulting in empty .partial file + .partial.mp4 exists
-            tempstat = os.stat(tempname)
-            if not tempstat.st_size:
-                tempname_with_ext = tempname + dot_ext
-                if os.path.isfile(tempname_with_ext):
-                    logger.debug('Youtubedl downloaded to %s instead of %s, moving',
-                                 os.path.basename(tempname),
-                                 os.path.basename(tempname_with_ext))
-                    os.remove(tempname)
-                    os.rename(tempname_with_ext, tempname)
         return headers, res.get('url', self._url)
 
     def _my_hook(self, d):
@@ -147,7 +152,7 @@ class YoutubeFeed(model.Feed):
         filtered_entries = []
         seen_guids = set()
         for i, e in enumerate(entries):  # consumes the generator!
-            if e.get('_type', 'video') == 'url' and e.get('ie_key') == 'Youtube':
+            if e.get('_type', 'video') in ('url', 'url_transparent') and e.get('ie_key') == 'Youtube':
                 guid = video_guid(e['id'])
                 e['guid'] = guid
                 if guid in seen_guids:
@@ -295,7 +300,7 @@ class gPodderYoutubeDL(download.CustomDownloader):
         #
         # See https://github.com/ytdl-org/youtube-dl#format-selection for details
         # about youtube-dl format specification.
-        fmt_ids = youtube.get_fmt_ids(gpodder_config.youtube)
+        fmt_ids = youtube.get_fmt_ids(gpodder_config.youtube, False)
         opts['format'] = '/'.join(str(fmt) for fmt in fmt_ids)
         if fallback:
             opts['format'] += '/' + fallback
@@ -389,11 +394,15 @@ class gPodderYoutubeDL(download.CustomDownloader):
         url = None
         m = CHANNEL_RE.match(channel.url)
         if m:
-            url = 'https://www.youtube.com/channel/{}'.format(m.group(1))
+            url = 'https://www.youtube.com/channel/{}/videos'.format(m.group(1))
         else:
-            m = PLAYLIST_RE.match(channel.url)
+            m = USER_RE.match(channel.url)
             if m:
-                url = 'https://www.youtube.com/playlist?list={}'.format(m.group(1))
+                url = 'https://www.youtube.com/user/{}/videos'.format(m.group(1))
+            else:
+                m = PLAYLIST_RE.match(channel.url)
+                if m:
+                    url = 'https://www.youtube.com/playlist?list={}'.format(m.group(1))
         if url:
             logger.info('Youtube-dl Handling %s => %s', channel.url, url)
             return self.refresh(url, channel.url, max_episodes)

share/gpodder/ui/gtk/gpodderpreferences.ui

@@ -184,6 +184,7 @@
                         <property name="label" translatable="yes">Preferred YouTube format:</property>
                       </object>
                       <packing>
+                        <property name="left_attach">0</property>
                         <property name="top_attach">0</property>
                       </packing>
                     </child>
@@ -199,6 +200,30 @@
                         <property name="top_attach">0</property>
                       </packing>
                     </child>
+                    <child>
+                      <object class="GtkLabel" id="label_preferred_youtube_hls_format">
+                        <property name="visible">True</property>
+                        <property name="can_focus">False</property>
+                        <property name="xalign">0</property>
+                        <property name="label" translatable="yes">Preferred YouTube HLS format:</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">0</property>
+                        <property name="top_attach">2</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkComboBox" id="combobox_preferred_youtube_hls_format">
+                        <property name="visible">True</property>
+                        <property name="can_focus">False</property>
+                        <property name="hexpand">True</property>
+                        <signal name="changed" handler="on_combobox_preferred_youtube_hls_format_changed" swapped="no"/>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="top_attach">2</property>
+                      </packing>
+                    </child>
                     <child>
                       <object class="GtkLabel" id="label_preferred_vimeo_format">
                         <property name="visible">True</property>
@@ -207,8 +232,8 @@
                         <property name="label" translatable="yes">Preferred Vimeo format:</property>
                       </object>
                       <packing>
-                        <property name="top_attach">2</property>
                         <property name="left_attach">0</property>
+                        <property name="top_attach">4</property>
                       </packing>
                     </child>
                     <child>
@@ -219,7 +244,7 @@
                       </object>
                       <packing>
                         <property name="left_attach">1</property>
-                        <property name="top_attach">2</property>
+                        <property name="top_attach">4</property>
                       </packing>
                     </child>
                   </object>

share/man/man1/gpo.1

@@ -1,4 +1,4 @@
-.TH GPO "1" "June 2020" "gpodder 3.10.16" "User Commands"
+.TH GPO "1" "November 2020" "gpodder 3.10.17" "User Commands"
 .SH NAME
 gpo \- Text mode interface of gPodder
 .SH SYNOPSIS

share/man/man1/gpodder.1

@@ -1,5 +1,5 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.15.
-.TH GPODDER "1" "June 2020" "gpodder 3.10.16" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.16.
+.TH GPODDER "1" "November 2020" "gpodder 3.10.17" "User Commands"
 .SH NAME
 gpodder \- Media aggregator and podcast client
 .SH SYNOPSIS

src/gpodder/__init__.py

@@ -20,8 +20,8 @@
 # This metadata block gets parsed by setup.py - use single quotes only
 __tagline__ = 'Media aggregator and podcast client'
 __author__ = 'Thomas Perl <thp@gpodder.org>'
-__version__ = '3.10.16'
-__date__ = '2020-06-21'
+__version__ = '3.10.17'
+__date__ = '2020-11-23'
 __copyright__ = '© 2005-2020 The gPodder Team'
 __license__ = 'GNU General Public License, version 3 or later'
 __url__ = 'http://gpodder.org/'

src/gpodder/config.py

@@ -203,6 +203,8 @@ defaults = {
     'youtube': {
         'preferred_fmt_id': 18,  # default fmt_id (see fallbacks in youtube.py)
         'preferred_fmt_ids': [],  # for advanced uses (custom fallback sequence)
+        'preferred_hls_fmt_id': 93,  # default fmt_id (see fallbacks in youtube.py)
+        'preferred_hls_fmt_ids': [],  # for advanced uses (custom fallback sequence)
     },
 
     'vimeo': {

src/gpodder/download.py

@@ -355,7 +355,7 @@ class DefaultDownloader(CustomDownloader):
     def custom_downloader(config, episode):
         url = episode.url
         # Resolve URL and start downloading the episode
-        res = registry.download_url.resolve(config, None, episode)
+        res = registry.download_url.resolve(config, None, episode, False)
         if res:
             url = res
         if url == episode.url:
@@ -796,7 +796,8 @@ class DownloadTask(object):
                 logger.info('Updating mime type: %s => %s', old_mimetype, new_mimetype)
                 old_extension = self.__episode.extension()
                 self.__episode.mime_type = new_mimetype
-                new_extension = self.__episode.extension()
+                # don't call local_filename because we'll get the old download name
+                new_extension = self.__episode.extension(may_call_local_filename=False)
 
                 # If the desired filename extension changed due to the new
                 # mimetype, we force an update of the local filename to fix the

src/gpodder/feedcore.py

@@ -206,6 +206,7 @@ class Fetcher(object):
             new_url = self._resolve_url(url)
             if new_url and new_url != url:
                 return Result(NEW_LOCATION, new_url)
+
         # xml documents specify the encoding inline so better pass encoded body.
         # Especially since requests will use ISO-8859-1 for content-type 'text/xml'
         # if the server doesn't specify a charset.

src/gpodder/gtkui/desktop/preferences.py

@@ -132,6 +132,32 @@ class YouTubeVideoFormatListModel(Gtk.ListStore):
         self._config.youtube.preferred_fmt_id = self[index][self.C_ID]
 
 
+class YouTubeVideoHLSFormatListModel(Gtk.ListStore):
+    C_CAPTION, C_ID = list(range(2))
+
+    def __init__(self, config):
+        Gtk.ListStore.__init__(self, str, int)
+        self._config = config
+
+        if self._config.youtube.preferred_hls_fmt_ids:
+            caption = _('Custom (%(format_ids)s)') % {
+                'format_ids': ', '.join(str(x) for x in self._config.youtube.preferred_hls_fmt_ids),
+            }
+            self.append((caption, 0))
+
+        for id, (fmt_id, path, description) in youtube.hls_formats:
+            self.append((description, id))
+
+    def get_index(self):
+        for index, row in enumerate(self):
+            if self._config.youtube.preferred_hls_fmt_id == row[self.C_ID]:
+                return index
+        return 0
+
+    def set_index(self, index):
+        self._config.youtube.preferred_hls_fmt_id = self[index][self.C_ID]
+
+
 class VimeoVideoFormatListModel(Gtk.ListStore):
     C_CAPTION, C_ID = list(range(2))
 
@@ -185,6 +211,13 @@ class gPodderPreferences(BuilderWidget):
         self.combobox_preferred_youtube_format.add_attribute(cellrenderer, 'text', self.preferred_youtube_format_model.C_CAPTION)
         self.combobox_preferred_youtube_format.set_active(self.preferred_youtube_format_model.get_index())
 
+        self.preferred_youtube_hls_format_model = YouTubeVideoHLSFormatListModel(self._config)
+        self.combobox_preferred_youtube_hls_format.set_model(self.preferred_youtube_hls_format_model)
+        cellrenderer = Gtk.CellRendererText()
+        self.combobox_preferred_youtube_hls_format.pack_start(cellrenderer, True)
+        self.combobox_preferred_youtube_hls_format.add_attribute(cellrenderer, 'text', self.preferred_youtube_hls_format_model.C_CAPTION)
+        self.combobox_preferred_youtube_hls_format.set_active(self.preferred_youtube_hls_format_model.get_index())
+
         self.preferred_vimeo_format_model = VimeoVideoFormatListModel(self._config)
         self.combobox_preferred_vimeo_format.set_model(self.preferred_vimeo_format_model)
         cellrenderer = Gtk.CellRendererText()
@@ -461,6 +494,10 @@ class gPodderPreferences(BuilderWidget):
         index = self.combobox_preferred_youtube_format.get_active()
         self.preferred_youtube_format_model.set_index(index)
 
+    def on_combobox_preferred_youtube_hls_format_changed(self, widget):
+        index = self.combobox_preferred_youtube_hls_format.get_active()
+        self.preferred_youtube_hls_format_model.set_index(index)
+
     def on_combobox_preferred_vimeo_format_changed(self, widget):
         index = self.combobox_preferred_vimeo_format.get_active()
         self.preferred_vimeo_format_model.set_index(index)

src/gpodder/gtkui/interface/common.py

@@ -296,7 +296,6 @@ class TreeViewHelper(object):
         see http://lazka.github.io/pgi-docs/#Gtk-3.0/classes/Menu.html#Gtk.Menu.popup
         """
         def position_func(menu, *unused_args):
-            print("POSITION(%r)" % (widget.get_bin_window().get_origin(),))
             _, x, y = widget.get_bin_window().get_origin()
 
             # If there's a selection, place the popup menu on top of

src/gpodder/gtkui/main.py

@@ -88,6 +88,9 @@ class gPodder(BuilderWidget, dbus.service.Object):
         self._search_episodes = None
         BuilderWidget.__init__(self, None, _builder_expose={'app': app})
 
+        self.last_episode_date_refresh = None
+        self.refresh_episode_dates()
+
     def new(self):
         if self.application.want_headerbar:
             self.header_bar = Gtk.HeaderBar()
@@ -2015,23 +2018,35 @@ class gPodder(BuilderWidget, dbus.service.Object):
         self.update_podcast_list_model(set(e.channel.url for e in episodes))
         self.db.commit()
 
-    def streaming_possible(self):
-        # User has to have a media player set on the Desktop, or else we
-        # would probably open the browser when giving a URL to xdg-open..
-        return (self.config.player and self.config.player != 'default')
+    def episode_player(self, episode):
+        file_type = episode.file_type()
+        if file_type == 'video' and self.config.videoplayer and \
+                self.config.videoplayer != 'default':
+            player = self.config.videoplayer
+        elif file_type == 'audio' and self.config.player and \
+                self.config.player != 'default':
+            player = self.config.player
+        else:
+            player = 'default'
+        return player
+
+    def streaming_possible(self, episode=None):
+        """
+        Don't try streaming if the user has not defined a player
+        or else we would probably open the browser when giving a URL to xdg-open.
+        If an episode is given, we look at the audio or video player depending on its file type.
+        :return bool: if streaming is possible
+        """
+        if episode:
+            player = self.episode_player(episode)
+        else:
+            player = self.config.player
+        return player and player != 'default'
 
     def playback_episodes_for_real(self, episodes):
         groups = collections.defaultdict(list)
         for episode in episodes:
-            file_type = episode.file_type()
-            if file_type == 'video' and self.config.videoplayer and \
-                    self.config.videoplayer != 'default':
-                player = self.config.videoplayer
-            elif file_type == 'audio' and self.config.player and \
-                    self.config.player != 'default':
-                player = self.config.player
-            else:
-                player = 'default'
+            player = self.episode_player(episode)
 
             # Mark episode as played in the database
             episode.playback_mark()
@@ -2103,7 +2118,7 @@ class gPodder(BuilderWidget, dbus.service.Object):
     def playback_episodes(self, episodes):
         # We need to create a list, because we run through it more than once
         episodes = list(Model.sort_episodes_by_pubdate(e for e in episodes if
-               e.was_downloaded(and_exists=True) or self.streaming_possible()))
+               e.was_downloaded(and_exists=True) or self.streaming_possible(e)))
 
         try:
             self.playback_episodes_for_real(episodes)
@@ -2129,6 +2144,7 @@ class gPodder(BuilderWidget, dbus.service.Object):
         selection = self.treeAvailable.get_selection()
         if selection.count_selected_rows() > 0:
             (model, paths) = selection.get_selected_rows()
+            streaming_possible = self.streaming_possible()
 
             for path in paths:
                 try:
@@ -2153,10 +2169,11 @@ class gPodder(BuilderWidget, dbus.service.Object):
                     if episode.downloading:
                         can_cancel = True
                     else:
+                        streaming_possible |= self.streaming_possible(episode)
                         can_download = True
 
             can_download = can_download and not can_cancel
-            can_play = self.streaming_possible() or (can_play and not can_cancel and not can_download)
+            can_play = streaming_possible or (can_play and not can_cancel and not can_download)
             can_delete = not can_cancel
 
         if open_instead_of_play:
@@ -2187,6 +2204,22 @@ class gPodder(BuilderWidget, dbus.service.Object):
         self.update_podcast_list_model()
         self.update_episode_list_icons(urls)
 
+    def refresh_episode_dates(self):
+        t = time.localtime()
+        current_day = t[:3]
+        if self.last_episode_date_refresh is not None and self.last_episode_date_refresh != current_day:
+            # update all episodes in current view
+            for row in self.episode_list_model:
+                row[EpisodeListModel.C_PUBLISHED_TEXT] = row[EpisodeListModel.C_EPISODE].cute_pubdate()
+
+        self.last_episode_date_refresh = current_day
+
+        remaining_seconds = 86400 - 3600 * t.tm_hour - 60 * t.tm_min - t.tm_sec
+        if remaining_seconds > 3600:
+            # timeout an hour early in the event daylight savings changes the clock forward
+            remaining_seconds = remaining_seconds - 3600
+        GObject.timeout_add(remaining_seconds * 1000, self.refresh_episode_dates)
+
     def update_podcast_list_model(self, urls=None, selected=False, select_url=None,
             sections_changed=False):
         """Update the podcast list treeview model
@@ -2631,7 +2664,10 @@ class gPodder(BuilderWidget, dbus.service.Object):
                 util.idle_add(update_progress, channel)
 
             if nr_update_errors > 0:
-                self.notification(_('%d channel(s) failed to update') % nr_update_errors,
+                self.notification(
+                    N_('%(count)d channel failed to update',
+                       '%(count)d channels failed to update',
+                       nr_update_errors) % {'count': nr_update_errors},
                     _('Error while updating feeds'), widget=self.treeChannels)
 
             def update_feed_cache_finish_callback():

src/gpodder/model.py

@@ -256,10 +256,11 @@ class PodcastEpisode(PodcastModelObject):
     # In theory, Linux can have 255 bytes (not characters!) in a filename, but
     # filesystems like eCryptFS store metadata in the filename, making the
     # effective number of characters less than that. eCryptFS recommends
-    # 140 chars, we use 120 here (140 - len(extension) - len(".partial")).
+    # 140 chars, we use 120 here (140 - len(extension) - len(".partial.webm"))
+    # (youtube-dl appends an extension after .partial, ".webm" is the longest).
     # References: gPodder bug 1898, http://unix.stackexchange.com/a/32834
     MAX_FILENAME_LENGTH = 120  # without extension
-    MAX_FILENAME_WITH_EXT_LENGTH = 140 - len(".partial")  # with extension
+    MAX_FILENAME_WITH_EXT_LENGTH = 140 - len(".partial.webm")  # with extension
 
     __slots__ = schema.EpisodeColumns
 
@@ -514,7 +515,7 @@ class PodcastEpisode(PodcastModelObject):
 
         if url is None or not os.path.exists(url):
             # FIXME: may custom downloaders provide the real url ?
-            url = registry.download_url.resolve(config, self.url, self)
+            url = registry.download_url.resolve(config, self.url, self, allow_partial)
         return url
 
     def find_unique_file_name(self, filename, extension):

src/gpodder/vimeo.py

@@ -50,7 +50,7 @@ class VimeoError(BaseException): pass
 
 
 @registry.download_url.register
-def vimeo_real_download_url(config, episode):
+def vimeo_real_download_url(config, episode, allow_partial):
     fmt = config.vimeo.fileformat if config else None
     res = get_real_download_url(episode.url, preferred_fileformat=fmt)
     return None if res == episode.url else res

src/gpodder/youtube.py

@@ -39,7 +39,7 @@ _ = gpodder.gettext
 
 
 # http://en.wikipedia.org/wiki/YouTube#Quality_and_formats
-# https://gist.github.com/Marco01809/34d47c65b1d28829bb17c24c04a0096f
+# https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L447
 
 # adaptive audio formats
 #   140  MP4   128k
@@ -130,6 +130,23 @@ formats = [
 ]
 formats_dict = dict(formats)
 
+# streaming formats and fallbacks to lower quality
+hls_144 = [91]
+hls_240 = [92] + hls_144
+hls_360 = [93] + hls_240
+hls_480 = [94] + hls_360
+hls_720 = [95] + hls_480
+hls_1080 = [96] + hls_720
+hls_formats = [
+    (96, (hls_1080, '9/1920x1080/9/0/115', 'MP4 1080p (1920x1080)')),   # N/A,       256 kbps
+    (95, (hls_720, '9/1280x720/9/0/115', 'MP4 720p (1280x720)')),       # N/A,       256 kbps
+    (94, (hls_480, '9/854x480/9/0/115', 'MP4 480p (854x480)')),         # N/A,       128 kbps
+    (93, (hls_360, '9/640x360/9/0/115', 'MP4 360p (640x360)')),         # N/A,       128 kbps
+    (92, (hls_240, '9/426x240/9/0/115', 'MP4 240p (426x240)')),         # N/A,        48 kbps
+    (91, (hls_144, '9/256x144/9/0/115', 'MP4 144p (256x144)')),         # N/A,        48 kbps
+]
+hls_formats_dict = dict(hls_formats)
+
 V3_API_ENDPOINT = 'https://www.googleapis.com/youtube/v3'
 CHANNEL_VIDEOS_XML = 'https://www.youtube.com/feeds/videos.xml'
 
@@ -138,27 +155,39 @@ class YouTubeError(Exception):
     pass
 
 
-def get_fmt_ids(youtube_config):
+def get_fmt_ids(youtube_config, allow_partial):
+    if allow_partial:
+        if youtube_config.preferred_hls_fmt_id == 0:
+            hls_fmt_ids = (youtube_config.preferred_hls_fmt_ids if youtube_config.preferred_hls_fmt_ids else [])
+        else:
+            format = hls_formats_dict.get(youtube_config.preferred_hls_fmt_id)
+            if format is None:
+                hls_fmt_ids = []
+            else:
+                hls_fmt_ids, path, description = format
+    else:
+        hls_fmt_ids = []
+
     if youtube_config.preferred_fmt_id == 0:
-        return (youtube_config.preferred_fmt_ids if youtube_config.preferred_fmt_ids else [])
+        return (youtube_config.preferred_fmt_ids + hls_fmt_ids if youtube_config.preferred_fmt_ids else hls_fmt_ids)
 
     format = formats_dict.get(youtube_config.preferred_fmt_id)
     if format is None:
-        return []
+        return hls_fmt_ids
     fmt_ids, path, description = format
-    return fmt_ids
+    return fmt_ids + hls_fmt_ids
 
 
 @registry.download_url.register
-def youtube_real_download_url(config, episode):
-    fmt_ids = get_fmt_ids(config.youtube) if config else None
-    res, duration = get_real_download_url(episode.url, fmt_ids)
+def youtube_real_download_url(config, episode, allow_partial):
+    fmt_ids = get_fmt_ids(config.youtube, allow_partial) if config else None
+    res, duration = get_real_download_url(episode.url, allow_partial, fmt_ids)
     if duration is not None:
         episode.total_time = int(int(duration) / 1000)
     return None if res == episode.url else res
 
 
-def get_real_download_url(url, preferred_fmt_ids=None):
+def get_real_download_url(url, allow_partial, preferred_fmt_ids=None):
     if not preferred_fmt_ids:
         preferred_fmt_ids, _, _ = formats_dict[22]  # MP4 720p
 
@@ -199,6 +228,25 @@ def get_real_download_url(url, preferred_fmt_ids=None):
                         and not playabilityStatus['liveStreamability'].get('liveStreamabilityRenderer', {}).get('displayEndscreen', False):
                     # playabilityStatus.liveStreamability -- video is or was a live stream
                     # playabilityStatus.liveStreamability.liveStreamabilityRenderer.displayEndscreen -- video has ended if present
+
+                    if allow_partial and 'streamingData' in player_response and 'hlsManifestUrl' in player_response['streamingData']:
+                        manifest = None
+                        url = player_response['streamingData']['hlsManifestUrl']
+                        while manifest is None:
+                            req = util.http_request(url, method='GET')
+                            if 'location' in req.msg:
+                                url = req.msg['location']
+                            else:
+                                manifest = req.read()
+                        manifest = manifest.decode().splitlines()
+
+                        urls = [line for line in manifest if line[0] != '#']
+                        itag_re = re.compile('/itag/([0-9]+)/')
+                        for url in urls:
+                            itag = itag_re.search(url).group(1)
+                            yield int(itag), [url, None]
+                        return
+
                     error_message = 'live stream'
                 elif 'streamingData' in player_response:
                     # DRM videos store url inside a cipher key - not supported
@@ -225,7 +273,7 @@ def get_real_download_url(url, preferred_fmt_ids=None):
         fmt_id_url_map = sorted(find_urls(page), reverse=True)
 
         if not fmt_id_url_map:
-            drm = re.search('%22cipher%22%3A', page)
+            drm = re.search('%22(cipher|signatureCipher)%22%3A', page)
             if drm is not None:
                 raise YouTubeError('Unsupported DRM content found for video ID "%s"' % vid)
             raise YouTubeError('No formats found for video ID "%s"' % vid)
@@ -239,7 +287,7 @@ def get_real_download_url(url, preferred_fmt_ids=None):
                 continue
             id = int(id)
             if id in formats_available:
-                format = formats_dict.get(id)
+                format = formats_dict.get(id) or hls_formats_dict.get(id)
                 if format is not None:
                     _, _, description = format
                 else:
@@ -438,7 +486,7 @@ def parse_youtube_url(url):
 
         if 'list=' in query:
             playlist_query = [query_value for query_value in query.split("&") if 'list=' in query_value][0]
-            playlist_id = playlist_query.strip("list=")
+            playlist_id = playlist_query[5:]
             query = 'playlist_id={playlist_id}'.format(playlist_id=playlist_id)
 
         path = '/feeds/videos.xml'

tools/win_installer/_base.sh

@@ -84,16 +84,15 @@ function extract_installer {
 }
 
 PIP_REQUIREMENTS="\
-podcastparser==0.6.5
+podcastparser==0.6.6
 mygpoclient==1.8
 git+https://github.com/enthought/pywin32-ctypes.git@f27d6a0
-html5lib==1.0.1
+html5lib==1.1
 webencodings==0.5.1
-six==1.12.0
-certifi==2020.6.20
-mutagen==1.44.0
-youtube_dl==2020.6.16.1
-requests==2.24.0
+certifi==2020.11.8
+mutagen==1.45.1
+youtube_dl==2020.11.21.1
+requests==2.25.0
 PySocks==1.7.1
 "
 
@@ -109,6 +108,7 @@ function install_deps {
         mingw-w64-"${ARCH}"-python3-gobject \
         mingw-w64-"${ARCH}"-python3-cairo \
         mingw-w64-"${ARCH}"-python3-pip \
+        mingw-w64-"${ARCH}"-python-six \
 		mingw-w64-"${ARCH}"-make
 
     build_pacman -S --noconfirm mingw-w64-"${ARCH}"-python3-setuptools