Implement import of externally-downloaded files (bug 902)

This only works for files that are still available in the
feed, and for which the filename can be determined easily.
For files that are supported (e.g. proper feeds and most
YouTube user channels), the import will happen automatically.

Command line users can use the new "gpo importfiles" command.

Other files that cannot be identified will be moved into
the 'Unknown' subfolder, so future versions or external
utilities can look at those files and import them.

bin/gpo

@@ -45,6 +45,8 @@
     pending [URL]              List new episodes (all or only from URL)
     episodes [URL]             List episodes (all or only from URL)
 
+    importfiles                Check download folders for external files
+
   - Other commands -
 
     youtube [URL]              Resolve the YouTube URL to a download URL
@@ -244,6 +246,20 @@ class gPodderCli(object):
         print count, 'episodes pending.'
         return True
 
+    def importfiles(self):
+        full_count = 0
+
+        for podcast in self.client.get_podcasts():
+            print 'Checking for files in', podcast.title
+            count = podcast._podcast.import_external_files()
+            if count:
+                print ' => found %d valid file(s)' % count
+            full_count += count
+
+        print '%d downloaded file(s) imported.' % full_count
+        self.client.finish()
+        return True
+
     def download(self, url=None):
         count = 0
         for podcast in self.client.get_podcasts():

src/gpodder/gui.py

@@ -407,6 +407,12 @@ class gPodder(BuilderWidget, dbus.service.Object):
         # Subscribed channels
         self.active_channel = None
         self.channels = Model.get_podcasts(self.db)
+
+        # Check if the user has downloaded any podcast with an external program
+        # and mark episodes as downloaded / move them away (bug 902)
+        for podcast in self.channels:
+            podcast.import_external_files()
+
         self.channel_list_changed = True
         self.update_podcasts_tab()
 

src/gpodder/model.py

@@ -416,7 +416,7 @@ class PodcastEpisode(PodcastModelObject):
         return current_try
 
     def local_filename(self, create, force_update=False, check_only=False,
-            template=None):
+            template=None, return_wanted_filename=False):
         """Get (and possibly generate) the local saving filename
 
         Pass create=True if you want this function to generate a
@@ -443,6 +443,10 @@ class PodcastEpisode(PodcastModelObject):
         be used as a template for generating the "real" filename.
 
         The generated filename is stored in the database for future access.
+
+        If return_wanted_filename is True, the filename will not be written to
+        the database, but simply returned by this function (for use by the
+        "import external downloads" feature).
         """
         ext = self.extension(may_call_local_filename=False).encode('utf-8', 'ignore')
 
@@ -498,6 +502,10 @@ class PodcastEpisode(PodcastModelObject):
             # Find a unique filename for this episode
             wanted_filename = self.find_unique_file_name(self.url, fn_template, ext)
 
+            if return_wanted_filename:
+                # return the calculated filename without updating the database
+                return wanted_filename
+
             # We populate the filename field the first time - does the old file still exist?
             if self.download_filename is None and os.path.exists(os.path.join(self.channel.save_dir, urldigest+ext)):
                 log('Found pre-0.15.0 downloaded file: %s', urldigest, sender=self)
@@ -682,6 +690,83 @@ class PodcastChannel(PodcastModelObject):
 
     feed_fetcher = gPodderFetcher()
 
+    def import_external_files(self):
+        """Check the download folder for externally-downloaded files
+
+        This will try to assign downloaded files with episodes in the
+        database and (failing that) will move downloaded files into
+        the "Unknown" subfolder in the download directory, so that
+        the user knows that gPodder doesn't know to which episode the
+        file belongs (the "Unknown" folder may be used by external
+        tools or future gPodder versions for better import support).
+        """
+        known_files = set(e.local_filename(create=False) \
+                for e in self.get_downloaded_episodes())
+        existing_files = set(filename for filename in \
+                glob.glob(os.path.join(self.save_dir, '*')))
+        external_files = existing_files.difference(known_files, \
+                [os.path.join(self.save_dir, x) \
+                for x in ('folder.jpg', 'Unknown')])
+        if not external_files:
+            return 0
+
+        all_episodes = self.get_all_episodes()
+
+        count = 0
+        for filename in external_files:
+            found = False
+
+            basename = os.path.basename(filename)
+            existing = self.get_episode_by_filename(basename)
+            if existing:
+                log('Importing external download: %s', filename)
+                existing.on_downloaded(filename)
+                count += 1
+                continue
+
+            for episode in all_episodes:
+                wanted_filename = episode.local_filename(create=True, \
+                        return_wanted_filename=True)
+                if basename == wanted_filename:
+                    log('Importing external download: %s', filename)
+                    episode.download_filename = basename
+                    episode.on_downloaded(filename)
+                    count += 1
+                    found = True
+                    break
+
+                wanted_base, wanted_ext = os.path.splitext(wanted_filename)
+                target_base, target_ext = os.path.splitext(basename)
+                if wanted_base == target_base:
+                    # Filenames only differ by the extension
+                    wanted_type = util.file_type_by_extension(wanted_ext)
+                    target_type = util.file_type_by_extension(target_ext)
+
+                    # If wanted type is None, assume that we don't know
+                    # the right extension before the download (e.g. YouTube)
+                    # if the wanted type is the same as the target type,
+                    # assume that it's the correct file
+                    if wanted_type is None or wanted_type == target_type:
+                        log('Importing external download: %s', filename)
+                        episode.download_filename = basename
+                        episode.on_downloaded(filename)
+                        found = True
+                        count += 1
+                        break
+
+            if not found:
+                log('Unknown external file: %s', filename)
+                target_dir = os.path.join(self.save_dir, 'Unknown')
+                if util.make_directory(target_dir):
+                    target_file = os.path.join(target_dir, basename)
+                    log('Moving %s => %s', filename, target_file)
+                    try:
+                        shutil.move(filename, target_file)
+                    except Exception, e:
+                        log('Could not move file: %s', e, sender=self)
+
+        return count
+
     @classmethod
     def load_from_db(cls, db):
         return db.load_podcasts(factory=cls.create_from_dict)
@@ -704,6 +789,10 @@ class PodcastChannel(PodcastModelObject):
                 tmp.auth_password = authentication_tokens[1]
 
             tmp.update(max_episodes, mimetype_prefs)
+
+            # Mark episodes as downloaded if files already exist (bug 902)
+            tmp.import_external_files()
+
             tmp.save()
             return tmp