User-agent header for all HTTP requests (bug 626)

This includes shownotes images (from the HTML widget),
OPML downloads, YouTube resolving and my.gpodder.org.

src/gpodder/gtkui/desktop/shownotes.py

@@ -20,7 +20,6 @@
 import gtk
 import gtk.gdk
 import pango
-import urllib2
 import threading
 
 from xml.sax import saxutils
@@ -96,7 +95,7 @@ class gPodderShownotes(gPodderShownotesBase):
             self.d.connect('link-clicked', lambda doc, url: util.open_website(url))
             def request_url(document, url, stream):
                 def opendata(url, stream):
-                    fp = urllib2.urlopen(url)
+                    fp = util.urlopen(url)
                     data = fp.read(1024*10)
                     while data != '':
                         stream.write(data)

src/gpodder/gtkui/services.py

@@ -35,7 +35,6 @@ from gpodder import youtube
 import gtk
 import os
 import threading
-import urllib2
 
 class DependencyModel(gtk.ListStore):
     C_NAME, C_DESCRIPTION, C_AVAILABLE_TEXT, C_AVAILABLE, C_MISSING = range(5)
@@ -155,7 +154,7 @@ class CoverDownloader(ObservableService):
                 try:
                     log('Trying to download: %s', url, sender=self)
 
-                    image_data = urllib2.urlopen(url).read()
+                    image_data = util.urlopen(url).read()
                 except:
                     log('Cannot get image from %s', url, sender=self)
 

src/gpodder/my.py

@@ -27,6 +27,8 @@
 import gpodder
 _ = gpodder.gettext
 
+from gpodder import util
+
 ########################################################################
 # Based on upload_test.py
 # Copyright Michael Foord, 2004 & 2005.
@@ -80,6 +82,7 @@ def build_request(theurl, fields, files, txheaders=None):
     if not txheaders: txheaders = {}
     txheaders['Content-type'] = content_type
     txheaders['Content-length'] = str(len(body))
+    txheaders['User-agent'] = gpodder.user_agent
     return urllib2.Request(theurl, body, txheaders)
 
 
@@ -94,7 +97,7 @@ class MygPodderClient(object):
         args = {'username': self.username, 'password': self.password}
         args = '&'.join(('%s=%s' % a for a in args.items()))
         url = theurl + '?' + args
-        opml_data = urllib2.urlopen(url).read()
+        opml_data = util.urlopen(url).read()
         return opml_data
 
     def upload_subscriptions(self, filename):

src/gpodder/opml.py

@@ -63,10 +63,6 @@ class Importer(object):
 
     VALID_TYPES = ( 'rss', 'link' )
 
-    def read_url( self, url):
-        request = urllib2.Request( url, headers = {'User-agent': gpodder.user_agent})
-        return urllib2.urlopen( request).read()
-
     def __init__( self, url):
         """
         Parses the OPML feed from the given URL into 
@@ -77,7 +73,7 @@ class Importer(object):
             if os.path.exists(url):
                 doc = xml.dom.minidom.parse(url)
             else:
-                doc = xml.dom.minidom.parseString(self.read_url(url))
+                doc = xml.dom.minidom.parseString(util.urlopen(url).read())
 
             for outline in doc.getElementsByTagName('outline'):
                 if outline.getAttribute('type') in self.VALID_TYPES and outline.getAttribute('xmlUrl') or outline.getAttribute('url'):

src/gpodder/util.py

@@ -825,11 +825,18 @@ def get_real_url(url):
             opener = urllib2.build_opener(handler)
             return opener.open(url).geturl()
         else:
-            return urllib2.urlopen(url).geturl()
+            return urlopen(url).geturl()
     except:
         log('Error getting real url for %s', url, traceback=True)
         return url
 
+def urlopen(url):
+    """
+    An URL opener with the User-agent set to gPodder (with version)
+    """
+    headers = {'User-agent': gpodder.user_agent}
+    request = urllib2.Request(url, headers=headers)
+    return urllib2.urlopen(request)
 
 def find_command( command):
     """

src/gpodder/youtube.py

@@ -127,7 +127,7 @@ def get_real_cover(url):
     if m is not None:
         username = m.group(1)
         api_url = 'http://gdata.youtube.com/feeds/api/users/%s?v=2' % username
-        data = urllib2.urlopen(api_url).read()
+        data = util.urlopen(api_url).read()
         match = re.search('<media:thumbnail url=[\'"]([^\'"]+)[\'"]/>', data)
         if match is not None:
             log('YouTube userpic for %s is: %s', url, match.group(1))
@@ -140,7 +140,7 @@ def get_real_episode_length(episode):
 
     if url != episode.url:
         try:
-            info = urllib2.urlopen(url).info()
+            info = util.urlopen(url).info()
             if 'content-length' in info:
                 return info['content-length']
         except urllib2.HTTPError:
@@ -154,7 +154,7 @@ def find_youtube_channels(string):
     url = 'http://www.youtube.com/results?search_query='+ urllib.quote(string, '') +'&search_type=search_users&aq=f'
 
     r = re.compile('>\s+<')
-    data = r.sub('><', urllib2.urlopen(url).read())
+    data = r.sub('><', util.urlopen(url).read())
 
     r1 = re.compile('<a href="/user/([^"]+)"[^>]*>([^<]+)</a>')
     m1 = r1.findall(data)