maryjane
/
gpodder
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Use slots and keep model instances in memory

This commit is contained in:
Thomas Perl 2011-07-16 17:26:04 +02:00
parent dde5272d9d
commit 7ddb2eb354
5 changed files with 145 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
src/gpodder/dbsqlite.py
View File

@ -44,44 +44,8 @@ import re
class Database(object):
UNICODE_TRANSLATE = {ord(u'ö'): u'o', ord(u'ä'): u'a', ord(u'ü'): u'u'}
# Column names, types, required and default values for the podcasts table
TABLE_PODCAST = 'podcast'
COLUMNS_PODCAST = (
'title',
'url',
'link',
'description',
'cover_url',
'auth_username',
'auth_password',
'http_last_modified',
'http_etag',
'auto_archive_episodes',
'download_folder',
'pause_subscription',
)
# Column names and types for the episodes table
TABLE_EPISODE = 'episode'
COLUMNS_EPISODE = (
'podcast_id',
'title',
'description',
'url',
'published',
'guid',
'link',
'file_size',
'mime_type',
'state',
'is_new',
'archive',
'download_filename',
'total_time',
'current_position',
'current_position_updated',
'last_playback',
)
def __init__(self, filename):
self.database_file = filename
@ -207,35 +171,35 @@ class Database(object):
return (total, deleted, new, downloaded, unplayed)
def load_podcasts(self, factory, cache_lookup):
"""
Returns podcast descriptions as a list of dictionaries or objects,
returned by the factory() function, which receives the dictionary
as the only argument.
def load_podcasts(self, factory):
logger.info('Loading podcasts')
The cache_lookup function takes a podcast ID and should return the
podcast object in case it is cached already in memory.
"""
logger.debug('load_podcasts')
sql = 'SELECT * FROM %s ORDER BY title COLLATE UNICODE' % self.TABLE_PODCAST
with self.lock:
cur = self.cursor()
cur.execute('SELECT * FROM %s ORDER BY title COLLATE UNICODE' % self.TABLE_PODCAST)
cur.execute(sql)
result = []
keys = [desc[0] for desc in cur.description]
id_index = keys.index('id')
def make_podcast(row):
o = cache_lookup(row[id_index])
if o is None:
o = factory(dict(zip(keys, row)), self)
# TODO: save in cache!
else:
logger.debug('Cache hit: podcast %d', o.id)
return o
result = map(lambda row: factory(dict(zip(keys, row)), self), cur)
cur.close()
result = map(make_podcast, cur)
return result
def load_episodes(self, podcast, factory):
assert podcast.id
logger.info('Loading episodes for podcast %d', podcast.id)
sql = 'SELECT * FROM %s WHERE podcast_id = ? ORDER BY published DESC' % self.TABLE_EPISODE
args = (podcast.id,)
with self.lock:
cur = self.cursor()
cur.execute(sql, args)
keys = [desc[0] for desc in cur.description]
result = map(lambda row: factory(dict(zip(keys, row))), cur)
cur.close()
return result
@ -251,49 +215,13 @@ class Database(object):
cur.execute("DELETE FROM %s WHERE podcast_id = ?" % self.TABLE_EPISODE, (podcast.id, ))
cur.close()
# Commit changes
self.db.commit()
# TODO: podcast.id -> remove from cache!
def load_episodes(self, podcast, factory, cache_lookup):
assert podcast.id
limit = 1000
logger.info('Loading episodes for podcast %d', podcast.id)
sql = 'SELECT * FROM %s WHERE podcast_id = ? ORDER BY published DESC LIMIT ?' % (self.TABLE_EPISODE,)
args = (podcast.id, limit)
with self.lock:
cur = self.cursor()
cur.execute(sql, args)
keys = [desc[0] for desc in cur.description]
id_index = keys.index('id')
def make_episode(row):
o = cache_lookup(row[id_index])
if o is None:
o = factory(dict(zip(keys, row)))
# TODO: save in cache!
else:
logger.debug('Cache hit: episode %d', o.id)
return o
result = map(make_episode, cur)
cur.close()
return result
def get_podcast_id_from_episode_url(self, url):
"""Return the (first) associated podcast ID given an episode URL"""
assert url
return self.get('SELECT podcast_id FROM %s WHERE url = ? LIMIT 1' % (self.TABLE_EPISODE,), (url,))
def save_podcast(self, podcast):
self._save_object(podcast, self.TABLE_PODCAST, self.COLUMNS_PODCAST)
self._save_object(podcast, self.TABLE_PODCAST, schema.PodcastColumns)
def save_episode(self, episode):
assert episode.podcast_id
assert episode.guid
self._save_object(episode, self.TABLE_EPISODE, self.COLUMNS_EPISODE)
self._save_object(episode, self.TABLE_EPISODE, schema.EpisodeColumns)
def _save_object(self, o, table, columns):
with self.lock:
@ -315,7 +243,6 @@ class Database(object):
logger.error('Cannot save %s: %s', o, e, exc_info=True)
cur.close()
# TODO: o -> into cache!
def get(self, sql, params=None):
"""
@ -337,6 +264,11 @@ class Database(object):
else:
return row[0]
def get_podcast_id_from_episode_url(self, url):
"""Return the (first) associated podcast ID given an episode URL"""
assert url
return self.get('SELECT podcast_id FROM %s WHERE url = ? LIMIT 1' % (self.TABLE_EPISODE,), (url,))
def podcast_download_folder_exists(self, foldername):
"""
Returns True if a foldername for a channel exists.
@ -367,5 +299,4 @@ class Database(object):
cur = self.cursor()
cur.execute('DELETE FROM %s WHERE podcast_id = ? AND guid = ?' % self.TABLE_EPISODE, \
(podcast_id, guid))
# TODO: Delete episode from cache

3
src/gpodder/gtkui/interface/shownotes.py
View File

@ -154,6 +154,9 @@ class gPodderShownotesBase(BuilderWidget):
def show(self, episode):
if self.main_window.get_property('visible'):
if episode == self.episode:
return
self.episode = None
self.task = None
self.on_hide_window()

15
src/gpodder/gtkui/model.py
View File

@ -50,6 +50,8 @@ except ImportError:
# ----------------------------------------------------------
class GEpisode(model.PodcastEpisode):
__slots__ = ()
@property
def title_markup(self):
return '%s\n<small>%s</small>' % (cgi.escape(self.title),
@ -88,6 +90,8 @@ class GEpisode(model.PodcastEpisode):
cgi.escape(self.channel.title))
class GPodcast(model.PodcastChannel):
__slots__ = ()
EpisodeClass = GEpisode
class Model(model.Model):
@ -413,7 +417,7 @@ class PodcastChannelProxy(object):
self.channels = channels
self.title = _('All episodes')
self.description = _('from all podcasts')
self.parse_error = ''
#self.parse_error = ''
self.url = ''
self.id = None
self.cover_file = os.path.join(gpodder.images_folder, 'podcast-all.png')
@ -624,10 +628,11 @@ class PodcastListModel(gtk.ListStore):
return ''.join(d)
def _format_error(self, channel):
if channel.parse_error:
return str(channel.parse_error)
else:
return None
#if channel.parse_error:
# return str(channel.parse_error)
#else:
# return None
return None
def set_channels(self, db, config, channels):
# Clear the model and update the list of podcasts

128
src/gpodder/model.py
View File

@ -27,6 +27,7 @@ import gpodder
from gpodder import util
from gpodder import feedcore
from gpodder import youtube
from gpodder import schema
import logging
logger = logging.getLogger(__name__)
@ -81,16 +82,24 @@ class gPodderFetcher(feedcore.Fetcher):
# The "register" method is exposed here for external usage
register_custom_handler = gPodderFetcher.register
# Our podcast model:
#
# database -> podcast -> episode -> download/playback
# podcast.parent == db
# podcast.children == [episode, ...]
# episode.parent == podcast
#
# - normally: episode.children = (None, None)
# - downloading: episode.children = (DownloadTask(), None)
# - playback: episode.children = (None, PlaybackTask())
class PodcastModelObject(object):
"""
A generic base class for our podcast model providing common helper
and utility functions.
"""
_cache = collections.defaultdict(weakref.WeakValueDictionary)
@classmethod
def _get_cached_object(cls, id):
return cls._cache[cls].get(id, None)
__slots__ = ('id', 'parent', 'children')
@classmethod
def create_from_dict(cls, d, *args):
@ -98,16 +107,11 @@ class PodcastModelObject(object):
Create a new object, passing "args" to the constructor
and then updating the object with the values from "d".
"""
o = cls._get_cached_object(d['id'])
o = cls(*args)
if o is None:
o = cls(*args)
# XXX: all(map(lambda k: hasattr(o, k), d))?
for k, v in d.iteritems():
setattr(o, k, v)
cls._cache[cls][o.id] = o
else:
logger.debug('Reusing reference to %s %d', cls.__name__, o.id)
# XXX: all(map(lambda k: hasattr(o, k), d))?
for k, v in d.iteritems():
setattr(o, k, v)
return o
@ -116,6 +120,8 @@ class PodcastEpisode(PodcastModelObject):
"""holds data for one object in a channel"""
MAX_FILENAME_LENGTH = 200
__slots__ = schema.EpisodeColumns
def _deprecated(self):
raise Exception('Property is deprecated!')
@ -267,8 +273,7 @@ class PodcastEpisode(PodcastModelObject):
return None
def __init__(self, channel):
self.db = channel.db
self.channel = channel
self.parent = channel
self.id = None
self.url = ''
@ -293,11 +298,18 @@ class PodcastEpisode(PodcastModelObject):
# Timestamp of last playback time
self.last_playback = 0
@property
def channel(self):
return self.parent
@property
def db(self):
return self.parent.db
def save(self):
if self.state != gpodder.STATE_DOWNLOADED and self.file_exists():
self.state = gpodder.STATE_DOWNLOADED
if gpodder.user_hooks is not None:
gpodder.user_hooks.on_episode_save(self)
self.db.save_episode(self)
def on_downloaded(self, filename):
@ -618,13 +630,43 @@ class PodcastEpisode(PodcastModelObject):
class PodcastChannel(PodcastModelObject):
"""holds data for a complete channel"""
__slots__ = schema.PodcastColumns
MAX_FOLDERNAME_LENGTH = 150
SECONDS_PER_WEEK = 7*24*60*60
EpisodeClass = PodcastEpisode
feed_fetcher = gPodderFetcher()
def __init__(self, db):
self.db = db
self.children = None
self.id = None
self.url = None
self.title = ''
self.link = ''
self.description = ''
self.cover_url = None
self.auth_username = ''
self.auth_password = ''
self.http_last_modified = None
self.http_etag = None
self.auto_archive_episodes = False
self.download_folder = None
self.pause_subscription = False
def _get_db(self):
return self.parent
def _set_db(self, db):
self.parent = db
db = property(_get_db, _set_db)
def import_external_files(self):
"""Check the download folder for externally-downloaded files
@ -706,7 +748,7 @@ class PodcastChannel(PodcastModelObject):
@classmethod
def load_from_db(cls, db):
return db.load_podcasts(cls.create_from_dict, cls._get_cached_object)
return db.load_podcasts(cls.create_from_dict)
@classmethod
def load(cls, db, url, create=True, authentication_tokens=None,\
@ -715,9 +757,7 @@ class PodcastChannel(PodcastModelObject):
if isinstance(url, unicode):
url = url.encode('utf-8')
existing = [podcast for podcast in
db.load_podcasts(cls.create_from_dict, cls._get_cached_object)
if podcast.url == url]
existing = filter(lambda p: p.url == url, self.load_from_db(db))
if existing:
return existing[0]
@ -764,7 +804,7 @@ class PodcastChannel(PodcastModelObject):
self.db.purge(max_episodes, self.id)
def _consume_updated_feed(self, feed, max_episodes=0, mimetype_prefs=''):
self.parse_error = feed.get('bozo_exception', None)
#self.parse_error = feed.get('bozo_exception', None)
# Replace multi-space and newlines with single space (Maemo bug 11173)
self.title = re.sub('\s+', ' ', feed.feed.get('title', self.url))
@ -938,9 +978,7 @@ class PodcastChannel(PodcastModelObject):
def save(self):
if gpodder.user_hooks is not None:
gpodder.user_hooks.on_podcast_save(self)
if self.download_folder is None:
# get_save_dir() finds a unique value for download_folder
self.get_save_dir()
self.db.save_podcast(self)
def get_statistics(self):
@ -962,43 +1000,11 @@ class PodcastChannel(PodcastModelObject):
def authenticate_url(self, url):
return util.url_add_authentication(url, self.auth_username, self.auth_password)
def __init__(self, db):
self.db = db
self.id = None
self.url = None
self.title = ''
self.link = ''
self.description = ''
self.cover_url = None
self.parse_error = None
self.auth_username = ''
self.auth_password = ''
self.http_last_modified = None
self.http_etag = None
self.auto_archive_episodes = False
self.download_folder = None
self.pause_subscription = False
def _get_cover_url(self):
return self.cover_url
image = property(_get_cover_url)
def get_title( self):
if not self.__title.strip():
return self.url
else:
return self.__title
def set_title( self, value):
self.__title = value.strip()
title = property(fget=get_title,
fset=set_title)
def set_custom_title( self, custom_title):
custom_title = custom_title.strip()
@ -1036,7 +1042,9 @@ class PodcastChannel(PodcastModelObject):
return filter(lambda e: e.was_downloaded(), self.get_all_episodes())
def get_all_episodes(self):
return self.db.load_episodes(self, self.episode_factory, self.EpisodeClass._get_cached_object)
if self.children is None:
self.children = self.db.load_episodes(self, self.episode_factory)
return self.children
def find_unique_folder_name(self, download_folder):
# Remove trailing dots to avoid errors on Windows (bug 600)

35
src/gpodder/schema.py
View File

@ -22,6 +22,41 @@
from sqlite3 import dbapi2 as sqlite
EpisodeColumns = (
'podcast_id',
'title',
'description',
'url',
'published',
'guid',
'link',
'file_size',
'mime_type',
'state',
'is_new',
'archive',
'download_filename',
'total_time',
'current_position',
'current_position_updated',
'last_playback',
)
PodcastColumns = (
'title',
'url',
'link',
'description',
'cover_url',
'auth_username',
'auth_password',
'http_last_modified',
'http_etag',
'auto_archive_episodes',
'download_folder',
'pause_subscription',
)
def initialize_database(db):
# Create table for podcasts