From 753396ac0c8a79f49bc1e6c8cb1ad6a9ac6e4c3a Mon Sep 17 00:00:00 2001 From: Lola Dam Date: Sun, 23 Jun 2019 14:21:50 +0200 Subject: [PATCH] Try and catch block for dbRebuild (#2047) * Try and catch block for dbRebuild * Use self.log.error and not logging * Use self.log.error and not logging in SiteStorage also * Check if the rebuild is working --- plugins/Sidebar/SidebarPlugin.py | 7 ++++--- plugins/Sidebar/media/all.js | 1 + src/Content/ContentDb.py | 12 +++++++++--- src/Db/Db.py | 11 +++++++++-- src/Site/Site.py | 5 ++++- src/Site/SiteStorage.py | 32 ++++++++++++++++++++------------ src/Test/TestSiteStorage.py | 3 +++ src/main.py | 7 +++++-- 8 files changed, 55 insertions(+), 23 deletions(-) diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index 45287d49..dc6a9e9c 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -786,9 +786,10 @@ class UiWebsocketPlugin(object): if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") - result = self.site.storage.rebuildDb() + try: + self.site.storage.rebuildDb() + except Exception as err: + return self.response(to, {"error": str(err)}) - if not result: - return self.response(to, {"error": "Failed to rebuild database"}) return self.response(to, "ok") diff --git a/plugins/Sidebar/media/all.js b/plugins/Sidebar/media/all.js index 55823738..69e1e198 100644 --- a/plugins/Sidebar/media/all.js +++ b/plugins/Sidebar/media/all.js @@ -814,6 +814,7 @@ window.initScrollable = function () { this.tag.find("#button-dbrebuild").off("click touchend").on("click touchend", (function(_this) { return function() { _this.wrapper.notifications.add("done-dbrebuild", "info", "Database rebuilding...."); + _this.wrapper.ws.cmd("dbRebuild", [], function(response) { if (response !== "ok") { diff --git a/src/Content/ContentDb.py b/src/Content/ContentDb.py index b42a6e27..aeb23fe1 100644 --- a/src/Content/ContentDb.py +++ b/src/Content/ContentDb.py @@ -1,7 +1,7 @@ import time import os -from Db.Db import Db +from Db.Db import Db, DbTableError from Config import config from Plugin import PluginManager from Debug import Debug @@ -14,7 +14,10 @@ class ContentDb(Db): self.foreign_keys = True try: self.schema = self.getSchema() - self.checkTables() + try: + self.checkTables() + except DbTableError: + pass self.log.debug("Checking foreign keys...") foreign_key_error = self.execute("PRAGMA foreign_key_check").fetchone() if foreign_key_error: @@ -26,7 +29,10 @@ class ContentDb(Db): Db.__init__(self, {"db_name": "ContentDb", "tables": {}}, path) self.foreign_keys = True self.schema = self.getSchema() - self.checkTables() + try: + self.checkTables() + except DbTableError: + pass self.site_ids = {} self.sites = {} diff --git a/src/Db/Db.py b/src/Db/Db.py index 4202e1c1..24c0e9f5 100644 --- a/src/Db/Db.py +++ b/src/Db/Db.py @@ -48,6 +48,10 @@ gevent.spawn(dbCleanup) gevent.spawn(dbCommitCheck) atexit.register(dbCloseAll) +class DbTableError(Exception): + def __init__(self, message, table): + super().__init__(message) + self.table = table class Db(object): @@ -256,15 +260,18 @@ class Db(object): # Check schema tables for table_name, table_settings in self.schema.get("tables", {}).items(): try: + indexes = table_settings.get("indexes", []) + version = table_settings.get("schema_changed", 0) changed = cur.needTable( table_name, table_settings["cols"], - table_settings.get("indexes", []), version=table_settings.get("schema_changed", 0) + indexes, version=version ) if changed: changed_tables.append(table_name) except Exception as err: self.log.error("Error creating table %s: %s" % (table_name, Debug.formatException(err))) - return False + raise DbTableError(err, table_name) + #return False self.log.debug("Db check done in %.3fs, changed tables: %s" % (time.time() - s, changed_tables)) if changed_tables: diff --git a/src/Site/Site.py b/src/Site/Site.py index b8620b56..23980d1b 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -708,7 +708,10 @@ class Site(object): # Rebuild DB if new_site.storage.isFile("dbschema.json"): new_site.storage.closeDb() - new_site.storage.rebuildDb() + try: + new_site.storage.rebuildDb() + except Exception as err: + self.log.error(err) return new_site diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index d25e5a3f..4948fadc 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -11,7 +11,7 @@ import gevent.event import util from util import SafeRe -from Db.Db import Db +from Db.Db import Db, DbTableError from Debug import Debug from Config import config from util import helper @@ -73,15 +73,21 @@ class SiteStorage(object): schema = self.getDbSchema() db_path = self.getPath(schema["db_file"]) if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: - self.rebuildDb() + try: + self.rebuildDb() + except Exception as err: + self.log.error(err) + pass if self.db: self.db.close() self.db = self.openDb(close_idle=True) - - changed_tables = self.db.checkTables() - if changed_tables: - self.rebuildDb(delete_db=False) # TODO: only update the changed table datas + try: + changed_tables = self.db.checkTables() + if changed_tables: + self.rebuildDb(delete_db=False) # TODO: only update the changed table datas + except sqlite3.OperationalError: + pass return self.db @@ -138,11 +144,10 @@ class SiteStorage(object): self.event_db_busy = gevent.event.AsyncResult() self.log.info("Creating tables...") - changed_tables = self.db.checkTables() - if not changed_tables: - # It failed - # "Error creating table..." - return False + + # raise DbTableError if not valid + self.db.checkTables() + cur = self.db.getCursor() cur.logging = False s = time.time() @@ -195,7 +200,10 @@ class SiteStorage(object): except sqlite3.DatabaseError as err: if err.__class__.__name__ == "DatabaseError": self.log.error("Database error: %s, query: %s, try to rebuilding it..." % (err, query)) - self.rebuildDb() + try: + self.rebuildDb() + except sqlite3.OperationalError: + pass res = self.db.cur.execute(query, params) else: raise err diff --git a/src/Test/TestSiteStorage.py b/src/Test/TestSiteStorage.py index e9977e8e..f11262bf 100644 --- a/src/Test/TestSiteStorage.py +++ b/src/Test/TestSiteStorage.py @@ -20,3 +20,6 @@ class TestSiteStorage: # Subdir assert set(site.storage.list("data-default")) == set(["data.json", "users"]) + + def testDbRebuild(self, site): + assert site.storage.rebuildDb() diff --git a/src/main.py b/src/main.py index 53e43965..dfc32cc9 100644 --- a/src/main.py +++ b/src/main.py @@ -238,8 +238,11 @@ class Actions(object): logging.info("Rebuilding site sql cache: %s..." % address) site = SiteManager.site_manager.get(address) s = time.time() - site.storage.rebuildDb() - logging.info("Done in %.3fs" % (time.time() - s)) + try: + site.storage.rebuildDb() + logging.info("Done in %.3fs" % (time.time() - s)) + except Exception as err: + logging.error(err) def dbQuery(self, address, query): from Site.Site import Site