421 lines
15 KiB
Python
Executable File
421 lines
15 KiB
Python
Executable File
#!/usr/bin/env python
|
|
import os
|
|
import sys
|
|
import yaml
|
|
import argparse
|
|
from itertools import chain
|
|
import psycopg2
|
|
import subprocess
|
|
import logging
|
|
import time
|
|
from enum import Enum
|
|
|
|
from blessings import Terminal
|
|
|
|
import trytond
|
|
from trytond.config import config as CONFIG, parse_uri
|
|
|
|
trytond_version = '.'.join(trytond.__version__.split('.')[:2])
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.DEBUG)
|
|
ch = logging.StreamHandler(sys.stdout)
|
|
ch.setLevel(logging.DEBUG)
|
|
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
ch.setFormatter(formatter)
|
|
logger.addHandler(ch)
|
|
|
|
|
|
class Step(Enum):
|
|
BEFORE = 'before'
|
|
UNINSTALL = 'uninstall'
|
|
FIRST_UPDATE = 'first-update'
|
|
AFTER = 'after'
|
|
SECOND_UPDATE = 'second-update'
|
|
|
|
|
|
t = Terminal()
|
|
|
|
def get_url():
|
|
if config_file:
|
|
CONFIG.update_etc(config_file)
|
|
url = parse_uri(CONFIG.get('database', 'uri'))
|
|
else:
|
|
url = parse_uri(os.environ.get('TRYTOND_DATABASE__URI', ''))
|
|
return url
|
|
|
|
def run(*args):
|
|
logger.info('RUNNING: %s' % ' '.join(args))
|
|
process = subprocess.Popen(args, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, bufsize=1, text=True, shell=False)
|
|
|
|
out, err = process.communicate()
|
|
|
|
summary = set()
|
|
in_traceback = False
|
|
for line in chain(out.split('\n'), err.split('\n')):
|
|
line = line.strip()
|
|
if 'ERROR' in line:
|
|
s = line[line.index('ERROR'):]
|
|
summary.add(s)
|
|
line = t.red(line)
|
|
elif 'WARNING' in line:
|
|
s = line[line.index('WARNING'):]
|
|
summary.add(s)
|
|
line = t.yellow(line)
|
|
elif 'Traceback' in line or in_traceback:
|
|
in_traceback = True
|
|
line = t.red(line)
|
|
if line.startswith('Exception'):
|
|
in_traceback = False
|
|
logger.info(line)
|
|
|
|
process.stdout.close()
|
|
process.stderr.close()
|
|
process.wait()
|
|
|
|
if summary:
|
|
summary = sorted(list(summary))
|
|
total = len(summary)
|
|
if cmd.show_ignored:
|
|
ignored = 0
|
|
else:
|
|
ignores = config['ignore']
|
|
summary = [x for x in summary if x.strip() not in ignores]
|
|
ignored = total - len(summary)
|
|
logger.error('WARNING AND ERROR SUMMARY (%d + %d ignored):' % (
|
|
len(summary), ignored))
|
|
logger.error(t.yellow('\n'.join(summary)))
|
|
|
|
return process.returncode
|
|
|
|
def execute(query, *args, **kwargs):
|
|
if not args:
|
|
args = kwargs
|
|
cursor.execute(query, args)
|
|
|
|
def run_trytond(to_install=None):
|
|
to_run = ['trytond/bin/trytond-admin', '-v']
|
|
if config_file:
|
|
to_run += ['-c', config_file]
|
|
if to_install:
|
|
to_run += ['-u'] + to_install
|
|
to_run.append('--all')
|
|
to_run.append('--activate-dependencies')
|
|
to_run.append('-d')
|
|
to_run.append(database_name)
|
|
returncode = run(*to_run)
|
|
if returncode:
|
|
logger.error('Trytond update failed. Upgrade aborted.')
|
|
return returncode
|
|
|
|
def table_exists(table):
|
|
execute('SELECT count(*) FROM information_schema.tables '
|
|
'WHERE table_name=%s', table)
|
|
return bool(cursor.fetchone()[0])
|
|
|
|
def field_exists(field):
|
|
table, field = field.split('.')
|
|
execute('SELECT count(*) FROM information_schema.columns '
|
|
'WHERE table_name=%s AND column_name=%s', table, field)
|
|
return bool(cursor.fetchone()[0])
|
|
|
|
def where_exists(query):
|
|
execute(query)
|
|
return bool(cursor.fetchone()[0])
|
|
|
|
|
|
class Upgrade:
|
|
|
|
def __init__(self, connection):
|
|
self.connection = connection
|
|
self.steps = None
|
|
self.before = None
|
|
self.after = None
|
|
self.to_install = None
|
|
self.to_uninstall = None
|
|
self.database_name = None
|
|
self.config_file = None
|
|
|
|
def uninstall_modules(self):
|
|
module_table = None
|
|
for table in ('ir_module_module', 'ir_module'):
|
|
if table_exists(table):
|
|
module_table = table
|
|
break
|
|
for module in self.to_uninstall:
|
|
logger.info('Module: %s' % module)
|
|
execute('DELETE FROM ' + module_table + '_dependency WHERE '
|
|
'module IN (SELECT id FROM ' + module_table + ' WHERE name=%s)',
|
|
module)
|
|
execute('DELETE FROM ' + module_table + ' WHERE name=%s', module)
|
|
|
|
execute('SELECT model, db_id FROM ir_model_data WHERE module=%s',
|
|
module)
|
|
for model, db_id in cursor.fetchall():
|
|
logger.info('DELETING %s %s' % (model, db_id))
|
|
if model == 'res.user':
|
|
continue
|
|
execute('DELETE FROM "' + model.replace('.', '_')
|
|
+ '" WHERE id=%s', db_id)
|
|
|
|
execute('DELETE FROM ir_model_data WHERE module=%s', module)
|
|
|
|
if table_exists('babi_report'):
|
|
execute('DELETE from babi_filter_parameter where filter in'
|
|
' (SELECT id FROM babi_filter WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
|
|
module_table)
|
|
execute('DELETE FROM babi_filter WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
|
|
module_table)
|
|
execute('DELETE from babi_order where report in'
|
|
' (SELECT id FROM babi_report WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
|
|
module_table)
|
|
execute('DELETE from babi_measure where report in'
|
|
' (SELECT id FROM babi_report WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
|
|
module_table)
|
|
execute('DELETE from babi_dimension where expression in'
|
|
' (SELECT id FROM babi_expression WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
|
|
module_table)
|
|
execute('DELETE FROM babi_expression WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
|
|
module_table)
|
|
execute('DELETE FROM babi_report WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
|
|
module_table)
|
|
|
|
if table_exists('mass_editing'):
|
|
execute('DELETE FROM mass_editing WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
|
|
module_table)
|
|
execute('DELETE FROM ir_trigger WHERE model IN (SELECT '
|
|
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
|
|
module_table)
|
|
execute('DELETE FROM ir_action_act_window WHERE res_model IN (SELECT '
|
|
'model FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
|
|
module_table)
|
|
execute('DELETE FROM ir_action_wizard WHERE model in (SELECT model FROM '
|
|
'ir_model WHERE module NOT IN (SELECT name FROM %s))' % module_table)
|
|
execute('DELETE FROM ir_model WHERE module NOT IN (SELECT name FROM '
|
|
'%s)' % module_table)
|
|
execute('DELETE FROM ir_model_field WHERE module NOT IN (SELECT name FROM '
|
|
'%s)' % module_table)
|
|
execute('DELETE FROM ir_ui_view WHERE module NOT IN (SELECT name FROM '
|
|
'%s)' % module_table)
|
|
|
|
def process_actions(self, actions):
|
|
if not actions:
|
|
return
|
|
for action in actions:
|
|
if isinstance(action, dict):
|
|
tables = action.get('tables', '')
|
|
fields = action.get('fields', '')
|
|
version = action.get('version', trytond_version)
|
|
query = action.get('query')
|
|
script = action.get('script')
|
|
where = action.get('where')
|
|
|
|
# Check version
|
|
if not (float(version) <= float(trytond_version) and float(version) > float(from_version)):
|
|
continue
|
|
|
|
# Check tables
|
|
found = True
|
|
tables = tables.split()
|
|
for table in tables:
|
|
if not table_exists(table):
|
|
logger.info("TABLE '%s' NOT FOUND" % table)
|
|
found = False
|
|
break
|
|
if not found:
|
|
continue
|
|
|
|
# Check fields
|
|
found = True
|
|
fields = fields.split()
|
|
for field in fields:
|
|
if not field_exists(field):
|
|
logger.info("FIELD '%s' NOT FOUND" % field)
|
|
found = False
|
|
break
|
|
if not found:
|
|
continue
|
|
|
|
# Check where
|
|
if where and not where_exists(where):
|
|
logger.info("WHERE '%s' NOT FOUND" % where)
|
|
continue
|
|
else:
|
|
query = action
|
|
script = None
|
|
|
|
if query:
|
|
logger.info(query)
|
|
query = query.replace('%', '%%')
|
|
execute(query)
|
|
|
|
if script:
|
|
if os.path.isfile(script):
|
|
# We must commit before executing the script so the script
|
|
# is not locked by our transaction
|
|
logger.info(t.green('Executing: %s' % script))
|
|
self.connection.commit()
|
|
res = run(script, self.database_name, self.config_file or
|
|
'')
|
|
self.cursor = self.connection.cursor()
|
|
|
|
if res:
|
|
logger.error(t.red('Script "%s" returned the following '
|
|
'error code: %d. Upgrade aborted.') % (script,
|
|
res))
|
|
return 1
|
|
else:
|
|
logger.warning("Not found script: %s" % script)
|
|
logger.error(t.red('Script "%s" not found. Upgrade '
|
|
'aborted.') % script)
|
|
return 1
|
|
|
|
def run(self):
|
|
|
|
|
|
for step in self.steps:
|
|
self.cursor = self.connection.cursor()
|
|
logger.info(t.green('Executing step %s...') % step.value)
|
|
if step == Step.BEFORE:
|
|
res = self.process_actions(self.before)
|
|
self.connection.commit()
|
|
elif step == Step.UNINSTALL:
|
|
res = self.uninstall_modules()
|
|
self.connection.commit()
|
|
elif step == Step.FIRST_UPDATE:
|
|
res = run_trytond(self.to_install)
|
|
elif step == Step.AFTER:
|
|
res = self.process_actions(self.after)
|
|
self.connection.commit()
|
|
elif step == Step.SECOND_UPDATE:
|
|
res = run_trytond()
|
|
|
|
if res:
|
|
return res
|
|
|
|
if not self.steps:
|
|
logger.error(t.red('No steps executed. Invalid from/until steps'))
|
|
return 1
|
|
|
|
logger.info(t.green('Finished executing steps: %s'
|
|
% ', '.join([x.value for x in steps_to_run])))
|
|
|
|
|
|
parser = argparse.ArgumentParser(description='Upgrade a Tryton database to the '
|
|
'version of the trytond library available.')
|
|
parser.add_argument('database', nargs=1, help='PostgreSQL database to upgrade')
|
|
parser.add_argument('from_version', nargs=1, help='Tryton version of the '
|
|
'database to be migrated')
|
|
parser.add_argument('-c', '--config', default=None,
|
|
help='path to the trytond configuration file')
|
|
parser.add_argument('--show-ignored', dest='show_ignored', default=False,
|
|
help='Show warnings that would otherwise be ignored')
|
|
parser.add_argument('--override', dest='override', default='upgrade.yml',
|
|
help='Search on the given filename for values to override default '
|
|
'configuration. Default: upgrade.yml')
|
|
|
|
steps = ', '.join([x.value for x in Step])
|
|
parser.add_argument('--until', default=None, help='Run the upgrade process '
|
|
'only until the given step. Possible steps include: %s' % steps)
|
|
parser.add_argument('--from', dest='from_', default=None, help='Run the '
|
|
'upgrade process from the given step. Possible steps include: %s' % steps)
|
|
|
|
cmd = parser.parse_args()
|
|
|
|
# Compute from steps_to_run
|
|
from_ = cmd.from_
|
|
steps_to_run = []
|
|
for step in Step:
|
|
if not from_ or step.value == from_ or steps_to_run:
|
|
steps_to_run.append(step)
|
|
if cmd.until and step.value == cmd.until:
|
|
break
|
|
|
|
|
|
database_name, = cmd.database
|
|
from_version, = cmd.from_version
|
|
if not cmd.config:
|
|
instance = os.path.basename(os.path.realpath(os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), '..')))
|
|
paths = (
|
|
'/etc/trytond/%s.conf' % instance,
|
|
os.environ.get('TRYTOND_CONFIG'),
|
|
)
|
|
for config_file in paths:
|
|
if not config_file:
|
|
continue
|
|
logger.info('Checking %s...' % config_file)
|
|
if os.path.exists(config_file):
|
|
break
|
|
if config_file:
|
|
logger.info("Configuration file: %s" % config_file)
|
|
else:
|
|
logger.info("No configuration file found.")
|
|
else:
|
|
config_file = cmd.config
|
|
if not os.path.isfile(config_file):
|
|
logger.info('File "%s" not found' % config_file)
|
|
sys.exit(1)
|
|
logger.info('Loading configuration from "%s"' % config_file)
|
|
|
|
url = get_url()
|
|
|
|
config = yaml.load(open('upgrades/config.yml', 'r').read(),
|
|
Loader=yaml.FullLoader)
|
|
config.setdefault('to_uninstall', [])
|
|
config.setdefault('to_install', [])
|
|
config.setdefault('ignore', [])
|
|
config.setdefault('vars', {})
|
|
if os.path.exists(cmd.override):
|
|
logger.info('Overriding configuration using "%s"' % cmd.override)
|
|
override = yaml.load(open(cmd.override, 'r').read(), Loader=yaml.FullLoader)
|
|
logger.info
|
|
if override:
|
|
config['to_install'] += override.get('to_install', [])
|
|
config['to_uninstall'] += override.get('to_uninstall', [])
|
|
config['before'] = (override.get('before_before', []) +
|
|
config.get('before', []) + override.get('before', []))
|
|
config['after'] = (override.get('before_after', []) +
|
|
config.get('after', []) + override.get('after', []))
|
|
config['ignore'] += override.get('ignore', [])
|
|
config['vars'].update(override.get('vars', {}))
|
|
else:
|
|
logger.info('Overriding file "%s" not found.' % cmd.override)
|
|
|
|
if url.username:
|
|
connection = psycopg2.connect(dbname=database_name, host=url.hostname,
|
|
port=url.port, user=url.username, password=url.password)
|
|
else:
|
|
connection = psycopg2.connect(dbname=database_name)
|
|
|
|
|
|
# Ensure variable values are of type str
|
|
config['vars'] = {k: str(v) for k, v in config['vars'].items()}
|
|
# Set environment variables
|
|
os.environ.update(config['vars'])
|
|
|
|
start = time.time()
|
|
cursor = connection.cursor()
|
|
upgrade = Upgrade(connection)
|
|
upgrade.before = config.get('before', [])
|
|
upgrade.after = config.get('after', [])
|
|
upgrade.to_install = config.get('to_install', [])
|
|
upgrade.to_uninstall = config.get('to_uninstall', [])
|
|
upgrade.database_name = database_name
|
|
upgrade.config_file = config_file
|
|
upgrade.steps = steps_to_run
|
|
returncode = upgrade.run()
|
|
end = time.time()
|
|
logger.info(t.cyan('Elapsed time: %.1fs' % (end - start)))
|
|
|
|
exit(returncode or 0)
|