tryton-upgrades/upgrade
2021-10-15 09:56:05 +02:00

420 lines
15 KiB
Python
Executable file

#!/usr/bin/env python
import os
import sys
import yaml
import argparse
from itertools import chain
import psycopg2
import subprocess
import logging
import time
from enum import Enum
from blessings import Terminal
import trytond
from trytond.config import config as CONFIG, parse_uri
trytond_version = '.'.join(trytond.__version__.split('.')[:2])
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
class Step(Enum):
BEFORE = 'before'
UNINSTALL = 'uninstall'
FIRST_UPDATE = 'first-update'
AFTER = 'after'
SECOND_UPDATE = 'second-update'
t = Terminal()
def get_url():
if config_file:
CONFIG.update_etc(config_file)
url = parse_uri(CONFIG.get('database', 'uri'))
else:
url = parse_uri(os.environ.get('TRYTOND_DATABASE__URI', ''))
return url
def run(*args):
logger.info('RUNNING: %s' % ' '.join(args))
process = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1, text=True, shell=False)
out, err = process.communicate()
summary = set()
in_traceback = False
for line in chain(out.split('\n'), err.split('\n')):
line = line.strip()
if 'ERROR' in line:
s = line[line.index('ERROR'):]
summary.add(s)
line = t.red(line)
elif 'WARNING' in line:
s = line[line.index('WARNING'):]
summary.add(s)
line = t.yellow(line)
elif 'Traceback' in line or in_traceback:
in_traceback = True
line = t.red(line)
if line.startswith('Exception'):
in_traceback = False
logger.info(line)
process.stdout.close()
process.stderr.close()
process.wait()
if summary:
summary = sorted(list(summary))
total = len(summary)
if cmd.show_ignored:
ignored = 0
else:
ignores = config['ignore']
summary = [x for x in summary if x.strip() not in ignores]
ignored = total - len(summary)
logger.error('WARNING AND ERROR SUMMARY (%d + %d ignored):' % (
len(summary), ignored))
logger.error(t.yellow('\n'.join(summary)))
return process.returncode
def execute(query, *args, **kwargs):
if not args:
args = kwargs
cursor.execute(query, args)
def run_trytond(to_install=None):
to_run = ['trytond/bin/trytond-admin', '-v']
if config_file:
to_run += ['-c', config_file]
if to_install:
to_run += ['-u'] + to_install
to_run.append('--all')
to_run.append('--activate-dependencies')
to_run.append('-d')
to_run.append(database_name)
returncode = run(*to_run)
if returncode:
logger.error('Trytond update failed. Upgrade aborted.')
return returncode
def table_exists(table):
execute('SELECT count(*) FROM information_schema.tables '
'WHERE table_name=%s', table)
return bool(cursor.fetchone()[0])
def field_exists(field):
table, field = field.split('.')
execute('SELECT count(*) FROM information_schema.columns '
'WHERE table_name=%s AND column_name=%s', table, field)
return bool(cursor.fetchone()[0])
def where_exists(query):
execute(query)
return bool(cursor.fetchone()[0])
class Upgrade:
def __init__(self, connection):
self.connection = connection
self.steps = None
self.before = None
self.after = None
self.to_install = None
self.to_uninstall = None
self.database_name = None
self.config_file = None
def uninstall_modules(self):
module_table = None
for table in ('ir_module_module', 'ir_module'):
if table_exists(table):
module_table = table
break
for module in self.to_uninstall:
logger.info('Module: %s' % module)
execute('DELETE FROM ' + module_table + '_dependency WHERE '
'module IN (SELECT id FROM ' + module_table + ' WHERE name=%s)',
module)
execute('DELETE FROM ' + module_table + ' WHERE name=%s', module)
execute('SELECT model, db_id FROM ir_model_data WHERE module=%s',
module)
for model, db_id in cursor.fetchall():
logger.info('DELETING %s %s' % (model, db_id))
if model == 'res.user':
continue
execute('DELETE FROM "' + model.replace('.', '_')
+ '" WHERE id=%s', db_id)
execute('DELETE FROM ir_model_data WHERE module=%s', module)
if table_exists('babi_report'):
execute('DELETE from babi_filter_parameter where filter in'
' (SELECT id FROM babi_filter WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
module_table)
execute('DELETE FROM babi_filter WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
module_table)
execute('DELETE from babi_order where report in'
' (SELECT id FROM babi_report WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
module_table)
execute('DELETE from babi_measure where report in'
' (SELECT id FROM babi_report WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
module_table)
execute('DELETE from babi_dimension where expression in'
' (SELECT id FROM babi_expression WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s)))' %
module_table)
execute('DELETE FROM babi_expression WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
module_table)
execute('DELETE FROM babi_report WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
module_table)
if table_exists('mass_editing'):
execute('DELETE FROM mass_editing WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
module_table)
execute('DELETE FROM ir_trigger WHERE model IN (SELECT '
'id FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
module_table)
execute('DELETE FROM ir_action_act_window WHERE res_model IN (SELECT '
'model FROM ir_model WHERE module NOT IN (SELECT name FROM %s))' %
module_table)
execute('DELETE FROM ir_action_wizard WHERE model in (SELECT model FROM '
'ir_model WHERE module NOT IN (SELECT name FROM %s))' % module_table)
execute('DELETE FROM ir_model WHERE module NOT IN (SELECT name FROM '
'%s)' % module_table)
execute('DELETE FROM ir_model_field WHERE module NOT IN (SELECT name FROM '
'%s)' % module_table)
execute('DELETE FROM ir_ui_view WHERE module NOT IN (SELECT name FROM '
'%s)' % module_table)
def process_actions(self, actions):
if not actions:
return
for action in actions:
if isinstance(action, dict):
tables = action.get('tables', '')
fields = action.get('fields', '')
version = action.get('version', trytond_version)
query = action.get('query')
script = action.get('script')
where = action.get('where')
# Check version
if not (float(version) <= float(trytond_version) and float(version) > float(from_version)):
continue
# Check tables
found = True
tables = tables.split()
for table in tables:
if not table_exists(table):
logger.info("TABLE '%s' NOT FOUND" % table)
found = False
break
if not found:
continue
# Check fields
found = True
fields = fields.split()
for field in fields:
if not field_exists(field):
logger.info("FIELD '%s' NOT FOUND" % field)
found = False
break
if not found:
continue
# Check where
if where and not where_exists(where):
logger.info("WHERE '%s' NOT FOUND" % where)
continue
else:
query = action
script = None
if query:
logger.info(query)
query = query.replace('%', '%%')
execute(query)
if script:
if os.path.isfile(script):
# We must commit before executing the script so the script
# is not locked by our transaction
logger.info(t.green('Executing: %s' % script))
self.connection.commit()
res = run(script, self.database_name, self.config_file or
'')
self.cursor = self.connection.cursor()
if res:
logger.error(t.red('Script "%s" returned the following '
'error code: %d. Upgrade aborted.') % (script,
res))
return 1
else:
logger.warning("Not found script: %s" % script)
logger.error(t.red('Script "%s" not found. Upgrade '
'aborted.') % script)
return 1
def run(self):
for step in self.steps:
self.cursor = self.connection.cursor()
logger.info(t.green('Executing step %s...') % step.value)
if step == Step.BEFORE:
res = self.process_actions(self.before)
self.connection.commit()
elif step == Step.UNINSTALL:
res = self.uninstall_modules()
self.connection.commit()
elif step == Step.FIRST_UPDATE:
res = run_trytond(self.to_install)
elif step == Step.AFTER:
res = self.process_actions(self.after)
self.connection.commit()
elif step == Step.SECOND_UPDATE:
res = run_trytond()
if res:
return res
if not self.steps:
logger.error(t.red('No steps executed. Invalid from/until steps'))
return 1
logger.info(t.green('Finished executing steps: %s'
% ', '.join([x.value for x in steps_to_run])))
parser = argparse.ArgumentParser(description='Upgrade a Tryton database to the '
'version of the trytond library available.')
parser.add_argument('database', nargs=1, help='PostgreSQL database to upgrade')
parser.add_argument('from_version', nargs=1, help='Tryton version of the '
'database to be migrated')
parser.add_argument('-c', '--config', default=None,
help='path to the trytond configuration file')
parser.add_argument('--show-ignored', dest='show_ignored', default=False,
help='Show warnings that would otherwise be ignored')
parser.add_argument('--override', dest='override', default='upgrade.yml',
help='Search on the given filename for values to override default '
'configuration. Default: upgrade.yml')
steps = ', '.join([x.value for x in Step])
parser.add_argument('--until', default=None, help='Run the upgrade process '
'only until the given step. Possible steps include: %s' % steps)
parser.add_argument('--from', dest='from_', default=None, help='Run the '
'upgrade process from the given step. Possible steps include: %s' % steps)
cmd = parser.parse_args()
# Compute from steps_to_run
from_ = cmd.from_
steps_to_run = []
for step in Step:
if not from_ or step.value == from_ or steps_to_run:
steps_to_run.append(step)
if cmd.until and step.value == cmd.until:
break
database_name, = cmd.database
from_version, = cmd.from_version
if not cmd.config:
instance = os.path.basename(os.path.realpath(os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..')))
paths = (
'/etc/trytond/%s.conf' % instance,
os.environ.get('TRYTOND_CONFIG'),
)
for config_file in paths:
if not config_file:
continue
logger.info('Checking %s...' % config_file)
if os.path.exists(config_file):
break
if config_file:
logger.info("Configuration file: %s" % config_file)
else:
logger.info("No configuration file found.")
else:
config_file = cmd.config
if not os.path.isfile(config_file):
logger.info('File "%s" not found' % config_file)
sys.exit(1)
logger.info('Loading configuration from "%s"' % config_file)
url = get_url()
config = yaml.load(open('upgrades/config.yml', 'r').read(),
Loader=yaml.FullLoader)
config.setdefault('to_uninstall', [])
config.setdefault('to_install', [])
config.setdefault('ignore', [])
config.setdefault('vars', {})
if os.path.exists(cmd.override):
logger.info('Overriding configuration using "%s"' % cmd.override)
override = yaml.load(open(cmd.override, 'r').read(), Loader=yaml.FullLoader)
logger.info
if override:
config['to_install'] += override.get('to_install', [])
config['to_uninstall'] += override.get('to_uninstall', [])
config['before'] = (override.get('before_before', []) +
config.get('before', []) + override.get('before', []))
config['after'] = (override.get('before_after', []) +
config.get('after', []) + override.get('after', []))
config['ignore'] += override.get('ignore', [])
config['vars'].update(override.get('vars', {}))
else:
logger.info('Overriding file "%s" not found.' % cmd.override)
if url.username:
connection = psycopg2.connect(dbname=database_name, host=url.hostname,
port=url.port, user=url.username, password=url.password)
else:
connection = psycopg2.connect(dbname=database_name)
# Ensure variable values are of type str
config['vars'] = {k: str(v) for k, v in config['vars'].items()}
# Set environment variables
os.environ.update(config['vars'])
start = time.time()
cursor = connection.cursor()
upgrade = Upgrade(connection)
upgrade.before = config.get('before', [])
upgrade.after = config.get('after', [])
upgrade.to_install = config.get('to_install', [])
upgrade.to_uninstall = config.get('to_uninstall', [])
upgrade.database_name = database_name
upgrade.config_file = config_file
upgrade.steps = steps_to_run
returncode = upgrade.run()
end = time.time()
logger.info(t.cyan('Elapsed time: %.1fs' % (end - start)))
exit(returncode or 0)