diff -r ad3cb7be7667 CHANGELOG --- a/trytond/CHANGELOG Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/CHANGELOG Mon Jan 08 14:40:45 2018 +0100 @@ -1,4 +1,7 @@ Version 3.8.16 - 2017-12-04 +======= +* Add filestore module + * Bug fixes (see mercurial logs for details) Version 3.8.15 - 2017-11-07 diff -r ad3cb7be7667 doc/ref/filestore.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trytond/doc/ref/filestore.rst Mon Jan 08 14:40:45 2018 +0100 @@ -0,0 +1,42 @@ +.. _ref-filestore: +.. module:: trytond.filestore + +========= +FileStore +========= + +.. class:: FileStore() + +The class is used to store and retrieve files from the directory defined in the +configuration `path` of `database` section. It uses a two levels of directory +composed of the 2 chars of the file hash. It is an append only storage. + +.. method:: get(id[, prefix]) + +Retrieve the content of the file referred by the id in the prefixed directory. + +.. method:: getmany(ids[, prefix]) + +Retrieve a list of contents for the sequence of ids. + +.. method:: size(id[, prefix]) + +Return the size of the file referred by the id in the prefixed directory. + +.. method:: sizemany(ids[, prefix]) + +Return a list of sizes for the sequence of ids. + +.. method:: set(data[, prefix]) + +Store the data in the prefixed directory and return the identifiers. + +.. method:: setmany(data[, prefix]) + +Store the sequence of data and return a list of identifiers. + +.. note:: + The class can be overridden by setting a fully qualified name of a + alternative class defined in the configuration `class` of the `database` + section. +.. diff -r ad3cb7be7667 doc/ref/index.rst --- a/trytond/doc/ref/index.rst Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/doc/ref/index.rst Mon Jan 08 14:40:45 2018 +0100 @@ -11,6 +11,7 @@ wizard pyson transaction - tools/singleton + tools/index pool rpc + filestore diff -r ad3cb7be7667 doc/ref/tools/index.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trytond/doc/ref/tools/index.rst Mon Jan 08 14:40:45 2018 +0100 @@ -0,0 +1,13 @@ +.. _ref-tools-index: + +===== +Tools +===== + +Tools API reference. + +.. toctree:: + :maxdepth: 1 + + misc + singleton diff -r ad3cb7be7667 doc/ref/tools/misc.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trytond/doc/ref/tools/misc.rst Mon Jan 08 14:40:45 2018 +0100 @@ -0,0 +1,10 @@ +.. _ref-tools: +.. module:: trytond.tools + +============= +Miscellaneous +============= + +.. method:: resolve(name) + +Resolve a dotted name to a global object. diff -r ad3cb7be7667 trytond/ir/attachment.py --- a/trytond/trytond/ir/attachment.py Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/trytond/ir/attachment.py Mon Jan 08 14:40:45 2018 +0100 @@ -1,7 +1,6 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -import os -import hashlib +from sql import Null from sql.operators import Concat from sql.conditionals import Coalesce @@ -10,7 +9,10 @@ from .. import backend from ..transaction import Transaction from ..pyson import Eval -from ..pool import Pool +from .resource import ResourceMixin +from ..filestore import filestore +from ..config import config + __all__ = [ 'Attachment', @@ -23,8 +25,15 @@ except StopIteration: return '' +if config.getboolean('attachment', 'filestore', default=True): + file_id = 'file_id' + store_prefix = config.get('attachment', 'store_prefix', default=None) +else: + file_id = None + store_prefix = None -class Attachment(ModelSQL, ModelView): + +class Attachment(ResourceMixin, ModelSQL, ModelView): "Attachment" __name__ = 'ir.attachment' name = fields.Char('Name', required=True) @@ -32,25 +41,20 @@ ('data', 'Data'), ('link', 'Link'), ], 'Type', required=True) - data = fields.Function(fields.Binary('Data', filename='name', states={ - 'invisible': Eval('type') != 'data', - }, depends=['type']), 'get_data', setter='set_data') + data = fields.Binary('Data', filename='name', + file_id=file_id, store_prefix=store_prefix, + states={ + 'invisible': Eval('type') != 'data', + }, depends=['type']) description = fields.Text('Description') summary = fields.Function(fields.Char('Summary'), 'on_change_with_summary') - resource = fields.Reference('Resource', selection='models_get', - select=True) link = fields.Char('Link', states={ 'invisible': Eval('type') != 'link', }, depends=['type']) - digest = fields.Char('Digest', size=32) - collision = fields.Integer('Collision') + file_id = fields.Char('File ID', readonly=True) data_size = fields.Function(fields.Integer('Data size', states={ 'invisible': Eval('type') != 'data', - }, depends=['type']), 'get_data') - last_modification = fields.Function(fields.DateTime('Last Modification'), - 'get_last_modification') - last_user = fields.Function(fields.Char('Last User'), - 'get_last_user') + }, depends=['type']), 'get_size') @classmethod def __setup__(cls): @@ -86,177 +90,33 @@ table.drop_column('res_model') table.drop_column('res_id') + # Migration from 4.0: merge digest and collision into file_id + if table.column_exist('digest') and table.column_exist('collision'): + cursor.execute(*attachment.update( + [attachment.file_id], + [attachment.digest], + where=(attachment.collision == 0) + | (attachment.collision == Null))) + cursor.execute(*attachment.update( + [attachment.file_id], + [Concat(Concat(attachment.digest, '-'), + attachment.collision)], + where=(attachment.collision != 0) + & (attachment.collision != Null))) + table.drop_column('digest') + table.drop_column('collision') + @staticmethod def default_type(): return 'data' - @staticmethod - def default_resource(): - return Transaction().context.get('resource') - - @staticmethod - def default_collision(): - return 0 - - @staticmethod - def models_get(): - pool = Pool() - Model = pool.get('ir.model') - ModelAccess = pool.get('ir.model.access') - models = Model.search([]) - access = ModelAccess.get_access([m.model for m in models]) - res = [] - for model in models: - if access[model.model]['read']: - res.append([model.model, model.name]) - return res - - def get_data(self, name): - db_name = Transaction().cursor.dbname - format_ = Transaction().context.pop('%s.%s' - % (self.__name__, name), '') - value = None - if name == 'data_size' or format_ == 'size': - value = 0 - if self.digest: - filename = self.digest - if self.collision: - filename = filename + '-' + str(self.collision) - filename = os.path.join(config.get('database', 'path'), db_name, - filename[0:2], filename[2:4], filename) - if name == 'data_size' or format_ == 'size': - try: - statinfo = os.stat(filename) - value = statinfo.st_size - except OSError: - pass - else: - try: - with open(filename, 'rb') as file_p: - value = fields.Binary.cast(file_p.read()) - except IOError: - pass - return value - - @classmethod - def set_data(cls, attachments, name, value): - if value is None: - return - cursor = Transaction().cursor - table = cls.__table__() - db_name = cursor.dbname - directory = os.path.join(config.get('database', 'path'), db_name) - if not os.path.isdir(directory): - os.makedirs(directory, 0770) - digest = hashlib.md5(value).hexdigest() - directory = os.path.join(directory, digest[0:2], digest[2:4]) - if not os.path.isdir(directory): - os.makedirs(directory, 0770) - filename = os.path.join(directory, digest) - collision = 0 - if os.path.isfile(filename): - with open(filename, 'rb') as file_p: - data = file_p.read() - if value != data: - cursor.execute(*table.select(table.collision, - where=(table.digest == digest) - & (table.collision != 0), - group_by=table.collision, - order_by=table.collision)) - collision2 = 0 - for row in cursor.fetchall(): - collision2 = row[0] - filename = os.path.join(directory, - digest + '-' + str(collision2)) - if os.path.isfile(filename): - with open(filename, 'rb') as file_p: - data = file_p.read() - if value == data: - collision = collision2 - break - if collision == 0: - collision = collision2 + 1 - filename = os.path.join(directory, - digest + '-' + str(collision)) - with open(filename, 'wb') as file_p: - file_p.write(value) - else: - with open(filename, 'wb') as file_p: - file_p.write(value) - cls.write(attachments, { - 'digest': digest, - 'collision': collision, - }) + def get_size(self, name): + with Transaction().set_context({ + '%s.%s' % (self.__name__, name): 'size', + }): + record = self.__class__(self.id) + return record.data @fields.depends('description') def on_change_with_summary(self, name=None): return firstline(self.description or '') - - def get_last_modification(self, name): - return (self.write_date if self.write_date else self.create_date - ).replace(microsecond=0) - - @staticmethod - def order_last_modification(tables): - table, _ = tables[None] - return [Coalesce(table.write_date, table.create_date)] - - def get_last_user(self, name): - return (self.write_uid.rec_name if self.write_uid - else self.create_uid.rec_name) - - @classmethod - def check_access(cls, ids, mode='read'): - pool = Pool() - ModelAccess = pool.get('ir.model.access') - if ((Transaction().user == 0) - or not Transaction().context.get('_check_access')): - return - model_names = set() - with Transaction().set_context(_check_access=False): - for attachment in cls.browse(ids): - if attachment.resource: - model_names.add(attachment.resource.__name__) - for model_name in model_names: - ModelAccess.check(model_name, mode=mode) - - @classmethod - def read(cls, ids, fields_names=None): - cls.check_access(ids, mode='read') - return super(Attachment, cls).read(ids, fields_names=fields_names) - - @classmethod - def delete(cls, attachments): - cls.check_access([a.id for a in attachments], mode='delete') - super(Attachment, cls).delete(attachments) - - @classmethod - def write(cls, attachments, values, *args): - all_attachments = [] - actions = iter((attachments, values) + args) - for records, _ in zip(actions, actions): - all_attachments += records - cls.check_access([a.id for a in all_attachments], mode='write') - super(Attachment, cls).write(attachments, values, *args) - cls.check_access(all_attachments, mode='write') - - @classmethod - def create(cls, vlist): - attachments = super(Attachment, cls).create(vlist) - cls.check_access([a.id for a in attachments], mode='create') - return attachments - - @classmethod - def view_header_get(cls, value, view_type='form'): - pool = Pool() - Model = pool.get('ir.model') - value = super(Attachment, cls).view_header_get(value, - view_type=view_type) - resource = Transaction().context.get('resource') - if resource: - model_name, record_id = resource.split(',', 1) - ir_model, = Model.search([('model', '=', model_name)]) - Resource = pool.get(model_name) - record = Resource(int(record_id)) - value = '%s - %s - %s' % (ir_model.name, record.rec_name, value) - return value diff -r ad3cb7be7667 trytond/ir/resource.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trytond/trytond/ir/resource.py Mon Jan 08 14:40:45 2018 +0100 @@ -0,0 +1,98 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. +from sql.conditionals import Coalesce + +from ..model import ModelSQL, ModelView, fields +from ..pool import Pool +from ..transaction import Transaction +from ..pyson import Eval + +__all__ = ['ResourceMixin'] + + +class ResourceMixin(ModelSQL, ModelView): + + resource = fields.Reference('Resource', selection='get_models', + required=True, select=True) + last_user = fields.Function(fields.Char('Last User', + states={ + 'invisible': ~Eval('last_user'), + }), + 'get_last_user') + last_modification = fields.Function(fields.DateTime('Last Modification', + states={ + 'invisible': ~Eval('last_modification'), + }), + 'get_last_modification') + + @classmethod + def __setup__(cls): + super(ResourceMixin, cls).__setup__() + cls._order.insert(0, ('last_modification', 'DESC')) + + @staticmethod + def default_resource(): + return Transaction().context.get('resource') + + @staticmethod + def get_models(): + pool = Pool() + Model = pool.get('ir.model') + ModelAccess = pool.get('ir.model.access') + models = Model.search([]) + access = ModelAccess.get_access([m.model for m in models]) + return [(m.model, m.name) for m in models if access[m.model]['read']] + + def get_last_user(self, name): + return (self.write_uid.rec_name if self.write_uid + else self.create_uid.rec_name) + + def get_last_modification(self, name): + return (self.write_date if self.write_date else self.create_date + ).replace(microsecond=0) + + @staticmethod + def order_last_modification(tables): + table, _ = tables[None] + return [Coalesce(table.write_date, table.create_date)] + + @classmethod + def check_access(cls, ids, mode='read'): + pool = Pool() + ModelAccess = pool.get('ir.model.access') + if ((Transaction().user == 0) + or not Transaction().context.get('_check_access')): + return + model_names = set() + with Transaction().set_context(_check_access=False): + for record in cls.browse(ids): + if record.resource: + model_names.add(str(record.resource).split(',')[0]) + for model_name in model_names: + ModelAccess.check(model_name, mode=mode) + + @classmethod + def read(cls, ids, fields_names=None): + cls.check_access(ids, mode='read') + return super(ResourceMixin, cls).read(ids, fields_names=fields_names) + + @classmethod + def delete(cls, records): + cls.check_access([a.id for a in records], mode='delete') + super(ResourceMixin, cls).delete(records) + + @classmethod + def write(cls, records, values, *args): + all_records = [] + actions = iter((records, values) + args) + for other_records, _ in zip(actions, actions): + all_records += other_records + cls.check_access([a.id for a in all_records], mode='write') + super(ResourceMixin, cls).write(records, values, *args) + cls.check_access(all_records, mode='write') + + @classmethod + def create(cls, vlist): + records = super(ResourceMixin, cls).create(vlist) + cls.check_access([r.id for r in records], mode='create') + return records diff -r ad3cb7be7667 trytond/model/fields/binary.py --- a/trytond/trytond/model/fields/binary.py Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/trytond/model/fields/binary.py Mon Jan 08 14:40:45 2018 +0100 @@ -1,10 +1,13 @@ -# This file is part of Tryton. The COPYRIGHT file at the top level of -# this repository contains the full copyright notices and license terms. -from sql import Query, Expression + +#This file is part of Tryton. The COPYRIGHT file at the top level of +#this repository contains the full copyright notices and license terms. +from sql import Query, Expression, Column, Null from .field import Field, SQLType from ...transaction import Transaction from ... import backend +from ...tools import grouped_slice, reduce_ids +from ...filestore import filestore class Binary(Field): @@ -17,20 +20,21 @@ def __init__(self, string='', help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, filename=None, context=None, - loading='lazy'): + loading='lazy', file_id=None, store_prefix=None): if filename is not None: self.filename = filename if depends is None: depends = [filename] else: depends.append(filename) + self.file_id = file_id + self.store_prefix = store_prefix super(Binary, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) - @classmethod - def get(cls, ids, model, name, values=None): + def get(self, ids, model, name, values=None): ''' Convert the binary value into ``bytes`` @@ -42,15 +46,45 @@ ''' if values is None: values = {} + transaction = Transaction() res = {} - converter = cls.cast + converter = self.cast default = None - format_ = Transaction().context.pop('%s.%s' % (model.__name__, name), - '') + format_ = Transaction().context.get( + '%s.%s' % (model.__name__, name), '') if format_ == 'size': converter = len default = 0 + + if self.file_id: + table = model.__table__() + cursor = transaction.cursor + + prefix = self.store_prefix + if prefix is None: + prefix = transaction.database.database_name + + if format_ == 'size': + store_func = filestore.size + else: + def store_func(id, prefix): + return self.cast(filestore.get(id, prefix=prefix)) + + for sub_ids in grouped_slice(ids): + cursor.execute(*table.select( + table.id, Column(table, self.file_id), + where=reduce_ids(table.id, sub_ids) + & (Column(table, self.file_id) != Null) + & (Column(table, self.file_id) != ''))) + for record_id, file_id in cursor.fetchall(): + try: + res[record_id] = store_func(file_id, prefix) + except (IOError, OSError): + pass + for i in values: + if i['id'] in res: + continue value = i[name] if value: if isinstance(value, unicode): @@ -63,6 +97,27 @@ res.setdefault(i, default) return res + def set(self, Model, name, ids, value, *args): + transaction = Transaction() + table = Model.__table__() + cursor = transaction.cursor + + prefix = self.store_prefix + if prefix is None: + prefix = transaction.database.database_name + + args = iter((ids, value) + args) + for ids, value in zip(args, args): + if self.file_id: + columns = [Column(table, self.file_id), Column(table, name)] + values = [ + filestore.set(value, prefix) if value else None, None] + else: + columns = [Column(table, name)] + values = [self.sql_format(value)] + cursor.execute(*table.update(columns, values, + where=reduce_ids(table.id, ids))) + @staticmethod def sql_format(value): if isinstance(value, (Query, Expression)): diff -r ad3cb7be7667 trytond/model/fields/field.py --- a/trytond/trytond/model/fields/field.py Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/trytond/model/fields/field.py Mon Jan 08 14:40:45 2018 +0100 @@ -253,7 +253,7 @@ return value def sql_type(self): - raise NotImplementedError + return None def sql_column(self, table): return Column(table, self.name) diff -r ad3cb7be7667 trytond/model/fields/function.py --- a/trytond/trytond/model/fields/function.py Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/trytond/model/fields/function.py Mon Jan 08 14:40:45 2018 +0100 @@ -62,6 +62,9 @@ def sql_type(self): raise AttributeError + def sql_type(self): + return None + def convert_domain(self, domain, tables, Model): name, operator, value = domain[:3] if not self.searcher: diff -r ad3cb7be7667 trytond/model/modelsql.py --- a/trytond/trytond/model/modelsql.py Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/trytond/model/modelsql.py Mon Jan 08 14:40:45 2018 +0100 @@ -146,10 +146,11 @@ for field_name, field in cls._fields.iteritems(): if field_name == 'id': continue + sql_type = field.sql_type() + if not sql_type: + continue default_fun = None - if hasattr(field, 'set'): - continue - sql_type = field.sql_type() + if field_name in cls._defaults: default_fun = cls._defaults[field_name] @@ -200,6 +201,10 @@ field_name, action=field.select and 'add' or 'remove') required = field.required + # Do not set 'NOT NULL' for Binary field stored in the filestore + # as the database column will be left empty. + if isinstance(field, fields.Binary) and field.file_id: + required = False table.not_null_action( field_name, action=required and 'add' or 'remove') @@ -228,7 +233,7 @@ cursor.execute(*history_table.select(history_table.id)) if not cursor.fetchone(): columns = [n for n, f in cls._fields.iteritems() - if not hasattr(f, 'set')] + if f.sql_type()] cursor.execute(*history_table.insert( [Column(history_table, c) for c in columns], table.select(*(Column(table, c) @@ -275,7 +280,7 @@ field = cls._fields[field_name] # Check required fields if (field.required - and not hasattr(field, 'set') + and field.sql_type() and field_name not in ('create_uid', 'create_date')): if values.get(field_name) is None: cls.raise_user_error('required_field', @@ -355,7 +360,7 @@ 'write_date': cls.write_date, } for fname, field in sorted(fields.iteritems()): - if hasattr(field, 'set'): + if not field.sql_type(): continue columns.append(Column(table, fname)) hcolumns.append(Column(history, fname)) @@ -385,7 +390,7 @@ columns = [] hcolumns = [] fnames = sorted(n for n, f in cls._fields.iteritems() - if not hasattr(f, 'set')) + if f.sql_type()) for fname in fnames: columns.append(Column(table, fname)) if fname == 'write_uid': @@ -650,7 +655,7 @@ columns = [] for f in fields_names + fields_related.keys() + datetime_fields: field = cls._fields.get(f) - if field and not hasattr(field, 'set'): + if field and field.sql_type(): columns.append(field.sql_column(table).as_(f)) elif f == '_timestamp' and not table_query: sql_type = fields.Char('timestamp').sql_type().base diff -r ad3cb7be7667 trytond/tools/misc.py --- a/trytond/trytond/tools/misc.py Mon Jan 08 13:34:08 2018 +0100 +++ b/trytond/trytond/tools/misc.py Mon Jan 08 14:40:45 2018 +0100 @@ -13,6 +13,7 @@ from itertools import islice import types import urllib +import importlib from sql import Literal from sql.operators import Or @@ -391,3 +392,17 @@ type_ = klass.__dict__.get(method) if type_ is not None: return isinstance(type_, types.FunctionType) + + +def resolve(name): + "Resolve a dotted name to a global object." + name = name.split('.') + used = name.pop(0) + found = importlib.import_module(used) + for n in name: + used = used + '.' + n + try: + found = getattr(found, n) + except AttributeError: + found = importlib.import_module(used) + return found