diff -r ab268bcaf539 trytond/doc/ref/models/fields.rst --- a/trytond/doc/ref/models/fields.rst Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/doc/ref/models/fields.rst Thu Mar 15 11:47:28 2018 +0100 @@ -165,7 +165,8 @@ .. method:: Field.sql_type() Return the namedtuple('SQLType', 'base type') which defines the SQL type to - use for creation and casting. + use for creation and casting. Or `None` if the field is not stored in the + database. .. method:: Field.sql_column(table) @@ -378,7 +379,7 @@ separated by a dot and its value is the string `size` then the read value is the size instead of the content. -:class:`Binary` has one extra optional argument: +:class:`Binary` has three extra optional arguments: .. attribute:: Binary.filename @@ -387,6 +388,21 @@ filename is hidden, and the "Open" button is hidden when the widget is set to "image"). +.. attribute:: Binary.file_id + + Name of the field that holds the `FileStore` identifier. Default value is + `None` which means the data is stored in the database. The field must be on + the same table and accept `char` values. + +.. warning:: + Switching from database to file-store is supported transparently. But + switching from file-store to database is not supported without manually + upload to the database all the files. + +.. attribute:: Binary.store_prefix + + The prefix to use with the `FileStore`. Default value is `None` which means + the database name is used. Selection --------- diff -r ab268bcaf539 trytond/doc/topics/configuration.rst --- a/trytond/doc/topics/configuration.rst Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/doc/topics/configuration.rst Thu Mar 15 11:47:28 2018 +0100 @@ -227,6 +227,24 @@ Default: `pipe,name=trytond;urp;StarOffice.ComponentContext` +attachment +---------- + +Defines how to store the attachments + +filestore +~~~~~~~~~ + +A boolean value to store attachment in the :ref:`FileStore `. + +Default: `True` + +store_prefix +~~~~~~~~~~~~ + +The prefix to use with the `FileStore`. + +Default: `None` .. _JSON-RPC: http://en.wikipedia.org/wiki/JSON-RPC .. _XML-RPC: http://en.wikipedia.org/wiki/XML-RPC diff -r ab268bcaf539 trytond/trytond/ir/attachment.py --- a/trytond/trytond/ir/attachment.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/ir/attachment.py Thu Mar 15 11:47:28 2018 +0100 @@ -1,15 +1,15 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -import os -import hashlib +from sql import Null from sql.operators import Concat from ..model import ModelView, ModelSQL, fields, Unique -from ..config import config from .. import backend from ..transaction import Transaction from ..pyson import Eval from .resource import ResourceMixin +from ..filestore import filestore +from ..config import config __all__ = [ 'Attachment', @@ -22,6 +22,13 @@ except StopIteration: return '' +if config.getboolean('attachment', 'filestore', default=True): + file_id = 'file_id' + store_prefix = config.get('attachment', 'store_prefix', default=None) +else: + file_id = None + store_prefix = None + class Attachment(ResourceMixin, ModelSQL, ModelView): "Attachment" @@ -39,8 +46,7 @@ link = fields.Char('Link', states={ 'invisible': Eval('type') != 'link', }, depends=['type']) - digest = fields.Char('Digest', size=32) - collision = fields.Integer('Collision') + file_id = fields.Char('File ID', readonly=True) data_size = fields.Function(fields.Integer('Data size', states={ 'invisible': Eval('type') != 'data', }, depends=['type']), 'get_data') @@ -77,14 +83,26 @@ table.drop_column('res_model') table.drop_column('res_id') + # Migration from 4.0: merge digest and collision into file_id + if table.column_exist('digest') and table.column_exist('collision'): + cursor.execute(*attachment.update( + [attachment.file_id], + [attachment.digest], + where=(attachment.collision == 0) + | (attachment.collision == Null))) + cursor.execute(*attachment.update( + [attachment.file_id], + [Concat(Concat(attachment.digest, '-'), + attachment.collision)], + where=(attachment.collision != 0) + & (attachment.collision != Null))) + table.drop_column('digest') + table.drop_column('collision') + @staticmethod def default_type(): return 'data' - @staticmethod - def default_collision(): - return 0 - def get_data(self, name): db_name = Transaction().database.name format_ = Transaction().context.get( @@ -92,22 +110,16 @@ value = None if name == 'data_size' or format_ == 'size': value = 0 - if self.digest: - filename = self.digest - if self.collision: - filename = filename + '-' + str(self.collision) - filename = os.path.join(config.get('database', 'path'), db_name, - filename[0:2], filename[2:4], filename) + if self.file_id: if name == 'data_size' or format_ == 'size': try: - statinfo = os.stat(filename) - value = statinfo.st_size + value = filestore.size(self.file_id, prefix=db_name) except OSError: pass else: try: - with open(filename, 'rb') as file_p: - value = fields.Binary.cast(file_p.read()) + value = fields.Binary.cast( + filestore.get(self.file_id, prefix=db_name)) except IOError: pass return value @@ -117,51 +129,11 @@ if value is None: return transaction = Transaction() - cursor = transaction.connection.cursor() - table = cls.__table__() db_name = transaction.database.name - directory = os.path.join(config.get('database', 'path'), db_name) - if not os.path.isdir(directory): - os.makedirs(directory, 0770) - digest = hashlib.md5(value).hexdigest() - directory = os.path.join(directory, digest[0:2], digest[2:4]) - if not os.path.isdir(directory): - os.makedirs(directory, 0770) - filename = os.path.join(directory, digest) - collision = 0 - if os.path.isfile(filename): - with open(filename, 'rb') as file_p: - data = file_p.read() - if value != data: - cursor.execute(*table.select(table.collision, - where=(table.digest == digest) - & (table.collision != 0), - group_by=table.collision, - order_by=table.collision)) - collision2 = 0 - for row in cursor.fetchall(): - collision2 = row[0] - filename = os.path.join(directory, - digest + '-' + str(collision2)) - if os.path.isfile(filename): - with open(filename, 'rb') as file_p: - data = file_p.read() - if value == data: - collision = collision2 - break - if collision == 0: - collision = collision2 + 1 - filename = os.path.join(directory, - digest + '-' + str(collision)) - with open(filename, 'wb') as file_p: - file_p.write(value) - else: - with open(filename, 'wb') as file_p: - file_p.write(value) + file_id = filestore.set(value, prefix=db_name) cls.write(attachments, { - 'digest': digest, - 'collision': collision, - }) + 'file_id': file_id, + }) @fields.depends('description') def on_change_with_summary(self, name=None): diff -r ab268bcaf539 trytond/trytond/model/fields/binary.py --- a/trytond/trytond/model/fields/binary.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/model/fields/binary.py Thu Mar 15 11:47:28 2018 +0100 @@ -1,10 +1,12 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from sql import Query, Expression +from sql import Query, Expression, Column, Null from .field import Field, SQLType from ...transaction import Transaction from ... import backend +from ...tools import grouped_slice, reduce_ids +from ...filestore import filestore class Binary(Field): @@ -16,21 +18,22 @@ def __init__(self, string='', help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, - on_change_with=None, depends=None, filename=None, context=None, - loading='lazy'): + on_change_with=None, depends=None, context=None, loading='lazy', + filename=None, file_id=None, store_prefix=None): if filename is not None: self.filename = filename if depends is None: depends = [filename] else: depends.append(filename) + self.file_id = file_id + self.store_prefix = store_prefix super(Binary, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) - @classmethod - def get(cls, ids, model, name, values=None): + def get(self, ids, model, name, values=None): ''' Convert the binary value into ``bytes`` @@ -42,15 +45,45 @@ ''' if values is None: values = {} + transaction = Transaction() res = {} - converter = cls.cast + converter = self.cast default = None format_ = Transaction().context.get( '%s.%s' % (model.__name__, name), '') if format_ == 'size': converter = len default = 0 + + if self.file_id: + table = model.__table__() + cursor = transaction.connection.cursor() + + prefix = self.store_prefix + if prefix is None: + prefix = transaction.database.name + + if format_ == 'size': + store_func = filestore.size + else: + def store_func(id, prefix): + return self.cast(filestore.get(id, prefix=prefix)) + + for sub_ids in grouped_slice(ids): + cursor.execute(*table.select( + table.id, Column(table, self.file_id), + where=reduce_ids(table.id, sub_ids) + & (Column(table, self.file_id) != Null) + & (Column(table, self.file_id) != ''))) + for record_id, file_id in cursor: + try: + res[record_id] = store_func(file_id, prefix) + except (IOError, OSError): + pass + for i in values: + if i['id'] in res: + continue value = i[name] if value: if isinstance(value, unicode): @@ -63,6 +96,27 @@ res.setdefault(i, default) return res + def set(self, Model, name, ids, value, *args): + transaction = Transaction() + table = Model.__table__() + cursor = transaction.connection.cursor() + + prefix = self.store_prefix + if prefix is None: + prefix = transaction.database.name + + args = iter((ids, value) + args) + for ids, value in zip(args, args): + if self.file_id: + columns = [Column(table, self.file_id), Column(table, name)] + values = [ + filestore.set(value, prefix) if value else None, None] + else: + columns = [Column(table, name)] + values = [self.sql_format(value)] + cursor.execute(*table.update(columns, values, + where=reduce_ids(table.id, ids))) + @staticmethod def sql_format(value): if isinstance(value, (Query, Expression)): diff -r ab268bcaf539 trytond/trytond/model/fields/field.py --- a/trytond/trytond/model/fields/field.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/model/fields/field.py Thu Mar 15 11:47:28 2018 +0100 @@ -255,7 +255,7 @@ return value def sql_type(self): - raise NotImplementedError + return None def sql_column(self, table): return Column(table, self.name) diff -r ab268bcaf539 trytond/trytond/model/fields/function.py --- a/trytond/trytond/model/fields/function.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/model/fields/function.py Thu Mar 15 11:47:28 2018 +0100 @@ -58,9 +58,8 @@ return setattr(self._field, name, value) - @property def sql_type(self): - raise AttributeError + return None def convert_domain(self, domain, tables, Model): name, operator, value = domain[:3] diff -r ab268bcaf539 trytond/trytond/model/modelsql.py --- a/trytond/trytond/model/modelsql.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/model/modelsql.py Thu Mar 15 11:47:28 2018 +0100 @@ -145,10 +145,10 @@ for field_name, field in cls._fields.iteritems(): if field_name == 'id': continue + sql_type = field.sql_type() + if not sql_type: + continue default_fun = None - if hasattr(field, 'set'): - continue - sql_type = field.sql_type() if field_name in cls._defaults: default_fun = cls._defaults[field_name] @@ -199,6 +199,10 @@ field_name, action=field.select and 'add' or 'remove') required = field.required + # Do not set 'NOT NULL' for Binary field stored in the filestore + # as the database column will be left empty. + if isinstance(field, fields.Binary) and field.file_id: + required = False table.not_null_action( field_name, action=required and 'add' or 'remove') @@ -228,7 +232,7 @@ cursor.execute(*history_table.select(history_table.id)) if not cursor.fetchone(): columns = [n for n, f in cls._fields.iteritems() - if not hasattr(f, 'set')] + if f.sql_type()] cursor.execute(*history_table.insert( [Column(history_table, c) for c in columns], sql_table.select(*(Column(sql_table, c) @@ -274,7 +278,7 @@ field = cls._fields[field_name] # Check required fields if (field.required - and not hasattr(field, 'set') + and field.sql_type() and field_name not in ('create_uid', 'create_date')): if values.get(field_name) is None: cls.raise_user_error('required_field', @@ -354,7 +358,7 @@ 'write_date': cls.write_date, } for fname, field in sorted(fields.iteritems()): - if hasattr(field, 'set'): + if not field.sql_type(): continue columns.append(Column(table, fname)) hcolumns.append(Column(history, fname)) @@ -384,7 +388,7 @@ columns = [] hcolumns = [] fnames = sorted(n for n, f in cls._fields.iteritems() - if not hasattr(f, 'set')) + if f.sql_type()) for fname in fnames: columns.append(Column(table, fname)) if fname == 'write_uid': @@ -663,7 +667,7 @@ columns = [] for f in fields_names + fields_related.keys() + datetime_fields: field = cls._fields.get(f) - if field and not hasattr(field, 'set'): + if field and field.sql_type(): columns.append(field.sql_column(table).as_(f)) elif f == '_timestamp' and not table_query: sql_type = fields.Char('timestamp').sql_type().base @@ -715,7 +719,7 @@ if field == '_timestamp': continue if (getattr(cls._fields[field], 'translate', False) - and not hasattr(field, 'set')): + and not hasattr(field, 'get')): translations = Translation.get_ids(cls.__name__ + ',' + field, 'model', Transaction().language, ids) for row in result: diff -r ab268bcaf539 trytond/trytond/tests/test.py --- a/trytond/trytond/tests/test.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/tests/test.py Thu Mar 15 11:47:28 2018 +0100 @@ -35,7 +35,7 @@ 'Property', 'Selection', 'SelectionRequired', 'DictSchema', 'Dict', 'DictDefault', 'DictRequired', - 'Binary', 'BinaryDefault', 'BinaryRequired', + 'Binary', 'BinaryDefault', 'BinaryRequired', 'BinaryFileStorage', 'Many2OneDomainValidation', 'Many2OneTarget', 'Many2OneOrderBy', 'Many2OneSearch', 'Many2OneTree', 'Many2OneMPTT', ] @@ -727,6 +727,13 @@ binary = fields.Binary('Binary Required', required=True) +class BinaryFileStorage(ModelSQL): + "Binary in FileStorage" + __name__ = 'test.binary_filestorage' + binary = fields.Binary('Binary', file_id='binary_id') + binary_id = fields.Char('Binary ID') + + class Many2OneTarget(ModelSQL): "Many2One Domain Validation Target" __name__ = 'test.many2one_target' diff -r ab268bcaf539 trytond/trytond/tests/test_fields.py --- a/trytond/trytond/tests/test_fields.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/tests/test_fields.py Thu Mar 15 11:47:28 2018 +0100 @@ -11,12 +11,16 @@ sys.modules['cdecimal'] = decimal import unittest import datetime +import shutil +import tempfile + from decimal import Decimal from trytond.tests.test_tryton import install_module, with_transaction from trytond.transaction import Transaction from trytond.exceptions import UserError from trytond.model import fields from trytond.pool import Pool +from trytond.config import config class FieldsTestCase(unittest.TestCase): @@ -26,6 +30,13 @@ def setUpClass(cls): install_module('tests') + def setUp(self): + path = config.get('database', 'path') + dtemp = tempfile.mkdtemp() + config.set('database', 'path', dtemp) + self.addCleanup(config.set, 'database', 'path', path) + self.addCleanup(shutil.rmtree, dtemp) + @with_transaction() def test_boolean(self): 'Test Boolean' @@ -3287,6 +3298,31 @@ [{'binary': fields.Binary.cast(b'')}]) @with_transaction() + def test_binary_filestorage(self): + "Test Binary FileStorage" + pool = Pool() + Binary = pool.get('test.binary_filestorage') + transaction = Transaction() + + bin1, = Binary.create([{ + 'binary': fields.Binary.cast(b'foo'), + }]) + self.assertEqual(bin1.binary, fields.Binary.cast(b'foo')) + self.assertTrue(bin1.binary_id) + + Binary.write([bin1], {'binary': fields.Binary.cast(b'bar')}) + self.assertEqual(bin1.binary, fields.Binary.cast(b'bar')) + + with transaction.set_context( + {'test.binary_filestorage.binary': 'size'}): + bin1_size = Binary(bin1.id) + self.assertEqual(bin1_size.binary, len(b'bar')) + + Binary.write([bin1], {'binary': None}) + self.assertEqual(bin1.binary, None) + self.assertEqual(bin1.binary_id, None) + + @with_transaction() def test_many2one(self): 'Test Many2One' pool = Pool() diff -r ab268bcaf539 trytond/trytond/tools/misc.py --- a/trytond/trytond/tools/misc.py Tue Mar 13 10:44:15 2018 +0100 +++ b/trytond/trytond/tools/misc.py Thu Mar 15 11:47:28 2018 +0100 @@ -12,6 +12,7 @@ import types import io import warnings +import importlib from sql import Literal from sql.operators import Or @@ -284,3 +285,17 @@ type_ = klass.__dict__.get(method) if type_ is not None: return isinstance(type_, types.FunctionType) + + +def resolve(name): + "Resolve a dotted name to a global object." + name = name.split('.') + used = name.pop(0) + found = importlib.import_module(used) + for n in name: + used = used + '.' + n + try: + found = getattr(found, n) + except AttributeError: + found = importlib.import_module(used) + return found diff -r ab268bcaf539 trytond/trytond/filestore.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trytond/trytond/filestore.py Thu Mar 15 12:00:05 2018 +0100 @@ -0,0 +1,67 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. +import os +import hashlib + +from trytond.config import config +from trytond.tools import resolve + +__all__ = ['filestore'] + + +class FileStore(object): + + def get(self, id, prefix=''): + filename = self._filename(id, prefix) + with open(filename, 'rb') as fp: + return fp.read() + + def getmany(self, ids, prefix=''): + return [self.get(id, prefix) for id in ids] + + def size(self, id, prefix=''): + filename = self._filename(id, prefix) + statinfo = os.stat(filename) + return statinfo.st_size + + def sizemany(self, ids, prefix=''): + return [self.size(id, prefix) for id in ids] + + def set(self, data, prefix=''): + id = self._id(data) + filename = self._filename(id, prefix) + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname, 0770) + + collision = 0 + while True: + basename = os.path.basename(filename) + if os.path.exists(filename): + if data != self.get(basename, prefix): + collision += 1 + filename = self._filename( + '%s-%s' % (id, collision), prefix) + continue + else: + with open(filename, 'wb')as fp: + fp.write(data) + return basename + + def setmany(self, data, prefix=''): + return [self.set(d, prefix) for d in data] + + def _filename(self, id, prefix): + path = os.path.normpath(config.get('database', 'path')) + filename = os.path.join(path, prefix, id[0:2], id[2:4], id) + filename = os.path.normpath(filename) + if not filename.startswith(path): + raise ValueError('Bad prefix') + return filename + + def _id(self, data): + return hashlib.md5(data).hexdigest() + +if config.get('database', 'class'): + FileStore = resolve(config.get('database', 'class')) +filestore = FileStore()