NaNtic
/
trytond-patches
mirror of https://github.com/NaN-tic/trytond-patches.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
trytond-patches/filestore.diff

651 lines
23 KiB
Diff
Raw Permalink Blame History

diff -r ab268bcaf539 trytond/doc/ref/models/fields.rst
--- a/trytond/doc/ref/models/fields.rst Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/doc/ref/models/fields.rst Thu Mar 15 11:47:28 2018 +0100
@@ -165,7 +165,8 @@
.. method:: Field.sql_type()
Return the namedtuple('SQLType', 'base type') which defines the SQL type to
- use for creation and casting.
+ use for creation and casting. Or `None` if the field is not stored in the
+ database.
.. method:: Field.sql_column(table)
@@ -378,7 +379,7 @@
separated by a dot and its value is the string `size` then the read value
is the size instead of the content.
-:class:`Binary` has one extra optional argument:
+:class:`Binary` has three extra optional arguments:
.. attribute:: Binary.filename
@@ -387,6 +388,21 @@
filename is hidden, and the "Open" button is hidden when the widget is set
to "image").
+.. attribute:: Binary.file_id
+
+ Name of the field that holds the `FileStore` identifier. Default value is
+ `None` which means the data is stored in the database. The field must be on
+ the same table and accept `char` values.
+
+.. warning::
+ Switching from database to file-store is supported transparently. But
+ switching from file-store to database is not supported without manually
+ upload to the database all the files.
+
+.. attribute:: Binary.store_prefix
+
+ The prefix to use with the `FileStore`. Default value is `None` which means
+ the database name is used.
Selection
---------
diff -r ab268bcaf539 trytond/doc/topics/configuration.rst
--- a/trytond/doc/topics/configuration.rst Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/doc/topics/configuration.rst Thu Mar 15 11:47:28 2018 +0100
@@ -227,6 +227,24 @@
Default: `pipe,name=trytond;urp;StarOffice.ComponentContext`
+attachment
+----------
+
+Defines how to store the attachments
+
+filestore
+~~~~~~~~~
+
+A boolean value to store attachment in the :ref:`FileStore <ref-filestore>`.
+
+Default: `True`
+
+store_prefix
+~~~~~~~~~~~~
+
+The prefix to use with the `FileStore`.
+
+Default: `None`
.. _JSON-RPC: http://en.wikipedia.org/wiki/JSON-RPC
.. _XML-RPC: http://en.wikipedia.org/wiki/XML-RPC
diff -r ab268bcaf539 trytond/trytond/ir/attachment.py
--- a/trytond/trytond/ir/attachment.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/ir/attachment.py Thu Mar 15 11:47:28 2018 +0100
@@ -1,15 +1,15 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-import os
-import hashlib
+from sql import Null
from sql.operators import Concat
from ..model import ModelView, ModelSQL, fields, Unique
-from ..config import config
from .. import backend
from ..transaction import Transaction
from ..pyson import Eval
from .resource import ResourceMixin
+from ..filestore import filestore
+from ..config import config
__all__ = [
'Attachment',
@@ -22,6 +22,13 @@
except StopIteration:
return ''
+if config.getboolean('attachment', 'filestore', default=True):
+ file_id = 'file_id'
+ store_prefix = config.get('attachment', 'store_prefix', default=None)
+else:
+ file_id = None
+ store_prefix = None
+
class Attachment(ResourceMixin, ModelSQL, ModelView):
"Attachment"
@@ -39,8 +46,7 @@
link = fields.Char('Link', states={
'invisible': Eval('type') != 'link',
}, depends=['type'])
- digest = fields.Char('Digest', size=32)
- collision = fields.Integer('Collision')
+ file_id = fields.Char('File ID', readonly=True)
data_size = fields.Function(fields.Integer('Data size', states={
'invisible': Eval('type') != 'data',
}, depends=['type']), 'get_data')
@@ -77,14 +83,26 @@
table.drop_column('res_model')
table.drop_column('res_id')
+ # Migration from 4.0: merge digest and collision into file_id
+ if table.column_exist('digest') and table.column_exist('collision'):
+ cursor.execute(*attachment.update(
+ [attachment.file_id],
+ [attachment.digest],
+ where=(attachment.collision == 0)
+ | (attachment.collision == Null)))
+ cursor.execute(*attachment.update(
+ [attachment.file_id],
+ [Concat(Concat(attachment.digest, '-'),
+ attachment.collision)],
+ where=(attachment.collision != 0)
+ & (attachment.collision != Null)))
+ table.drop_column('digest')
+ table.drop_column('collision')
+
@staticmethod
def default_type():
return 'data'
- @staticmethod
- def default_collision():
- return 0
-
def get_data(self, name):
db_name = Transaction().database.name
format_ = Transaction().context.get(
@@ -92,22 +110,16 @@
value = None
if name == 'data_size' or format_ == 'size':
value = 0
- if self.digest:
- filename = self.digest
- if self.collision:
- filename = filename + '-' + str(self.collision)
- filename = os.path.join(config.get('database', 'path'), db_name,
- filename[0:2], filename[2:4], filename)
+ if self.file_id:
if name == 'data_size' or format_ == 'size':
try:
- statinfo = os.stat(filename)
- value = statinfo.st_size
+ value = filestore.size(self.file_id, prefix=db_name)
except OSError:
pass
else:
try:
- with open(filename, 'rb') as file_p:
- value = fields.Binary.cast(file_p.read())
+ value = fields.Binary.cast(
+ filestore.get(self.file_id, prefix=db_name))
except IOError:
pass
return value
@@ -117,51 +129,11 @@
if value is None:
return
transaction = Transaction()
- cursor = transaction.connection.cursor()
- table = cls.__table__()
db_name = transaction.database.name
- directory = os.path.join(config.get('database', 'path'), db_name)
- if not os.path.isdir(directory):
- os.makedirs(directory, 0770)
- digest = hashlib.md5(value).hexdigest()
- directory = os.path.join(directory, digest[0:2], digest[2:4])
- if not os.path.isdir(directory):
- os.makedirs(directory, 0770)
- filename = os.path.join(directory, digest)
- collision = 0
- if os.path.isfile(filename):
- with open(filename, 'rb') as file_p:
- data = file_p.read()
- if value != data:
- cursor.execute(*table.select(table.collision,
- where=(table.digest == digest)
- & (table.collision != 0),
- group_by=table.collision,
- order_by=table.collision))
- collision2 = 0
- for row in cursor.fetchall():
- collision2 = row[0]
- filename = os.path.join(directory,
- digest + '-' + str(collision2))
- if os.path.isfile(filename):
- with open(filename, 'rb') as file_p:
- data = file_p.read()
- if value == data:
- collision = collision2
- break
- if collision == 0:
- collision = collision2 + 1
- filename = os.path.join(directory,
- digest + '-' + str(collision))
- with open(filename, 'wb') as file_p:
- file_p.write(value)
- else:
- with open(filename, 'wb') as file_p:
- file_p.write(value)
+ file_id = filestore.set(value, prefix=db_name)
cls.write(attachments, {
- 'digest': digest,
- 'collision': collision,
- })
+ 'file_id': file_id,
+ })
@fields.depends('description')
def on_change_with_summary(self, name=None):
diff -r ab268bcaf539 trytond/trytond/model/fields/binary.py
--- a/trytond/trytond/model/fields/binary.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/model/fields/binary.py Thu Mar 15 11:47:28 2018 +0100
@@ -1,10 +1,12 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-from sql import Query, Expression
+from sql import Query, Expression, Column, Null
from .field import Field, SQLType
from ...transaction import Transaction
from ... import backend
+from ...tools import grouped_slice, reduce_ids
+from ...filestore import filestore
class Binary(Field):
@@ -16,21 +18,22 @@
def __init__(self, string='', help='', required=False, readonly=False,
domain=None, states=None, select=False, on_change=None,
- on_change_with=None, depends=None, filename=None, context=None,
- loading='lazy'):
+ on_change_with=None, depends=None, context=None, loading='lazy',
+ filename=None, file_id=None, store_prefix=None):
if filename is not None:
self.filename = filename
if depends is None:
depends = [filename]
else:
depends.append(filename)
+ self.file_id = file_id
+ self.store_prefix = store_prefix
super(Binary, self).__init__(string=string, help=help,
required=required, readonly=readonly, domain=domain, states=states,
select=select, on_change=on_change, on_change_with=on_change_with,
depends=depends, context=context, loading=loading)
- @classmethod
- def get(cls, ids, model, name, values=None):
+ def get(self, ids, model, name, values=None):
'''
Convert the binary value into ``bytes``
@@ -42,15 +45,45 @@
'''
if values is None:
values = {}
+ transaction = Transaction()
res = {}
- converter = cls.cast
+ converter = self.cast
default = None
format_ = Transaction().context.get(
'%s.%s' % (model.__name__, name), '')
if format_ == 'size':
converter = len
default = 0
+
+ if self.file_id:
+ table = model.__table__()
+ cursor = transaction.connection.cursor()
+
+ prefix = self.store_prefix
+ if prefix is None:
+ prefix = transaction.database.name
+
+ if format_ == 'size':
+ store_func = filestore.size
+ else:
+ def store_func(id, prefix):
+ return self.cast(filestore.get(id, prefix=prefix))
+
+ for sub_ids in grouped_slice(ids):
+ cursor.execute(*table.select(
+ table.id, Column(table, self.file_id),
+ where=reduce_ids(table.id, sub_ids)
+ & (Column(table, self.file_id) != Null)
+ & (Column(table, self.file_id) != '')))
+ for record_id, file_id in cursor:
+ try:
+ res[record_id] = store_func(file_id, prefix)
+ except (IOError, OSError):
+ pass
+
for i in values:
+ if i['id'] in res:
+ continue
value = i[name]
if value:
if isinstance(value, unicode):
@@ -63,6 +96,27 @@
res.setdefault(i, default)
return res
+ def set(self, Model, name, ids, value, *args):
+ transaction = Transaction()
+ table = Model.__table__()
+ cursor = transaction.connection.cursor()
+
+ prefix = self.store_prefix
+ if prefix is None:
+ prefix = transaction.database.name
+
+ args = iter((ids, value) + args)
+ for ids, value in zip(args, args):
+ if self.file_id:
+ columns = [Column(table, self.file_id), Column(table, name)]
+ values = [
+ filestore.set(value, prefix) if value else None, None]
+ else:
+ columns = [Column(table, name)]
+ values = [self.sql_format(value)]
+ cursor.execute(*table.update(columns, values,
+ where=reduce_ids(table.id, ids)))
+
@staticmethod
def sql_format(value):
if isinstance(value, (Query, Expression)):
diff -r ab268bcaf539 trytond/trytond/model/fields/field.py
--- a/trytond/trytond/model/fields/field.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/model/fields/field.py Thu Mar 15 11:47:28 2018 +0100
@@ -255,7 +255,7 @@
return value
def sql_type(self):
- raise NotImplementedError
+ return None
def sql_column(self, table):
return Column(table, self.name)
diff -r ab268bcaf539 trytond/trytond/model/fields/function.py
--- a/trytond/trytond/model/fields/function.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/model/fields/function.py Thu Mar 15 11:47:28 2018 +0100
@@ -58,9 +58,8 @@
return
setattr(self._field, name, value)
- @property
def sql_type(self):
- raise AttributeError
+ return None
def convert_domain(self, domain, tables, Model):
name, operator, value = domain[:3]
diff -r ab268bcaf539 trytond/trytond/model/modelsql.py
--- a/trytond/trytond/model/modelsql.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/model/modelsql.py Thu Mar 15 11:47:28 2018 +0100
@@ -145,10 +145,10 @@
for field_name, field in cls._fields.iteritems():
if field_name == 'id':
continue
+ sql_type = field.sql_type()
+ if not sql_type:
+ continue
default_fun = None
- if hasattr(field, 'set'):
- continue
- sql_type = field.sql_type()
if field_name in cls._defaults:
default_fun = cls._defaults[field_name]
@@ -199,6 +199,10 @@
field_name, action=field.select and 'add' or 'remove')
required = field.required
+ # Do not set 'NOT NULL' for Binary field stored in the filestore
+ # as the database column will be left empty.
+ if isinstance(field, fields.Binary) and field.file_id:
+ required = False
table.not_null_action(
field_name, action=required and 'add' or 'remove')
@@ -228,7 +232,7 @@
cursor.execute(*history_table.select(history_table.id))
if not cursor.fetchone():
columns = [n for n, f in cls._fields.iteritems()
- if not hasattr(f, 'set')]
+ if f.sql_type()]
cursor.execute(*history_table.insert(
[Column(history_table, c) for c in columns],
sql_table.select(*(Column(sql_table, c)
@@ -274,7 +278,7 @@
field = cls._fields[field_name]
# Check required fields
if (field.required
- and not hasattr(field, 'set')
+ and field.sql_type()
and field_name not in ('create_uid', 'create_date')):
if values.get(field_name) is None:
cls.raise_user_error('required_field',
@@ -354,7 +358,7 @@
'write_date': cls.write_date,
}
for fname, field in sorted(fields.iteritems()):
- if hasattr(field, 'set'):
+ if not field.sql_type():
continue
columns.append(Column(table, fname))
hcolumns.append(Column(history, fname))
@@ -384,7 +388,7 @@
columns = []
hcolumns = []
fnames = sorted(n for n, f in cls._fields.iteritems()
- if not hasattr(f, 'set'))
+ if f.sql_type())
for fname in fnames:
columns.append(Column(table, fname))
if fname == 'write_uid':
@@ -663,7 +667,7 @@
columns = []
for f in fields_names + fields_related.keys() + datetime_fields:
field = cls._fields.get(f)
- if field and not hasattr(field, 'set'):
+ if field and field.sql_type():
columns.append(field.sql_column(table).as_(f))
elif f == '_timestamp' and not table_query:
sql_type = fields.Char('timestamp').sql_type().base
@@ -715,7 +719,7 @@
if field == '_timestamp':
continue
if (getattr(cls._fields[field], 'translate', False)
- and not hasattr(field, 'set')):
+ and not hasattr(field, 'get')):
translations = Translation.get_ids(cls.__name__ + ',' + field,
'model', Transaction().language, ids)
for row in result:
diff -r ab268bcaf539 trytond/trytond/tests/test.py
--- a/trytond/trytond/tests/test.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/tests/test.py Thu Mar 15 11:47:28 2018 +0100
@@ -35,7 +35,7 @@
'Property',
'Selection', 'SelectionRequired',
'DictSchema', 'Dict', 'DictDefault', 'DictRequired',
- 'Binary', 'BinaryDefault', 'BinaryRequired',
+ 'Binary', 'BinaryDefault', 'BinaryRequired', 'BinaryFileStorage',
'Many2OneDomainValidation', 'Many2OneTarget', 'Many2OneOrderBy',
'Many2OneSearch', 'Many2OneTree', 'Many2OneMPTT',
]
@@ -727,6 +727,13 @@
binary = fields.Binary('Binary Required', required=True)
+class BinaryFileStorage(ModelSQL):
+ "Binary in FileStorage"
+ __name__ = 'test.binary_filestorage'
+ binary = fields.Binary('Binary', file_id='binary_id')
+ binary_id = fields.Char('Binary ID')
+
+
class Many2OneTarget(ModelSQL):
"Many2One Domain Validation Target"
__name__ = 'test.many2one_target'
diff -r ab268bcaf539 trytond/trytond/tests/test_fields.py
--- a/trytond/trytond/tests/test_fields.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/tests/test_fields.py Thu Mar 15 11:47:28 2018 +0100
@@ -11,12 +11,16 @@
sys.modules['cdecimal'] = decimal
import unittest
import datetime
+import shutil
+import tempfile
+
from decimal import Decimal
from trytond.tests.test_tryton import install_module, with_transaction
from trytond.transaction import Transaction
from trytond.exceptions import UserError
from trytond.model import fields
from trytond.pool import Pool
+from trytond.config import config
class FieldsTestCase(unittest.TestCase):
@@ -26,6 +30,13 @@
def setUpClass(cls):
install_module('tests')
+ def setUp(self):
+ path = config.get('database', 'path')
+ dtemp = tempfile.mkdtemp()
+ config.set('database', 'path', dtemp)
+ self.addCleanup(config.set, 'database', 'path', path)
+ self.addCleanup(shutil.rmtree, dtemp)
+
@with_transaction()
def test_boolean(self):
'Test Boolean'
@@ -3287,6 +3298,31 @@
[{'binary': fields.Binary.cast(b'')}])
@with_transaction()
+ def test_binary_filestorage(self):
+ "Test Binary FileStorage"
+ pool = Pool()
+ Binary = pool.get('test.binary_filestorage')
+ transaction = Transaction()
+
+ bin1, = Binary.create([{
+ 'binary': fields.Binary.cast(b'foo'),
+ }])
+ self.assertEqual(bin1.binary, fields.Binary.cast(b'foo'))
+ self.assertTrue(bin1.binary_id)
+
+ Binary.write([bin1], {'binary': fields.Binary.cast(b'bar')})
+ self.assertEqual(bin1.binary, fields.Binary.cast(b'bar'))
+
+ with transaction.set_context(
+ {'test.binary_filestorage.binary': 'size'}):
+ bin1_size = Binary(bin1.id)
+ self.assertEqual(bin1_size.binary, len(b'bar'))
+
+ Binary.write([bin1], {'binary': None})
+ self.assertEqual(bin1.binary, None)
+ self.assertEqual(bin1.binary_id, None)
+
+ @with_transaction()
def test_many2one(self):
'Test Many2One'
pool = Pool()
diff -r ab268bcaf539 trytond/trytond/tools/misc.py
--- a/trytond/trytond/tools/misc.py Tue Mar 13 10:44:15 2018 +0100
+++ b/trytond/trytond/tools/misc.py Thu Mar 15 11:47:28 2018 +0100
@@ -12,6 +12,7 @@
import types
import io
import warnings
+import importlib
from sql import Literal
from sql.operators import Or
@@ -284,3 +285,17 @@
type_ = klass.__dict__.get(method)
if type_ is not None:
return isinstance(type_, types.FunctionType)
+
+
+def resolve(name):
+ "Resolve a dotted name to a global object."
+ name = name.split('.')
+ used = name.pop(0)
+ found = importlib.import_module(used)
+ for n in name:
+ used = used + '.' + n
+ try:
+ found = getattr(found, n)
+ except AttributeError:
+ found = importlib.import_module(used)
+ return found
diff -r ab268bcaf539 trytond/trytond/filestore.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/trytond/filestore.py Thu Mar 15 12:00:05 2018 +0100
@@ -0,0 +1,67 @@
+# This file is part of Tryton. The COPYRIGHT file at the top level of
+# this repository contains the full copyright notices and license terms.
+import os
+import hashlib
+
+from trytond.config import config
+from trytond.tools import resolve
+
+__all__ = ['filestore']
+
+
+class FileStore(object):
+
+ def get(self, id, prefix=''):
+ filename = self._filename(id, prefix)
+ with open(filename, 'rb') as fp:
+ return fp.read()
+
+ def getmany(self, ids, prefix=''):
+ return [self.get(id, prefix) for id in ids]
+
+ def size(self, id, prefix=''):
+ filename = self._filename(id, prefix)
+ statinfo = os.stat(filename)
+ return statinfo.st_size
+
+ def sizemany(self, ids, prefix=''):
+ return [self.size(id, prefix) for id in ids]
+
+ def set(self, data, prefix=''):
+ id = self._id(data)
+ filename = self._filename(id, prefix)
+ dirname = os.path.dirname(filename)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname, 0770)
+
+ collision = 0
+ while True:
+ basename = os.path.basename(filename)
+ if os.path.exists(filename):
+ if data != self.get(basename, prefix):
+ collision += 1
+ filename = self._filename(
+ '%s-%s' % (id, collision), prefix)
+ continue
+ else:
+ with open(filename, 'wb')as fp:
+ fp.write(data)
+ return basename
+
+ def setmany(self, data, prefix=''):
+ return [self.set(d, prefix) for d in data]
+
+ def _filename(self, id, prefix):
+ path = os.path.normpath(config.get('database', 'path'))
+ filename = os.path.join(path, prefix, id[0:2], id[2:4], id)
+ filename = os.path.normpath(filename)
+ if not filename.startswith(path):
+ raise ValueError('Bad prefix')
+ return filename
+
+ def _id(self, data):
+ return hashlib.md5(data).hexdigest()
+
+if config.get('database', 'class'):
+ FileStore = resolve(config.get('database', 'class'))
+filestore = FileStore()
Reference in New Issue View Git Blame Copy Permalink