NaNtic
/
trytond-patches
mirror of https://github.com/NaN-tic/trytond-patches.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
trytond-patches/filestore.diff

727 lines
26 KiB
Diff
Raw Permalink Blame History

diff -r ad3cb7be7667 CHANGELOG
--- a/trytond/CHANGELOG Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/CHANGELOG Mon Jan 08 14:40:45 2018 +0100
@@ -1,4 +1,7 @@
Version 3.8.16 - 2017-12-04
+=======
+* Add filestore module
+
* Bug fixes (see mercurial logs for details)
Version 3.8.15 - 2017-11-07
diff -r ad3cb7be7667 doc/ref/filestore.rst
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/doc/ref/filestore.rst Mon Jan 08 14:40:45 2018 +0100
@@ -0,0 +1,42 @@
+.. _ref-filestore:
+.. module:: trytond.filestore
+
+=========
+FileStore
+=========
+
+.. class:: FileStore()
+
+The class is used to store and retrieve files from the directory defined in the
+configuration `path` of `database` section. It uses a two levels of directory
+composed of the 2 chars of the file hash. It is an append only storage.
+
+.. method:: get(id[, prefix])
+
+Retrieve the content of the file referred by the id in the prefixed directory.
+
+.. method:: getmany(ids[, prefix])
+
+Retrieve a list of contents for the sequence of ids.
+
+.. method:: size(id[, prefix])
+
+Return the size of the file referred by the id in the prefixed directory.
+
+.. method:: sizemany(ids[, prefix])
+
+Return a list of sizes for the sequence of ids.
+
+.. method:: set(data[, prefix])
+
+Store the data in the prefixed directory and return the identifiers.
+
+.. method:: setmany(data[, prefix])
+
+Store the sequence of data and return a list of identifiers.
+
+.. note::
+ The class can be overridden by setting a fully qualified name of a
+ alternative class defined in the configuration `class` of the `database`
+ section.
+..
diff -r ad3cb7be7667 doc/ref/index.rst
--- a/trytond/doc/ref/index.rst Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/doc/ref/index.rst Mon Jan 08 14:40:45 2018 +0100
@@ -11,6 +11,7 @@
wizard
pyson
transaction
- tools/singleton
+ tools/index
pool
rpc
+ filestore
diff -r ad3cb7be7667 doc/ref/tools/index.rst
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/doc/ref/tools/index.rst Mon Jan 08 14:40:45 2018 +0100
@@ -0,0 +1,13 @@
+.. _ref-tools-index:
+
+=====
+Tools
+=====
+
+Tools API reference.
+
+.. toctree::
+ :maxdepth: 1
+
+ misc
+ singleton
diff -r ad3cb7be7667 doc/ref/tools/misc.rst
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/doc/ref/tools/misc.rst Mon Jan 08 14:40:45 2018 +0100
@@ -0,0 +1,10 @@
+.. _ref-tools:
+.. module:: trytond.tools
+
+=============
+Miscellaneous
+=============
+
+.. method:: resolve(name)
+
+Resolve a dotted name to a global object.
diff -r ad3cb7be7667 trytond/ir/attachment.py
--- a/trytond/trytond/ir/attachment.py Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/trytond/ir/attachment.py Mon Jan 08 14:40:45 2018 +0100
@@ -1,7 +1,6 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-import os
-import hashlib
+from sql import Null
from sql.operators import Concat
from sql.conditionals import Coalesce
@@ -10,7 +9,10 @@
from .. import backend
from ..transaction import Transaction
from ..pyson import Eval
-from ..pool import Pool
+from .resource import ResourceMixin
+from ..filestore import filestore
+from ..config import config
+
__all__ = [
'Attachment',
@@ -23,8 +25,15 @@
except StopIteration:
return ''
+if config.getboolean('attachment', 'filestore', default=True):
+ file_id = 'file_id'
+ store_prefix = config.get('attachment', 'store_prefix', default=None)
+else:
+ file_id = None
+ store_prefix = None
-class Attachment(ModelSQL, ModelView):
+
+class Attachment(ResourceMixin, ModelSQL, ModelView):
"Attachment"
__name__ = 'ir.attachment'
name = fields.Char('Name', required=True)
@@ -32,25 +41,20 @@
('data', 'Data'),
('link', 'Link'),
], 'Type', required=True)
- data = fields.Function(fields.Binary('Data', filename='name', states={
- 'invisible': Eval('type') != 'data',
- }, depends=['type']), 'get_data', setter='set_data')
+ data = fields.Binary('Data', filename='name',
+ file_id=file_id, store_prefix=store_prefix,
+ states={
+ 'invisible': Eval('type') != 'data',
+ }, depends=['type'])
description = fields.Text('Description')
summary = fields.Function(fields.Char('Summary'), 'on_change_with_summary')
- resource = fields.Reference('Resource', selection='models_get',
- select=True)
link = fields.Char('Link', states={
'invisible': Eval('type') != 'link',
}, depends=['type'])
- digest = fields.Char('Digest', size=32)
- collision = fields.Integer('Collision')
+ file_id = fields.Char('File ID', readonly=True)
data_size = fields.Function(fields.Integer('Data size', states={
'invisible': Eval('type') != 'data',
- }, depends=['type']), 'get_data')
- last_modification = fields.Function(fields.DateTime('Last Modification'),
- 'get_last_modification')
- last_user = fields.Function(fields.Char('Last User'),
- 'get_last_user')
+ }, depends=['type']), 'get_size')
@classmethod
def __setup__(cls):
@@ -86,177 +90,33 @@
table.drop_column('res_model')
table.drop_column('res_id')
+ # Migration from 4.0: merge digest and collision into file_id
+ if table.column_exist('digest') and table.column_exist('collision'):
+ cursor.execute(*attachment.update(
+ [attachment.file_id],
+ [attachment.digest],
+ where=(attachment.collision == 0)
+ | (attachment.collision == Null)))
+ cursor.execute(*attachment.update(
+ [attachment.file_id],
+ [Concat(Concat(attachment.digest, '-'),
+ attachment.collision)],
+ where=(attachment.collision != 0)
+ & (attachment.collision != Null)))
+ table.drop_column('digest')
+ table.drop_column('collision')
+
@staticmethod
def default_type():
return 'data'
- @staticmethod
- def default_resource():
- return Transaction().context.get('resource')
-
- @staticmethod
- def default_collision():
- return 0
-
- @staticmethod
- def models_get():
- pool = Pool()
- Model = pool.get('ir.model')
- ModelAccess = pool.get('ir.model.access')
- models = Model.search([])
- access = ModelAccess.get_access([m.model for m in models])
- res = []
- for model in models:
- if access[model.model]['read']:
- res.append([model.model, model.name])
- return res
-
- def get_data(self, name):
- db_name = Transaction().cursor.dbname
- format_ = Transaction().context.pop('%s.%s'
- % (self.__name__, name), '')
- value = None
- if name == 'data_size' or format_ == 'size':
- value = 0
- if self.digest:
- filename = self.digest
- if self.collision:
- filename = filename + '-' + str(self.collision)
- filename = os.path.join(config.get('database', 'path'), db_name,
- filename[0:2], filename[2:4], filename)
- if name == 'data_size' or format_ == 'size':
- try:
- statinfo = os.stat(filename)
- value = statinfo.st_size
- except OSError:
- pass
- else:
- try:
- with open(filename, 'rb') as file_p:
- value = fields.Binary.cast(file_p.read())
- except IOError:
- pass
- return value
-
- @classmethod
- def set_data(cls, attachments, name, value):
- if value is None:
- return
- cursor = Transaction().cursor
- table = cls.__table__()
- db_name = cursor.dbname
- directory = os.path.join(config.get('database', 'path'), db_name)
- if not os.path.isdir(directory):
- os.makedirs(directory, 0770)
- digest = hashlib.md5(value).hexdigest()
- directory = os.path.join(directory, digest[0:2], digest[2:4])
- if not os.path.isdir(directory):
- os.makedirs(directory, 0770)
- filename = os.path.join(directory, digest)
- collision = 0
- if os.path.isfile(filename):
- with open(filename, 'rb') as file_p:
- data = file_p.read()
- if value != data:
- cursor.execute(*table.select(table.collision,
- where=(table.digest == digest)
- & (table.collision != 0),
- group_by=table.collision,
- order_by=table.collision))
- collision2 = 0
- for row in cursor.fetchall():
- collision2 = row[0]
- filename = os.path.join(directory,
- digest + '-' + str(collision2))
- if os.path.isfile(filename):
- with open(filename, 'rb') as file_p:
- data = file_p.read()
- if value == data:
- collision = collision2
- break
- if collision == 0:
- collision = collision2 + 1
- filename = os.path.join(directory,
- digest + '-' + str(collision))
- with open(filename, 'wb') as file_p:
- file_p.write(value)
- else:
- with open(filename, 'wb') as file_p:
- file_p.write(value)
- cls.write(attachments, {
- 'digest': digest,
- 'collision': collision,
- })
+ def get_size(self, name):
+ with Transaction().set_context({
+ '%s.%s' % (self.__name__, name): 'size',
+ }):
+ record = self.__class__(self.id)
+ return record.data
@fields.depends('description')
def on_change_with_summary(self, name=None):
return firstline(self.description or '')
-
- def get_last_modification(self, name):
- return (self.write_date if self.write_date else self.create_date
- ).replace(microsecond=0)
-
- @staticmethod
- def order_last_modification(tables):
- table, _ = tables[None]
- return [Coalesce(table.write_date, table.create_date)]
-
- def get_last_user(self, name):
- return (self.write_uid.rec_name if self.write_uid
- else self.create_uid.rec_name)
-
- @classmethod
- def check_access(cls, ids, mode='read'):
- pool = Pool()
- ModelAccess = pool.get('ir.model.access')
- if ((Transaction().user == 0)
- or not Transaction().context.get('_check_access')):
- return
- model_names = set()
- with Transaction().set_context(_check_access=False):
- for attachment in cls.browse(ids):
- if attachment.resource:
- model_names.add(attachment.resource.__name__)
- for model_name in model_names:
- ModelAccess.check(model_name, mode=mode)
-
- @classmethod
- def read(cls, ids, fields_names=None):
- cls.check_access(ids, mode='read')
- return super(Attachment, cls).read(ids, fields_names=fields_names)
-
- @classmethod
- def delete(cls, attachments):
- cls.check_access([a.id for a in attachments], mode='delete')
- super(Attachment, cls).delete(attachments)
-
- @classmethod
- def write(cls, attachments, values, *args):
- all_attachments = []
- actions = iter((attachments, values) + args)
- for records, _ in zip(actions, actions):
- all_attachments += records
- cls.check_access([a.id for a in all_attachments], mode='write')
- super(Attachment, cls).write(attachments, values, *args)
- cls.check_access(all_attachments, mode='write')
-
- @classmethod
- def create(cls, vlist):
- attachments = super(Attachment, cls).create(vlist)
- cls.check_access([a.id for a in attachments], mode='create')
- return attachments
-
- @classmethod
- def view_header_get(cls, value, view_type='form'):
- pool = Pool()
- Model = pool.get('ir.model')
- value = super(Attachment, cls).view_header_get(value,
- view_type=view_type)
- resource = Transaction().context.get('resource')
- if resource:
- model_name, record_id = resource.split(',', 1)
- ir_model, = Model.search([('model', '=', model_name)])
- Resource = pool.get(model_name)
- record = Resource(int(record_id))
- value = '%s - %s - %s' % (ir_model.name, record.rec_name, value)
- return value
diff -r ad3cb7be7667 trytond/ir/resource.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/trytond/ir/resource.py Mon Jan 08 14:40:45 2018 +0100
@@ -0,0 +1,98 @@
+# This file is part of Tryton. The COPYRIGHT file at the top level of
+# this repository contains the full copyright notices and license terms.
+from sql.conditionals import Coalesce
+
+from ..model import ModelSQL, ModelView, fields
+from ..pool import Pool
+from ..transaction import Transaction
+from ..pyson import Eval
+
+__all__ = ['ResourceMixin']
+
+
+class ResourceMixin(ModelSQL, ModelView):
+
+ resource = fields.Reference('Resource', selection='get_models',
+ required=True, select=True)
+ last_user = fields.Function(fields.Char('Last User',
+ states={
+ 'invisible': ~Eval('last_user'),
+ }),
+ 'get_last_user')
+ last_modification = fields.Function(fields.DateTime('Last Modification',
+ states={
+ 'invisible': ~Eval('last_modification'),
+ }),
+ 'get_last_modification')
+
+ @classmethod
+ def __setup__(cls):
+ super(ResourceMixin, cls).__setup__()
+ cls._order.insert(0, ('last_modification', 'DESC'))
+
+ @staticmethod
+ def default_resource():
+ return Transaction().context.get('resource')
+
+ @staticmethod
+ def get_models():
+ pool = Pool()
+ Model = pool.get('ir.model')
+ ModelAccess = pool.get('ir.model.access')
+ models = Model.search([])
+ access = ModelAccess.get_access([m.model for m in models])
+ return [(m.model, m.name) for m in models if access[m.model]['read']]
+
+ def get_last_user(self, name):
+ return (self.write_uid.rec_name if self.write_uid
+ else self.create_uid.rec_name)
+
+ def get_last_modification(self, name):
+ return (self.write_date if self.write_date else self.create_date
+ ).replace(microsecond=0)
+
+ @staticmethod
+ def order_last_modification(tables):
+ table, _ = tables[None]
+ return [Coalesce(table.write_date, table.create_date)]
+
+ @classmethod
+ def check_access(cls, ids, mode='read'):
+ pool = Pool()
+ ModelAccess = pool.get('ir.model.access')
+ if ((Transaction().user == 0)
+ or not Transaction().context.get('_check_access')):
+ return
+ model_names = set()
+ with Transaction().set_context(_check_access=False):
+ for record in cls.browse(ids):
+ if record.resource:
+ model_names.add(str(record.resource).split(',')[0])
+ for model_name in model_names:
+ ModelAccess.check(model_name, mode=mode)
+
+ @classmethod
+ def read(cls, ids, fields_names=None):
+ cls.check_access(ids, mode='read')
+ return super(ResourceMixin, cls).read(ids, fields_names=fields_names)
+
+ @classmethod
+ def delete(cls, records):
+ cls.check_access([a.id for a in records], mode='delete')
+ super(ResourceMixin, cls).delete(records)
+
+ @classmethod
+ def write(cls, records, values, *args):
+ all_records = []
+ actions = iter((records, values) + args)
+ for other_records, _ in zip(actions, actions):
+ all_records += other_records
+ cls.check_access([a.id for a in all_records], mode='write')
+ super(ResourceMixin, cls).write(records, values, *args)
+ cls.check_access(all_records, mode='write')
+
+ @classmethod
+ def create(cls, vlist):
+ records = super(ResourceMixin, cls).create(vlist)
+ cls.check_access([r.id for r in records], mode='create')
+ return records
diff -r ad3cb7be7667 trytond/model/fields/binary.py
--- a/trytond/trytond/model/fields/binary.py Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/trytond/model/fields/binary.py Mon Jan 08 14:40:45 2018 +0100
@@ -1,10 +1,13 @@
-# This file is part of Tryton. The COPYRIGHT file at the top level of
-# this repository contains the full copyright notices and license terms.
-from sql import Query, Expression
+
+#This file is part of Tryton. The COPYRIGHT file at the top level of
+#this repository contains the full copyright notices and license terms.
+from sql import Query, Expression, Column, Null
from .field import Field, SQLType
from ...transaction import Transaction
from ... import backend
+from ...tools import grouped_slice, reduce_ids
+from ...filestore import filestore
class Binary(Field):
@@ -17,20 +20,21 @@
def __init__(self, string='', help='', required=False, readonly=False,
domain=None, states=None, select=False, on_change=None,
on_change_with=None, depends=None, filename=None, context=None,
- loading='lazy'):
+ loading='lazy', file_id=None, store_prefix=None):
if filename is not None:
self.filename = filename
if depends is None:
depends = [filename]
else:
depends.append(filename)
+ self.file_id = file_id
+ self.store_prefix = store_prefix
super(Binary, self).__init__(string=string, help=help,
required=required, readonly=readonly, domain=domain, states=states,
select=select, on_change=on_change, on_change_with=on_change_with,
depends=depends, context=context, loading=loading)
- @classmethod
- def get(cls, ids, model, name, values=None):
+ def get(self, ids, model, name, values=None):
'''
Convert the binary value into ``bytes``
@@ -42,15 +46,45 @@
'''
if values is None:
values = {}
+ transaction = Transaction()
res = {}
- converter = cls.cast
+ converter = self.cast
default = None
- format_ = Transaction().context.pop('%s.%s' % (model.__name__, name),
- '')
+ format_ = Transaction().context.get(
+ '%s.%s' % (model.__name__, name), '')
if format_ == 'size':
converter = len
default = 0
+
+ if self.file_id:
+ table = model.__table__()
+ cursor = transaction.cursor
+
+ prefix = self.store_prefix
+ if prefix is None:
+ prefix = transaction.database.database_name
+
+ if format_ == 'size':
+ store_func = filestore.size
+ else:
+ def store_func(id, prefix):
+ return self.cast(filestore.get(id, prefix=prefix))
+
+ for sub_ids in grouped_slice(ids):
+ cursor.execute(*table.select(
+ table.id, Column(table, self.file_id),
+ where=reduce_ids(table.id, sub_ids)
+ & (Column(table, self.file_id) != Null)
+ & (Column(table, self.file_id) != '')))
+ for record_id, file_id in cursor.fetchall():
+ try:
+ res[record_id] = store_func(file_id, prefix)
+ except (IOError, OSError):
+ pass
+
for i in values:
+ if i['id'] in res:
+ continue
value = i[name]
if value:
if isinstance(value, unicode):
@@ -63,6 +97,27 @@
res.setdefault(i, default)
return res
+ def set(self, Model, name, ids, value, *args):
+ transaction = Transaction()
+ table = Model.__table__()
+ cursor = transaction.cursor
+
+ prefix = self.store_prefix
+ if prefix is None:
+ prefix = transaction.database.database_name
+
+ args = iter((ids, value) + args)
+ for ids, value in zip(args, args):
+ if self.file_id:
+ columns = [Column(table, self.file_id), Column(table, name)]
+ values = [
+ filestore.set(value, prefix) if value else None, None]
+ else:
+ columns = [Column(table, name)]
+ values = [self.sql_format(value)]
+ cursor.execute(*table.update(columns, values,
+ where=reduce_ids(table.id, ids)))
+
@staticmethod
def sql_format(value):
if isinstance(value, (Query, Expression)):
diff -r ad3cb7be7667 trytond/model/fields/field.py
--- a/trytond/trytond/model/fields/field.py Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/trytond/model/fields/field.py Mon Jan 08 14:40:45 2018 +0100
@@ -253,7 +253,7 @@
return value
def sql_type(self):
- raise NotImplementedError
+ return None
def sql_column(self, table):
return Column(table, self.name)
diff -r ad3cb7be7667 trytond/model/fields/function.py
--- a/trytond/trytond/model/fields/function.py Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/trytond/model/fields/function.py Mon Jan 08 14:40:45 2018 +0100
@@ -62,6 +62,9 @@
def sql_type(self):
raise AttributeError
+ def sql_type(self):
+ return None
+
def convert_domain(self, domain, tables, Model):
name, operator, value = domain[:3]
if not self.searcher:
diff -r ad3cb7be7667 trytond/model/modelsql.py
--- a/trytond/trytond/model/modelsql.py Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/trytond/model/modelsql.py Mon Jan 08 14:40:45 2018 +0100
@@ -146,10 +146,11 @@
for field_name, field in cls._fields.iteritems():
if field_name == 'id':
continue
+ sql_type = field.sql_type()
+ if not sql_type:
+ continue
default_fun = None
- if hasattr(field, 'set'):
- continue
- sql_type = field.sql_type()
+
if field_name in cls._defaults:
default_fun = cls._defaults[field_name]
@@ -200,6 +201,10 @@
field_name, action=field.select and 'add' or 'remove')
required = field.required
+ # Do not set 'NOT NULL' for Binary field stored in the filestore
+ # as the database column will be left empty.
+ if isinstance(field, fields.Binary) and field.file_id:
+ required = False
table.not_null_action(
field_name, action=required and 'add' or 'remove')
@@ -228,7 +233,7 @@
cursor.execute(*history_table.select(history_table.id))
if not cursor.fetchone():
columns = [n for n, f in cls._fields.iteritems()
- if not hasattr(f, 'set')]
+ if f.sql_type()]
cursor.execute(*history_table.insert(
[Column(history_table, c) for c in columns],
table.select(*(Column(table, c)
@@ -275,7 +280,7 @@
field = cls._fields[field_name]
# Check required fields
if (field.required
- and not hasattr(field, 'set')
+ and field.sql_type()
and field_name not in ('create_uid', 'create_date')):
if values.get(field_name) is None:
cls.raise_user_error('required_field',
@@ -355,7 +360,7 @@
'write_date': cls.write_date,
}
for fname, field in sorted(fields.iteritems()):
- if hasattr(field, 'set'):
+ if not field.sql_type():
continue
columns.append(Column(table, fname))
hcolumns.append(Column(history, fname))
@@ -385,7 +390,7 @@
columns = []
hcolumns = []
fnames = sorted(n for n, f in cls._fields.iteritems()
- if not hasattr(f, 'set'))
+ if f.sql_type())
for fname in fnames:
columns.append(Column(table, fname))
if fname == 'write_uid':
@@ -650,7 +655,7 @@
columns = []
for f in fields_names + fields_related.keys() + datetime_fields:
field = cls._fields.get(f)
- if field and not hasattr(field, 'set'):
+ if field and field.sql_type():
columns.append(field.sql_column(table).as_(f))
elif f == '_timestamp' and not table_query:
sql_type = fields.Char('timestamp').sql_type().base
diff -r ad3cb7be7667 trytond/tools/misc.py
--- a/trytond/trytond/tools/misc.py Mon Jan 08 13:34:08 2018 +0100
+++ b/trytond/trytond/tools/misc.py Mon Jan 08 14:40:45 2018 +0100
@@ -13,6 +13,7 @@
from itertools import islice
import types
import urllib
+import importlib
from sql import Literal
from sql.operators import Or
@@ -391,3 +392,17 @@
type_ = klass.__dict__.get(method)
if type_ is not None:
return isinstance(type_, types.FunctionType)
+
+
+def resolve(name):
+ "Resolve a dotted name to a global object."
+ name = name.split('.')
+ used = name.pop(0)
+ found = importlib.import_module(used)
+ for n in name:
+ used = used + '.' + n
+ try:
+ found = getattr(found, n)
+ except AttributeError:
+ found = importlib.import_module(used)
+ return found
Reference in New Issue View Git Blame Copy Permalink