Add tables.

__init__.py

@@ -6,6 +6,7 @@ from . import cron
 from . import babi
 from . import test_model
 from . import report
+from . import table
 
 def register():
     Pool.register(
@@ -32,6 +33,8 @@ def register():
         babi.UpdateDataWizardUpdated,
         babi.CleanExecutionsStart,
         test_model.TestBabiModel,
+        table.Table,
+        table.Field,
         module='babi', type_='model')
     Pool.register(
         babi.OpenChart,

babi.py

@@ -323,9 +323,12 @@ class TimeoutChecker:
         self._callback = callback
         self._start = datetime.now()
 
+    @property
+    def elapsed(self):
+        return (datetime.now() - self._start).seconds
+
     def check(self):
-        elapsed = (datetime.now() - self._start).seconds
-        if elapsed > self._timeout:
+        if self.elapsed > self._timeout:
             self._callback()
 
 
@@ -663,7 +666,7 @@ class Report(ModelSQL, ModelView):
     def default_timeout():
         Config = Pool().get('babi.configuration')
         config = Config(1)
-        return config.default_timeout
+        return config.default_timeout or 30
 
     @staticmethod
     def default_report_cell_level():

babi.xml

@@ -459,7 +459,7 @@ contains the full copyright notices and license terms. -->
         </record>
 
         <record model="ir.action.act_window" id="act_babi_report">
-            <field name="name">Report</field>
+            <field name="name">Reports</field>
             <field name="res_model">babi.report</field>
             <field name="search_value"></field>
             <!-- <field name="domain">[]</field> -->

cron.py

@@ -3,42 +3,65 @@
 from trytond.model import fields, dualmethod
 from trytond.pool import Pool, PoolMeta
 from trytond.transaction import Transaction
+from trytond.pyson import Eval
 
 
 class Cron(metaclass=PoolMeta):
     __name__ = "ir.cron"
-    babi_report = fields.Many2One('babi.report', 'Babi Report')
+    babi_report = fields.Many2One('babi.report', 'Babi Report', states={
+            'invisible': Eval('method') != 'babi.report|calculate_babi_report',
+            }, depends=['method'])
+    babi_table = fields.Many2One('babi.table', 'Babi Table', states={
+            'invisible': Eval('method') != 'babi.table|calculate_babi_table',
+            }, depends=['method'])
 
     @classmethod
     def __setup__(cls):
         super(Cron, cls).__setup__()
         cls.method.selection.extend([
-            ('babi.report|calculate_babi_report', 'Calculate Babi Report'),
-            ('babi.report.execution|clean', 'Clean Babi Excutions'),
-        ])
+                ('babi.report|calculate_babi_report', 'Calculate Babi Report'),
+                ('babi.table|calculate_babi_table', 'Calculate Babi Table'),
+                ('babi.report.execution|clean', 'Clean Babi Excutions'),
+                ])
 
     @classmethod
     def default_get(cls, fields, with_rec_name=True):
-        User = Pool().get('res.user')
         res = super(Cron, cls).default_get(fields, with_rec_name)
-        admin_user, = User.search([('login', '=', 'admin')])
         context = Transaction().context
-        if context.get('babi_report', False):
-            res['user'] = admin_user.id
+        if context.get('babi_report'):
             res['interval_type'] = 'days'
-            res['repeat_missed'] = False
-            res['function'] = 'babi.report|calculate_babi_report'
+            res['interval_number'] = 1
+            res['minute'] = 0
+            res['hour'] = 5
+            res['method'] = 'babi.report|calculate_babi_report'
+        if context.get('babi_table'):
+            res['interval_type'] = 'days'
+            res['interval_number'] = 1
+            res['minute'] = 0
+            res['hour'] = 5
+            res['method'] = 'babi.table|calculate_babi_table'
         return res
 
     @dualmethod
     def run_once(cls, crons):
-        BabiReport = Pool().get('babi.report')
+        pool = Pool()
+        BabiReport = pool.get('babi.report')
+        BabiTable = pool.get('babi.table')
 
-        babi_crons = [cron for cron in crons if cron.babi_report]
-        for cron in babi_crons:
+        report_crons = [cron for cron in crons if cron.babi_report]
+        for cron in report_crons:
             # babi execution require company. Run calculate when has a company
             for company in cron.companies:
                 with Transaction().set_context(company=company.id,
                         queue_name='babi'):
                     BabiReport.__queue__.compute(cron.babi_report)
-        return super(Cron, cls).run_once(list(set(crons) - set(babi_crons)))
+
+        table_crons = [cron for cron in crons if cron.babi_table]
+        for cron in table_crons:
+            # babi execution require company. Run calculate when has a company
+            for company in cron.companies:
+                with Transaction().set_context(company=company.id,
+                        queue_name='babi'):
+                    BabiTable.__queue__.compute(cron.babi_table)
+        return super(Cron, cls).run_once(list(
+                set(crons) - set(report_crons) - set(table_crons)))

messages.xml

@@ -65,5 +65,30 @@ Exception: %(error)s</field>
         <record model="ir.message" id="msg_render_report_columns">
             <field name="text">Print report "%(report)s" has dimensions in columns and is not supported.</field>
         </record>
+
+        <record model="ir.message" id="msg_invalid_table_internal_name_first_character">
+            <field name="text">Invalid first character in internal name "%(internal_name)s" in table "%(table)s".</field>
+        </record>
+        <record model="ir.message" id="msg_invalid_table_internal_name">
+            <field name="text">Invalid internal name "%(internal_name)s" in table "%(table)s".</field>
+        </record>
+        <record model="ir.message" id="msg_invalid_field_internal_name_first_character">
+            <field name="text">Invalid first character in internal name "%(internal_name)s" in field "%(field)s".</field>
+        </record>
+        <record model="ir.message" id="msg_invalid_field_internal_name">
+            <field name="text">Invalid internal name "%(internal_name)s" in field "%(field)s".</field>
+        </record>
+        <record model="ir.message" id="msg_filter_with_parameters">
+            <field name="text">Cannot use filters with parameters in table "%(table)s".</field>
+        </record>
+        <record model="ir.message" id="msg_table_no_fields">
+            <field name="text">Table "%(table)s" cannot be computed because it has no fields.</field>
+        </record>
+
+        <record model="ir.message" id="msg_compute_table_exception">
+            <field name="text">An exception occurred while computing the value for field "%(field)s" in table "%(table)s", record "%(record)s". The error was:
+
+%(error)s</field>
+        </record>
     </data>
 </tryton>

table.py

@@ -0,0 +1,295 @@
+import datetime as mdatetime
+from datetime import datetime
+import logging
+import sql
+import unidecode
+from simpleeval import EvalWithCompoundTypes
+from trytond.bus import notify
+from trytond.transaction import Transaction
+from trytond.pool import Pool
+from trytond.model import ModelView, ModelSQL, fields, Unique, DeactivableMixin
+from trytond.exceptions import UserError
+from trytond.i18n import gettext
+from trytond.pyson import Eval, PYSONDecoder
+from .babi import TimeoutChecker, TimeoutException
+from .babi_eval import babi_eval
+
+VALID_FIRST_SYMBOLS = 'abcdefghijklmnopqrstuvwxyz'
+VALID_NEXT_SYMBOLS = '_0123456789'
+VALID_SYMBOLS = VALID_FIRST_SYMBOLS + VALID_NEXT_SYMBOLS
+
+logger = logging.getLogger(__name__)
+
+def convert_to_symbol(text):
+    if not text:
+        return 'x'
+    text = unidecode.unidecode(text)
+    text = text.lower()
+    if text[0] not in VALID_FIRST_SYMBOLS:
+        symbol = '_'
+    else:
+        symbol = ''
+    for x in text:
+        if not x in VALID_SYMBOLS:
+            if symbol[-1] == '_':
+                continue
+            symbol += '_'
+        else:
+            symbol += x
+
+    if len(symbol) > 1 and symbol[-1] == '_':
+        symbol = symbol[:-1]
+    return symbol
+
+
+class Table(DeactivableMixin, ModelSQL, ModelView):
+    'BABI Table'
+    __name__ = 'babi.table'
+    name = fields.Char('Name', required=True)
+    internal_name = fields.Char('Internal Name', required=True)
+    model = fields.Many2One('ir.model', 'Model', required=True,
+        domain=[('babi_enabled', '=', True)])
+    filter = fields.Many2One('babi.filter', 'Filter', domain=[
+            ('model', '=', Eval('model')),
+            ], depends=['model'])
+    fields_ = fields.One2Many('babi.field', 'table', 'Fields')
+    timeout = fields.Integer('Timeout', required=True, help='If table '
+        'calculation should take more than the specified timeout (in seconds) '
+        'the process will be stopped automatically.')
+    babi_raise_user_error = fields.Boolean('Raise User Error',
+        help='Will raise a UserError in case of an error in the table.')
+    crons = fields.One2Many('ir.cron', 'babi_table', 'Schedulers', context={
+            'babi_table': Eval('id'),
+            }, depends=['id'])
+
+    @staticmethod
+    def default_timeout():
+        Config = Pool().get('babi.configuration')
+        config = Config(1)
+        return config.default_timeout or 30
+
+    @classmethod
+    def __setup__(cls):
+        super(Table, cls).__setup__()
+        cls._buttons.update({
+                'compute': {},
+                })
+
+    @classmethod
+    def validate(cls, tables):
+        super(Table, cls).validate(tables)
+        for table in tables:
+            table.check_internal_name()
+            table.check_filter()
+
+    def check_internal_name(self):
+        if not self.internal_name[0] in VALID_FIRST_SYMBOLS:
+            raise UserError(gettext(
+                'babi.msg_invalid_internal_name_first_character',
+                table=self.rec_name, internal_name=self.internal_name))
+        for symbol in self.internal_name:
+            if not symbol in VALID_SYMBOLS:
+                raise UserError(gettext('babi.msg_invalid_internal_name',
+                        table=self.rec_name, internal_name=self.internal_name))
+
+    def check_filter(self):
+        if self.filter and self.filter.parameters:
+            raise UserError(gettext('babi.msg_filter_with_parameters',
+                    table=self.rec_name))
+
+    @fields.depends('name')
+    def on_change_name(self):
+        self.internal_name = convert_to_symbol(self.name)
+
+    def get_python_filter(self):
+        if self.filter and self.filter.python_expression:
+            return self.filter.python_expression
+
+    def get_domain_filter(self):
+        domain = '[]'
+        if self.filter and self.filter.domain:
+            domain = self.filter.domain
+            if '__' in domain:
+                domain = str(PYSONDecoder().decode(domain))
+        return eval(domain, {
+                'datetime': mdatetime,
+                'false': False,
+                'true': True,
+                })
+
+    def get_context(self):
+        if self.filter and self.filter.context:
+            context = self.replace_parameters(self.filter.context)
+            ev = EvalWithCompoundTypes(names={}, functions={
+                'date': lambda x: datetime.strptime(x, '%Y-%m-%d').date(),
+                'datetime': lambda x: datetime.strptime(x, '%Y-%m-%d'),
+                })
+            context = ev.eval(context)
+            return context
+
+    @classmethod
+    @ModelView.button
+    def compute(cls, tables):
+        for table in tables:
+            cls.__queue__._compute(table)
+
+    def _compute(self):
+        if not self.fields_:
+            raise UserError(gettext('babi.msg_table_no_fields',
+                    table=self.name))
+
+        if self.filter and self.filter.parameters:
+            raise UserError(gettext('babi.msg_filter_with_parameters',
+                    table=self.rec_name))
+
+        Model = Pool().get(self.model.model)
+
+        cursor = Transaction().connection.cursor()
+
+        # Create table
+        cursor.execute('DROP TABLE IF EXISTS "%s"' % self.internal_name)
+        fields = []
+        for field in self.fields_:
+            fields.append('"%s" %s' % (field.internal_name, field.sql_type()))
+        cursor.execute('CREATE TABLE IF NOT EXISTS "%s" (%s);' % (
+                self.internal_name, ', '.join(fields)))
+
+        checker = TimeoutChecker(self.timeout, TimeoutException)
+        domain = self.get_domain_filter()
+
+        context = self.get_context()
+        if not context:
+            context = {}
+        else:
+            assert isinstance(context, dict)
+        context['_datetime'] = None
+        # This is needed when execute the wizard to calculate the report, to
+        # ensure the company rule is used.
+        context['_check_access'] = True
+
+        python_filter = self.get_python_filter()
+
+        table = sql.Table(self.internal_name)
+        columns = [sql.Column(table, x.internal_name) for x in self.fields_]
+        expressions = [x.expression.expression for x in self.fields_]
+        index = 0
+        count = 0
+        offset = 2000
+
+        with Transaction().set_context(**context):
+            try:
+                records = Model.search(domain, offset=index * offset,
+                    limit=offset)
+            except Exception as message:
+                if self.babi_raise_user_error:
+                    raise UserError(gettext(
+                        'babi.create_data_exception',
+                        error=repr(message)))
+                raise
+
+        while records:
+            checker.check()
+            logger.info('Calculated %s, %s records in %s seconds'
+                % (self.model.model, count, checker.elapsed))
+
+            to_insert = []
+            for record in records:
+                if python_filter:
+                    if not babi_eval(python_filter, record, convert_none=None):
+                        continue
+                values = []
+                for expression in expressions:
+                    try:
+                        values.append(babi_eval(expression, record,
+                                convert_none=None))
+                    except Exception as message:
+                        notify(gettext('babi.msg_compute_table_exception',
+                                table=self.name, field=field.name,
+                                record=record.id, error=repr(message)),
+                            priority=1)
+                        if self.babi_raise_user_error:
+                            raise UserError(gettext(
+                                'babi.msg_compute_table_exception',
+                                table=self.name,
+                                field=field.name,
+                                record=record.id,
+                                error=repr(message)))
+                        raise
+
+                to_insert.append(values)
+
+            cursor.execute(*table.insert(columns=columns, values=to_insert))
+
+            index += 1
+            count += len(records)
+            with Transaction().set_context(**context):
+                records = Model.search(domain, offset=index * offset,
+                    limit=offset)
+
+        logger.info('Calculated %s, %s records in %s seconds'
+            % (self.model.model, count, checker.elapsed))
+
+
+class Field(ModelSQL, ModelView):
+    'BABI Field'
+    __name__ = 'babi.field'
+    table = fields.Many2One('babi.table', 'Table', required=True)
+    name = fields.Char('Name', required=True)
+    internal_name = fields.Char('Internal Name', required=True)
+    expression = fields.Many2One('babi.expression', 'Expression', required=True,
+        domain=[
+            ('model', '=', Eval('model')),
+            ], depends=['model'])
+    model = fields.Function(fields.Many2One('ir.model', 'Model'),
+        'on_change_with_model')
+
+    @classmethod
+    def __setup__(cls):
+        super().__setup__()
+        t = cls.__table__()
+        cls._sql_constraints += [
+            ('table_internal_name_uniq', Unique(t, t.table, t.internal_name),
+                'Field must be unique per Table'),
+            ]
+        cls.__access__.add('table')
+
+    @classmethod
+    def validate(cls, tables):
+        super().validate(tables)
+        for table in tables:
+            table.check_internal_name()
+
+    def sql_type(self):
+        mapping = {
+            'char': 'VARCHAR',
+            'integer': 'INTEGER',
+            'float': 'FLOAT',
+            'numeric': 'NUMERIC',
+            'boolean': 'BOOLEAN',
+            'many2one': 'INTEGER',
+            }
+        return mapping[self.expression.ttype]
+
+    def check_internal_name(self):
+        if not self.internal_name[0] in VALID_FIRST_SYMBOLS:
+            raise UserError(gettext('babi.msg_invalid_field_internal_name',
+                    field=self.name, internal_name=self.internal_name))
+        for symbol in self.internal_name:
+            if not symbol in VALID_SYMBOLS:
+                raise UserError(gettext('babi.msg_invalid_field_internal_name',
+                        field=self.name, internal_name=self.internal_name))
+
+    @fields.depends('name')
+    def on_change_name(self):
+        self.internal_name = convert_to_symbol(self.name)
+
+    @fields.depends('name', 'expression', methods=['on_change_name'])
+    def on_change_expression(self):
+        if self.expression:
+            self.name = self.expression.name
+            self.on_change_name()
+
+    @fields.depends('table')
+    def on_change_with_model(self, name=None):
+        if self.table:
+            return self.table.model.id

table.xml

@@ -0,0 +1,87 @@
+<tryton>
+    <data>
+        <!-- babi.table -->
+        <record model="ir.ui.view" id="babi_table_form_view">
+            <field name="model">babi.table</field>
+            <field name="type">form</field>
+            <field name="name">table_form</field>
+        </record>
+
+        <record model="ir.ui.view" id="babi_table_tree_view">
+            <field name="model">babi.table</field>
+            <field name="type">tree</field>
+            <field name="name">table_list</field>
+        </record>
+
+        <record model="ir.action.act_window" id="act_babi_table">
+            <field name="name">Tables</field>
+            <field name="res_model">babi.table</field>
+        </record>
+        <record model="ir.action.act_window.view" id="act_babi_table_view1">
+            <field name="sequence" eval="10"/>
+            <field name="view" ref="babi_table_tree_view"/>
+            <field name="act_window" ref="act_babi_table"/>
+        </record>
+        <record model="ir.action.act_window.view" id="act_babi_table_view2">
+            <field name="sequence" eval="20"/>
+            <field name="view" ref="babi_table_form_view"/>
+            <field name="act_window" ref="act_babi_table"/>
+        </record>
+
+        <record model="ir.model.access" id="access_babi_table">
+            <field name="model" search="[('model', '=', 'babi.table')]"/>
+            <field name="perm_read" eval="False"/>
+            <field name="perm_write" eval="False"/>
+            <field name="perm_create" eval="False"/>
+            <field name="perm_delete" eval="False"/>
+        </record>
+
+        <record model="ir.model.access" id="access_babi_table_babi">
+            <field name="model" search="[('model', '=', 'babi.table')]"/>
+            <field name="group" ref="group_babi"/>
+            <field name="perm_read" eval="True"/>
+            <field name="perm_write" eval="True"/>
+            <field name="perm_create" eval="True"/>
+            <field name="perm_delete" eval="True"/>
+        </record>
+
+        <!-- Create button -->
+        <record model="ir.model.button" id="babi_table_calculate_button">
+            <field name="name">compute</field>
+            <field name="string">Compute</field>
+            <field name="model" search="[('model', '=', 'babi.table')]"/>
+        </record>
+
+        <menuitem id="menu_babi_table" parent="menu_babi" action="act_babi_table" sequence="10"/>
+
+        <!-- babi.field -->
+        <record model="ir.ui.view" id="babi_field_form_view">
+            <field name="model">babi.field</field>
+            <field name="type">form</field>
+            <field name="name">field_form</field>
+        </record>
+
+        <record model="ir.ui.view" id="babi_field_tree_view">
+            <field name="model">babi.field</field>
+            <field name="type">tree</field>
+            <field name="name">field_list</field>
+        </record>
+
+        <record model="ir.action.act_window" id="act_babi_field">
+            <field name="name">Fields</field>
+            <field name="res_model">babi.field</field>
+        </record>
+        <record model="ir.action.act_window.view" id="act_babi_field_view1">
+            <field name="sequence" eval="10"/>
+            <field name="view" ref="babi_field_tree_view"/>
+            <field name="act_window" ref="act_babi_field"/>
+        </record>
+        <record model="ir.action.act_window.view" id="act_babi_field_view2">
+            <field name="sequence" eval="20"/>
+            <field name="view" ref="babi_field_form_view"/>
+            <field name="act_window" ref="act_babi_field"/>
+        </record>
+
+        <menuitem id="menu_babi_field" parent="menu_babi_table" action="act_babi_field" sequence="10"/>
+    </data>
+</tryton>

tests/test_module.py

@@ -706,4 +706,42 @@ class BabiTestCase(CompanyTestMixin, ModuleTestCase):
         executions = Execution.search([])
         self.assertEqual(len(executions), 0)
 
+    @with_transaction()
+    def test_table(self):
+        pool = Pool()
+        Table = pool.get('babi.table')
+        Field = pool.get('babi.field')
+        Model = pool.get('ir.model')
+        Expression = pool.get('babi.expression')
+
+        self.create_data()
+
+        table = Table()
+        table.name = 'Table'
+        table.on_change_name()
+        self.assertEqual(table.internal_name, 'table')
+        table.model, = Model.search([('model', '=', 'babi.test')])
+
+        fields = []
+        names = set([])
+        for expression in Expression.search([], order=[('name', 'ASC')]):
+            field = Field()
+            field.expression = expression
+            field.on_change_expression()
+            field.on_change_name()
+            if field.name in names:
+                continue
+            names.add(field.name)
+            fields.append(field)
+
+        table.fields_ = fields
+        table.save()
+        table._compute()
+
+        cursor = Transaction().connection.cursor()
+        cursor.execute('SELECT count(*) FROM "%s"' % table.internal_name)
+        count = cursor.fetchall()[0][0]
+        self.assertNotEqual(count, 0)
+
+
 del ModuleTestCase

tryton.cfg

@@ -8,6 +8,7 @@ depends:
     smtp
 xml:
     babi.xml
+    table.xml
     configuration.xml
     cron.xml
     messages.xml

view/cron_form.xml

@@ -1,7 +1,12 @@
 <data>
-    <xpath expr="/form" position="inside">
+    <xpath expr="/form/field[@name='active']" position="after">
+        <newline/>
         <label name="babi_report"/>
         <field name="babi_report"/>
+        <newline/>
+        <label name="babi_table"/>
+        <field name="babi_table"/>
+        <newline/>
     </xpath>
 </data>
 

view/cron_list.xml

@@ -1,5 +1,6 @@
 <data>
     <xpath expr="/tree" position="inside">
         <field name="babi_report"/>
+        <field name="babi_table"/>
     </xpath>
 </data>

view/field_form.xml

@@ -0,0 +1,11 @@
+<form col="6">
+    <label name="table"/>
+    <field name="table"/>
+    <newline/>
+    <label name="name"/>
+    <field name="name"/>
+    <label name="internal_name"/>
+    <field name="internal_name"/>
+    <label name="expression"/>
+    <field name="expression"/>
+</form>

view/field_list.xml

@@ -0,0 +1,6 @@
+<tree editable="1">
+    <field name="table"/>
+    <field name="expression"/>
+    <field name="name"/>
+    <field name="internal_name"/>
+</tree>

view/table_form.xml

@@ -0,0 +1,25 @@
+<form col="6">
+    <label name="name"/>
+    <field name="name"/>
+    <label name="internal_name"/>
+    <field name="internal_name"/>
+    <label name="model"/>
+    <field name="model"/>
+    <label name="filter"/>
+    <field name="filter" colspan="3"/>
+    <button name="compute" colspan="2"/>
+    <notebook colspan="6">
+        <page name="fields_">
+            <field name="fields_" colspan="4"/>
+        </page>
+        <page name="crons">
+            <field name="crons" colspan="4"/>
+        </page>
+        <page id="configuration" string="Configuration">
+            <label name="timeout"/>
+            <field name="timeout"/>
+            <label name="babi_raise_user_error"/>
+            <field name="babi_raise_user_error"/>
+        </page>
+    </notebook>
+</form>

view/table_list.xml

@@ -0,0 +1,7 @@
+<tree>
+    <field name="name"/>
+    <field name="internal_name"/>
+    <field name="model"/>
+    <field name="filter"/>
+    <button name="compute"/>
+</tree>