From a1f4e9f46385c8c884f4475196b9dc9c92f1ac0a Mon Sep 17 00:00:00 2001 From: Raimon Esteve Date: Wed, 28 Oct 2020 15:22:05 +0100 Subject: [PATCH] Upgrade to 5.4 --- __init__.py | 10 +-- csv_import.py | 129 ++++++++++++---------------- csv_import.xml | 17 +++- message.xml | 32 +++++++ tests/scenario_csv_import.rst | 28 ++---- tests/test_csv_import.py | 2 +- tryton.cfg | 3 +- view/base_external_mapping_form.xml | 3 +- view/base_external_mapping_tree.xml | 3 +- view/csv_archive_form.xml | 12 +-- view/csv_archive_tree.xml | 2 +- view/csv_profile_form.xml | 2 +- view/csv_profile_tree.xml | 2 +- 13 files changed, 130 insertions(+), 115 deletions(-) create mode 100644 message.xml diff --git a/__init__.py b/__init__.py index beabe8e..801853a 100644 --- a/__init__.py +++ b/__init__.py @@ -2,13 +2,13 @@ # The COPYRIGHT file at the top level of this repository contains # the full copyright notices and license terms. from trytond.pool import Pool -from .csv_import import * +from . import csv_import def register(): Pool.register( - CSVProfile, - CSVProfileBaseExternalMapping, - CSVArchive, - BaseExternalMapping, + csv_import.CSVProfile, + csv_import.CSVProfileBaseExternalMapping, + csv_import.CSVArchive, + csv_import.BaseExternalMapping, module='csv_import', type_='model') diff --git a/csv_import.py b/csv_import.py index e34a1ea..3f1a403 100644 --- a/csv_import.py +++ b/csv_import.py @@ -1,39 +1,33 @@ # This file is part of csv_import module for Tryton. # The COPYRIGHT file at the top level of this repository contains # the full copyright notices and license terms. -try: - import cStringIO as StringIO -except ImportError: - from io import StringIO -from csv import reader -from datetime import datetime -from trytond.config import config -from trytond.model import ModelSQL, ModelView, fields, Workflow -from trytond.pool import Pool, PoolMeta -from trytond.pyson import Eval, If -from trytond.transaction import Transaction import os import re import unicodedata import string +import csv +from io import StringIO +from datetime import datetime +from trytond.config import config +from trytond.model import ModelSQL, ModelView, fields, Workflow +from trytond.pool import Pool, PoolMeta +from trytond.pyson import Eval +from trytond.transaction import Transaction +from trytond.i18n import gettext +from trytond.exceptions import UserError __all__ = ['BaseExternalMapping', 'CSVProfile', 'CSVProfileBaseExternalMapping', 'CSVArchive'] -_slugify_strip_re = re.compile(r'[^\w\s-]') -_slugify_hyphenate_re = re.compile(r'[-\s]+') def slugify(value): - if not isinstance(value, unicode): - value = unicode(value) value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') - value = unicode(_slugify_strip_re.sub('', value).strip().lower()) - return _slugify_hyphenate_re.sub('-', value) + value = re.sub('[^\w\s-]', '', value.decode('utf-8')).strip().lower() + return re.sub('[-\s]+', '-', value) -class BaseExternalMapping: - __metaclass__ = PoolMeta +class BaseExternalMapping(metaclass=PoolMeta): __name__ = 'base.external.mapping' csv_mapping = fields.Many2One('base.external.mapping', 'CSV Mapping') csv_rel_field = fields.Many2One('ir.model.field', 'CSV Field related') @@ -137,8 +131,10 @@ class CSVArchive(Workflow, ModelSQL, ModelView): @classmethod def __setup__(cls): super(CSVArchive, cls).__setup__() - cls._order.insert(0, ('date_archive', 'DESC')) - cls._order.insert(1, ('id', 'DESC')) + cls._order = [ + ('date_archive', 'DESC'), + ('id', 'DESC'), + ] cls._transitions |= set(( ('draft', 'done'), ('draft', 'canceled'), @@ -147,59 +143,43 @@ class CSVArchive(Workflow, ModelSQL, ModelView): cls._buttons.update({ 'cancel': { 'invisible': Eval('state') != 'draft', + 'depends': ['state'], }, 'draft': { 'invisible': Eval('state') != 'canceled', - 'icon': If(Eval('state') == 'canceled', 'tryton-clear', - 'tryton-go-previous'), + 'depends': ['state'], }, 'import_csv': { 'invisible': Eval('state') != 'draft', + 'depends': ['state'], }, }) - cls._error_messages.update({ - 'error': 'CSV Import Error!', - 'reading_error': 'Error reading file %s.', - 'read_error': 'Error reading file: %s.\nError %s.', - 'success_simulation': 'Simulation successfully.', - 'record_saved': 'Record ID %s saved successfully!', - 'record_error': 'Error saving records.', - 'not_create_update': 'Not create or update line %s', - }) def get_data(self, name): - cursor = Transaction().connection.cursor() path = os.path.join(config.get('database', 'path'), - cursor.database_name, 'csv_import') + Transaction().database.name, 'csv_import') archive = '%s/%s' % (path, self.archive_name.replace(' ', '_')) try: - with open(archive, 'r') as f: + with open(archive, 'rb') as f: return fields.Binary.cast(f.read()) except IOError: - self.raise_user_error('error', - error_description='reading_error', - error_description_args=(self.archive_name.replace(' ', '_'),), - raise_exception=True) + pass @classmethod def set_data(cls, archives, name, value): - cursor = Transaction().connection.cursor() path = os.path.join(config.get('database', 'path'), - cursor.database_name, 'csv_import') + Transaction().database.name, 'csv_import') if not os.path.exists(path): - os.makedirs(path, mode=0777) + os.makedirs(path, mode=0o777) for archive in archives: archive = '%s/%s' % (path, archive.archive_name.replace(' ', '_')) try: - with open(archive, 'w') as f: + with open(archive, 'wb') as f: f.write(value) - except IOError, e: - cls.raise_user_error('error', - error_description='save_error', - error_description_args=(e,), - raise_exception=True) + except IOError: + raise UserError(gettext('csv_import.msg_error')) - @fields.depends('profile') + @fields.depends('profile', '_parent_profile.rec_name') def on_change_profile(self): if self.profile: today = Pool().get('ir.date').today() @@ -235,7 +215,7 @@ class CSVArchive(Workflow, ModelSQL, ModelView): if hasattr(cls, method_data): import_data = getattr(cls, method_data) record = import_data(record, values, parent_values) - for k, v in values.iteritems(): + for k, v in values.items(): setattr(record, k, v) return record @@ -260,24 +240,22 @@ class CSVArchive(Workflow, ModelSQL, ModelView): quote = profile.csv_quote header = profile.csv_header - data = StringIO(archive.data) + data = StringIO(archive.data.decode('ascii', errors='replace')) try: - rows = reader(data, delimiter=str(separator), + reader = csv.reader(data, delimiter=str(separator), quotechar=str(quote)) - except TypeError, e: + except TypeError: cls.write([archive], {'logs': 'Error - %s' % ( - cls.raise_user_error('error', - error_description='read_error', - error_description_args=(archive.archive_name, e), - raise_exception=False), + gettext('csv_import.msg_read_error', + filename=archive.archive_name.replace(' ', '_')) )}) return - if header: # TODO. Know why some header columns get "" - headers = [filter(lambda x: x in string.printable, x - ).replace('"', '') - for x in next(rows)] - return rows, headers + if header: + # TODO. Know why some header columns get "" + headers = ["".join(list(filter(lambda x: x in string.printable, + x.replace('"', '')))) for x in next(reader)] + return reader, headers @classmethod @ModelView.button @@ -298,7 +276,7 @@ class CSVArchive(Workflow, ModelSQL, ModelView): if not profile.create_record and not profile.update_record: continue - data, headers = cls._read_csv_file(archive) + reader, headers = cls._read_csv_file(archive) base_model = profile.model.model @@ -308,10 +286,14 @@ class CSVArchive(Workflow, ModelSQL, ModelView): child_mappings.append(mapping) else: base_mapping = mapping + if not base_mapping: + logs.append(gettext('csv_import.msg_not_mapping', + profile=profile.rec_name)) + continue new_records = [] new_lines = [] - rows = list(data) + rows = list(reader) Base = pool.get(base_model) for i in range(len(rows)): row = rows[i] @@ -325,13 +307,17 @@ class CSVArchive(Workflow, ModelSQL, ModelView): if not new_lines: base_values = ExternalMapping.map_external_to_tryton( base_mapping.name, vals) - if not base_values.values()[0] == '': + if not list(base_values.values())[0] == '': new_lines = [] #get values child models child_values = None child_rel_field = None for child in child_mappings: + if not child.csv_rel_field: + logs.append(gettext('csv_import.msg_missing_rel_field', + mapping=child.rec_name)) + continue child_rel_field = child.csv_rel_field.name child_values = ExternalMapping.map_external_to_tryton( child.name, vals) @@ -345,7 +331,7 @@ class CSVArchive(Workflow, ModelSQL, ModelView): if child_rel_field: base_values[child_rel_field] = new_lines - #next row is empty first value, is a new line. Continue + # next row is empty first value, is a new line. Continue if i < len(rows) - 1: if rows[i + 1]: if rows[i + 1][0] == '': @@ -367,8 +353,8 @@ class CSVArchive(Workflow, ModelSQL, ModelView): record = Base() if not record: - logs.append(cls.raise_user_error('not_create_update', - error_args=(i + 1,), raise_exception=False)) + logs.append(gettext('csv_import.msg_not_create_update', + line=i + 1)) continue #get default values from base model @@ -377,13 +363,12 @@ class CSVArchive(Workflow, ModelSQL, ModelView): #save - not testing if not profile.testing: record.save() # save or update - logs.append(cls.raise_user_error('record_saved', - error_args=(record.id,), raise_exception=False)) + logs.append(gettext('csv_import.msg_record_saved', + record=record.id)) new_records.append(record.id) if profile.testing: - logs.append(cls.raise_user_error('success_simulation', - raise_exception=False)) + logs.append(gettext('csv_import.msg_success_simulation')) cls.post_import(profile, new_records) cls.write([archive], {'logs': '\n'.join(logs)}) diff --git a/csv_import.xml b/csv_import.xml index 93e8869..2672344 100644 --- a/csv_import.xml +++ b/csv_import.xml @@ -78,6 +78,22 @@ copyright notices and license terms. --> + + cancel + Cancel + + + + draft + Draft + + + + import_csv + Import CSV + + + base.external.mapping @@ -138,4 +154,3 @@ copyright notices and license terms. --> - diff --git a/message.xml b/message.xml new file mode 100644 index 0000000..447d451 --- /dev/null +++ b/message.xml @@ -0,0 +1,32 @@ + + + + + + CSV Import Error! + + + Error reading file: %(filename)s. +%(error)s + + + Simulation successfully + + + Record ID "%(record)s" saved successfully! + + + Error saving records + + + Not create or update line %(line)s + + + Not found mapping at "%(profile)s" + + + Missing relation field at "%(mapping)s" + + + diff --git a/tests/scenario_csv_import.rst b/tests/scenario_csv_import.rst index ed557a6..8f54ba1 100644 --- a/tests/scenario_csv_import.rst +++ b/tests/scenario_csv_import.rst @@ -12,9 +12,11 @@ Imports:: >>> from dateutil.relativedelta import relativedelta >>> from decimal import Decimal >>> from operator import attrgetter - >>> today = datetime.date.today() + >>> from proteus import config, Model, Wizard + >>> from trytond.tests.tools import activate_modules >>> from trytond.config import config >>> from trytond.tests.test_tryton import DB_NAME as db_name + >>> today = datetime.date.today() >>> module_path = os.path.dirname(__file__) + '/' @@ -28,27 +30,14 @@ Imports:: >>> if not os.path.exists(data_path + db_name + module_name): ... os.makedirs(data_path + db_name + module_name) - >>> from proteus import config, Model, Wizard +Install csv_import:: -Create database:: - - >>> config = config.set_trytond() - >>> config.pool.test = True - -Install modules:: - - >>> Module = Model.get('ir.module') - >>> modules = Module.find([ - ... ('name', 'in', ('party', 'csv_import')), - ... ]) - >>> Module.install([x.id for x in modules], config.context) - >>> Wizard('ir.module.install_upgrade').execute('upgrade') + >>> config = activate_modules(['csv_import', 'party']) Init models:: >>> Model = Model.get('ir.model') >>> Field = Model.get('ir.model.field') - >>> Group = Model.get('res.group') >>> BaseExternalMapping = Model.get('base.external.mapping') >>> BaseExternalMappingLine = Model.get('base.external.mapping.line') >>> CSVProfile = Model.get('csv.profile') @@ -122,7 +111,6 @@ Create profile:: >>> profile = CSVProfile() >>> profile.name = 'Parties' >>> profile.model = Model.find([('model', '=', 'party.party')])[0] - >>> profile.group = Group.find([('name', '=', 'Administration')])[0] >>> profile.create_record = True >>> profile.csv_header = True >>> profile.csv_archive_separator = ',' @@ -136,7 +124,7 @@ Create CSV archive:: >>> srcfile = '%s/%s' % (module_path, 'import_party.csv') >>> dstfile = '%s/%s/%s/%s' % (data_path, db_name, module_name, ... 'import_party.csv') - >>> shutil.copy(srcfile, dstfile) + >>> _ = shutil.copy(srcfile, dstfile) >>> CSVArchive = Model.get('csv.archive') >>> archive = CSVArchive() >>> archive.profile = profile @@ -156,7 +144,7 @@ Create Parties and multi Addresses:: >>> srcfile = '%s/%s' % (module_path, 'import_party_multiaddress.csv') >>> dstfile = '%s/%s/%s/%s' % (data_path, db_name, module_name, ... 'import_party_multiaddress.csv') - >>> shutil.copy(srcfile, dstfile) + >>> _ = shutil.copy(srcfile, dstfile) >>> CSVArchive = Model.get('csv.archive') >>> archive = CSVArchive() >>> archive.profile = profile @@ -190,7 +178,7 @@ Create CSV Update archive:: >>> srcfile = '%s/%s' % (module_path, 'update_party.csv') >>> dstfile = '%s/%s/%s/%s' % (data_path, db_name, module_name, ... 'update_party.csv') - >>> shutil.copy(srcfile, dstfile) + >>> _ = shutil.copy(srcfile, dstfile) >>> CSVArchive = Model.get('csv.archive') >>> archive = CSVArchive() >>> archive.profile = CSVProfile.find([])[0] diff --git a/tests/test_csv_import.py b/tests/test_csv_import.py index d0eab91..f7e0e5b 100644 --- a/tests/test_csv_import.py +++ b/tests/test_csv_import.py @@ -10,7 +10,7 @@ from trytond.tests.test_tryton import doctest_checker class CsvImportTestCase(ModuleTestCase): - 'Test Csv Import module' + 'Test CSV Import module' module = 'csv_import' diff --git a/tryton.cfg b/tryton.cfg index 07010fa..aec693f 100644 --- a/tryton.cfg +++ b/tryton.cfg @@ -1,8 +1,9 @@ [tryton] -version=3.9.0 +version=5.4.0 depends: ir res base_external_mapping xml: csv_import.xml + message.xml diff --git a/view/base_external_mapping_form.xml b/view/base_external_mapping_form.xml index 032df09..18775c7 100644 --- a/view/base_external_mapping_form.xml +++ b/view/base_external_mapping_form.xml @@ -3,8 +3,7 @@ The COPYRIGHT file at the top level of this repository contains the full copyright notices and license terms. --> - +