NaNtic
/
trytond-csv_import
mirror of https://github.com/NaN-tic/trytond-csv_import.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Upgrade to 5.4

This commit is contained in:
Raimon Esteve 2020-10-28 15:22:05 +01:00
parent aa37f4fa6c
commit a1f4e9f463
13 changed files with 130 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
__init__.py
View File

@ -2,13 +2,13 @@
# The COPYRIGHT file at the top level of this repository contains
# the full copyright notices and license terms.
from trytond.pool import Pool
from .csv_import import *
from . import csv_import
def register():
Pool.register(
CSVProfile,
CSVProfileBaseExternalMapping,
CSVArchive,
BaseExternalMapping,
csv_import.CSVProfile,
csv_import.CSVProfileBaseExternalMapping,
csv_import.CSVArchive,
csv_import.BaseExternalMapping,
module='csv_import', type_='model')

129
csv_import.py
View File

@ -1,39 +1,33 @@
# This file is part of csv_import module for Tryton.
# The COPYRIGHT file at the top level of this repository contains
# the full copyright notices and license terms.
try:
import cStringIO as StringIO
except ImportError:
from io import StringIO
from csv import reader
from datetime import datetime
from trytond.config import config
from trytond.model import ModelSQL, ModelView, fields, Workflow
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval, If
from trytond.transaction import Transaction
import os
import re
import unicodedata
import string
import csv
from io import StringIO
from datetime import datetime
from trytond.config import config
from trytond.model import ModelSQL, ModelView, fields, Workflow
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval
from trytond.transaction import Transaction
from trytond.i18n import gettext
from trytond.exceptions import UserError
__all__ = ['BaseExternalMapping',
'CSVProfile', 'CSVProfileBaseExternalMapping', 'CSVArchive']
_slugify_strip_re = re.compile(r'[^\w\s-]')
_slugify_hyphenate_re = re.compile(r'[-\s]+')
def slugify(value):
if not isinstance(value, unicode):
value = unicode(value)
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
value = unicode(_slugify_strip_re.sub('', value).strip().lower())
return _slugify_hyphenate_re.sub('-', value)
value = re.sub('[^\w\s-]', '', value.decode('utf-8')).strip().lower()
return re.sub('[-\s]+', '-', value)
class BaseExternalMapping:
__metaclass__ = PoolMeta
class BaseExternalMapping(metaclass=PoolMeta):
__name__ = 'base.external.mapping'
csv_mapping = fields.Many2One('base.external.mapping', 'CSV Mapping')
csv_rel_field = fields.Many2One('ir.model.field', 'CSV Field related')
@ -137,8 +131,10 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
@classmethod
def __setup__(cls):
super(CSVArchive, cls).__setup__()
cls._order.insert(0, ('date_archive', 'DESC'))
cls._order.insert(1, ('id', 'DESC'))
cls._order = [
('date_archive', 'DESC'),
('id', 'DESC'),
]
cls._transitions |= set((
('draft', 'done'),
('draft', 'canceled'),
@ -147,59 +143,43 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
cls._buttons.update({
'cancel': {
'invisible': Eval('state') != 'draft',
'depends': ['state'],
},
'draft': {
'invisible': Eval('state') != 'canceled',
'icon': If(Eval('state') == 'canceled', 'tryton-clear',
'tryton-go-previous'),
'depends': ['state'],
},
'import_csv': {
'invisible': Eval('state') != 'draft',
'depends': ['state'],
},
})
cls._error_messages.update({
'error': 'CSV Import Error!',
'reading_error': 'Error reading file %s.',
'read_error': 'Error reading file: %s.\nError %s.',
'success_simulation': 'Simulation successfully.',
'record_saved': 'Record ID %s saved successfully!',
'record_error': 'Error saving records.',
'not_create_update': 'Not create or update line %s',
})
def get_data(self, name):
cursor = Transaction().connection.cursor()
path = os.path.join(config.get('database', 'path'),
cursor.database_name, 'csv_import')
Transaction().database.name, 'csv_import')
archive = '%s/%s' % (path, self.archive_name.replace(' ', '_'))
try:
with open(archive, 'r') as f:
with open(archive, 'rb') as f:
return fields.Binary.cast(f.read())
except IOError:
self.raise_user_error('error',
error_description='reading_error',
error_description_args=(self.archive_name.replace(' ', '_'),),
raise_exception=True)
pass
@classmethod
def set_data(cls, archives, name, value):
cursor = Transaction().connection.cursor()
path = os.path.join(config.get('database', 'path'),
cursor.database_name, 'csv_import')
Transaction().database.name, 'csv_import')
if not os.path.exists(path):
os.makedirs(path, mode=0777)
os.makedirs(path, mode=0o777)
for archive in archives:
archive = '%s/%s' % (path, archive.archive_name.replace(' ', '_'))
try:
with open(archive, 'w') as f:
with open(archive, 'wb') as f:
f.write(value)
except IOError, e:
cls.raise_user_error('error',
error_description='save_error',
error_description_args=(e,),
raise_exception=True)
except IOError:
raise UserError(gettext('csv_import.msg_error'))
@fields.depends('profile')
@fields.depends('profile', '_parent_profile.rec_name')
def on_change_profile(self):
if self.profile:
today = Pool().get('ir.date').today()
@ -235,7 +215,7 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
if hasattr(cls, method_data):
import_data = getattr(cls, method_data)
record = import_data(record, values, parent_values)
for k, v in values.iteritems():
for k, v in values.items():
setattr(record, k, v)
return record
@ -260,24 +240,22 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
quote = profile.csv_quote
header = profile.csv_header
data = StringIO(archive.data)
data = StringIO(archive.data.decode('ascii', errors='replace'))
try:
rows = reader(data, delimiter=str(separator),
reader = csv.reader(data, delimiter=str(separator),
quotechar=str(quote))
except TypeError, e:
except TypeError:
cls.write([archive], {'logs': 'Error - %s' % (
cls.raise_user_error('error',
error_description='read_error',
error_description_args=(archive.archive_name, e),
raise_exception=False),
gettext('csv_import.msg_read_error',
filename=archive.archive_name.replace(' ', '_'))
)})
return
if header: # TODO. Know why some header columns get ""
headers = [filter(lambda x: x in string.printable, x
).replace('"', '')
for x in next(rows)]
return rows, headers
if header:
# TODO. Know why some header columns get ""
headers = ["".join(list(filter(lambda x: x in string.printable,
x.replace('"', '')))) for x in next(reader)]
return reader, headers
@classmethod
@ModelView.button
@ -298,7 +276,7 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
if not profile.create_record and not profile.update_record:
continue
data, headers = cls._read_csv_file(archive)
reader, headers = cls._read_csv_file(archive)
base_model = profile.model.model
@ -308,10 +286,14 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
child_mappings.append(mapping)
else:
base_mapping = mapping
if not base_mapping:
logs.append(gettext('csv_import.msg_not_mapping',
profile=profile.rec_name))
continue
new_records = []
new_lines = []
rows = list(data)
rows = list(reader)
Base = pool.get(base_model)
for i in range(len(rows)):
row = rows[i]
@ -325,13 +307,17 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
if not new_lines:
base_values = ExternalMapping.map_external_to_tryton(
base_mapping.name, vals)
if not base_values.values()[0] == '':
if not list(base_values.values())[0] == '':
new_lines = []
#get values child models
child_values = None
child_rel_field = None
for child in child_mappings:
if not child.csv_rel_field:
logs.append(gettext('csv_import.msg_missing_rel_field',
mapping=child.rec_name))
continue
child_rel_field = child.csv_rel_field.name
child_values = ExternalMapping.map_external_to_tryton(
child.name, vals)
@ -345,7 +331,7 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
if child_rel_field:
base_values[child_rel_field] = new_lines
#next row is empty first value, is a new line. Continue
# next row is empty first value, is a new line. Continue
if i < len(rows) - 1:
if rows[i + 1]:
if rows[i + 1][0] == '':
@ -367,8 +353,8 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
record = Base()
if not record:
logs.append(cls.raise_user_error('not_create_update',
error_args=(i + 1,), raise_exception=False))
logs.append(gettext('csv_import.msg_not_create_update',
line=i + 1))
continue
#get default values from base model
@ -377,13 +363,12 @@ class CSVArchive(Workflow, ModelSQL, ModelView):
#save - not testing
if not profile.testing:
record.save() # save or update
logs.append(cls.raise_user_error('record_saved',
error_args=(record.id,), raise_exception=False))
logs.append(gettext('csv_import.msg_record_saved',
record=record.id))
new_records.append(record.id)
if profile.testing:
logs.append(cls.raise_user_error('success_simulation',
raise_exception=False))
logs.append(gettext('csv_import.msg_success_simulation'))
cls.post_import(profile, new_records)
cls.write([archive], {'logs': '\n'.join(logs)})

17
csv_import.xml
View File

@ -78,6 +78,22 @@ copyright notices and license terms. -->
<field name="group" ref="group_csv_import_admin"/>
</record>
<record model="ir.model.button" id="cancel_button">
<field name="name">cancel</field>
<field name="string">Cancel</field>
<field name="model" search="[('model', '=', 'csv.archive')]"/>
</record>
<record model="ir.model.button" id="draft_button">
<field name="name">draft</field>
<field name="string">Draft</field>
<field name="model" search="[('model', '=', 'csv.archive')]"/>
</record>
<record model="ir.model.button" id="import_csv_button">
<field name="name">import_csv</field>
<field name="string">Import CSV</field>
<field name="model" search="[('model', '=', 'csv.archive')]"/>
</record>
<!-- base.external.mapping -->
<record model="ir.ui.view" id="base_external_mapping_tree_view">
<field name="model">base.external.mapping</field>
@ -138,4 +154,3 @@ copyright notices and license terms. -->
</record>
</data>
</tryton>

32
message.xml Normal file
View File

@ -0,0 +1,32 @@
<?xml version="1.0"?>
<!-- This file is part of Tryton. The COPYRIGHT file at the top level of
this repository contains the full copyright notices and license terms. -->
<tryton>
<data grouped="1">
<record model="ir.message" id="msg_error">
<field name="text">CSV Import Error!</field>
</record>
<record model="ir.message" id="msg_read_error">
<field name="text">Error reading file: %(filename)s.
%(error)s</field>
</record>
<record model="ir.message" id="msg_success_simulation">
<field name="text">Simulation successfully</field>
</record>
<record model="ir.message" id="msg_record_saved">
<field name="text">Record ID "%(record)s" saved successfully!</field>
</record>
<record model="ir.message" id="msg_record_error">
<field name="text">Error saving records</field>
</record>
<record model="ir.message" id="msg_not_create_update">
<field name="text">Not create or update line %(line)s</field>
</record>
<record model="ir.message" id="msg_not_mapping">
<field name="text">Not found mapping at "%(profile)s"</field>
</record>
<record model="ir.message" id="msg_missing_rel_field">
<field name="text">Missing relation field at "%(mapping)s"</field>
</record>
</data>
</tryton>

28
tests/scenario_csv_import.rst
View File

@ -12,9 +12,11 @@ Imports::
>>> from dateutil.relativedelta import relativedelta
>>> from decimal import Decimal
>>> from operator import attrgetter
>>> today = datetime.date.today()
>>> from proteus import config, Model, Wizard
>>> from trytond.tests.tools import activate_modules
>>> from trytond.config import config
>>> from trytond.tests.test_tryton import DB_NAME as db_name
>>> today = datetime.date.today()
>>> module_path = os.path.dirname(__file__) + '/'
@ -28,27 +30,14 @@ Imports::
>>> if not os.path.exists(data_path + db_name + module_name):
... os.makedirs(data_path + db_name + module_name)
>>> from proteus import config, Model, Wizard
Install csv_import::
Create database::
>>> config = config.set_trytond()
>>> config.pool.test = True
Install modules::
>>> Module = Model.get('ir.module')
>>> modules = Module.find([
... ('name', 'in', ('party', 'csv_import')),
... ])
>>> Module.install([x.id for x in modules], config.context)
>>> Wizard('ir.module.install_upgrade').execute('upgrade')
>>> config = activate_modules(['csv_import', 'party'])
Init models::
>>> Model = Model.get('ir.model')
>>> Field = Model.get('ir.model.field')
>>> Group = Model.get('res.group')
>>> BaseExternalMapping = Model.get('base.external.mapping')
>>> BaseExternalMappingLine = Model.get('base.external.mapping.line')
>>> CSVProfile = Model.get('csv.profile')
@ -122,7 +111,6 @@ Create profile::
>>> profile = CSVProfile()
>>> profile.name = 'Parties'
>>> profile.model = Model.find([('model', '=', 'party.party')])[0]
>>> profile.group = Group.find([('name', '=', 'Administration')])[0]
>>> profile.create_record = True
>>> profile.csv_header = True
>>> profile.csv_archive_separator = ','
@ -136,7 +124,7 @@ Create CSV archive::
>>> srcfile = '%s/%s' % (module_path, 'import_party.csv')
>>> dstfile = '%s/%s/%s/%s' % (data_path, db_name, module_name,
... 'import_party.csv')
>>> shutil.copy(srcfile, dstfile)
>>> _ = shutil.copy(srcfile, dstfile)
>>> CSVArchive = Model.get('csv.archive')
>>> archive = CSVArchive()
>>> archive.profile = profile
@ -156,7 +144,7 @@ Create Parties and multi Addresses::
>>> srcfile = '%s/%s' % (module_path, 'import_party_multiaddress.csv')
>>> dstfile = '%s/%s/%s/%s' % (data_path, db_name, module_name,
... 'import_party_multiaddress.csv')
>>> shutil.copy(srcfile, dstfile)
>>> _ = shutil.copy(srcfile, dstfile)
>>> CSVArchive = Model.get('csv.archive')
>>> archive = CSVArchive()
>>> archive.profile = profile
@ -190,7 +178,7 @@ Create CSV Update archive::
>>> srcfile = '%s/%s' % (module_path, 'update_party.csv')
>>> dstfile = '%s/%s/%s/%s' % (data_path, db_name, module_name,
... 'update_party.csv')
>>> shutil.copy(srcfile, dstfile)
>>> _ = shutil.copy(srcfile, dstfile)
>>> CSVArchive = Model.get('csv.archive')
>>> archive = CSVArchive()
>>> archive.profile = CSVProfile.find([])[0]

2
tests/test_csv_import.py
View File

@ -10,7 +10,7 @@ from trytond.tests.test_tryton import doctest_checker
class CsvImportTestCase(ModuleTestCase):
'Test Csv Import module'
'Test CSV Import module'
module = 'csv_import'

3
tryton.cfg
View File

@ -1,8 +1,9 @@
[tryton]
version=3.9.0
version=5.4.0
depends:
ir
res
base_external_mapping
xml:
csv_import.xml
message.xml

3
view/base_external_mapping_form.xml
View File

@ -3,8 +3,7 @@
The COPYRIGHT file at the top level of this repository contains the full
copyright notices and license terms. -->
<data>
<xpath expr="/form/field[@name=&quot;mapping_lines&quot;]"
position="after">
<xpath expr="/form/field[@name='mapping_lines']" position="after">
<group col="6" colspan="6" id="csv_import">
<separator id="csv_import" string="CSV Import" colspan="6"/>
<label name="csv_mapping"/>

3
view/base_external_mapping_tree.xml
View File

@ -3,8 +3,7 @@
The COPYRIGHT file at the top level of this repository contains the full
copyright notices and license terms. -->
<data>
<xpath expr="/tree/field[@name=&quot;state&quot;]"
position="after">
<xpath expr="/tree/field[@name='state']" position="after">
<field name="csv_mapping"/>
<field name="csv_rel_field"/>
</xpath>

12
view/csv_archive_form.xml
View File

@ -2,7 +2,7 @@
<!-- This file is part of csv_import module for Tryton.
The COPYRIGHT file at the top level of this repository contains the full
copyright notices and license terms. -->
<form string="CSV Archive" col="4">
<form col="4">
<label name="profile"/>
<field name="profile"/>
<label name="date_archive"/>
@ -14,13 +14,9 @@ copyright notices and license terms. -->
<separator name="logs" colspan="4"/>
<field name="logs" colspan="4"/>
<group col="4" colspan="4" id="csv_buttons">
<button name="cancel"
string="Cancel Import Data from CSV"
icon="tryton-cancel"/>
<button name="draft" string="Draft"/>
<button name="import_csv"
string="Import Data from CSV"
icon="tryton-go-next"/>
<button name="cancel" icon="tryton-cancel"/>
<button name="draft"/>
<button name="import_csv" icon="tryton-go-next"/>
</group>
<group col="4" colspan="4" id="states">
<label name="state"/>

2
view/csv_archive_tree.xml
View File

@ -2,7 +2,7 @@
<!-- This file is part of csv_field module for Tryton.
The COPYRIGHT file at the top level of this repository contains the full
copyright notices and license terms. -->
<tree string="CSV Archive">
<tree>
<field name="profile"/>
<field name="date_archive" widget="date"/>
<field name="date_archive" widget="time" string="Time"/>

2
view/csv_profile_form.xml
View File

@ -2,7 +2,7 @@
<!-- This file is part of csv_import module for Tryton.
The COPYRIGHT file at the top level of this repository contains the full
copyright notices and license terms. -->
<form string="CSV Import">
<form>
<label name="name"/>
<field name="name"/>
<notebook colspan="4">

2
view/csv_profile_tree.xml
View File

@ -2,7 +2,7 @@
<!-- This file is part of csv_import module for Tryton.
The COPYRIGHT file at the top level of this repository contains the full
copyright notices and license terms. -->
<tree string="CSV Profiles">
<tree>
<field name="name"/>
<field name="model"/>
<field name="create_record"/>