diff --git a/connector_importer/README.rst b/connector_importer/README.rst new file mode 100644 index 000000000..57e19f980 --- /dev/null +++ b/connector_importer/README.rst @@ -0,0 +1,54 @@ +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 + +================== +Connector importer +================== + +This module allows to import / update records from files using the connector +framework (i.e. mappers) and job queues. + +Known issues / Roadmap +====================== + +N/A + + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +`here `_. + + +Credits +======= + +Contributors +------------ + +Simone Orsi (Camptocamp) for the original implementation. + + +Other contributors include: + +* Guewen Baconnier (Camptocamp) +* Mykhailo Panarin (Camptocamp) + +Maintainer +---------- + +.. image:: http://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: http://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. diff --git a/connector_importer/__init__.py b/connector_importer/__init__.py new file mode 100644 index 000000000..5f713a85c --- /dev/null +++ b/connector_importer/__init__.py @@ -0,0 +1,3 @@ +from . import models +from . import components +from . import controllers diff --git a/connector_importer/__manifest__.py b/connector_importer/__manifest__.py new file mode 100644 index 000000000..e0371d771 --- /dev/null +++ b/connector_importer/__manifest__.py @@ -0,0 +1,30 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +{ + 'name': 'Connector Importer', + 'summary': """This module takes care of import sessions.""", + 'version': '11.0.1.0.0', + 'depends': [ + 'connector', + 'queue_job', + ], + 'author': 'Camptocamp, Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Connector', + 'website': 'https://github.com/OCA/connector-interfaces', + 'data': [ + 'data/ir_cron.xml', + 'security/security.xml', + 'security/ir.model.access.csv', + 'views/backend_views.xml', + 'views/recordset_views.xml', + 'views/source_views.xml', + 'views/report_template.xml', + 'views/docs_template.xml', + 'views/source_config_template.xml', + 'menuitems.xml', + ], + 'external_dependencies': {'python': ['chardet']}, +} diff --git a/connector_importer/components/__init__.py b/connector_importer/components/__init__.py new file mode 100644 index 000000000..03ec7686a --- /dev/null +++ b/connector_importer/components/__init__.py @@ -0,0 +1,5 @@ +from . import base +from . import tracker +from . import odoorecord +from . import importer +from . import mapper diff --git a/connector_importer/components/base.py b/connector_importer/components/base.py new file mode 100644 index 000000000..cd2ae7355 --- /dev/null +++ b/connector_importer/components/base.py @@ -0,0 +1,12 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.addons.component.core import AbstractComponent + + +class ImporterComponent(AbstractComponent): + + _name = 'importer.base.component' + _inherit = 'base.connector' + _collection = 'import.backend' diff --git a/connector_importer/components/importer.py b/connector_importer/components/importer.py new file mode 100644 index 000000000..c90b987d1 --- /dev/null +++ b/connector_importer/components/importer.py @@ -0,0 +1,363 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import fields +from odoo.addons.component.core import Component +from ..log import logger +from ..log import LOGGER_NAME + + +class RecordSetImporter(Component): + """Importer for recordsets.""" + + _name = 'importer.recordset' + _inherit = 'importer.base.component' + _usage = 'recordset.importer' + _apply_on = 'import.recordset' + + def run(self, recordset, **kw): + """Run recordset job. + + Steps: + + * update last start date on recordset + * read source + * process all source lines in chunks + * create an import record per each chunk + * schedule import for each record + """ + # update recordset report + recordset.set_report({ + '_last_start': fields.Datetime.now(), + }, reset=True) + msg = 'START RECORDSET {0}({1})'.format(recordset.name, + recordset.id) + logger.info(msg) + + # flush existing records as we are going to re-create them + recordset.record_ids.unlink() + source = recordset.get_source() + for chunk in source.get_lines(): + # create chuncked records and run their imports + record = self.env['import.record'].create( + {'recordset_id': recordset.id}) + # store data + record.set_data(chunk) + record.run_import() + + +class RecordImporter(Component): + """Importer for records. + + This importer is actually the one that does the real import work. + It loads each import records and tries to import them + and keep tracks of errored, skipped, etc. + See `run` method for detailed information on what it does. + """ + + _name = 'importer.record' + _inherit = ['importer.base.component'] + _usage = 'record.importer' + _apply_on = 'import.record' + # log and report errors + # do not make the whole import fail + _break_on_error = False + # a unique key (field name) to retrieve the odoo record + odoo_unique_key = '' + + def _init_importer(self, recordset): + self.recordset = recordset + # record handler is responsible for create/write on odoo records + self.record_handler = self.component(usage='odoorecord.handler') + self.record_handler._init_handler( + importer=self, + unique_key=self.odoo_unique_key + ) + # tracking handler is responsible for logging and chunk reports + self.tracker = self.component(usage='tracking.handler') + self.tracker._init_handler( + model_name=self.model._name, + logger_name=LOGGER_NAME, + log_prefix=self.recordset.import_type_id.key + ' ', + ) + + _mapper = None + + @property + def mapper(self): + if not self._mapper: + self._mapper = self.component(usage='importer.mapper') + return self._mapper + + def required_keys(self, create=False): + """Keys that are mandatory to import a line.""" + req = self.mapper.required_keys() + all_values = [] + for k, v in req.items(): + # make sure values are always tuples + # as we support multiple dest keys + if not isinstance(v, (tuple, list)): + req[k] = (v, ) + all_values.extend(req[k]) + unique_key = self.odoo_unique_key + if (unique_key and + unique_key not in list(req.keys()) and + unique_key not in all_values): + # this one is REALLY required :) + req[unique_key] = (unique_key, ) + return req + + # mostly for auto-documentation in UI + def default_values(self): + """Values that are automatically assigned.""" + return self.mapper.default_values() + + def translatable_keys(self, create=False): + """Keys that are translatable.""" + return self.mapper.translatable_keys() + + def translatable_langs(self): + return self.env['res.lang'].search([ + ('translatable', '=', True)]).mapped('code') + + def make_translation_key(self, key, lang): + return '{}:{}'.format(key, lang) + + def collect_translatable(self, values, orig_values): + """Get translations values for `mapper.translatable_keys`. + + We assume that the source contains translatable columns in the form: + + `mapper_key:lang` + + whereas `mapper_key` is an odoo record field to translate + and lang matches one of the installed languages. + + Translatable keys must be declared on the mapper + within the attribute `translatable`. + """ + translatable = {} + if not self.translatable_keys(): + return translatable + for lang in self.translatable_langs(): + for key in self.translatable_keys(): + # eg: name:fr_FR + tkey = self.make_translation_key(key, lang) + if tkey in orig_values and values.get(key): + if lang not in translatable: + translatable[lang] = {} + # we keep only translation for existing values + translatable[lang][key] = orig_values.get(tkey) + return translatable + + def _check_missing(self, source_key, dest_key, values, orig_values): + """Check for required keys missing.""" + missing = (not source_key.startswith('__') and + orig_values.get(source_key) is None) + unique_key = self.odoo_unique_key + if missing: + msg = 'MISSING REQUIRED SOURCE KEY={}'.format(source_key) + if unique_key and values.get(unique_key): + msg += ': {}={}'.format( + unique_key, values[unique_key]) + return { + 'message': msg, + } + missing = (not dest_key.startswith('__') and + values.get(dest_key) is None) + if missing: + msg = 'MISSING REQUIRED DESTINATION KEY={}'.format(dest_key) + if unique_key and values.get(unique_key): + msg += ': {}={}'.format( + unique_key, values[unique_key]) + return { + 'message': msg, + } + return False + + def skip_it(self, values, orig_values): + """Skip item import conditionally... if you want ;). + + You can return back `False` to not skip + or a dictionary containing info about skip reason. + """ + msg = '' + required = self.required_keys() + for source_key, dest_key in required.items(): + # we support multiple destination keys + for _dest_key in dest_key: + missing = self._check_missing( + source_key, _dest_key, values, orig_values) + if missing: + return missing + + if self.record_handler.odoo_exists(values, orig_values) \ + and not self.recordset.override_existing: + msg = 'ALREADY EXISTS' + if self.odoo_unique_key: + msg += ': {}={}'.format( + self.odoo_unique_key, values[self.odoo_unique_key]) + return { + 'message': msg, + 'odoo_record': + self.record_handler.odoo_find(values, orig_values).id, + } + return False + + def _cleanup_line(self, line): + """Apply basic cleanup on lines.""" + # we cannot alter dict keys while iterating + res = {} + for k, v in line.items(): + # skip internal tech keys if any + if not k.startswith('_'): + k = self.clean_line_key(k) + if isinstance(v, str): + v = v.strip() + res[k] = v + return res + + def clean_line_key(self, key): + """Clean record key. + + Sometimes your CSV source do not have proper keys, + they can contain a lot of crap or they can change + lower/uppercase from import to importer. + You can override this method to normalize keys + and make your import mappers work reliably. + """ + return key.strip() + + def prepare_line(self, line): + """Pre-manipulate a line if needed. + + For instance: you might want to fix some field names. + Sometimes in CSV you have mispelled names + (upper/lowercase, spaces, etc) all chars that might break your mappers. + + Here you can adapt the source line before the mapper is called + so that the logic in the mapper will be always the same. + """ + return self._cleanup_line(line) + + def _do_report(self): + """Update recordset report using the tracker.""" + previous = self.recordset.get_report() + report = self.tracker.get_report(previous) + self.recordset.set_report({self.model._name: report}) + + def _record_lines(self): + """Get lines from import record.""" + return self.record.get_data() + + def _load_mapper_options(self): + """Retrieve mapper options.""" + return { + 'override_existing': self.recordset.override_existing + } + + # TODO: make these contexts customizable via recordset settings + def _odoo_create_context(self): + """Inject context variables on create, merged by odoorecord handler.""" + return { + 'tracking_disable': True, + } + + def _odoo_write_context(self): + """Inject context variables on write, merged by odoorecord handler.""" + return { + 'tracking_disable': True, + } + + def run(self, record, is_last_importer=True, **kw): + """Run record job. + + Steps: + + * check if record is still available + * initialize the import + * read each line to be imported + * clean them up + * manipulate them (field names fixes and such) + * retrieve a mapper and convert values + * check and skip record if needed + * if record exists: update it, else, create it + * produce a report and store it on recordset + """ + + self.record = record + if not self.record: + # maybe deleted??? + msg = 'NO RECORD FOUND, maybe deleted? Check your jobs!' + logger.error(msg) + return + + self._init_importer(self.record.recordset_id) + for line in self._record_lines(): + line = self.prepare_line(line) + options = self._load_mapper_options() + + odoo_record = None + + try: + with self.env.cr.savepoint(): + values = self.mapper.map_record(line).values(**options) + logger.debug(values) + except Exception as err: + values = {} + self.tracker.log_error(values, line, odoo_record, message=err) + if self._break_on_error: + raise err + continue + + # handle forced skipping + skip_info = self.skip_it(values, line) + if skip_info: + self.tracker.log_skipped(values, line, skip_info) + continue + + try: + with self.env.cr.savepoint(): + if self.record_handler.odoo_exists(values, line): + odoo_record = \ + self.record_handler.odoo_write(values, line) + self.tracker.log_updated(values, line, odoo_record) + else: + odoo_record = \ + self.record_handler.odoo_create(values, line) + self.tracker.log_created(values, line, odoo_record) + except Exception as err: + self.tracker.log_error(values, line, odoo_record, message=err) + if self._break_on_error: + raise err + continue + + # TODO: fix this later, for now we need to remove it, + # otherwise we won't be able to run multiple import in same time + # self._do_report() + + # log chunk finished + msg = ' '.join([ + 'CHUNK FINISHED', + '[created: {created}]', + '[updated: {updated}]', + '[skipped: {skipped}]', + '[errored: {errored}]', + ]).format(**self.tracker.get_counters()) + self.tracker._log(msg) + + # TODO + # chunk_finished_event.fire( + # self.env, self.model._name, self.record) + return 'ok' + + # TODO + def after_all(self, recordset): + """Get something done after all the children jobs have completed. + + This should be triggered by `chunk_finished_event`. + """ + # TODO: needed for logger and other stuff. Can be simplified. + # self._init_importer(recordset) + pass diff --git a/connector_importer/components/mapper.py b/connector_importer/components/mapper.py new file mode 100644 index 000000000..4bacd2b3b --- /dev/null +++ b/connector_importer/components/mapper.py @@ -0,0 +1,94 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.addons.component.core import Component +from odoo.addons.connector.components.mapper import mapping + + +class ImportMapper(Component): + _name = 'importer.base.mapper' + _inherit = ['importer.base.component', 'base.import.mapper'] + _usage = 'importer.mapper' + + required = { + # source key: dest key + # You can declare here the keys the importer must have + # to import a record. + # `source key` means a key in the source record + # either a line in a csv file or a lien from an sql table. + # `dest key` is the destination the for the source one. + + # Eg: in your mapper you could have a mapping like + # direct = [ + # ('title', 'name'), + # (concat(('title', 'foo', ), separator=' - '), 'baz'), + # ] + + # You want the record to be skipped if: + + # 1. title or name are not valued in the source + # 2. title is valued but the conversion gives an empty value for name + # 3. title or foo are not valued in the source + # 4. title and foo are valued but the conversion + # gives an empty value for baz + + # You can achieve this like: + # required = { + # 'title': ('name', 'baz'), + # 'foo': 'baz', + # } + + # If you want to check only the source or the destination key + # use the same name and prefix it w/ double underscore, like: + + # {'__foo': 'baz', 'foo': '__baz'} + } + + def required_keys(self, create=False): + """Return required keys for this mapper. + + The importer can use this to determine if a line + has to be skipped. + + The recordset will use this to show required fields to users. + """ + return self.required + + translatable = [] + + def translatable_keys(self, create=False): + """Return translatable keys for this mapper. + + The importer can use this to translate specific fields + if the are found in the csv in the form `field_name:lang_code`. + + The recordset will use this to show translatable fields to users. + """ + return self.translatable + + defaults = [ + # odoo field, value + # ('sale_ok', True), + + # defaults can be also retrieved via xmlid to other records. + # The format is: `_xmlid::$record_xmlid::$record_field_value` + # whereas `$record_xmlid` is the xmlid to retrieve + # and ``$record_field_value` is the field to be used as value. + # Example: + # ('company_id', '_xmlid:base.main_company:id'), + ] + + @mapping + def default_values(self, record=None): + """Return default values for this mapper. + + The recordset will use this to show default values to users. + """ + values = {} + for k, v in self.defaults: + if isinstance(v, str) and v.startswith('_xmlid::'): + xmlid, field_value = v.split('::')[1:] + v = self.env.ref(xmlid)[field_value] + values[k] = v + return values diff --git a/connector_importer/components/odoorecord.py b/connector_importer/components/odoorecord.py new file mode 100644 index 000000000..882fc3e49 --- /dev/null +++ b/connector_importer/components/odoorecord.py @@ -0,0 +1,133 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.addons.component.core import Component + + +class OdooRecordHandler(Component): + """Interact w/ odoo importable records.""" + + _name = 'importer.odoorecord.handler' + _inherit = 'importer.base.component' + _usage = 'odoorecord.handler' + + unique_key = '' + importer = None + # By default odoo ignores create_uid/write_uid in vals. + # If you enable this flags and `create_uid` and/or `write_uid` + # are found in values they gonna be used for sudo. + # Same for `create_date`. + override_create_uid = False + override_create_date = False + override_write_uid = False + + def _init_handler(self, importer=None, unique_key=None): + self.importer = importer + self.unique_key = unique_key + + def odoo_find_domain(self, values, orig_values): + """Domain to find the record in odoo.""" + return [(self.unique_key, '=', values[self.unique_key])] + + def odoo_find(self, values, orig_values): + """Find any existing item in odoo.""" + if not self.unique_key: + return self.model + item = self.model.search( + self.odoo_find_domain(values, orig_values), + order='create_date desc', limit=1) + return item + + def odoo_exists(self, values, orig_values): + """Return true if the items exists.""" + return bool(self.odoo_find(values, orig_values)) + + def update_translations(self, odoo_record, translatable, ctx=None): + """Write translations on given record.""" + ctx = ctx or {} + for lang, values in translatable.items(): + odoo_record.with_context( + lang=lang, **self.write_context()).write(values.copy()) + + def odoo_pre_create(self, values, orig_values): + """Do some extra stuff before creating a missing record.""" + pass + + def odoo_post_create(self, odoo_record, values, orig_values): + """Do some extra stuff after creating a missing record.""" + pass + + def create_context(self): + """Inject context variables on create.""" + return dict( + self.importer._odoo_create_context(), + # mark each action w/ this flag + connector_importer_session=True, + ) + + def odoo_create(self, values, orig_values): + """Create a new odoo record.""" + self.odoo_pre_create(values, orig_values) + # TODO: remove keys that are not model's fields + odoo_record = self.model.with_context( + **self.create_context()).create(values.copy()) + # force uid + if self.override_create_uid and values.get('create_uid'): + self._force_value(odoo_record, values, 'create_uid') + # force create date + if self.override_create_date and values.get('create_date'): + self._force_value(odoo_record, values, 'create_date') + self.odoo_post_create(odoo_record, values, orig_values) + translatable = self.importer.collect_translatable(values, orig_values) + self.update_translations(odoo_record, translatable) + return odoo_record + + def odoo_pre_write(self, odoo_record, values, orig_values): + """Do some extra stuff before updating an existing object.""" + pass + + def odoo_post_write(self, odoo_record, values, orig_values): + """Do some extra stuff after updating an existing object.""" + pass + + def write_context(self): + """Inject context variables on write.""" + return dict( + self.importer._odoo_write_context(), + # mark each action w/ this flag + connector_importer_session=True, + ) + + def odoo_write(self, values, orig_values): + """Update an existing odoo record.""" + # pass context here to be applied always on retrieved record + odoo_record = self.odoo_find( + values, orig_values + ).with_context(**self.write_context()) + self.odoo_pre_write(odoo_record, values, orig_values) + # TODO: remove keys that are not model's fields + odoo_record.write(values.copy()) + # force uid + if self.override_write_uid and values.get('write_uid'): + self._force_value(odoo_record, values, 'write_uid') + # force create date + if self.override_create_date and values.get('create_date'): + self._force_value(odoo_record, values, 'create_date') + self.odoo_post_write(odoo_record, values, orig_values) + translatable = self.importer.collect_translatable(values, orig_values) + self.update_translations(odoo_record, translatable) + return odoo_record + + def _force_value(self, record, values, fname): + # the query construction is not vulnerable to SQL injection, as we are + # replacing the table and column names here. + # pylint: disable=sql-injection + query = 'UPDATE {} SET {} = %s WHERE id = %s'.format( + record._table, fname + ) + self.env.cr.execute( + query, + (values[fname], record.id, ) + ) + record.invalidate_cache([fname, ]) diff --git a/connector_importer/components/tracker.py b/connector_importer/components/tracker.py new file mode 100644 index 000000000..041480f06 --- /dev/null +++ b/connector_importer/components/tracker.py @@ -0,0 +1,133 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.addons.component.core import Component +import logging + + +class ChunkReport(dict): + """A smarter dict for chunk reports.""" + + chunk_report_keys = ( + 'created', + 'updated', + 'errored', + 'skipped', + ) + + def __init__(self, **kwargs): + super(ChunkReport, self).__init__(**kwargs) + for k in self.chunk_report_keys: + self[k] = [] + + def track_error(self, item): + self['errored'].append(item) + + def track_skipped(self, item): + self['skipped'].append(item) + + def track_updated(self, item): + self['updated'].append(item) + + def track_created(self, item): + self['created'].append(item) + + def counters(self): + res = {} + for k, v in self.items(): + res[k] = len(v) + return res + + +class Tracker(Component): + """Track what happens during importer jobs.""" + + _name = 'importer.tracking.handler' + _inherit = 'importer.base.component' + _usage = 'tracking.handler' + + model_name = '' + logger_name = '' + log_prefix = '' + _chunk_report_klass = ChunkReport + + def _init_handler(self, model_name='', logger_name='', log_prefix=''): + self.model_name = model_name + self.logger_name = logger_name + self.log_prefix = log_prefix + + _logger = None + _chunk_report = None + + @property + def logger(self): + if not self._logger: + self._logger = logging.getLogger(self.logger_name) + return self._logger + + @property + def chunk_report(self): + if not self._chunk_report: + self._chunk_report = self._chunk_report_klass() + return self._chunk_report + + def chunk_report_item(self, line, odoo_record=None, message=''): + return { + 'line_nr': line['_line_nr'], + 'message': message, + 'model': self.model_name, + 'odoo_record': odoo_record.id if odoo_record else None, + } + + def _log(self, msg, line=None, level='info'): + handler = getattr(self.logger, level) + msg = '{prefix}{line}[model: {model}] {msg}'.format( + prefix=self.log_prefix, + line='[line: {}]'.format(line['_line_nr']) if line else '', + model=self.model_name, + msg=msg + ) + handler(msg) + + def log_updated(self, values, line, odoo_record, message=''): + self._log('UPDATED [id: {}]'.format(odoo_record.id), line=line) + self.chunk_report.track_updated(self.chunk_report_item( + line, odoo_record=odoo_record, message=message + )) + + def log_error(self, values, line, odoo_record, message=''): + if isinstance(message, Exception): + message = str(message) + self._log(message, line=line, level='error') + self.chunk_report.track_error(self.chunk_report_item( + line, odoo_record=odoo_record, message=message + )) + + def log_created(self, values, line, odoo_record, message=''): + self._log('CREATED [id: {}]'.format(odoo_record.id), line=line) + self.chunk_report.track_created(self.chunk_report_item( + line, odoo_record=odoo_record, message=message + )) + + def log_skipped(self, values, line, skip_info): + # `skip_it` could contain a msg + self._log('SKIPPED ' + skip_info.get('message'), + line=line, level='warn') + + item = self.chunk_report_item(line) + item.update(skip_info) + self.chunk_report.track_skipped(item) + + def get_report(self, previous=None): + previous = previous or {} + # init a new report + report = self._chunk_report_klass() + # merge previous and current + for k, v in report.items(): + prev = previous.get(self.model_name, {}).get(k, []) + report[k] = prev + self.chunk_report[k] + return report + + def get_counters(self): + return self.chunk_report.counters() diff --git a/connector_importer/controllers/__init__.py b/connector_importer/controllers/__init__.py new file mode 100644 index 000000000..12a7e529b --- /dev/null +++ b/connector_importer/controllers/__init__.py @@ -0,0 +1 @@ +from . import main diff --git a/connector_importer/controllers/main.py b/connector_importer/controllers/main.py new file mode 100644 index 000000000..777c2e92d --- /dev/null +++ b/connector_importer/controllers/main.py @@ -0,0 +1,24 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import http +from odoo.http import request +from ..utils.report_html import Reporter + + +class ReportController(http.Controller): + """Controller to display import reports.""" + + # TODO: refactor this to use qweb template only + @http.route( + '/importer/import-recordset/', + type='http', auth="user", website=False) + def full_report(self, recordset, **kwargs): + reporter = Reporter(recordset.jsondata, detailed=1) + values = { + 'recordset': recordset, + 'report': reporter.html(wrapped=0), + + } + return request.render("connector_importer.recordset_report", values) diff --git a/connector_importer/data/ir_cron.xml b/connector_importer/data/ir_cron.xml new file mode 100644 index 000000000..3adfaf125 --- /dev/null +++ b/connector_importer/data/ir_cron.xml @@ -0,0 +1,17 @@ + + + + + Importer backend: cleanup old recordsets + + code + model.cron_cleanup_recordsets() + + + 1 + weeks + -1 + + + + diff --git a/connector_importer/events.py b/connector_importer/events.py new file mode 100644 index 000000000..5ab96821c --- /dev/null +++ b/connector_importer/events.py @@ -0,0 +1,28 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.addons.connector.event import Event + +chunk_finished_event = Event() + + +@chunk_finished_event +def chunk_finished_subscriber(env, dest_model_name, last_record): + """Run `import_record_after_all` after last record has been imported.""" + if not last_record.job_id: + # ok... we are not running in cron mode..my job here has finished! + return + # TODO + # backend = last_record.backend_id + # recordset = last_record.recordset_id + # other_records_completed = [ + # r.job_id.state == 'done' + # for r in recordset.record_ids + # if r != last_record + # ] + # if all(other_records_completed): + # job_method = last_record.with_delay().import_record_after_all + # if backend.debug_mode(): + # job_method = last_record.import_record_after_all + # job_method(last_record_id=record_id) diff --git a/connector_importer/log.py b/connector_importer/log.py new file mode 100644 index 000000000..44c270138 --- /dev/null +++ b/connector_importer/log.py @@ -0,0 +1,26 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import os +import logging +from logging.handlers import RotatingFileHandler + +LOGGER_NAME = '[importer]' +logger = logging.getLogger(LOGGER_NAME) +logger.setLevel(logging.INFO) + +if os.getenv('IMPORTER_LOG_PATH'): + # use separated log file when developing + FNAME = 'import.log' + + base_path = os.environ.get('IMPORTER_LOG_PATH') + if not os.path.exists(base_path): + os.makedirs(base_path) + + # add a rotating handler + handler = RotatingFileHandler(base_path + '/' + FNAME, + maxBytes=1024 * 5, + backupCount=5) + logger.addHandler(handler) + logging.info('logging to {}'.format(base_path + '/' + FNAME)) diff --git a/connector_importer/menuitems.xml b/connector_importer/menuitems.xml new file mode 100644 index 000000000..03f4d8aa3 --- /dev/null +++ b/connector_importer/menuitems.xml @@ -0,0 +1,25 @@ + + + + + + + + + + diff --git a/connector_importer/models/__init__.py b/connector_importer/models/__init__.py new file mode 100644 index 000000000..f79d6ef03 --- /dev/null +++ b/connector_importer/models/__init__.py @@ -0,0 +1,7 @@ +from . import cron_mixin +from . import backend +from . import import_type +from . import sources +from . import recordset +from . import record +from . import reporter diff --git a/connector_importer/models/backend.py b/connector_importer/models/backend.py new file mode 100644 index 000000000..8be57bc34 --- /dev/null +++ b/connector_importer/models/backend.py @@ -0,0 +1,154 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api, exceptions, _ +import logging + +cleanup_logger = logging.getLogger('[recordset-cleanup]') + +BACKEND_VERSIONS = [ + ('1.0', 'Version 1.0'), +] + + +class ImporterBackend(models.Model): + _name = 'import.backend' + _description = 'Importer Backend' + _inherit = [ + 'connector.backend', + 'cron.mixin', + ] + + @api.model + def _select_version(self): + """ Available versions + + Can be inherited to add custom versions. + """ + return BACKEND_VERSIONS + + name = fields.Char(required=True) + version = fields.Selection( + selection='_select_version', + string='Version', + required=True, + ) + recordset_ids = fields.One2many( + 'import.recordset', + 'backend_id', + string='Record Sets' + ) + # cron stuff + cron_master_recordset_id = fields.Many2one( + 'import.recordset', + string='Master recordset', + help=('If an existing recordset is selected ' + 'it will be used to create a new recordset ' + 'each time the cron runs. ' + '\nIn this way you can keep every import session isolated. ' + '\nIf none, all recordsets will run.') + ) + cron_cleanup_keep = fields.Integer( + string='Cron cleanup keep', + help=('If this value is greater than 0 ' + 'a cron will cleanup old recordsets ' + 'and keep only the latest N records matching this value.'), + ) + notes = fields.Text('Notes') + debug_mode = fields.Boolean( + 'Debug mode?', + help=("Enabling debug mode causes the import to run " + "in real time, without using any job queue. " + "Make sure you don't do this in production!") + ) + job_running = fields.Boolean( + 'Job running', + compute='_compute_job_running', + help="Tells you if a job is running for this backend.", + readonly=True + ) + + @api.multi + def unlink(self): + """Prevent delete if jobs are running.""" + for item in self: + item.check_delete() + return super(ImporterBackend, self).unlink() + + @api.multi + def check_delete(self): + if not self.debug_mode and self.job_running: + raise exceptions.Warning(_('You must complete the job first!')) + + @api.multi + def _compute_job_running(self): + for item in self: + running = False + for recordset in self.recordset_ids: + if recordset.has_job() and not recordset.job_done(): + running = True + break + for record in recordset.record_ids: + if record.has_job() and not record.job_done(): + running = True + break + item.job_running = running + + @api.model + def run_cron(self, backend_id): + # required by cron mixin + self.browse(backend_id).run_all() + + @api.multi + def run_all(self): + """Run all recordset imports.""" + self.ensure_one() + recordsets = self.recordset_ids + if self.cron_master_recordset_id: + # clone and use it to run + recordsets = self.cron_master_recordset_id.copy() + for recordset in recordsets: + recordset.run_import() + + @api.model + def cron_cleanup_recordsets(self): + """Delete obsolete recordsets. + + If you are running imports via cron and you create one recorset + per each run then you might end up w/ tons of old recordsets. + + You can use `cron_cleanup_keep` to enable auto-cleanup. + Here we lookup for backends w/ this settings + and keep only latest recordsets. + """ + cleanup_logger.info('Looking for recorsets to cleanup.') + backends = self.search([('cron_cleanup_keep', '>', 0)]) + to_clean = self.env['import.recordset'] + for backend in backends: + if len(backend.recordset_ids) <= backend.cron_cleanup_keep: + continue + to_keep = backend.recordset_ids.sorted( + lambda x: x.create_date, + reverse=True + )[:backend.cron_cleanup_keep] + # always keep this + to_keep |= backend.cron_master_recordset_id + to_clean = backend.recordset_ids - to_keep + if to_clean: + msg = 'Cleaning up {}'.format(','.join(to_clean.mapped('name'))) + cleanup_logger.info(msg) + to_clean.unlink() + else: + cleanup_logger.info('Nothing to do.') + + @api.multi + def button_complete_jobs(self): + """Set all jobs to "completed" state.""" + self.ensure_one() + for recordset in self.recordset_ids: + for record in recordset.record_ids: + if record.has_job() and not record.job_done(): + record.job_id.button_done() + if recordset.has_job() and not recordset.job_done(): + recordset.job_id.button_done() diff --git a/connector_importer/models/cron_mixin.py b/connector_importer/models/cron_mixin.py new file mode 100644 index 000000000..ed077fc3a --- /dev/null +++ b/connector_importer/models/cron_mixin.py @@ -0,0 +1,85 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api + + +class CronMixin(models.AbstractModel): + """Add cron-related features to your models. + + On inheriting models you can: + + * enable cron mode + * configure a cron + * save and get a specific cron to run something on your model + + You have to implement the method `run_cron`. + """ + _name = 'cron.mixin' + _description = 'Cron Mixin' + + cron_mode = fields.Boolean('Cron mode?') + cron_start_date = fields.Datetime('Start date') + cron_interval_number = fields.Integer('Interval number') + cron_interval_type = fields.Selection( + selection='_select_interval_type', + string='Interval type', + ) + cron_id = fields.Many2one( + 'ir.cron', + string='Related cron', + domain=lambda self: [ + ('model_id', '=', self.env['ir.model']._get_id(self._name)) + ], + ) + + @api.model + def _select_interval_type(self): + return [ + ('hours', 'Hours'), + ('work_days', 'Work Days'), + ('days', 'Days'), + ('weeks', 'Weeks'), + ('months', 'Months') + ] + + @api.model + def get_cron_vals(self): + model_id = self.env['ir.model']._get_id(self._name) + return { + 'name': 'Cron for import backend %s' % self.name, + 'model_id': model_id, + 'state': 'code', + 'code': 'model.run_cron(%d)' % self.id, + 'interval_number': self.cron_interval_number, + 'interval_type': self.cron_interval_type, + 'nextcall': self.cron_start_date, + } + + def _update_or_create_cron(self): + """Update or create cron record if needed.""" + if self.cron_mode: + cron_model = self.env['ir.cron'] + cron_vals = self.get_cron_vals() + if not self.cron_id: + self.cron_id = cron_model.create(cron_vals) + else: + self.cron_id.write(cron_vals) + + @api.model + def create(self, vals): + rec = super().create(vals) + rec._update_or_create_cron() + return rec + + @api.multi + def write(self, vals): + res = super().write(vals) + for backend in self: + backend._update_or_create_cron() + return res + + @api.model + def run_cron(self): + raise NotImplementedError() diff --git a/connector_importer/models/import_type.py b/connector_importer/models/import_type.py new file mode 100644 index 000000000..2f3c6bc21 --- /dev/null +++ b/connector_importer/models/import_type.py @@ -0,0 +1,74 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api, _ + + +class ImportType(models.Model): + """Define an import. + + An import type describes what an recordset should do. + You can describe an import using the `settings` field. + Here you can declare what you want to import (model) and how (importer). + + Settings example: + + product.template::template.importer.component.name + product.product::product.importer.component.name + + Each line contains a couple model::importer. + The model is what you want to import, the importer states + the name of the connector component to handle the import for that model. + + The importer machinery will run the imports for all the models declared + and will retrieve their specific importerts to execute them. + """ + _name = 'import.type' + _description = 'Import type' + + name = fields.Char( + required=True, + help='A meaningful human-friendly name' + ) + key = fields.Char( + required=True, + help='Unique mnemonic identifier' + ) + settings = fields.Text( + string='Settings', + required=True, + help=""" + # comment me + product.template::template.importer.component.name + product.product::product.importer.component.name + # another one + product.supplierinfo::supplierinfo.importer.component.name + """ + ) + _sql_constraints = [ + ('key_uniq', 'unique (key)', _("Import type `key` must be unique!")) + ] + # TODO: provide default source and configuration policy + # for an import type to ease bootstrapping recordsets from UI. + # default_source_model_id = fields.Many2one() + + @api.multi + def available_models(self): + """Retrieve available import models and their importers. + + Parse `settings` and yield a tuple + `(model, importer, is_last_importer)`. + """ + self.ensure_one() + lines = self.settings.strip().splitlines() + for _line in lines: + line = _line.strip() + if line and not line.startswith('#'): + model_name, importer = line.split('::') + is_last_importer = False + if _line == lines[-1]: + is_last_importer = True + yield ( + model_name.strip(), importer.strip(), is_last_importer + ) diff --git a/connector_importer/models/job_mixin.py b/connector_importer/models/job_mixin.py new file mode 100644 index 000000000..3d0747557 --- /dev/null +++ b/connector_importer/models/job_mixin.py @@ -0,0 +1,47 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import fields, api, exceptions, _ + +from odoo.addons.queue_job.job import DONE, STATES + + +class JobRelatedMixin(object): + """Mixin klass for queue.job relationship. + + We do not use an abstract model to be able to not re-define + the relation on each inheriting model. + """ + + job_id = fields.Many2one( + 'queue.job', + string='Job', + readonly=True, + ) + job_state = fields.Selection( + STATES, + string='Job State', + readonly=True, + index=True, + related='job_id.state' + ) + + @api.model + def has_job(self): + return bool(self.job_id) + + @api.model + def job_done(self): + return self.job_state == DONE + + @api.model + def check_delete(self): + if self.has_job() and not self.job_done(): + raise exceptions.Warning(_('You must complete the job first!')) + + @api.multi + def unlink(self): + for item in self: + item.check_delete() + return super(JobRelatedMixin, self).unlink() diff --git a/connector_importer/models/record.py b/connector_importer/models/record.py new file mode 100644 index 000000000..dde1b481a --- /dev/null +++ b/connector_importer/models/record.py @@ -0,0 +1,134 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import base64 +import json +import os + +from odoo import models, fields, api +from odoo.addons.queue_job.job import job +from .job_mixin import JobRelatedMixin +from ..log import logger + + +class ImportRecord(models.Model, JobRelatedMixin): + """Data to be imported. + + An import record contains what you are actually importing. + + Depending on backend settings you gonna have one or more source records + stored as JSON data into `jsondata` field. + + No matter where you are importing from (CSV, SQL, etc) + the importer machinery will: + + * retrieve the models to import and their importer + * process all records and import them + * update recordset info + + When the importer will run, it will read all the records, + convert them using connector mappers and do the import. + """ + _name = 'import.record' + _description = 'Import record' + _order = 'id' + _backend_type = 'import_backend' + + date = fields.Datetime( + 'Import date', + default=fields.Date.context_today, + ) + # This field holds the whole bare data to import from the external source + # hence it can be huge. For this reason we store it in an attachment. + jsondata_file = fields.Binary( + attachment=True, + ) + recordset_id = fields.Many2one( + 'import.recordset', + string='Recordset' + ) + backend_id = fields.Many2one( + 'import.backend', + string='Backend', + related='recordset_id.backend_id', + readonly=True, + ) + + @api.multi + def unlink(self): + # inheritance of non-model mixin does not work w/out this + return super(ImportRecord, self).unlink() + + @api.multi + @api.depends('date') + def _compute_name(self): + for item in self: + names = [ + item.date, + ] + item.name = ' / '.join([_f for _f in names if _f]) + + @api.multi + def set_data(self, adict): + self.ensure_one() + jsondata = json.dumps(adict) + self.jsondata_file = base64.b64encode(bytes(jsondata, 'utf-8')) + + @api.multi + def get_data(self): + self.ensure_one() + jsondata = None + if self.jsondata_file: + raw_data = base64.b64decode(self.jsondata_file).decode('utf-8') + jsondata = json.loads(raw_data) + return jsondata or {} + + @api.multi + def debug_mode(self): + self.ensure_one() + return self.backend_id.debug_mode or \ + os.environ.get('IMPORTER_DEBUG_MODE') + + @api.multi + @job + def import_record(self, component_name, model_name, is_last_importer=True): + """This job will import a record. + + :param component_name: name of the importer component to use + :param model_name: name of the model to import + :param is_last_importer: flag for last importer of the recordset + """ + with self.backend_id.work_on(self._name) as work: + importer = work.component_by_name( + component_name, model_name=model_name) + return importer.run(self, is_last_importer=is_last_importer) + + @api.multi + def run_import(self): + """ queue a job for importing data stored in to self + """ + job_method = self.with_delay().import_record + if self.debug_mode(): + logger.warn('### DEBUG MODE ACTIVE: WILL NOT USE QUEUE ###') + job_method = self.import_record + _result = {} + for item in self: + # we create a record and a job for each model name + # that needs to be imported + for ( + model, importer, is_last_importer + ) in item.recordset_id.available_models(): + # TODO: grab component from config + result = job_method( + importer, model, is_last_importer=is_last_importer) + _result[model] = result + if self.debug_mode(): + # debug mode, no job here: reset it! + item.write({'job_id': False}) + else: + # FIXME: we should have a o2m here otherwise + # w/ multiple importers for the same record + # we keep the reference on w/ the last job. + item.write({'job_id': result.db_record().id}) + return _result diff --git a/connector_importer/models/recordset.py b/connector_importer/models/recordset.py new file mode 100644 index 000000000..25a052a1f --- /dev/null +++ b/connector_importer/models/recordset.py @@ -0,0 +1,333 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import base64 +import os +from collections import OrderedDict + +from odoo import models, fields, api +from odoo.addons.queue_job.job import ( + DONE, STATES, job) +from odoo.addons.base_sparse_field.models.fields import Serialized +from .job_mixin import JobRelatedMixin +from ..log import logger + + +class ImportRecordset(models.Model, JobRelatedMixin): + """Set of records, together with their configuration. + + A recordset can be considered as an "import session". + Here you declare: + + * what you want to import (via "Import type") + * where you get records from (via "Source" configuration) + + A recordset is also responsible to hold and display some meaningful + information about imports: + + * required fields, translatable fields, defaults + * import stats (created|updated|skipped|errored counters, latest run) + * fully customizable HTML report to provide more details + * downloadable report file (via reporters) + * global states of running jobs + + When you run the import of a recordset this is what it does: + + * ask the source to provide all the records (chunked) + * create an import record for each chunk + * schedule the import job for each import record + """ + _name = 'import.recordset' + _inherit = 'import.source.consumer.mixin' + _description = 'Import recordset' + _order = 'sequence ASC, create_date DESC' + _backend_type = 'import_backend' + + backend_id = fields.Many2one( + 'import.backend', + string='Import Backend' + ) + sequence = fields.Integer( + 'Sequence', + help="Sequence for the handle.", + default=10 + ) + import_type_id = fields.Many2one( + string='Import type', + comodel_name='import.type', + required=True, + ) + override_existing = fields.Boolean( + string='Override existing items', + help='Enable to update existing items w/ new values. ' + 'If disabled, matching records will be skipped.', + default=True, + ) + name = fields.Char( + string='Name', + compute='_compute_name', + ) + create_date = fields.Datetime('Create date') + record_ids = fields.One2many( + 'import.record', + 'recordset_id', + string='Records', + ) + # store info about imports report + report_data = Serialized(oldname='jsondata') + shared_data = Serialized() + report_html = fields.Html( + 'Report summary', compute='_compute_report_html') + full_report_url = fields.Char( + 'Full report url', compute='_compute_full_report_url') + jobs_global_state = fields.Selection( + string='Jobs global state', + selection=STATES, + compute='_compute_jobs_global_state', + help=( + "Tells you if a job is running for this recordset. " + "If any of the sub jobs is not DONE or FAILED " + "we assume the global state is PENDING." + ), + readonly=True + ) + report_file = fields.Binary('Report file') + report_filename = fields.Char('Report filename') + docs_html = fields.Html( + string='Docs', + compute='_compute_docs_html' + ) + notes = fields.Html('Notes', help="Useful info for your users") + + @api.multi + def unlink(self): + # inheritance of non-model mixin - like JobRelatedMixin - + # does not work w/out this + return super().unlink() + + @api.multi + @api.depends('backend_id.name') + def _compute_name(self): + for item in self: + names = [ + item.backend_id.name.strip(), + '#' + str(item.id), + ] + item.name = ' '.join(names) + + def get_records(self): + """Retrieve importable records and keep ordering.""" + return self.env['import.record'].search([ + ('recordset_id', '=', self.id)]) + + def _set_serialized(self, fname, values, reset=False): + """Update seriazed data.""" + _values = {} + if not reset: + _values = self[fname] + _values.update(values) + self[fname] = _values + # Without invalidating cache we will have a bug because of Serialized + # field in odoo. It uses json.loads on convert_to_cache, which leads + # to all of our int dict keys converted to strings. Except for the + # first value get, where we get not from cache yet. + # SO if you plan on using integers as your dict keys for a serialized + # field beware that they will be converted to strings. + # In order to streamline this I invalidate cache right away so the + # values are converted right away + # TL/DR integer dict keys will always be converted to strings, beware + self.invalidate_cache((fname,)) + + @api.multi + def set_report(self, values, reset=False): + """Update import report values.""" + self.ensure_one() + self._set_serialized('report_data', values, reset=reset) + + @api.multi + def get_report(self): + self.ensure_one() + return self.report_data or {} + + @api.multi + def set_shared(self, values, reset=False): + """Update import report values.""" + self.ensure_one() + self._set_serialized('shared_data', values, reset=reset) + + @api.multi + def get_shared(self): + self.ensure_one() + return self.shared_data or {} + + def _get_report_html_data(self): + """Prepare data for HTML report. + + :return dict: containing data for HTML report. + + Keys: + ``recordset``: current recordset + ``last_start``: last time import ran + ``report_by_model``: report data grouped by model. Like: + data['report_by_model'] = { + ir.model(res.parner): { + 'errored': 1, + 'skipped': 4, + 'created': 10, + 'updated': 8, + } + } + """ + report = self.get_report() + data = { + 'recordset': self, + 'last_start': report.pop('_last_start'), + 'report_by_model': OrderedDict(), + } + # count keys by model + for item in self.available_models(): + _model = item[0] + model = self.env['ir.model']._get(_model) + data['report_by_model'][model] = {} + # be defensive here. At some point + # we could decide to skip models on demand. + for k, v in report.get(_model, {}).items(): + data['report_by_model'][model][k] = len(v) + return data + + @api.depends('report_data') + def _compute_report_html(self): + template = self.env.ref('connector_importer.recordset_report') + for item in self: + if not item.report_data: + continue + data = item._get_report_html_data() + item.report_html = template.render(data) + + @api.multi + def _compute_full_report_url(self): + for item in self: + item.full_report_url = \ + '/importer/import-recordset/{}'.format(item.id) + + def debug_mode(self): + return self.backend_id.debug_mode or \ + os.getenv('IMPORTER_DEBUG_MODE') + + @api.multi + @api.depends('job_id.state', 'record_ids.job_id.state') + def _compute_jobs_global_state(self): + for item in self: + item.jobs_global_state = item._get_global_state() + + @api.model + def _get_global_state(self): + if not self.job_id: + return DONE + res = DONE + for item in self.record_ids: + if not item.job_id: + # TODO: investigate how this is possible + continue + # TODO: check why `item.job_state` does not reflect the job state + if item.job_id.state != DONE: + res = item.job_id.state + break + return res + + def available_models(self): + return self.import_type_id.available_models() + + @api.multi + @job + def import_recordset(self): + """This job will import a recordset.""" + with self.backend_id.work_on(self._name) as work: + importer = work.component(usage='recordset.importer') + return importer.run(self) + + @api.multi + def run_import(self): + """ queue a job for creating records (import.record items) + """ + job_method = self.with_delay().import_recordset + if self.debug_mode(): + logger.warn('### DEBUG MODE ACTIVE: WILL NOT USE QUEUE ###') + job_method = self.import_recordset + + for item in self: + result = job_method() + if self.debug_mode(): + # debug mode, no job here: reset it! + item.write({'job_id': False}) + else: + # link the job + item.write({'job_id': result.db_record().id}) + if self.debug_mode(): + # TODO: port this + # the "after_all" job needs to be fired manually when in debug mode + # since the event handler in .events.chunk_finished_subscriber + # cannot estimate when all the chunks have been processed. + # for model, importer in self.import_type_id.available_models(): + # import_record_after_all( + # session, + # self.backend_id.id, + # model, + # ) + pass + + @api.multi + def generate_report(self): + self.ensure_one() + reporter = self.get_source().get_reporter() + if reporter is None: + logger.debug('No reporter found...') + return + metadata, content = reporter.report_get(self) + self.write({ + 'report_file': base64.encodestring(content.encode()), + 'report_filename': metadata['complete_filename'] + }) + logger.info(( + 'Report file updated on recordset={}. ' + 'Filename: {}' + ).format(self.id, metadata['complete_filename'])) + + def _get_importers(self): + importers = OrderedDict() + for model_name, importer, __ in self.available_models(): + model = self.env['ir.model']._get(model_name) + with self.backend_id.work_on(self._name) as work: + importers[model] = work.component_by_name( + importer, model_name=model_name) + return importers + + @api.depends('import_type_id') + def _compute_docs_html(self): + template = self.env.ref('connector_importer.recordset_docs') + for item in self: + if isinstance(item.id, models.NewId): + continue + importers = item._get_importers() + data = { + 'recordset': item, + 'importers': importers, + } + item.docs_html = template.render(data) + + +# TODO +# @job +# def import_record_after_all( +# session, backend_id, model_name, last_record_id=None, **kw): +# """This job will import a record.""" +# # TODO: check this +# model = 'import.record' +# env = get_environment(session, model, backend_id) +# # recordset = None +# # if last_record_id: +# # record = env[model].browse(last_record_id) +# # recordset = record.recordset_id +# importer = get_record_importer(env) +# return importer.after_all() diff --git a/connector_importer/models/reporter.py b/connector_importer/models/reporter.py new file mode 100644 index 000000000..15c3d30b5 --- /dev/null +++ b/connector_importer/models/reporter.py @@ -0,0 +1,253 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +from odoo import models, api +import csv +import io +import time +import base64 + + +class ReporterMixin(models.AbstractModel): + """Base mixin for reporters. + + A reporter can be used to produce a file with a summary of the import + that the user can generate and download on each recordset. + + The summary can be anything you like: you are in total control of it. + See the CSV example for a real case. + """ + _name = 'reporter.mixin' + + report_extension = '.txt' + + @api.model + def report_get(self, recordset, **options): + """Create and return a report for given recordset.""" + fileout = io.StringIO() + self.report_do(recordset, fileout, **options) + self.report_finalize(recordset, fileout, **options) + metadata = self.report_get_metadata(recordset, **options) + return metadata, fileout.getvalue() + + def report_do(self, recordset, fileout, **options): + """Override me to generate the report.""" + raise NotImplementedError() + + def report_finalize(self, recordset, fileout, **options): + """Apply late updates to report.""" + + def report_get_metadata(self, recordset, **options): + """Retrieve report file's metadata.""" + fname = str(time.time()) + ext = self.report_extension + return { + 'filename': fname, + 'ext': ext, + 'complete_filename': fname + ext, + } + + +class CSVReporter(models.AbstractModel): + """Produce a CSV report. + + Very often is not easy to let the customer know what went wrong. + Here a CSV report is generated based on the initial CSV + provided by the customer/user. + + Basically we: + + * compute the stats of errored and skipped _get_lines + * clone the original CSV file + * add new columns at the end + + The new columns number is controlled by the flag `report_group_by_status`: + + * True: 2 new columns per each model imported. For instance: + * [R] res.partner skipped + * [R] res.partner errored + * [R] res.partner.category skipped + * [R] res.partner.category errored + * False: errors are grouped by state in 2 columns: + * [R] skipped + * [R] errored + + In this way the end user can check side by side which lines went wrong. + """ + _name = 'reporter.csv' + _inherit = 'reporter.mixin' + + report_extension = '.csv' + # columns to track/add + report_keys = ['skipped', 'errored'] + # Flag to determine if status report must be grouped by status. + # If `True` report result will be merged by status (errored, skipped, ...) + report_group_by_status = True + + def report_get_writer(self, fileout, columns, + delimiter=';', quotechar='"'): + writer = csv.DictWriter( + fileout, columns, + delimiter=delimiter, + quoting=csv.QUOTE_NONNUMERIC, + quotechar=quotechar) + writer.writeheader() + return writer + + def report_add_line(self, writer, item): + writer.writerow(item) + + def report_get_columns(self, recordset, orig_content, + extra_keys=None, delimiter=';'): + """Retrieve columns by recordset. + + :param recordset: instance of recordset. + :param orig_content: original csv content list of line. + :param extra_keys: report-related extra columns. + """ + extra_keys = extra_keys or [] + # read only the 1st line of the original file + if orig_content: + line1 = orig_content[0].split(delimiter) + return line1 + extra_keys + return extra_keys + + def report_do(self, recordset, fileout, **options): + """Produce report.""" + json_report = recordset.get_report() + report_keys = options.get('report_keys', self.report_keys) + group_by_status = options.get( + 'group_by_status', self.report_group_by_status) + + model_keys = [ + x for x in json_report.keys() if not x.startswith('_')] + + extra_keys = [ + self._report_make_key(x) for x in report_keys + ] + if not group_by_status: + # we produce one column per-model per-status + for model in model_keys: + for key in report_keys: + extra_keys.append(self._report_make_key(key, model=model)) + + source = recordset.get_source() + orig_content = base64.b64decode(source.csv_file).decode().splitlines() + delimiter = source.csv_delimiter + quotechar = source.csv_quotechar + + columns = self.report_get_columns( + recordset, orig_content, + extra_keys=extra_keys, delimiter=delimiter) + + writer = self.report_get_writer( + fileout, columns, delimiter=delimiter, quotechar=quotechar) + + reader = csv.DictReader( + orig_content, delimiter=delimiter, quotechar=quotechar) + + self._report_do( + json_report=json_report, + reader=reader, + writer=writer, + model_keys=model_keys, + report_keys=report_keys, + group_by_status=group_by_status + ) + + def _report_do( + self, + json_report=None, + reader=None, + writer=None, + model_keys=None, + report_keys=None, + group_by_status=True): + + line_handler = self._report_line_by_model_and_status + if group_by_status: + line_handler = self._report_line_by_status + + grouped = self._report_group_by_line( + json_report, model_keys, report_keys) + + for line in reader: + line_handler(line, reader.line_num, grouped, model_keys) + self.report_add_line(writer, line) + + def _report_make_key(self, key, model=''): + if model: + return '[R] {}: {}'.format(model, key) + return '[R] {}'.format(key) + + def _report_group_by_line(self, json_report, model_keys, report_keys): + """Group report items by line number. + + Return something like: + + { + 'errored': {}, + 'skipped': { + 2: [ + { + u'line_nr': 2, + u'message': u'MISSING REQUIRED KEY=foo', + u'model': u'product.supplierinfo', + u'odoo_record': None + }, + { + u'line_nr': 2, + u'message': u'MISSING REQUIRED KEY=bla', + u'model': u'product.product', + u'odoo_record': None + }, + ], + 3: [ + { + u'line_nr': 3, + u'message': u'MISSING REQUIRED KEY=foo', + u'model': u'product.template', + u'odoo_record': None + }, + { + u'line_nr': 3, + u'message': u'ALREADY_EXISTS code=XXXX', + u'model': u'product.product', + u'odoo_record': None + }, + ], + } + """ + by_line = {} + for model in model_keys: + # list of messages + by_model = json_report.get(model, {}) + for key in report_keys: + by_line.setdefault(key, {}) + for item in by_model.get(key, []): + by_line[key].setdefault( + item['line_nr'], [] + ).append(item) + return by_line + + def _report_line_by_model_and_status( + self, line, line_num, grouped, model_keys): + """Get one column per each pair model-status.""" + for model in model_keys: + for status, lines in grouped.items(): + # get info on current line if any + line_info = lines.get(line_num, {}) + # add the extra report column anyway + line[self._report_make_key(model, status)] = \ + line_info.get('message') + + def _report_line_by_status( + self, line, line_num, grouped, model_keys): + """Get one column per each status containing all modelss messages.""" + for status, by_line in grouped.items(): + line_info = by_line.get(line_num, []) + line[self._report_make_key(status)] = '\n'.join([ + '{model}: {message}'.format(**item) for item in line_info + ]) diff --git a/connector_importer/models/sources/__init__.py b/connector_importer/models/sources/__init__.py new file mode 100644 index 000000000..45a20d979 --- /dev/null +++ b/connector_importer/models/sources/__init__.py @@ -0,0 +1,2 @@ +from . import source_mixin +from . import source_csv diff --git a/connector_importer/models/sources/source_csv.py b/connector_importer/models/sources/source_csv.py new file mode 100644 index 000000000..a32c6b6ce --- /dev/null +++ b/connector_importer/models/sources/source_csv.py @@ -0,0 +1,108 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api +from ...utils.import_utils import CSVReader, guess_csv_metadata +import base64 + + +class CSVSource(models.Model): + _name = 'import.source.csv' + _inherit = 'import.source' + _description = 'CSV import source' + _source_type = 'csv' + _reporter_model = 'reporter.csv' + + csv_file = fields.Binary('CSV file') + # use these to load file from an FS path + csv_filename = fields.Char('CSV filename') + csv_filesize = fields.Char( + string='CSV filesize', + compute='_compute_csv_filesize', + readonly=True, + ) + # This is for scheduled import via FS path (FTP, sFTP, etc) + csv_path = fields.Char('CSV path') + csv_delimiter = fields.Char( + string='CSV delimiter', + default=';', + ) + csv_quotechar = fields.Char( + string='CSV quotechar', + default='"', + ) + + _csv_reader_klass = CSVReader + + @property + def _config_summary_fields(self): + _fields = super()._config_summary_fields + return _fields + [ + 'csv_filename', 'csv_filesize', + 'csv_delimiter', 'csv_quotechar', + ] + + def _binary_csv_content(self): + return base64.b64decode(self.csv_file) + + @api.onchange('csv_file') + def _onchance_csv_file(self): + if self.csv_file: + # auto-guess CSV details + meta = guess_csv_metadata(self._binary_csv_content()) + if meta: + self.csv_delimiter = meta['delimiter'] + self.csv_quotechar = meta['quotechar'] + + @api.depends('csv_file') + def _compute_csv_filesize(self): + for item in self: + if item.csv_file: + # in v11 binary fields now can return the size of the file + item.csv_filesize = self.with_context(bin_size=True).csv_file + + def _get_lines(self): + # read CSV + reader_args = { + 'delimiter': self.csv_delimiter, + } + if self.csv_path: + # TODO: join w/ filename + reader_args['filepath'] = self.csv_path + else: + reader_args['filedata'] = base64.decodestring(self.csv_file) + + reader = self._csv_reader_klass(**reader_args) + return reader.read_lines() + + # TODO: this stuff is now unrelated from backend version must be refactored + # # handy fields to make the example attachment + # # downloadable within recordset view + # example_file_xmlid = fields.Char() + # example_file_url = fields.Char( + # string='Download example file', + # compute='_compute_example_file_url', + # readonly=True, + # ) + # + # def _get_example_attachment(self): + # # You can define example file by creating attachments + # # with an xmlid matching the import type/key + # # `connector_importer.example_file_$version_key` + # if not self.backend_id.version or not self.import_type_id: + # return + # xmlid = self.example_file_xmlid + # if not xmlid: + # xmlid = u'connector_importer.examplefile_{}_{}'.format( + # self.backend_id.version.replace('.', '_'), + # self.import_type_id.key) + # return self.env.ref(xmlid, raise_if_not_found=0) + # + # @api.depends( + # 'backend_id.version', 'import_type_id', 'example_file_xmlid') + # def _compute_example_file_url(self): + # att = self._get_example_attachment() + # if att: + # self.example_file_url = u'/web/content/{}/{}'.format( + # att.id, att.name) diff --git a/connector_importer/models/sources/source_mixin.py b/connector_importer/models/sources/source_mixin.py new file mode 100644 index 000000000..856a511df --- /dev/null +++ b/connector_importer/models/sources/source_mixin.py @@ -0,0 +1,204 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api, tools + +from ...utils.import_utils import gen_chunks + + +class ImportSourceConsumerMixin(models.AbstractModel): + """Source consumer mixin. + + Inheriting models can setup, configure and use import sources. + + Relation towards source records is generic to grant maximum freedom + on which source type to use. + """ + _name = 'import.source.consumer.mixin' + _description = 'Import source consumer' + + source_id = fields.Integer( + string='Source ID', + required=False, + ondelete='cascade', + ) + source_model = fields.Selection( + string='Source type', + selection='_selection_source_ref_id', + ) + source_ref_id = fields.Reference( + string='Source', + compute='_compute_source_ref_id', + selection='_selection_source_ref_id', + store=True, + ) + source_config_summary = fields.Html( + compute='_compute_source_config_summary', + readonly=True, + ) + + @api.multi + @api.depends('source_model', 'source_id') + def _compute_source_ref_id(self): + for item in self: + if not item.source_id or not item.source_model: + continue + item.source_ref_id = '{0.source_model},{0.source_id}'.format(item) + + @api.model + @tools.ormcache('self') + def _selection_source_ref_id(self): + return [ + ('import.source.csv', 'CSV'), + ] + + @api.multi + @api.depends('source_ref_id', ) + def _compute_source_config_summary(self): + for item in self: + if not item.source_ref_id: + continue + item.source_config_summary = item.source_ref_id.config_summary + + @api.multi + def open_source_config(self): + self.ensure_one() + action = self.env[self.source_model].get_formview_action() + action.update({ + 'views': [ + (self.env[self.source_model].get_config_view_id(), 'form'), + ], + 'res_id': self.source_id, + 'target': 'new', + }) + return action + + def get_source(self): + """Return the source to the consumer.""" + return self.source_ref_id + + +class ImportSource(models.AbstractModel): + """Define a source for an import. + + A source model is responsible for: + + * storing specific settings (chunk size, source params, etc) + * retrieve source lines (connect to an external service, or db or read CSV) + * yield lines in chunks + * display configuration summary on the recordset (via config summary) + * optionally, provide a reporter to create an extensive report for users. + """ + _name = 'import.source' + _description = 'Import source' + _source_type = 'none' + _reporter_model = '' + + name = fields.Char( + compute='_compute_name', + readony=True, + ) + chunk_size = fields.Integer( + required=True, + default=500, + string='Chunks Size' + ) + config_summary = fields.Html( + compute='_compute_config_summary', + readonly=True, + ) + + # tmpl that renders configuration summary + _config_summary_template = 'connector_importer.source_config_summary' + + @api.multi + def _compute_name(self): + self.name = self._source_type + + @property + def _config_summary_fields(self): + """Fields automatically included in the summary. + + Override it to add your custom fields automatically to the summary. + """ + return ['chunk_size', ] + + @api.depends() + def _compute_config_summary(self): + """Generate configuration summary HTML. + + Configurations parameters can vary depending on the kind of source. + To display meaningful information on the recordset + w/out hacking the recordset view each time + we generate a short HTML summary. + + For instance, if you are connecting to an external db + you might want to show DSN, if you are loading a CSV + you might want to show delimiter, quotechar and so on. + + To add your fields automatically to the summary, + just override `_config_summary_fields`. + They'll be automatically included in the summary. + """ + template = self.env.ref(self._config_summary_template) + for item in self: + item.config_summary = template.render(item._config_summary_data()) + + def _config_summary_data(self): + """Collect data for summary.""" + info = [] + for fname in self._config_summary_fields: + info.append((fname, self[fname])) + return { + 'source': self, + 'summary_fields': self._config_summary_fields, + 'fields_info': self.fields_get(self._config_summary_fields), + } + + @api.model + def create(self, vals): + """Override to update reference to source on the consumer.""" + res = super(ImportSource, self).create(vals) + if self.env.context.get('active_model'): + # update reference on consumer + self.env[self.env.context['active_model']].browse( + self.env.context['active_id']).source_id = res.id + return res + + @api.multi + def get_lines(self): + """Retrieve lines to import.""" + self.ensure_one() + # retrieve lines + lines = self._get_lines() + + # sort them + lines_sorted = self._sort_lines(lines) + + for i, chunk in enumerate(gen_chunks(lines_sorted, + chunksize=self.chunk_size)): + # get out of chunk iterator + yield list(chunk) + + def _get_lines(self): + """Your duty here...""" + raise NotImplementedError() + + def _sort_lines(self, lines): + """Override to customize sorting.""" + return lines + + def get_config_view_id(self): + """Retrieve configuration view.""" + return self.env['ir.ui.view'].search([ + ('model', '=', self._name), + ('type', '=', 'form')], limit=1).id + + def get_reporter(self): + """Retrieve a specific reporter for this source. + + A report can be used to produce and extensive report for the end user. + See `reporter` models. + """ + return self.env.get(self._reporter_model) diff --git a/connector_importer/security/ir.model.access.csv b/connector_importer/security/ir.model.access.csv new file mode 100644 index 000000000..ae646c509 --- /dev/null +++ b/connector_importer/security/ir.model.access.csv @@ -0,0 +1,10 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_import_recordset,connector_importer.access_import_recordset,model_import_recordset,connector.group_connector_manager,1,1,1,1 +access_import_record,connector_importer.access_import_record,model_import_record,connector.group_connector_manager,1,1,1,1 +access_import_type,connector_importer.access_import_type,model_import_type,connector.group_connector_manager,1,1,1,1 +access_import_souce_csv,connector_importer.access_import_source_csv,model_import_source_csv,connector.group_connector_manager,1,1,1,1 +access_import_backend_user,connector_importer.access_import_backend_user,model_import_backend,connector_importer.group_importer_user,1,0,0,0 +access_import_recordset_user,connector_importer.access_import_recordset_user,model_import_recordset,connector_importer.group_importer_user,1,0,0,0 +access_import_type_user,connector_importer.access_import_type_user,model_import_type,connector_importer.group_importer_user,1,0,0,0 +access_import_souce_csv_user,connector_importer.access_import_source_csv_user,model_import_source_csv,connector_importer.group_importer_user,1,0,0,0 +access_connector_queue_job_user,connector job user,connector.model_queue_job,connector_importer.group_importer_user,1,0,0,0 diff --git a/connector_importer/security/security.xml b/connector_importer/security/security.xml new file mode 100644 index 000000000..29fce945d --- /dev/null +++ b/connector_importer/security/security.xml @@ -0,0 +1,9 @@ + + + + + Connector importer user + + + + diff --git a/connector_importer/static/description/icon.png b/connector_importer/static/description/icon.png new file mode 100644 index 000000000..3a0328b51 Binary files /dev/null and b/connector_importer/static/description/icon.png differ diff --git a/connector_importer/tests/__init__.py b/connector_importer/tests/__init__.py new file mode 100644 index 000000000..80105c9d1 --- /dev/null +++ b/connector_importer/tests/__init__.py @@ -0,0 +1,5 @@ +from . import test_backend +from . import test_cron +from . import test_import_type +from . import test_recordset +from . import test_source diff --git a/connector_importer/tests/common.py b/connector_importer/tests/common.py new file mode 100644 index 000000000..5084ad532 --- /dev/null +++ b/connector_importer/tests/common.py @@ -0,0 +1,40 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.modules.module import get_resource_path +import odoo.tests.common as common +from .fake_models import ( + setup_test_model, + teardown_test_model, +) +import io + + +def _load_filecontent(module, filepath, mode='r'): + path = get_resource_path(module, filepath) + with io.open(path, mode) as fd: + return fd.read() + + +class BaseTestCase(common.SavepointCase): + + load_filecontent = _load_filecontent + + +class FakeModelTestCase(BaseTestCase): + + # override this in your test case to inject new models on the fly + TEST_MODELS_KLASSES = [] + + @classmethod + def _setup_models(cls): + """Setup new fake models for testing.""" + for kls in cls.TEST_MODELS_KLASSES: + setup_test_model(cls.env, kls) + + @classmethod + def _teardown_models(cls): + """Wipe fake models once tests have finished.""" + for kls in cls.TEST_MODELS_KLASSES: + teardown_test_model(cls.env, kls) diff --git a/connector_importer/tests/fake_components.py b/connector_importer/tests/fake_components.py new file mode 100644 index 000000000..7c8cd9644 --- /dev/null +++ b/connector_importer/tests/fake_components.py @@ -0,0 +1,38 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.addons.component.core import Component + + +class PartnerMapper(Component): + _name = 'fake.partner.mapper' + _inherit = 'importer.base.mapper' + _apply_on = 'res.partner' + + required = { + 'fullname': 'name', + 'id': 'ref', + } + + defaults = [ + ('is_company', False), + ] + + direct = [ + ('id', 'ref'), + ('fullname', 'name'), + ] + + +class PartnerRecordImporter(Component): + _name = 'fake.partner.importer' + _inherit = 'importer.record' + _apply_on = 'res.partner' + + odoo_unique_key = 'ref' + + def create_context(self): + return {'tracking_disable': True} + + write_context = create_context diff --git a/connector_importer/tests/fake_models.py b/connector_importer/tests/fake_models.py new file mode 100644 index 000000000..e094b1b91 --- /dev/null +++ b/connector_importer/tests/fake_models.py @@ -0,0 +1,57 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import fields, models + + +def setup_test_model(env, model_cls): + """Pass a test model class and initialize it. + + Courtesy of SBidoul from https://github.com/OCA/mis-builder :) + """ + model_cls._build_model(env.registry, env.cr) + env.registry.setup_models(env.cr) + env.registry.init_models( + env.cr, [model_cls._name], + dict(env.context, update_custom_fields=True) + ) + + +def teardown_test_model(env, model_cls): + """Pass a test model class and deinitialize it. + + Courtesy of SBidoul from https://github.com/OCA/mis-builder :) + """ + del env.registry.models[model_cls._name] + env.registry.setup_models(env.cr) + + +class FakeSourceConsumer(models.Model): + + _name = 'fake.source.consumer' + _inherit = 'import.source.consumer.mixin' + + +class FakeSourceStatic(models.Model): + + _name = 'fake.source.static' + _inherit = 'import.source' + _source_type = 'static' + + fake_param = fields.Char(summary_field=True) + + @property + def _config_summary_fields(self): + return super()._config_summary_fields + ['fake_param'] + + def _get_lines(self): + for i in range(1, 21): + yield { + 'id': i, + 'fullname': 'Fake line #{}'.format(i), + 'address': 'Some fake place, {}'.format(i), + } + + def _sort_lines(self, lines): + return reversed(list(lines)) diff --git a/connector_importer/tests/fixtures/csv_source_test1.csv b/connector_importer/tests/fixtures/csv_source_test1.csv new file mode 100644 index 000000000..88c3cc993 --- /dev/null +++ b/connector_importer/tests/fixtures/csv_source_test1.csv @@ -0,0 +1,6 @@ +id,fullname +1,Marty McFly +2,Biff Tannen +3,Emmet Brown +4,Clara Clayton +5,George McFly diff --git a/connector_importer/tests/test_backend.py b/connector_importer/tests/test_backend.py new file mode 100644 index 000000000..9f4ed5912 --- /dev/null +++ b/connector_importer/tests/test_backend.py @@ -0,0 +1,52 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import odoo.tests.common as common + + +class TestBackend(common.SavepointCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend_model = cls.env['import.backend'] + + def test_backend_create(self): + bknd = self.backend_model.create({ + 'name': 'Foo', + 'version': '1.0', + }) + self.assertTrue(bknd) + + def test_backend_cron_cleanup_recordsets(self): + # create a backend + bknd = self.backend_model.create({ + 'name': 'Foo', + 'version': '1.0', + 'cron_cleanup_keep': 3, + }) + itype = self.env['import.type'].create({ + 'name': 'Fake', + 'key': 'fake', + 'settings': '# nothing to do' + }) + # and 5 recorsets + for x in range(5): + rec = self.env['import.recordset'].create({ + 'backend_id': bknd.id, + 'import_type_id': itype.id, + }) + # make sure create date is increased + rec.create_date = '2018-01-01 00:00:0' + str(x) + self.assertEqual(len(bknd.recordset_ids), 5) + # clean them up + bknd.cron_cleanup_recordsets() + recsets = bknd.recordset_ids.mapped('name') + # we should find only 3 records and #1 and #2 gone + self.assertEqual(len(recsets), 3) + self.assertNotIn('Foo #1', recsets) + self.assertNotIn('Foo #2', recsets) + + # TODO + # def test_job_running_unlink_lock(self): diff --git a/connector_importer/tests/test_cron.py b/connector_importer/tests/test_cron.py new file mode 100644 index 000000000..dce4b07f8 --- /dev/null +++ b/connector_importer/tests/test_cron.py @@ -0,0 +1,40 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +import odoo.tests.common as common + + +class TestBackendCron(common.SavepointCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend_model = cls.env['import.backend'] + cls.bknd = cls.backend_model.create({ + 'name': 'Croned one', + 'version': '1.0', + 'cron_mode': True, + 'cron_start_date': '2018-01-01', + 'cron_interval_type': 'days', + 'cron_interval_number': 2, + }) + + def test_backend_cron_create(self): + cron = self.bknd.cron_id + self.assertTrue(cron) + self.assertEqual(cron.nextcall, '2018-01-01 00:00:00') + self.assertEqual(cron.interval_type, 'days') + self.assertEqual(cron.interval_number, 2) + self.assertEqual(cron.code, 'model.run_cron(%d)' % self.bknd.id) + + def test_backend_cron_update(self): + self.bknd.write({ + 'cron_start_date': '2018-05-01', + 'cron_interval_type': 'weeks', + }) + cron = self.bknd.cron_id + self.assertTrue(cron) + self.assertEqual(cron.nextcall, '2018-05-01 00:00:00') + self.assertEqual(cron.interval_type, 'weeks') diff --git a/connector_importer/tests/test_import_type.py b/connector_importer/tests/test_import_type.py new file mode 100644 index 000000000..5f0ee606b --- /dev/null +++ b/connector_importer/tests/test_import_type.py @@ -0,0 +1,52 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +import odoo.tests.common as common +from odoo.tools import mute_logger + +from psycopg2 import IntegrityError + + +class TestImportType(common.SavepointCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.type_model = cls.env['import.type'] + + @mute_logger('odoo.sql_db') + def test_unique_constrain(self): + self.type_model.create({ + 'name': 'Ok', + 'key': 'ok', + 'settings': '', + }) + with self.assertRaises(IntegrityError): + self.type_model.create({ + 'name': 'Duplicated Ok', + 'key': 'ok', + 'settings': '', + }) + + def test_available_models(self): + itype = self.type_model.create({ + 'name': 'Ok', + 'key': 'ok', + 'settings': """ + # skip this pls + res.partner::partner.importer + res.users::user.importer + + # this one as well + another.one :: import.withspaces + """, + }) + models = tuple(itype.available_models()) + self.assertEqual(models, ( + # model, importer, is_last_importer + ('res.partner', 'partner.importer', False), + ('res.users', 'user.importer', False), + ('another.one', 'import.withspaces', True), + )) diff --git a/connector_importer/tests/test_record_importer.py b/connector_importer/tests/test_record_importer.py new file mode 100644 index 000000000..1e211e294 --- /dev/null +++ b/connector_importer/tests/test_record_importer.py @@ -0,0 +1,109 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.tools import mute_logger +from .test_recordset_importer import TestImporterBase +import logging + +# TODO: really annoying when running tests. Remove or find a better way +logging.getLogger('PIL.PngImagePlugin').setLevel(logging.ERROR) +logging.getLogger('passlib.registry').setLevel(logging.ERROR) + +MOD_PATH = 'odoo.addons.connector_importer' +RECORD_MODEL = MOD_PATH + '.models.record.ImportRecord' + + +class TestRecordImporter(TestImporterBase): + + def _setup_records(self): + super()._setup_records() + self.record = self.env['import.record'].create({ + 'recordset_id': self.recordset.id, + }) + # no jobs thanks (I know, we should test this too at some point :)) + self.backend.debug_mode = True + + @mute_logger('[importer]') + def test_importer_create(self): + # generate 10 records + lines = self._fake_lines(10, keys=('id', 'fullname')) + # set them on record + self.record.set_data(lines) + res = self.record.run_import() + # in any case we'll get this per each model if the import is not broken + self.assertEqual(res, {'res.partner': 'ok'}) + report = self.recordset.get_report() + self.assertEqual(len(report['res.partner']['created']), 10) + self.assertEqual(len(report['res.partner']['errored']), 0) + self.assertEqual(len(report['res.partner']['updated']), 0) + self.assertEqual(len(report['res.partner']['skipped']), 0) + self.assertEqual( + self.env['res.partner'].search_count([('ref', 'like', 'id_%')]), + 10 + ) + + @mute_logger('[importer]') + def test_importer_skip(self): + # generate 10 records + lines = self._fake_lines(10, keys=('id', 'fullname')) + # make a line skip + lines[0].pop('fullname') + lines[1].pop('id') + # set them on record + self.record.set_data(lines) + res = self.record.run_import() + # in any case we'll get this per each model if the import is not broken + self.assertEqual(res, {'res.partner': 'ok'}) + report = self.recordset.get_report() + self.assertEqual(len(report['res.partner']['created']), 8) + self.assertEqual(len(report['res.partner']['errored']), 0) + self.assertEqual(len(report['res.partner']['updated']), 0) + self.assertEqual(len(report['res.partner']['skipped']), 2) + skipped_msg1 = report['res.partner']['skipped'][0]['message'] + skipped_msg2 = report['res.partner']['skipped'][1]['message'] + self.assertEqual( + skipped_msg1, 'MISSING REQUIRED SOURCE KEY=fullname: ref=id_1' + ) + # `id` missing, so the destination key `ref` is missing + # so we don't see it in the message + self.assertEqual( + skipped_msg2, 'MISSING REQUIRED SOURCE KEY=id' + ) + self.assertEqual( + self.env['res.partner'].search_count([('ref', 'like', 'id_%')]), + 8 + ) + + @mute_logger('[importer]') + def test_importer_update(self): + # generate 10 records + lines = self._fake_lines(10, keys=('id', 'fullname')) + self.record.set_data(lines) + res = self.record.run_import() + # in any case we'll get this per each model if the import is not broken + self.assertEqual(res, {'res.partner': 'ok'}) + report = self.recordset.get_report() + self.assertEqual(len(report['res.partner']['created']), 10) + self.assertEqual(len(report['res.partner']['updated']), 0) + # now run it a second time + # but we must flush the old report which is usually done + # by the recordset importer + self.recordset.set_report({}, reset=True) + res = self.record.run_import() + report = self.recordset.get_report() + self.assertEqual(len(report['res.partner']['created']), 0) + self.assertEqual(len(report['res.partner']['updated']), 10) + # now run it a second time + # but we set `override existing` false + self.recordset.set_report({}, reset=True) + report = self.recordset.override_existing = False + res = self.record.run_import() + report = self.recordset.get_report() + self.assertEqual(len(report['res.partner']['created']), 0) + self.assertEqual(len(report['res.partner']['updated']), 0) + self.assertEqual(len(report['res.partner']['skipped']), 10) + skipped_msg1 = report['res.partner']['skipped'][0]['message'] + self.assertEqual( + skipped_msg1, 'ALREADY EXISTS: ref=id_1' + ) diff --git a/connector_importer/tests/test_record_importer_basic.py b/connector_importer/tests/test_record_importer_basic.py new file mode 100644 index 000000000..a6be96517 --- /dev/null +++ b/connector_importer/tests/test_record_importer_basic.py @@ -0,0 +1,120 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.tools import mute_logger +from .test_recordset_importer import TestImporterBase +import logging + +# TODO: really annoying when running tests. Remove or find a better way +logging.getLogger('PIL.PngImagePlugin').setLevel(logging.ERROR) +logging.getLogger('passlib.registry').setLevel(logging.ERROR) + +MOD_PATH = 'odoo.addons.connector_importer' +RECORD_MODEL = MOD_PATH + '.models.record.ImportRecord' + + +class TestRecordImporter(TestImporterBase): + + def _setup_records(self): + super()._setup_records() + self.record = self.env['import.record'].create({ + 'recordset_id': self.recordset.id, + }) + # no jobs thanks (I know, we should test this too at some point :)) + self.backend.debug_mode = True + + def _get_importer(self): + with self.backend.work_on(self.record._name) as work: + return work.component( + usage='record.importer', model_name='res.partner') + + @mute_logger('[importer]') + def test_importer_lookup(self): + importer = self._get_importer() + self.assertEqual(importer._name, 'fake.partner.importer') + + @mute_logger('[importer]') + def test_importer_required_keys(self): + importer = self._get_importer() + required = importer.required_keys() + self.assertDictEqual(required, {'fullname': ('name',), 'id': ('ref',)}) + + @mute_logger('[importer]') + def test_importer_check_missing_none(self): + importer = self._get_importer() + values = { + 'name': 'John Doe', + 'ref': 'doe', + } + orig_values = { + 'fullname': 'john doe', + 'id': '#doe', + } + missing = importer._check_missing( + 'id', 'ref', values, orig_values) + self.assertFalse(missing) + + @mute_logger('[importer]') + def test_importer_check_missing_source(self): + importer = self._get_importer() + values = { + 'name': 'John Doe', + 'ref': 'doe', + } + orig_values = { + 'fullname': 'john doe', + 'id': '#doe', + } + fullname = orig_values.pop('fullname') + missing = importer._check_missing( + 'fullname', 'name', values, orig_values) + # name is missing now + self.assertDictEqual( + missing, + {'message': 'MISSING REQUIRED SOURCE KEY=fullname: ref=doe'} + ) + # drop ref + orig_values['fullname'] = fullname + orig_values.pop('id') + missing = importer._check_missing( + 'id', 'ref', values, orig_values) + # name is missing now + # `id` missing, so the destination key `ref` is missing + # so we don't see it in the message + self.assertDictEqual( + missing, + {'message': 'MISSING REQUIRED SOURCE KEY=id: ref=doe'} + ) + + @mute_logger('[importer]') + def test_importer_check_missing_destination(self): + importer = self._get_importer() + values = { + 'name': 'John Doe', + 'ref': 'doe', + } + orig_values = { + 'fullname': 'john doe', + 'id': '#doe', + } + name = values.pop('name') + missing = importer._check_missing( + 'fullname', 'name', values, orig_values) + # name is missing now + self.assertDictEqual( + missing, + {'message': 'MISSING REQUIRED DESTINATION KEY=name: ref=doe'} + ) + # drop ref + values['name'] = name + values.pop('ref') + missing = importer._check_missing( + 'id', 'ref', values, orig_values) + # name is missing now + # `id` missing, so the destination key `ref` is missing + # so we don't see it in the message + self.assertDictEqual( + missing, + {'message': 'MISSING REQUIRED DESTINATION KEY=ref'} + ) diff --git a/connector_importer/tests/test_recordset.py b/connector_importer/tests/test_recordset.py new file mode 100644 index 000000000..b478e2790 --- /dev/null +++ b/connector_importer/tests/test_recordset.py @@ -0,0 +1,96 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +import odoo.tests.common as common + + +class TestRecordset(common.SavepointCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.recordset_model = cls.env['import.recordset'] + cls.backend_model = cls.env['import.backend'] + cls.type_model = cls.env['import.type'] + cls.bknd = cls._create_backend() + cls.itype = cls._create_type() + cls.recordset = cls._create_recordset() + + @classmethod + def _create_backend(cls): + return cls.backend_model.create({ + 'name': 'Foo', + 'version': '1.0', + }) + + @classmethod + def _create_type(cls): + return cls.type_model.create({ + 'name': 'Ok', + 'key': 'ok', + 'settings': """ + res.partner::partner.importer + """ + }) + + @classmethod + def _create_recordset(cls): + return cls.recordset_model.create({ + 'backend_id': cls.bknd.id, + 'import_type_id': cls.itype.id, + }) + + def test_recordset_name(self): + self.assertEqual( + self.recordset.name, + self.recordset.backend_id.name + ' #' + str(self.recordset.id)) + + def test_available_models(self): + """Available models are propagated from import type.""" + models = tuple(self.recordset.available_models()) + self.assertEqual(models, ( + # model, importer, is_last_importer + ('res.partner', 'partner.importer', True), + )) + + def test_get_set_raw_report(self): + val = {'baz': 'bar'} + # store report + self.recordset.set_report(val) + # retrieve it, should be the same + self.assertEqual(self.recordset.get_report(), val) + new_val = {'foo': 'boo'} + # set a new value + self.recordset.set_report(new_val) + merged = val.copy() + merged.update(new_val) + # by default previous value is preserved and merged w/ the new one + self.assertDictEqual( + self.recordset.get_report(), merged) + # unless we use `reset` + val = {'goo': 'gle'} + # store report + self.recordset.set_report(val, reset=True) + self.assertDictEqual( + self.recordset.get_report(), val) + + def test_get_report_html_data(self): + val = { + '_last_start': '2018-01-20', + 'res.partner': { + 'errored': list(range(10)), + 'skipped': list(range(4)), + 'updated': list(range(20)), + 'created': list(range(2)), + } + } + self.recordset.set_report(val) + data = self.recordset._get_report_html_data() + self.assertEqual(data['recordset'], self.recordset) + self.assertEqual(data['last_start'], '2018-01-20') + by_model = data['report_by_model'] + key = list(by_model.keys())[0] + self.assertEqual(key._name, 'ir.model') + self.assertEqual(key.model, 'res.partner') diff --git a/connector_importer/tests/test_recordset_importer.py b/connector_importer/tests/test_recordset_importer.py new file mode 100644 index 000000000..e2091ecd3 --- /dev/null +++ b/connector_importer/tests/test_recordset_importer.py @@ -0,0 +1,112 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from .fake_components import PartnerMapper, PartnerRecordImporter +from ..utils.import_utils import gen_chunks +from odoo.addons.component.tests import common +import mock + +MOD_PATH = 'odoo.addons.connector_importer' +RECORD_MODEL = MOD_PATH + '.models.record.ImportRecord' + + +class MockedSource(object): + """A fake source for recordsets.""" + + lines = [] + chunks_size = 5 + + def __init__(self, lines, chunk_size=5): + self.lines = lines + self.chunks_size = chunk_size + + def get_lines(self): + return gen_chunks(self.lines, self.chunks_size) + + +def fake_lines(count, keys): + """Generate importable fake lines.""" + res = [] + _item = {}.fromkeys(keys, '') + for i in range(1, count + 1): + item = _item.copy() + for k in keys: + item[k] = '{}_{}'.format(k, i) + item['_line_nr'] = i + res.append(item) + return res + + +class TestImporterBase(common.TransactionComponentRegistryCase): + + def setUp(self): + super().setUp() + self._setup_records() + self._load_module_components('connector_importer') + self._build_components(PartnerMapper, PartnerRecordImporter) + + def _setup_records(self): + self.backend = self.env['import.backend'].create({ + 'name': 'Foo', + 'version': '1.0', + }) + itype = self.env['import.type'].create({ + 'name': 'Fake', + 'key': 'fake', + 'settings': 'res.partner::fake.partner.importer' + }) + self.recordset = self.env['import.recordset'].create({ + 'backend_id': self.backend.id, + 'import_type_id': itype.id, + }) + + def _patch_get_source(self, lines, chunk_size=5): + self.env['import.recordset']._patch_method( + 'get_source', + lambda x: MockedSource(lines, chunk_size=chunk_size), + ) + + def _fake_lines(self, count, keys=None): + return fake_lines(count, keys=keys or []) + + +class TestRecordsetImporter(TestImporterBase): + + @mock.patch('%s.run_import' % RECORD_MODEL) + def test_recordset_importer(self, mocked_run_inport): + # generate 100 records + lines = self._fake_lines(100, keys=('id', 'fullname')) + # source will provide 5x20 chunks + self._patch_get_source(lines, chunk_size=20) + # run the recordset importer + with self.backend.work_on( + 'import.recordset', + components_registry=self.comp_registry + ) as work: + importer = work.component(usage='recordset.importer') + self.assertTrue(importer) + importer.run(self.recordset) + mocked_run_inport.assert_called() + # we expect 5 records w/ 20 lines each + records = self.recordset.get_records() + self.assertEqual(len(records), 5) + for rec in records: + data = rec.get_data() + self.assertEqual(len(data), 20) + # order is preserved + data1 = records[0].get_data() + self.assertEqual(data1[0]['id'], 'id_1') + self.assertEqual(data1[0]['fullname'], 'fullname_1') + # run it twice and make sure old records are wiped + # run the recordset importer + with self.backend.work_on( + 'import.recordset', + components_registry=self.comp_registry + ) as work: + importer = work.component(usage='recordset.importer') + self.assertTrue(importer) + importer.run(self.recordset) + # we expect 5 records w/ 20 lines each + records = self.recordset.get_records() + self.assertEqual(len(records), 5) diff --git a/connector_importer/tests/test_source.py b/connector_importer/tests/test_source.py new file mode 100644 index 000000000..3fceea7b9 --- /dev/null +++ b/connector_importer/tests/test_source.py @@ -0,0 +1,81 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +from .common import FakeModelTestCase +from .fake_models import ( + FakeSourceStatic, + FakeSourceConsumer, +) +import mock + +MOD_PATH = 'odoo.addons.connector_importer.models' +SOURCE_MODEL = MOD_PATH + '.sources.source_mixin.ImportSourceConsumerMixin' + + +class TestSource(FakeModelTestCase): + + TEST_MODELS_KLASSES = [FakeSourceStatic, FakeSourceConsumer] + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._setup_models() + cls.source = cls._create_source() + cls.consumer = cls._create_consumer() + + @classmethod + def tearDownClass(cls): + cls._teardown_models() + super().tearDownClass() + + @classmethod + def _create_source(cls): + return cls.env['fake.source.static'].create({ + 'fake_param': 'some_condition', + 'chunk_size': 5, + }) + + @classmethod + def _create_consumer(cls): + return cls.env['fake.source.consumer'].create({ + 'name': 'Foo', + 'version': '1.0', + }) + + def test_source_basic(self): + source = self.source + self.assertEqual(source.name, 'static') + self.assertItemsEqual( + source._config_summary_fields, ['chunk_size', 'fake_param']) + + def test_source_get_lines(self): + source = self.source + lines = list(source.get_lines()) + # 20 records, chunk size 5 + self.assertEqual(len(lines), 4) + # custom sorting: reversed + self.assertEqual(lines[0][0]['id'], 20) + + def test_source_summary_data(self): + source = self.source + data = source._config_summary_data() + self.assertEqual(data['source'], source) + self.assertEqual( + sorted(data['summary_fields']), + sorted(['chunk_size', 'fake_param'])) + self.assertIn('chunk_size', data['fields_info']) + self.assertIn('fake_param', data['fields_info']) + + @mock.patch(SOURCE_MODEL + '._selection_source_ref_id') + def test_consumer_basic(self, _selection_source_ref_id): + # enable our fake source + _selection_source_ref_id.return_value = [(self.source._name, 'Fake')] + consumer = self.consumer + self.assertFalse(consumer.get_source()) + consumer.update({ + 'source_id': self.source.id, + 'source_model': self.source._name + }) + self.assertEqual(consumer.get_source(), self.source) diff --git a/connector_importer/tests/test_source_csv.py b/connector_importer/tests/test_source_csv.py new file mode 100644 index 000000000..b1ce6553b --- /dev/null +++ b/connector_importer/tests/test_source_csv.py @@ -0,0 +1,81 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from .common import FakeModelTestCase +from .fake_models import ( + FakeSourceConsumer, +) +import base64 + + +class TestSourceCSV(FakeModelTestCase): + + TEST_MODELS_KLASSES = [FakeSourceConsumer] + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._setup_models() + cls.source = cls._create_source() + cls.consumer = cls._create_consumer() + + @classmethod + def tearDownClass(cls): + cls._teardown_models() + super().tearDownClass() + + @classmethod + def _create_source(cls): + filecontent = cls.load_filecontent( + 'connector_importer', + 'tests/fixtures/csv_source_test1.csv', mode='rb') + source = cls.env['import.source.csv'].create({ + 'csv_file': base64.encodestring(filecontent), + }) + source._onchance_csv_file() + return source + + @classmethod + def _create_consumer(cls): + return cls.env['fake.source.consumer'].create({ + 'name': 'Foo', + 'version': '1.0', + }) + + extra_fields = [ + 'chunk_size', 'csv_filesize', 'csv_filename', + 'csv_delimiter', 'csv_quotechar', + ] + + def test_source_basic(self): + source = self.source + self.assertEqual(source.name, 'csv') + self.assertItemsEqual( + source._config_summary_fields, self.extra_fields) + self.assertEqual(source.csv_delimiter, ',') + self.assertEqual(source.csv_quotechar, '"') + + def test_source_get_lines(self): + source = self.source + # call private method to skip chunking, pointless here + lines = list(source._get_lines()) + self.assertEqual(len(lines), 5) + self.assertDictEqual( + lines[0], {'id': '1', 'fullname': 'Marty McFly', '_line_nr': 2}) + self.assertDictEqual( + lines[1], {'id': '2', 'fullname': 'Biff Tannen', '_line_nr': 3}) + self.assertDictEqual( + lines[2], {'id': '3', 'fullname': 'Emmet Brown', '_line_nr': 4}) + self.assertDictEqual( + lines[3], {'id': '4', 'fullname': 'Clara Clayton', '_line_nr': 5}) + self.assertDictEqual( + lines[4], {'id': '5', 'fullname': 'George McFly', '_line_nr': 6}) + + def test_source_summary_data(self): + source = self.source + data = source._config_summary_data() + self.assertEqual(data['source'], source) + self.assertItemsEqual(data['summary_fields'], self.extra_fields) + self.assertItemsEqual( + sorted(self.extra_fields), sorted(data['fields_info'].keys())) diff --git a/connector_importer/utils/__init__.py b/connector_importer/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/connector_importer/utils/import_utils.py b/connector_importer/utils/import_utils.py new file mode 100644 index 000000000..ec2da55ef --- /dev/null +++ b/connector_importer/utils/import_utils.py @@ -0,0 +1,122 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from ..log import logger +import csv +import time +import io +try: + from chardet.universaldetector import UniversalDetector +except ImportError: + import logging + _logger = logging.getLogger(__name__) + _logger.debug('`chardet` lib is missing') + + +def get_encoding(data): + """Try to get encoding incrementally. + + See http://chardet.readthedocs.org/en/latest/usage.html#example-detecting-encoding-incrementally # noqa + """ + start = time.time() + msg = 'detecting file encoding...' + logger.info(msg) + file_like = io.BytesIO(data) + detector = UniversalDetector() + for i, line in enumerate(file_like): + detector.feed(line) + if detector.done: + break + detector.close() + msg = 'encoding found in %s sec' % str(time.time() - start) + msg += str(detector.result) + logger.info(msg) + return detector.result + + +def csv_content_to_file(data): + """Odoo binary fields spit out b64 data.""" + # guess encoding via chardet (LOVE IT! :)) + encoding_info = get_encoding(data) + encoding = encoding_info['encoding'] + if encoding is None or encoding != 'utf-8': + try: + data_str = data.decode(encoding) + except (UnicodeDecodeError, TypeError): + # dirty fallback in case + # we don't spot the right encoding above + for enc in ('utf-16le', 'latin-1', 'ascii', ): + try: + data_str = data.decode(enc) + break + except UnicodeDecodeError: + data_str = data + data_str = data_str.encode('utf-8') + else: + data_str = data + return data_str + + +def guess_csv_metadata(filecontent): + with io.StringIO(str(filecontent, 'utf-8')) as ff: + try: + dialect = csv.Sniffer().sniff(ff.readline(), "\t,;") + ff.seek(0) + meta = { + 'delimiter': dialect.delimiter, + 'quotechar': dialect.quotechar, + } + except BaseException: + meta = {} + return meta + + +def read_path(path): + with open(path, 'r') as thefile: + return thefile.read() + + +class CSVReader(object): + """Advanced CSV reader.""" + + def __init__(self, + filepath=None, + filedata=None, + delimiter='|', + quotechar='"', + fieldnames=None): + assert filedata or filepath, 'Provide a file path or some file data!' + if filepath: + filedata = read_path(filepath) + self.bdata = csv_content_to_file(filedata) + self.data = str(self.bdata, 'utf-8') + self.delimiter = delimiter + self.quotechar = quotechar + self.fieldnames = fieldnames + + def read_lines(self): + """Yields lines and add info to them (like line nr).""" + reader = csv.DictReader( + self.data.splitlines(), + delimiter=str(self.delimiter), + quotechar=str(self.quotechar), + fieldnames=self.fieldnames, + ) + for line in reader: + line['_line_nr'] = reader.line_num + yield line + + +def gen_chunks(iterable, chunksize=10): + """Chunk generator. + + Take an iterable and yield `chunksize` sized slices. + """ + chunk = [] + for i, line in enumerate(iterable): + if (i % chunksize == 0 and i > 0): + yield chunk + del chunk[:] + chunk.append(line) + yield chunk diff --git a/connector_importer/utils/mapper_utils.py b/connector_importer/utils/mapper_utils.py new file mode 100644 index 000000000..7a6d1ac11 --- /dev/null +++ b/connector_importer/utils/mapper_utils.py @@ -0,0 +1,314 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import pytz +from datetime import datetime + +from odoo import fields + +from ..log import logger + +FMTS = ( + '%d/%m/%Y', +) + +FMTS_DT = ( + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%d %H:%M:%S.000' +) + + +def to_date(value, formats=FMTS): + """Convert date strings to odoo format.""" + + for fmt in formats: + try: + value = datetime.strptime(value, fmt).date() + break + except ValueError: + pass + if not isinstance(value, str): + try: + return fields.Date.to_string(value) + except ValueError: + pass + # the value has not been converted, + # maybe because is like 00/00/0000 + # or in another bad format + return None + + +def to_utc_datetime(orig_value, tz='Europe/Rome'): + """Convert date strings to odoo format respecting TZ.""" + value = orig_value + local_tz = pytz.timezone('Europe/Rome') + for fmt in FMTS_DT: + try: + naive = datetime.strptime(orig_value, fmt) + local_dt = local_tz.localize(naive, is_dst=None) + value = local_dt.astimezone(pytz.utc) + break + except ValueError: + pass + if not isinstance(value, str): + return fields.Datetime.to_string(value) + # the value has not been converted, + # maybe because is like 00/00/0000 + # or in another bad format + return None + + +def to_safe_float(value): + """Safely convert to float.""" + if isinstance(value, float): + return value + if not value: + return 0.0 + try: + return float(value.replace(',', '.')) + except ValueError: + return 0.0 + + +def to_safe_int(value): + """Safely convert to integer.""" + if isinstance(value, int): + return value + if not value: + return 0 + try: + return int(value.replace(',', '').replace('.', '')) + except ValueError: + return 0 + + +CONV_MAPPING = { + 'date': to_date, + 'utc_date': to_utc_datetime, + 'safe_float': to_safe_float, + 'safe_int': to_safe_int, +} + + +def convert(field, conv_type, + fallback_field=None, + pre_value_handler=None, + **kw): + """ Convert the source field to a defined ``conv_type`` + (ex. str) before returning it. + You can also use predefined converters like 'date'. + Use ``fallback_field`` to provide a field of the same type + to be used in case the base field has no value. + """ + if conv_type in CONV_MAPPING: + conv_type = CONV_MAPPING[conv_type] + + def modifier(self, record, to_attr): + if field not in record: + # be gentle + logger.warn( + 'Field `%s` missing in line `%s`', field, record['_line_nr']) + return None + value = record.get(field) + if not value and fallback_field: + value = record[fallback_field] + if pre_value_handler: + value = pre_value_handler(value) + # do not use `if not value` otherwise you override all zero values + if value is None: + return None + return conv_type(value, **kw) + + return modifier + + +def from_mapping(field, mapping, default_value=None): + """ Convert the source value using a ``mapping`` of values. + """ + + def modifier(self, record, to_attr): + value = record.get(field) + return mapping.get(value, default_value) + + return modifier + + +def concat(field, separator=' ', handler=None): + """Concatenate values from different fields.""" + + # TODO: `field` is actually a list of fields. + # `field` attribute is required ATM by the base connector mapper and + # `_direct_source_field_name` raises and error if you don't specify it. + # Check if we can get rid of it. + + def modifier(self, record, to_attr): + value = [ + record.get(_field, '') + for _field in field if record.get(_field, '').strip() + ] + return separator.join(value) + + return modifier + + +def xmlid_to_rel(field): + """ Convert xmlids source values to ids. + """ + + def modifier(self, record, to_attr): + value = record.get(field) + if value is None: + return None + if isinstance(value, str): + # m2o + rec = self.env.ref(value, raise_if_not_found=False) + if rec: + return rec.id + return None + # x2m + return [ + (6, 0, self.env.ref(x).ids) for x in value + if self.env.ref(x, raise_if_not_found=False) + ] + + return modifier + +# TODO: consider to move this to mapper base klass +# to ease maintanability and override + + +def backend_to_rel(field, + search_field=None, + search_operator=None, + value_handler=None, + default_search_value=None, + default_search_field=None, + search_value_handler=None, + allowed_length=None, + create_missing=False, + create_missing_handler=None,): + """A modifier intended to be used on the ``direct`` mappings. + + Example:: + + direct = [(backend_to_rel('country', + search_field='code', + default_search_value='IT', + allowed_length=2), 'country_id'),] + + :param field: name of the source field in the record + :param search_field: name of the field to be used for searching + :param search_operator: operator to be used for searching + :param value_handler: a function to manipulate the raw value + before using it. You can use it to strip out none values + that are not none, like '0' instead of an empty string. + :param default_search_value: if the value is none you can provide + a default value to look up + :param default_search_field: if the value is none you can provide + a different field to look up for the default value + :param search_value_handler: a callable to use + to manipulate value before searching + :param allowed_length: enforce a check on the search_value length + :param create_missing: create a new record if not found + :param create_missing_handler: provide an handler + for getting new values for a new record to be created. + """ + + def modifier(self, record, to_attr): + search_value = record.get(field) + + if search_value and value_handler: + search_value = value_handler(self, record, search_value) + + # handle defaults if no search value here + if not search_value and default_search_value: + search_value = default_search_value + if default_search_field: + modifier.search_field = default_search_field + + # get the real column and the model + column = self.model._fields[to_attr] + rel_model = \ + self.env[column.comodel_name].with_context(active_test=False) + + if allowed_length and len(search_value) != allowed_length: + return None + + # alter search value if handler is given + if search_value and search_value_handler: + search_value = search_value_handler(search_value) + + if not search_value: + return None + + search_operator = '=' + if column.type.endswith('2many'): + # we need multiple values + search_operator = 'in' + if not isinstance(search_value, (list, tuple)): + search_value = [search_value] + + if modifier.search_operator: + # override by param + search_operator = modifier.search_operator + + # finally search it + search_args = [(modifier.search_field, + search_operator, + search_value)] + + value = rel_model.search(search_args) + + if (column.type.endswith('2many') and + isinstance(search_value, (list, tuple)) and + not len(search_value) == len(value or [])): + # make sure we consider all the values and related records + # that we pass here. + # If one of them is missing we have to create them all before. + # If `create_missing_handler` is given, it must make sure + # to create all the missing records and return existing ones too. + # Typical use case is: product categories. + # If we pass ['Categ1', 'Categ2', 'Categ3'] we want them all, + # and if any of them is missing we might want to create them + # using a `create_missing_handler`. + value = None + + # create if missing + if not value and create_missing: + try: + if create_missing_handler: + value = create_missing_handler(self, rel_model, record) + else: + value = rel_model.create({'name': record[field]}) + except Exception as e: + msg = ( + '`backend_to_rel` failed creation. ' + '[model: %s] [line: %s] [to_attr: %s] ' + 'Error: %s' + ) + logger.error( + msg, rel_model._name, record['_line_nr'], to_attr, str(e) + ) + # raise error to make importer's savepoint ctx manager catch it + raise + + # handle the final value based on col type + if value: + if column.type == 'many2one': + value = value[0].id + if column.type in ('one2many', 'many2many'): + value = [(6, 0, [x.id for x in value])] + else: + return None + + return value + + # use method attributes to not mess up the variables' scope. + # If we change the var inside modifier, without this trick + # you get UnboundLocalError, as the variable was never defined. + # Trick tnx to http://stackoverflow.com/a/27910553/647924 + modifier.search_field = search_field or 'name' + modifier.search_operator = search_operator or None + + return modifier diff --git a/connector_importer/utils/report_html.py b/connector_importer/utils/report_html.py new file mode 100644 index 000000000..4a6fe369a --- /dev/null +++ b/connector_importer/utils/report_html.py @@ -0,0 +1,140 @@ +# Author: Simone Orsi +# Copyright 2018 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import json + +EXAMPLEDATA = { + "last_summary": { + "updated": 0, "skipped": 584, "errors": 0, "created": 414 + }, + "errors": [], + "last_start": "08/03/2018 13:46", + "skipped": [ + {"model": "product.template", + "line": 3, + "message": "ALREADY EXISTS code: 8482", + "odoo_record": 6171}, + {"model": "product.template", + "line": 4, + "message": "ALREADY EXISTS code: 8482", + "odoo_record": 6171}, + {"model": "product.template", + "line": 5, + "message": "ALREADY EXISTS code: 8482", + "odoo_record": 6171}, + ], +} +JSONDATA = json.dumps(EXAMPLEDATA) + + +def link_record(record_id, model='', record=None, + name_field='name', target='_new'): + """Link an existing odoo record.""" + name = 'View' + if record: + default = getattr(record, '_rec_name', 'Unknown') + name = getattr(record, name_field, default) + model = record._name + link = ( + """{name}""" + ).format( + id=record_id, + model=model, + name=name, + target=target, + ) + return link + + +class Reporter(object): + """Produce a formatted HTML report from importer json data.""" + + def __init__(self, jsondata, detailed=False, full_url=''): + self._jsondata = jsondata + self._data = json.loads(self._jsondata) + self._html = [] + self._detailed = detailed + self._full_url = full_url + + def html(self, wrapped=True): + """Return HTML report.""" + self._produce() + content = ''.join(self._html) + if wrapped: + return self._wrap( + 'html', self._wrap('body', content) + ) + return content + + def _add(self, el): + self._html.append(el) + + def _wrap(self, tag, content): + return '<{tag}>{content}'.format(tag=tag, content=content) + + def _line(self, content): + return self._wrap('p', content) + + def _value(self, key, value): + return self._wrap('strong', key.capitalize() + ': ') + str(value) + + def _value_line(self, key, value): + return self._line( + self._value(key, value) + ) + + def _line_to_msg(self, line): + res = [] + if line.get('line'): + res.append('CSV line: {}, '.format(line['line'])) + if line.get('message'): + res.append(line['message']) + if 'odoo_record' in line and 'model' in line: + res.append( + link_record(line['odoo_record'], model=line['model']) + ) + return ' '.join(res) + + def _listing(self, lines, list_type='ol'): + _lines = [] + for line in lines: + _lines.append(self._wrap('li', self._line_to_msg(line))) + return self._wrap( + list_type, ''.join(_lines) + ) + + def _produce(self): + if not self._data.get('last_summary'): + return + # header + self._add(self._wrap('h2', 'Last summary')) + # start date + self._add(self._value_line('Last start', self._data['last_start'])) + # global counters + summary_items = self._data['last_summary'].items() + for key, value in summary_items: + last = key == summary_items[-1][0] + self._add(self._value(key, value) + (' - ' if not last else '')) + if self._detailed: + self._add(self._wrap('h3', 'Details')) + if self._data['skipped']: + self._add(self._wrap('h4', 'Skipped')) + # skip messages + self._add(self._listing(self._data['skipped'])) + if self._data['errors']: + self._add(self._wrap('h4', 'Errors')) + # skip messages + self._add(self._listing(self._data['errors'])) + if self._full_url: + link = ( + 'View full report' + ).format(self._full_url) + self._add(self._line(link)) + + +if __name__ == '__main__': + reporter = Reporter(JSONDATA, detailed=1) + # pylint: disable=print-used + print(reporter.html()) diff --git a/connector_importer/views/backend_views.xml b/connector_importer/views/backend_views.xml new file mode 100644 index 000000000..4cda76f08 --- /dev/null +++ b/connector_importer/views/backend_views.xml @@ -0,0 +1,101 @@ + + + + + + import.backend + +
+ +

Import

+ + + + + + + + + + + + + + + + +