From e1977b4ab6eec9eac80a92eb660c91c34c61ba19 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Wed, 2 May 2018 11:59:58 +0200 Subject: [PATCH 01/28] Rename a column in the Souper: 'path' --> 'remote_path'. --- src/senaite/sync/complementstep.py | 9 ++--- src/senaite/sync/importstep.py | 56 ++++++++++++++++-------------- src/senaite/sync/souphandler.py | 39 +++++++++++++-------- src/senaite/sync/syncstep.py | 3 +- src/senaite/sync/utils.py | 7 ++-- 5 files changed, 64 insertions(+), 50 deletions(-) diff --git a/src/senaite/sync/complementstep.py b/src/senaite/sync/complementstep.py index 6527cc4..0d531bf 100644 --- a/src/senaite/sync/complementstep.py +++ b/src/senaite/sync/complementstep.py @@ -9,7 +9,8 @@ from senaite.sync.importstep import ImportStep from senaite.sync import logger, utils -from senaite.sync.souphandler import SoupHandler, REMOTE_UID, LOCAL_UID +from senaite.sync.souphandler import SoupHandler, REMOTE_UID, LOCAL_UID, \ + REMOTE_PATH class ComplementStep(ImportStep): @@ -82,8 +83,8 @@ def _fetch_data(self): # If remote UID is in the souper table already, just check if # remote path of the object has been updated if existing_rec: - rem_path = data_dict.get('path') - if rem_path != existing_rec.get('path'): + rem_path = data_dict.get(REMOTE_PATH) + if rem_path != existing_rec.get(REMOTE_PATH): self.sh.update_by_remote_uid(**data_dict) rec_id = existing_rec.get("rec_int_id") else: @@ -122,7 +123,7 @@ def _update_objects(self): row = self.sh.get_record_by_id(rec_id, as_dict=True) if not row: continue - logger.debug("Handling: {} ".format(row["path"])) + logger.debug("Handling: {} ".format(row[REMOTE_PATH])) self._handle_obj(row, handle_dependencies=False) # Log.info every 50 objects imported diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 2761109..8d31cef 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -25,7 +25,8 @@ from senaite.sync import logger from senaite.sync import _ from senaite.sync.souphandler import SoupHandler -from senaite.sync.souphandler import REMOTE_UID, LOCAL_UID, PORTAL_TYPE +from senaite.sync.souphandler import REMOTE_UID, LOCAL_UID, REMOTE_PATH,\ + PORTAL_TYPE from senaite.sync import utils COMMIT_INTERVAL = 1000 @@ -212,7 +213,7 @@ def _import_data(self): for item_index, r_uid in enumerate(ordered_uids): row = self.sh.find_unique(REMOTE_UID, r_uid) - logger.debug("Handling: {} ".format(row["path"])) + logger.debug("Handling: {} ".format(row[REMOTE_PATH])) self._handle_obj(row) # Handling object means there is a chunk containing several objects @@ -297,47 +298,48 @@ def _do_obj_creation(self, row): :param row: A row dictionary from the souper :type row: dict """ - path = row.get("path") + remote_path = row.get(REMOTE_PATH) - remote_parent_path = utils.get_parent_path(path) + remote_parent_path = utils.get_parent_path(remote_path) # If parent creation failed previously, do not try to create the object if remote_parent_path in self.skipped: logger.warning("Parent creation failed previously, skipping: {}" - .format(path)) + .format(remote_path)) return None - existing = self.portal.unrestrictedTraverse(self.translate_path(path), + existing = self.portal.unrestrictedTraverse(self.translate_path(remote_path), None) if existing: - local_uid = self.sh.find_unique("path", path).get(LOCAL_UID, + local_uid = self.sh.find_unique(REMOTE_PATH, remote_path).get(LOCAL_UID, None) if not local_uid: local_uid = api.get_uid(existing) - self.sh.update_by_path(path, local_uid=local_uid) + self.sh.update_by_remote_path(remote_path, local_uid=local_uid) return existing - if not self._parents_created(path): - logger.warning("Parent creation failed, skipping: {}".format(path)) + if not self._parents_created(remote_path): + logger.warning("Parent creation failed, skipping: {}".format( + remote_path)) return None - parent = self.translate_path(utils.get_parent_path(path)) + parent = self.translate_path(utils.get_parent_path(remote_path)) container = self.portal.unrestrictedTraverse(str(parent), None) obj_data = { - "id": utils.get_id_from_path(path), + "id": utils.get_id_from_path(remote_path), "portal_type": row.get(PORTAL_TYPE)} obj = self._create_object_slug(container, obj_data) if obj is not None: local_uid = api.get_uid(obj) - self.sh.update_by_path(path, local_uid=local_uid) + self.sh.update_by_remote_path(remote_path, local_uid=local_uid) return obj - def _parents_created(self, path): + def _parents_created(self, remote_path): """ Check if parents have been already created and create all non-existing parents and updates local UIDs for the existing ones. :param path: object path in the remote :return: True if ALL the parents were created successfully """ - p_path = utils.get_parent_path(path) + p_path = utils.get_parent_path(remote_path) if p_path == "/": return True @@ -346,42 +348,42 @@ def _parents_created(self, path): return True # Incoming path was remote path, translate it into local one - local_path = self.translate_path(p_path) + local_p_path = self.translate_path(p_path) # Check if the parent already exists. If yes, make sure it has # 'local_uid' value set in the soup table. - existing = self.portal.unrestrictedTraverse(local_path, None) + existing = self.portal.unrestrictedTraverse(local_p_path, None) if existing: - p_row = self.sh.find_unique("path", p_path) + p_row = self.sh.find_unique(REMOTE_PATH, p_path) if p_row is None: + # This should never happen return False - p_local_uid = self.sh.find_unique("path", p_path).get( - LOCAL_UID, None) + p_local_uid = p_row.get(LOCAL_UID, None) if not p_local_uid: if hasattr(existing, "UID") and existing.UID(): p_local_uid = existing.UID() - self.sh.update_by_path(p_path, local_uid=p_local_uid) + self.sh.update_by_remote_path(p_path, local_uid=p_local_uid) return True # Before creating an object's parent, make sure grand parents are # already ready. if not self._parents_created(p_path): return False - parent = self.sh.find_unique("path", p_path) + parent = self.sh.find_unique(REMOTE_PATH, p_path) grand_parent = self.translate_path(utils.get_parent_path(p_path)) container = self.portal.unrestrictedTraverse(str(grand_parent), None) parent_data = { "id": utils.get_id_from_path(p_path), - "path": p_path, + "remote_path": p_path, "portal_type": parent.get(PORTAL_TYPE)} parent_obj = self._create_object_slug(container, parent_data) if parent_obj is None: - logger.warning("Couldn't create parent of {}".format(path)) + logger.warning("Couldn't create parent of {}".format(remote_path)) return False # Parent is created, update it in the soup table. p_local_uid = api.get_uid(parent_obj) - self.sh.update_by_path(p_path, local_uid=p_local_uid) + self.sh.update_by_remote_path(p_path, local_uid=p_local_uid) return True def _create_dependencies(self, obj, data): @@ -537,7 +539,7 @@ def _create_object_slug(self, container, data, *args, **kwargs): """Create an content object slug for the given data """ id = data.get("id") - remote_path = data.get("path") + remote_path = data.get("remote_path") portal_type = data.get("portal_type") types_tool = api.get_tool("portal_types") fti = types_tool.getTypeInfo(portal_type) @@ -615,7 +617,7 @@ def _recover_failed_objects(self): if existing is None: continue logger.info('Recovering {0}/{1} : {2} '.format( - idx+1, total, existing["path"])) + idx+1, total, existing[REMOTE_PATH])) # Mark that update failed previously existing['updated'] = '0' self._handle_obj(existing, handle_dependencies=False) diff --git a/src/senaite/sync/souphandler.py b/src/senaite/sync/souphandler.py index 4adeee2..c3117af 100644 --- a/src/senaite/sync/souphandler.py +++ b/src/senaite/sync/souphandler.py @@ -1,5 +1,4 @@ - from senaite import api from senaite.sync import logger @@ -66,9 +65,10 @@ def insert(self, data): return False record = Record() record.attrs[REMOTE_UID] = data[REMOTE_UID] - record.attrs['path'] = data['path'] - record.attrs[PORTAL_TYPE] = data[PORTAL_TYPE] record.attrs[LOCAL_UID] = data.get(LOCAL_UID, "") + record.attrs[REMOTE_PATH] = data[REMOTE_PATH] + record.attrs[LOCAL_PATH] = data[LOCAL_PATH] + record.attrs[PORTAL_TYPE] = data[PORTAL_TYPE] record.attrs[UPDATED] = data.get(UPDATED, "0") r_id = self.soup.add(record) logger.info("Record {} inserted: {}".format(r_id, data)) @@ -82,11 +82,13 @@ def _already_exists(self, data): """ r_uid = data.get(REMOTE_UID, False) or '-1' l_uid = data.get(LOCAL_UID, False) or '-1' - path = data.get("path", False) or '-1' + r_path = data.get(REMOTE_PATH, False) or '-1' + l_path = data.get(LOCAL_PATH, False) or '-1' r_uid_q = Eq(REMOTE_UID, r_uid) l_uid_q = Eq(LOCAL_UID, l_uid) - p_q = Eq('path', path) - ret = [r for r in self.soup.query(Or(r_uid_q, l_uid_q, p_q))] + r_p_q = Eq(REMOTE_PATH, r_path) + l_p_q = Eq(REMOTE_PATH, l_path) + ret = [r for r in self.soup.query(Or(r_uid_q, l_uid_q, r_p_q, l_p_q))] return ret != [] def get_record_by_id(self, rec_id, as_dict=False): @@ -137,16 +139,16 @@ def update_by_remote_uid(self, remote_uid, **kwargs): self.soup.reindex([recs[0]]) return True - def update_by_path(self, path, **kwargs): + def update_by_remote_path(self, remote_path, **kwargs): """ Update the row by path column. :param path: path of the record :param kwargs: columns and their values to be updated. """ - recs = [r for r in self.soup.query(Eq('path', path))] + recs = [r for r in self.soup.query(Eq(REMOTE_PATH, remote_path))] if not recs: logger.error("Could not find any record with path: '{}'" - .format(path)) + .format(REMOTE_PATH)) return False for k, v in kwargs.iteritems(): recs[0].attrs[k] = v @@ -174,7 +176,7 @@ def reset_updated_flags(self): for intid in self.soup.data: rec = self.soup.get(intid) rec.attrs[UPDATED] = "0" - self.soup.reindex(rec) + self.soup.reindex() return True def _create_domain_catalog(self): @@ -187,11 +189,17 @@ class DomainSoupCatalogFactory(object): def __call__(self, context=None): catalog = Catalog() r_uid_indexer = NodeAttributeIndexer(REMOTE_UID) - catalog[u'remote_uid'] = CatalogFieldIndex(r_uid_indexer) - path_indexer = NodeAttributeIndexer('path') - catalog[u'path'] = CatalogFieldIndex(path_indexer) + catalog[unicode(REMOTE_UID)] = CatalogFieldIndex(r_uid_indexer) + l_uid_indexer = NodeAttributeIndexer(LOCAL_UID) - catalog[u'local_uid'] = CatalogFieldIndex(l_uid_indexer) + catalog[unicode(LOCAL_UID)] = CatalogFieldIndex(l_uid_indexer) + + r_path_indexer = NodeAttributeIndexer(REMOTE_PATH) + catalog[unicode(REMOTE_PATH)] = CatalogFieldIndex(r_path_indexer) + + l_path_indexer = NodeAttributeIndexer(LOCAL_PATH) + catalog[unicode(LOCAL_PATH)] = CatalogFieldIndex(l_path_indexer) + return catalog provideUtility(DomainSoupCatalogFactory(), name=self.domain_name) @@ -226,7 +234,8 @@ def record_to_dict(record): 'rec_int_id': record.intid, REMOTE_UID: record.attrs.get(REMOTE_UID, ""), LOCAL_UID: record.attrs.get(LOCAL_UID, ""), - 'path': record.attrs.get('path', ""), + REMOTE_PATH: record.attrs.get(REMOTE_PATH, ""), + LOCAL_PATH: record.attrs.get(LOCAL_PATH, ""), UPDATED: record.attrs.get(UPDATED, "0"), PORTAL_TYPE: record.attrs.get(PORTAL_TYPE, "") } diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 6bd24d2..17935fb 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -15,6 +15,7 @@ from senaite.sync import logger from senaite.sync import utils from senaite.sync.syncerror import SyncError +from senaite.sync.souphandler import REMOTE_PATH SYNC_STORAGE = "senaite.sync" API_BASE_URL = "API/senaite/v1" @@ -227,7 +228,7 @@ def _parents_fetched(self, item): if self.is_portal_path(parent_path): return True # Skip if already exists - if self.sh.find_unique("path", parent_path): + if self.sh.find_unique(REMOTE_PATH, parent_path): return True logger.debug("Inserting missing parent: {}".format(parent_path)) parent = self.get_first_item(item.get("parent_url")) diff --git a/src/senaite/sync/utils.py b/src/senaite/sync/utils.py index d8ca84d..8b4ab72 100644 --- a/src/senaite/sync/utils.py +++ b/src/senaite/sync/utils.py @@ -7,11 +7,12 @@ from senaite.sync import logger from DateTime import DateTime from datetime import datetime +from senaite.sync.souphandler import REMOTE_UID, REMOTE_PATH, PORTAL_TYPE -SOUPER_REQUIRED_FIELDS = {"uid": "remote_uid", - "path": "path", - "portal_type": "portal_type"} +SOUPER_REQUIRED_FIELDS = {"uid": REMOTE_UID, + "path": REMOTE_PATH, + "portal_type": PORTAL_TYPE} SYNC_CREDENTIALS = "senaite.sync.credentials" From 0822a514f81a788f948fec047c11ffe6e142135d Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Wed, 2 May 2018 13:03:42 +0200 Subject: [PATCH 02/28] Do not fail if local path is not set while inserting to Souper. --- src/senaite/sync/souphandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/senaite/sync/souphandler.py b/src/senaite/sync/souphandler.py index c3117af..4004eca 100644 --- a/src/senaite/sync/souphandler.py +++ b/src/senaite/sync/souphandler.py @@ -67,7 +67,7 @@ def insert(self, data): record.attrs[REMOTE_UID] = data[REMOTE_UID] record.attrs[LOCAL_UID] = data.get(LOCAL_UID, "") record.attrs[REMOTE_PATH] = data[REMOTE_PATH] - record.attrs[LOCAL_PATH] = data[LOCAL_PATH] + record.attrs[LOCAL_PATH] = data.get(LOCAL_PATH, "") record.attrs[PORTAL_TYPE] = data[PORTAL_TYPE] record.attrs[UPDATED] = data.get(UPDATED, "0") r_id = self.soup.add(record) From 95bd9924b2c5c54bb9268fd8bb652fae42ebfcfc Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Wed, 2 May 2018 15:15:18 +0200 Subject: [PATCH 03/28] Path Translator with prefix. --- src/senaite/sync/syncstep.py | 42 ++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 17935fb..8375e3a 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -15,7 +15,7 @@ from senaite.sync import logger from senaite.sync import utils from senaite.sync.syncerror import SyncError -from senaite.sync.souphandler import REMOTE_PATH +from senaite.sync.souphandler import REMOTE_PATH, LOCAL_PATH, PORTAL_TYPE SYNC_STORAGE = "senaite.sync" API_BASE_URL = "API/senaite/v1" @@ -44,6 +44,7 @@ def __init__(self, data): self.password = data.get("ac_password", None) # Import configuration self.content_types = data.get("content_types", None) + self.prefix = "" self.import_settings = data.get("import_settings", False) self.import_users = data.get("import_users", False) self.import_registry = data.get("import_registry", False) @@ -52,12 +53,49 @@ def __init__(self, data): self.fail("Missing parameter in Sync Step: {}".format(data)) def translate_path(self, path): - """Translate the physical path to a local path + """ Translate the physical path to a local path """ portal_id = self.portal.getId() remote_portal_id = path.split("/")[1] return str(path.replace(remote_portal_id, portal_id)) + def translate_path_with_prefix(self, remote_path): + """ + """ + portal_id = self.portal.getId() + remote_portal_id = remote_path.split("/")[1] + if not self.prefix: + return str(remote_path.replace(remote_portal_id, portal_id)) + + rem_id = utils.get_id_from_path(remote_path) + rec = self.sh.find_unique(REMOTE_PATH, remote_path) + if rec[LOCAL_PATH]: + return rec[LOCAL_PATH] + + # Get parent's local path + remote_parent_path = utils.get_parent_path(remote_path) + parent_path = self.translate_path_with_prefix(remote_parent_path) + + # Will check whether prefix needed by portal type + portal_type = rec[PORTAL_TYPE] + prefix = self.get_prefix(portal_type) + + res = "{0}/{1}{2}".format(parent_path, prefix, rem_id) + res = res.replace(remote_portal_id, portal_id) + self.sh.update_by_remote_path(remote_path, LOCAL_PATH = res) + return res + + def get_prefix(self, portal_type): + """ + + :param portal_type: + :return: + """ + if self.prefix and portal_type == 'Analysis': + return "PR_" + + return "" + def is_portal_path(self, path): """ Check if the given path is the path of any portal object. :return: From 522d9d2493e5b1e372e6fde4a37e478c6a89a517 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Wed, 2 May 2018 15:29:57 +0200 Subject: [PATCH 04/28] Use prefixes in object and parent creation. --- src/senaite/sync/importstep.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 8d31cef..4bdc79f 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -307,8 +307,8 @@ def _do_obj_creation(self, row): .format(remote_path)) return None - existing = self.portal.unrestrictedTraverse(self.translate_path(remote_path), - None) + local_path = self.translate_path_with_prefix(remote_path) + existing = self.portal.unrestrictedTraverse(local_path, None) if existing: local_uid = self.sh.find_unique(REMOTE_PATH, remote_path).get(LOCAL_UID, None) @@ -322,10 +322,10 @@ def _do_obj_creation(self, row): remote_path)) return None - parent = self.translate_path(utils.get_parent_path(remote_path)) - container = self.portal.unrestrictedTraverse(str(parent), None) + parent_path = utils.get_parent_path(local_path) + container = self.portal.unrestrictedTraverse(str(parent_path), None) obj_data = { - "id": utils.get_id_from_path(remote_path), + "id": utils.get_id_from_path(local_path), "portal_type": row.get(PORTAL_TYPE)} obj = self._create_object_slug(container, obj_data) if obj is not None: @@ -348,7 +348,7 @@ def _parents_created(self, remote_path): return True # Incoming path was remote path, translate it into local one - local_p_path = self.translate_path(p_path) + local_p_path = self.translate_path_with_prefix(p_path) # Check if the parent already exists. If yes, make sure it has # 'local_uid' value set in the soup table. @@ -369,13 +369,15 @@ def _parents_created(self, remote_path): # already ready. if not self._parents_created(p_path): return False + parent = self.sh.find_unique(REMOTE_PATH, p_path) - grand_parent = self.translate_path(utils.get_parent_path(p_path)) - container = self.portal.unrestrictedTraverse(str(grand_parent), None) + grand_parent = utils.get_parent_path(local_p_path) + container = self.portal.unrestrictedTraverse(grand_parent, None) parent_data = { "id": utils.get_id_from_path(p_path), "remote_path": p_path, "portal_type": parent.get(PORTAL_TYPE)} + parent_obj = self._create_object_slug(container, parent_data) if parent_obj is None: logger.warning("Couldn't create parent of {}".format(remote_path)) From ecb6c18f7f6ac35cd0a677fd998332a446e59cbd Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 3 May 2018 10:26:41 +0200 Subject: [PATCH 05/28] Use new local ID while creating parents. --- src/senaite/sync/importstep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 4bdc79f..6513995 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -374,7 +374,7 @@ def _parents_created(self, remote_path): grand_parent = utils.get_parent_path(local_p_path) container = self.portal.unrestrictedTraverse(grand_parent, None) parent_data = { - "id": utils.get_id_from_path(p_path), + "id": utils.get_id_from_path(local_p_path), "remote_path": p_path, "portal_type": parent.get(PORTAL_TYPE)} From e62d580019f024877555c1f3b5a871333b942865 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 3 May 2018 11:38:06 +0200 Subject: [PATCH 06/28] Separated 'Add' and 'Show' views of Remotes. --- src/senaite/sync/browser/add.py | 86 +++++++++ src/senaite/sync/browser/configure.zcml | 7 + src/senaite/sync/browser/templates/add.pt | 201 +++++++++++++++++++++ src/senaite/sync/browser/templates/sync.pt | 178 +----------------- src/senaite/sync/browser/views.py | 44 ----- src/senaite/sync/fetchstep.py | 2 +- 6 files changed, 301 insertions(+), 217 deletions(-) create mode 100644 src/senaite/sync/browser/add.py create mode 100644 src/senaite/sync/browser/templates/add.pt diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py new file mode 100644 index 0000000..ce2a3c0 --- /dev/null +++ b/src/senaite/sync/browser/add.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2017-2018 SENAITE SYNC. + +from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile +from plone import protect +from senaite import api +from senaite.sync import _ +from senaite.sync.browser.interfaces import ISync +from senaite.sync.browser.views import Sync +from senaite.sync.fetchstep import FetchStep +from zope.interface import implements + +SYNC_STORAGE = "senaite.sync" + + +class Add(Sync): + """ Add new sync instance view + """ + implements(ISync) + + template = ViewPageTemplateFile("templates/add.pt") + + def __init__(self, context, request): + super(Sync, self).__init__(context, request) + + def __call__(self): + protect.CheckAuthenticator(self.request.form) + + self.portal = api.get_portal() + self.request.set('disable_plone.rightcolumn', 1) + self.request.set('disable_border', 1) + + # Handle form submit + form = self.request.form + fetchform = form.get("fetchform", False) + dataform = form.get("dataform", False) + if not any([fetchform, dataform]): + return self.template() + + # Handle "Fetch" action + if form.get("fetch", False): + + url = form.get("url", "") + if not url.startswith("http"): + url = "http://{}".format(url) + domain_name = form.get("domain_name", None) + username = form.get("ac_name", None) + password = form.get("ac_password", None) + # check if all mandatory fields have values + if not all([domain_name, url, username, password]): + message = _("Please fill in all required fields") + self.add_status_message(message, "error") + return self.template() + + import_settings = True if form.get("import_settings") == 'on' else False + import_users = True if form.get("import_users") == 'on' else False + import_registry = True if form.get("import_registry") == 'on' else False + content_types = form.get("content_types", None) + if content_types is not None: + content_types = [t.strip() for t in content_types.split(",")] + portal_types = api.get_tool("portal_types") + content_types = filter(lambda ct: ct in portal_types, + content_types) + + data = { + "url": url, + "domain_name": domain_name, + "ac_name": username, + "ac_password": password, + "content_types": content_types, + "import_settings": import_settings, + "import_users": import_users, + "import_registry": import_registry, + } + + fs = FetchStep(data) + verified, message = fs.verify() + if verified: + fs.run() + self.add_status_message(message, "info") + else: + self.add_status_message(message, "error") + + # render the template + return self.template() diff --git a/src/senaite/sync/browser/configure.zcml b/src/senaite/sync/browser/configure.zcml index c0200e3..33c85de 100644 --- a/src/senaite/sync/browser/configure.zcml +++ b/src/senaite/sync/browser/configure.zcml @@ -9,6 +9,13 @@ permission="cmf.ManagePortal" /> + + + + + + + + + +

+ Add new Remote +

+
+ +

+

+
+ +
+ +
+ + + + + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+
+
+ +
+
    +
  • +
    + + +
    +
  • +
  • +
    + + +
    +
  • +
  • +
    + + +
    +
  • +
  • + +
    + +
    + +
    +
    +
  • +
+
+
+
+
+ + + +

+
+ +
+
+ + + diff --git a/src/senaite/sync/browser/templates/sync.pt b/src/senaite/sync/browser/templates/sync.pt index 96a4c45..99d5c4b 100644 --- a/src/senaite/sync/browser/templates/sync.pt +++ b/src/senaite/sync/browser/templates/sync.pt @@ -12,7 +12,7 @@

- SENAITE SYNC + SENAITE SYNC REMOTES

@@ -23,175 +23,9 @@
-
- - - - - -
- -
- - -
-
- - -
- -
- - -
-
- - -
- -
- - -
-
- - -
- -
- - -
-
- - -
-
-
- -
-
    -
  • -
    - - -
    -
  • -
  • -
    - - -
    -
  • -
  • -
    - - -
    -
  • -
  • - -
    - -
    - -
    -
    -
  • -
-
-
-
-
- - - - +
+ +


@@ -202,7 +36,7 @@
- Data for + ( Items)
@@ -229,7 +63,7 @@
- Complement Step + Run Complement Step
diff --git a/src/senaite/sync/browser/views.py b/src/senaite/sync/browser/views.py index fb0ef39..40ac0fc 100644 --- a/src/senaite/sync/browser/views.py +++ b/src/senaite/sync/browser/views.py @@ -122,50 +122,6 @@ def __call__(self): self.add_status_message(message, "info") return self.template() - # Handle "Fetch" action - if form.get("fetch", False): - - url = form.get("url", "") - if not url.startswith("http"): - url = "http://{}".format(url) - domain_name = form.get("domain_name", None) - username = form.get("ac_name", None) - password = form.get("ac_password", None) - # check if all mandatory fields have values - if not all([domain_name, url, username, password]): - message = _("Please fill in all required fields") - self.add_status_message(message, "error") - return self.template() - - import_settings = True if form.get("import_settings") == 'on' else False - import_users = True if form.get("import_users") == 'on' else False - import_registry = True if form.get("import_registry") == 'on' else False - content_types = form.get("content_types", None) - if content_types is not None: - content_types = [t.strip() for t in content_types.split(",")] - portal_types = api.get_tool("portal_types") - content_types = filter(lambda ct: ct in portal_types, - content_types) - - data = { - "url": url, - "domain_name": domain_name, - "ac_name": username, - "ac_password": password, - "content_types": content_types, - "import_settings": import_settings, - "import_users": import_users, - "import_registry": import_registry, - } - - fs = FetchStep(data) - verified, message = fs.verify() - if verified: - fs.run() - self.add_status_message(message, "info") - else: - self.add_status_message(message, "error") - # always render the template return self.template() diff --git a/src/senaite/sync/fetchstep.py b/src/senaite/sync/fetchstep.py index c65ec09..5441cd7 100644 --- a/src/senaite/sync/fetchstep.py +++ b/src/senaite/sync/fetchstep.py @@ -69,7 +69,7 @@ def verify(self): storage["configuration"]["import_users"] = self.import_users storage["last_fetch_time"] = DateTime() - message = "Fetching Data started for {}".format(self.domain_name) + message = "Data fetched and saved: {}".format(self.domain_name) return True, message def get_version(self): From d182d3b3cdd7996162d95e7f1841449b4d768e9e Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 3 May 2018 13:08:20 +0200 Subject: [PATCH 07/28] Introduction to Prefix Logic in import Step. --- src/senaite/sync/browser/add.py | 13 ++++++++++ src/senaite/sync/browser/templates/add.pt | 30 ++++++++++++++++++----- src/senaite/sync/browser/views.py | 4 +++ src/senaite/sync/fetchstep.py | 1 + src/senaite/sync/syncstep.py | 9 ++++--- src/senaite/sync/utils.py | 4 ++- 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index ce2a3c0..2e17fcc 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -57,12 +57,24 @@ def __call__(self): import_users = True if form.get("import_users") == 'on' else False import_registry = True if form.get("import_registry") == 'on' else False content_types = form.get("content_types", None) + prefix = form.get("prefix", None) + + # Content Type Validation if content_types is not None: content_types = [t.strip() for t in content_types.split(",")] portal_types = api.get_tool("portal_types") content_types = filter(lambda ct: ct in portal_types, content_types) + # Prefix Validation + if prefix: + prefix = prefix.strip(u"*.!$%&/()=-+:'`´^") + if not prefix: + self.add_status_message("Invalid Prefix!", "error") + return self.template() + if len(prefix) > 3: + self.add_status_message("Long Prefix!", "warning") + data = { "url": url, "domain_name": domain_name, @@ -72,6 +84,7 @@ def __call__(self): "import_settings": import_settings, "import_users": import_users, "import_registry": import_registry, + "prefix": prefix, } fs = FetchStep(data) diff --git a/src/senaite/sync/browser/templates/add.pt b/src/senaite/sync/browser/templates/add.pt index 6be80bd..e18f908 100644 --- a/src/senaite/sync/browser/templates/add.pt +++ b/src/senaite/sync/browser/templates/add.pt @@ -47,7 +47,6 @@
@@ -69,7 +68,6 @@
@@ -93,7 +91,6 @@ autocomplete="new-username" class="form-control" id="ac_name" - tal:attributes="value view/username|nothing" name="ac_name"/> @@ -117,7 +114,6 @@ size="30" class="form-control" id="ac_password" - tal:attributes="value view/password|nothing" name="ac_password"/> @@ -128,7 +124,7 @@
@@ -174,11 +170,33 @@ size="45" class="form-control" id="content_types" - tal:attributes="value view/content_types|nothing" name="content_types"/>
+ +
  • + +
    + +
    + +
    +
    +
  • + diff --git a/src/senaite/sync/browser/views.py b/src/senaite/sync/browser/views.py index 40ac0fc..7810496 100644 --- a/src/senaite/sync/browser/views.py +++ b/src/senaite/sync/browser/views.py @@ -66,12 +66,14 @@ def __call__(self): username = storage["credentials"]["username"] password = storage["credentials"]["password"] content_types = storage["configuration"].get("content_types", None) + prefix = storage["configuration"].get("prefix", None) data = { "url": url, "domain_name": domain_name, "ac_name": username, "ac_password": password, "content_types": content_types, + "prefix": prefix, } step = ImportStep(data) step.run() @@ -101,6 +103,7 @@ def __call__(self): username = storage["credentials"]["username"] password = storage["credentials"]["password"] content_types = storage["configuration"].get("content_types", None) + prefix = storage["configuration"].get("prefix", None) data = { "url": url, "domain_name": domain_name, @@ -108,6 +111,7 @@ def __call__(self): "ac_password": password, "fetch_time": fetch_time, "content_types": content_types, + "prefix": prefix, } step = ComplementStep(data) step.run() diff --git a/src/senaite/sync/fetchstep.py b/src/senaite/sync/fetchstep.py index 5441cd7..02f3274 100644 --- a/src/senaite/sync/fetchstep.py +++ b/src/senaite/sync/fetchstep.py @@ -64,6 +64,7 @@ def verify(self): storage["credentials"]["password"] = self.password # remember import configuration in the storage storage["configuration"]["content_types"] = self.content_types + storage["configuration"]["prefix"] = self.prefix storage["configuration"]["import_settings"] = self.import_settings storage["configuration"]["import_registry"] = self.import_registry storage["configuration"]["import_users"] = self.import_users diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 8375e3a..9e6188e 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -44,7 +44,7 @@ def __init__(self, data): self.password = data.get("ac_password", None) # Import configuration self.content_types = data.get("content_types", None) - self.prefix = "" + self.prefix = data.get("prefix", None) self.import_settings = data.get("import_settings", False) self.import_users = data.get("import_users", False) self.import_registry = data.get("import_registry", False) @@ -62,6 +62,9 @@ def translate_path(self, path): def translate_path_with_prefix(self, remote_path): """ """ + if self.is_portal_path(remote_path): + return api.get_path(self.portal) + portal_id = self.portal.getId() remote_portal_id = remote_path.split("/")[1] if not self.prefix: @@ -91,8 +94,8 @@ def get_prefix(self, portal_type): :param portal_type: :return: """ - if self.prefix and portal_type == 'Analysis': - return "PR_" + if self.prefix and portal_type == 'Client': + return self.prefix return "" diff --git a/src/senaite/sync/utils.py b/src/senaite/sync/utils.py index 8b4ab72..5118e2e 100644 --- a/src/senaite/sync/utils.py +++ b/src/senaite/sync/utils.py @@ -57,7 +57,9 @@ def get_parent_path(path): if path.endswith("/"): path = path[:-1] parts = path.split("/") - return "/".join(parts[:-1]) + if len(parts) < 3: + return "/" + return str("/".join(parts[:-1])) def get_id_from_path(path): From d26222db5f637c9df9efb050110a6e527540d464 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 3 May 2018 13:38:05 +0200 Subject: [PATCH 08/28] Return string values while getting local paths. --- src/senaite/sync/syncstep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 9e6188e..28bae28 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -73,7 +73,7 @@ def translate_path_with_prefix(self, remote_path): rem_id = utils.get_id_from_path(remote_path) rec = self.sh.find_unique(REMOTE_PATH, remote_path) if rec[LOCAL_PATH]: - return rec[LOCAL_PATH] + return str(rec[LOCAL_PATH]) # Get parent's local path remote_parent_path = utils.get_parent_path(remote_path) @@ -86,7 +86,7 @@ def translate_path_with_prefix(self, remote_path): res = "{0}/{1}{2}".format(parent_path, prefix, rem_id) res = res.replace(remote_portal_id, portal_id) self.sh.update_by_remote_path(remote_path, LOCAL_PATH = res) - return res + return str(res) def get_prefix(self, portal_type): """ From d34b423ae68dd549e5b0346d52d619dd923bc0d4 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 3 May 2018 14:13:12 +0200 Subject: [PATCH 09/28] Do not touch ID while updating object data. --- src/senaite/sync/importstep.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 6513995..da99e6f 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -60,7 +60,8 @@ class ImportStep(SyncStep): update objects based on previously fetched data. """ - fields_to_skip = ['excludeFromNav', 'constrainTypesMode', 'allowDiscussion'] + fields_to_skip = ['id', # Overriding ID's can remove prefixes + 'excludeFromNav', 'constrainTypesMode', 'allowDiscussion'] def __init__(self, data): SyncStep.__init__(self, data) From 183ac9779624fa02a09a9d345f6683ad8a7995ed Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 3 May 2018 14:35:53 +0200 Subject: [PATCH 10/28] Content Types to be created with prefix in the 'Add Remote' View. --- src/senaite/sync/browser/add.py | 13 +++++++++++++ src/senaite/sync/browser/templates/add.pt | 22 ++++++++++++++++++++++ src/senaite/sync/browser/views.py | 4 ++++ src/senaite/sync/fetchstep.py | 1 + src/senaite/sync/syncstep.py | 4 ++-- 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index 2e17fcc..9df3a1e 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -58,6 +58,7 @@ def __call__(self): import_registry = True if form.get("import_registry") == 'on' else False content_types = form.get("content_types", None) prefix = form.get("prefix", None) + prefixable_types = form.get("prefixable_types", None) # Content Type Validation if content_types is not None: @@ -75,6 +76,17 @@ def __call__(self): if len(prefix) > 3: self.add_status_message("Long Prefix!", "warning") + # Content Type Validation + if prefixable_types is not None: + prefixable_types = [t.strip() for t in prefixable_types.split(",")] + portal_types = api.get_tool("portal_types") + prefixable_types = filter(lambda ct: ct in portal_types, + prefixable_types) + if not prefixable_types: + self.add_status_message("Invalid Content Types to be" + "Prefixified!", "error") + return self.template() + data = { "url": url, "domain_name": domain_name, @@ -85,6 +97,7 @@ def __call__(self): "import_users": import_users, "import_registry": import_registry, "prefix": prefix, + "prefixable_types": prefixable_types, } fs = FetchStep(data) diff --git a/src/senaite/sync/browser/templates/add.pt b/src/senaite/sync/browser/templates/add.pt index e18f908..ff846bc 100644 --- a/src/senaite/sync/browser/templates/add.pt +++ b/src/senaite/sync/browser/templates/add.pt @@ -197,6 +197,28 @@ +
  • + +
    + +
    + +
    +
    +
  • + diff --git a/src/senaite/sync/browser/views.py b/src/senaite/sync/browser/views.py index 7810496..3256e66 100644 --- a/src/senaite/sync/browser/views.py +++ b/src/senaite/sync/browser/views.py @@ -67,6 +67,7 @@ def __call__(self): password = storage["credentials"]["password"] content_types = storage["configuration"].get("content_types", None) prefix = storage["configuration"].get("prefix", None) + prefixable_types = storage["configuration"].get("prefixable_types", None) data = { "url": url, "domain_name": domain_name, @@ -74,6 +75,7 @@ def __call__(self): "ac_password": password, "content_types": content_types, "prefix": prefix, + "prefixable_types": prefixable_types, } step = ImportStep(data) step.run() @@ -104,6 +106,7 @@ def __call__(self): password = storage["credentials"]["password"] content_types = storage["configuration"].get("content_types", None) prefix = storage["configuration"].get("prefix", None) + prefixable_types = storage["configuration"].get("prefixable_types", None) data = { "url": url, "domain_name": domain_name, @@ -112,6 +115,7 @@ def __call__(self): "fetch_time": fetch_time, "content_types": content_types, "prefix": prefix, + "prefixable_types": prefixable_types, } step = ComplementStep(data) step.run() diff --git a/src/senaite/sync/fetchstep.py b/src/senaite/sync/fetchstep.py index 02f3274..6009189 100644 --- a/src/senaite/sync/fetchstep.py +++ b/src/senaite/sync/fetchstep.py @@ -65,6 +65,7 @@ def verify(self): # remember import configuration in the storage storage["configuration"]["content_types"] = self.content_types storage["configuration"]["prefix"] = self.prefix + storage["configuration"]["prefixable_types"] = self.prefixable_types storage["configuration"]["import_settings"] = self.import_settings storage["configuration"]["import_registry"] = self.import_registry storage["configuration"]["import_users"] = self.import_users diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 28bae28..74399c4 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -45,6 +45,7 @@ def __init__(self, data): # Import configuration self.content_types = data.get("content_types", None) self.prefix = data.get("prefix", None) + self.prefixable_types = data.get("prefixable_types", None) self.import_settings = data.get("import_settings", False) self.import_users = data.get("import_users", False) self.import_registry = data.get("import_registry", False) @@ -94,9 +95,8 @@ def get_prefix(self, portal_type): :param portal_type: :return: """ - if self.prefix and portal_type == 'Client': + if self.prefix and portal_type in self.prefixable_types: return self.prefix - return "" def is_portal_path(self, path): From ffc621979fb070b19f7c946928d8e26db2107827 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 09:41:29 +0200 Subject: [PATCH 11/28] Raise SyncError if record not found in the Souper table. --- src/senaite/sync/syncstep.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 74399c4..dab74a5 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -73,6 +73,9 @@ def translate_path_with_prefix(self, remote_path): rem_id = utils.get_id_from_path(remote_path) rec = self.sh.find_unique(REMOTE_PATH, remote_path) + if rec is None: + raise SyncError("Missing Remote path in Soup table: {}".format( + remote_path)) if rec[LOCAL_PATH]: return str(rec[LOCAL_PATH]) From 8c640348ea0264c85cd4a48d02842ad83641d8e2 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 10:13:15 +0200 Subject: [PATCH 12/28] Validate Prefix if Prefixable Types Introduced. --- src/senaite/sync/browser/add.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index 9df3a1e..58bc9b8 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -78,6 +78,11 @@ def __call__(self): # Content Type Validation if prefixable_types is not None: + if not prefix: + self.add_status_message("Please enter a valid Prefix.", + "error") + return self.template() + prefixable_types = [t.strip() for t in prefixable_types.split(",")] portal_types = api.get_tool("portal_types") prefixable_types = filter(lambda ct: ct in portal_types, From ea669446a015637571febcd8a3b3c120ddd20891 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 10:24:47 +0200 Subject: [PATCH 13/28] Polish. --- src/senaite/sync/browser/add.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index 58bc9b8..a26defe 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -12,7 +12,7 @@ from zope.interface import implements SYNC_STORAGE = "senaite.sync" - +PREFIX_SPECIAL_CHARACTERS = u"*.!$%&/()=-+:'`´^" class Add(Sync): """ Add new sync instance view @@ -69,7 +69,7 @@ def __call__(self): # Prefix Validation if prefix: - prefix = prefix.strip(u"*.!$%&/()=-+:'`´^") + prefix = prefix.strip(PREFIX_SPECIAL_CHARACTERS) if not prefix: self.add_status_message("Invalid Prefix!", "error") return self.template() @@ -87,10 +87,11 @@ def __call__(self): portal_types = api.get_tool("portal_types") prefixable_types = filter(lambda ct: ct in portal_types, prefixable_types) - if not prefixable_types: - self.add_status_message("Invalid Content Types to be" - "Prefixified!", "error") - return self.template() + + if prefix and not prefixable_types: + self.add_status_message("Please enter valid Content Types to be" + " prefixified.", "error") + return self.template() data = { "url": url, From b5af958ec723bd0527bb83cb5b647e827feb3135 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 10:42:01 +0200 Subject: [PATCH 14/28] Unwanted Portal Types in 'Add Remote' View. --- src/senaite/sync/browser/add.py | 12 +++++++++-- src/senaite/sync/browser/templates/add.pt | 26 +++++++++++++++++++++-- src/senaite/sync/browser/views.py | 4 ++++ src/senaite/sync/fetchstep.py | 6 +++++- src/senaite/sync/syncstep.py | 1 + 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index a26defe..954c377 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -57,15 +57,23 @@ def __call__(self): import_users = True if form.get("import_users") == 'on' else False import_registry = True if form.get("import_registry") == 'on' else False content_types = form.get("content_types", None) + unwanted_content_types = form.get("unwanted_content_types", None) prefix = form.get("prefix", None) prefixable_types = form.get("prefixable_types", None) + portal_types = api.get_tool("portal_types") + # Content Type Validation if content_types is not None: content_types = [t.strip() for t in content_types.split(",")] - portal_types = api.get_tool("portal_types") content_types = filter(lambda ct: ct in portal_types, content_types) + # Content Type Validation + if unwanted_content_types is not None: + unwanted_content_types = [t.strip() for t + in unwanted_content_types.split(",")] + unwanted_content_types = filter(lambda ct: ct in portal_types, + unwanted_content_types) # Prefix Validation if prefix: @@ -84,7 +92,6 @@ def __call__(self): return self.template() prefixable_types = [t.strip() for t in prefixable_types.split(",")] - portal_types = api.get_tool("portal_types") prefixable_types = filter(lambda ct: ct in portal_types, prefixable_types) @@ -99,6 +106,7 @@ def __call__(self): "ac_name": username, "ac_password": password, "content_types": content_types, + "unwanted_content_types": unwanted_content_types, "import_settings": import_settings, "import_users": import_users, "import_registry": import_registry, diff --git a/src/senaite/sync/browser/templates/add.pt b/src/senaite/sync/browser/templates/add.pt index ff846bc..05a0676 100644 --- a/src/senaite/sync/browser/templates/add.pt +++ b/src/senaite/sync/browser/templates/add.pt @@ -162,7 +162,7 @@ Content Types - If filled, only indicated Content Types (separated by comma) will be imported. E.g: 'Sample, Client, Worksheet' + If filled, ONLY indicated Content Types (separated by comma) will be imported. E.g: 'Sample, Client, Worksheet'
    @@ -175,6 +175,28 @@
    +
  • + +
    + +
    + +
    +
    +
  • +
  • @@ -206,7 +228,7 @@ Prefixable Content Types - Please specify exactly which content types must contain Remote's Prefix in their ID's. E.g: 'Sample, Client, Worksheet' + Please specify exactly which content types must contain Remote's Prefix in their ID's. Must be filled if prefix is enabled. E.g: 'Sample, Client, Worksheet'
    diff --git a/src/senaite/sync/browser/views.py b/src/senaite/sync/browser/views.py index 3256e66..393024d 100644 --- a/src/senaite/sync/browser/views.py +++ b/src/senaite/sync/browser/views.py @@ -66,6 +66,7 @@ def __call__(self): username = storage["credentials"]["username"] password = storage["credentials"]["password"] content_types = storage["configuration"].get("content_types", None) + unwanted_content_types = storage["configuration"].get("unwanted_content_types", None) prefix = storage["configuration"].get("prefix", None) prefixable_types = storage["configuration"].get("prefixable_types", None) data = { @@ -74,6 +75,7 @@ def __call__(self): "ac_name": username, "ac_password": password, "content_types": content_types, + "unwanted_content_types": unwanted_content_types, "prefix": prefix, "prefixable_types": prefixable_types, } @@ -105,6 +107,7 @@ def __call__(self): username = storage["credentials"]["username"] password = storage["credentials"]["password"] content_types = storage["configuration"].get("content_types", None) + unwanted_content_types = storage["configuration"].get("unwanted_content_types", None) prefix = storage["configuration"].get("prefix", None) prefixable_types = storage["configuration"].get("prefixable_types", None) data = { @@ -114,6 +117,7 @@ def __call__(self): "ac_password": password, "fetch_time": fetch_time, "content_types": content_types, + "unwanted_content_types": unwanted_content_types, "prefix": prefix, "prefixable_types": prefixable_types, } diff --git a/src/senaite/sync/fetchstep.py b/src/senaite/sync/fetchstep.py index 6009189..fe5eb8e 100644 --- a/src/senaite/sync/fetchstep.py +++ b/src/senaite/sync/fetchstep.py @@ -64,6 +64,7 @@ def verify(self): storage["credentials"]["password"] = self.password # remember import configuration in the storage storage["configuration"]["content_types"] = self.content_types + storage["configuration"]["unwanted_content_types"] = self.unwanted_content_types storage["configuration"]["prefix"] = self.prefix storage["configuration"]["prefixable_types"] = self.prefixable_types storage["configuration"]["import_settings"] = self.import_settings @@ -125,7 +126,10 @@ def _fetch_data(self, window=1000, overlap=10): start_from, start_from+window)) for item in items: # skip object or extract the required data for the import - if not item or not item.get("portal_type", True): + if not item: + continue + portal_type = item.get("portal_type", None) + if not portal_type or portal_type in self.unwanted_content_types: continue data_dict = utils.get_soup_format(item) rec_id = self.sh.insert(data_dict) diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index dab74a5..6e9f590 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -44,6 +44,7 @@ def __init__(self, data): self.password = data.get("ac_password", None) # Import configuration self.content_types = data.get("content_types", None) + self.unwanted_content_types = data.get("unwanted_content_types", None) self.prefix = data.get("prefix", None) self.prefixable_types = data.get("prefixable_types", None) self.import_settings = data.get("import_settings", False) From 5e6bea60b1f15ea00d0c9af281bf0a745cbba6b1 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 11:12:51 +0200 Subject: [PATCH 15/28] Generic 'is_item_allowed' method. --- src/senaite/sync/complementstep.py | 2 +- src/senaite/sync/fetchstep.py | 5 +---- src/senaite/sync/importstep.py | 2 +- src/senaite/sync/syncstep.py | 19 +++++++++++++++---- src/senaite/sync/utils.py | 2 +- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/senaite/sync/complementstep.py b/src/senaite/sync/complementstep.py index 0d531bf..ac88aba 100644 --- a/src/senaite/sync/complementstep.py +++ b/src/senaite/sync/complementstep.py @@ -71,7 +71,7 @@ def _fetch_data(self): for item in items: # skip object or extract the required data for the import - if not item or not item.get("portal_type", True): + if not self.is_item_allowed(item): continue modified = DateTime(item.get('modification_date')) if modified < self.fetch_time: diff --git a/src/senaite/sync/fetchstep.py b/src/senaite/sync/fetchstep.py index fe5eb8e..292c2e4 100644 --- a/src/senaite/sync/fetchstep.py +++ b/src/senaite/sync/fetchstep.py @@ -126,10 +126,7 @@ def _fetch_data(self, window=1000, overlap=10): start_from, start_from+window)) for item in items: # skip object or extract the required data for the import - if not item: - continue - portal_type = item.get("portal_type", None) - if not portal_type or portal_type in self.unwanted_content_types: + if not self.is_item_allowed(item): continue data_dict = utils.get_soup_format(item) rec_id = self.sh.insert(data_dict) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index da99e6f..ee87176 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -428,7 +428,7 @@ def _create_dependencies(self, obj, data): logger.error("Remote UID not found in fetched data: {}". format(r_uid)) continue - if not utils.is_item_allowed(dep_item): + if not utils.has_valid_portal_type(dep_item): logger.error("Skipping dependency with unknown portal type:" " {}".format(dep_item)) continue diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 6e9f590..bc22cfe 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -43,10 +43,10 @@ def __init__(self, data): self.username = data.get("ac_name", None) self.password = data.get("ac_password", None) # Import configuration - self.content_types = data.get("content_types", None) - self.unwanted_content_types = data.get("unwanted_content_types", None) + self.content_types = data.get("content_types", []) + self.unwanted_content_types = data.get("unwanted_content_types", []) self.prefix = data.get("prefix", None) - self.prefixable_types = data.get("prefixable_types", None) + self.prefixable_types = data.get("prefixable_types", []) self.import_settings = data.get("import_settings", False) self.import_users = data.get("import_users", False) self.import_registry = data.get("import_registry", False) @@ -266,7 +266,7 @@ def _parents_fetched(self, item): :return: True if ALL parents are fetched """ # Never fetch parents of an unnecessary objects - if not utils.is_item_allowed(item): + if not utils.has_valid_portal_type(item): return False parent_path = item.get("parent_path") # Skip if the parent is portal object @@ -281,3 +281,14 @@ def _parents_fetched(self, item): self.sh.insert(par_dict) # Recursively import grand parents too return self._parents_fetched(parent) + + def is_item_allowed(self, item): + """ Checks if item is allowed based on its portal_type + :param item: object data dict + """ + if not utils.has_valid_portal_type(item): + return False + if item.get("portal_type") in self.unwanted_content_types: + return False + + return True diff --git a/src/senaite/sync/utils.py b/src/senaite/sync/utils.py index 5118e2e..1c727b5 100644 --- a/src/senaite/sync/utils.py +++ b/src/senaite/sync/utils.py @@ -31,7 +31,7 @@ def to_review_history_format(review_history): return review_history -def is_item_allowed(item): +def has_valid_portal_type(item): """ Check if an item can be handled based on its portal type. :return: True if the item can be handled """ From 8c3cb5f94ab002c0d30bbff0979cf78475a59d72 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 11:28:10 +0200 Subject: [PATCH 16/28] 'None' --> '[]' --- src/senaite/sync/browser/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/senaite/sync/browser/views.py b/src/senaite/sync/browser/views.py index 393024d..7a5d3d9 100644 --- a/src/senaite/sync/browser/views.py +++ b/src/senaite/sync/browser/views.py @@ -65,10 +65,10 @@ def __call__(self): url = storage["credentials"]["url"] username = storage["credentials"]["username"] password = storage["credentials"]["password"] - content_types = storage["configuration"].get("content_types", None) - unwanted_content_types = storage["configuration"].get("unwanted_content_types", None) + content_types = storage["configuration"].get("content_types", []) + unwanted_content_types = storage["configuration"].get("unwanted_content_types", []) prefix = storage["configuration"].get("prefix", None) - prefixable_types = storage["configuration"].get("prefixable_types", None) + prefixable_types = storage["configuration"].get("prefixable_types", []) data = { "url": url, "domain_name": domain_name, @@ -106,10 +106,10 @@ def __call__(self): url = storage["credentials"]["url"] username = storage["credentials"]["username"] password = storage["credentials"]["password"] - content_types = storage["configuration"].get("content_types", None) - unwanted_content_types = storage["configuration"].get("unwanted_content_types", None) + content_types = storage["configuration"].get("content_types", []) + unwanted_content_types = storage["configuration"].get("unwanted_content_types", []) prefix = storage["configuration"].get("prefix", None) - prefixable_types = storage["configuration"].get("prefixable_types", None) + prefixable_types = storage["configuration"].get("prefixable_types", []) data = { "url": url, "domain_name": domain_name, From 54ad0e1b6dc24848c05e7b523ac1f75f990c25be Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 11:43:43 +0200 Subject: [PATCH 17/28] Quick Fix. --- src/senaite/sync/souphandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/senaite/sync/souphandler.py b/src/senaite/sync/souphandler.py index 4004eca..9c313a4 100644 --- a/src/senaite/sync/souphandler.py +++ b/src/senaite/sync/souphandler.py @@ -87,7 +87,7 @@ def _already_exists(self, data): r_uid_q = Eq(REMOTE_UID, r_uid) l_uid_q = Eq(LOCAL_UID, l_uid) r_p_q = Eq(REMOTE_PATH, r_path) - l_p_q = Eq(REMOTE_PATH, l_path) + l_p_q = Eq(LOCAL_PATH, l_path) ret = [r for r in self.soup.query(Or(r_uid_q, l_uid_q, r_p_q, l_p_q))] return ret != [] From 7fb93f976d9da8347894c4ef23af30547d965414 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 12:06:33 +0200 Subject: [PATCH 18/28] Polish & Comments --- src/senaite/sync/browser/add.py | 11 +++++------ src/senaite/sync/importstep.py | 4 ++-- src/senaite/sync/syncstep.py | 24 ++++++++++++------------ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index 954c377..6158cf9 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -61,14 +61,13 @@ def __call__(self): prefix = form.get("prefix", None) prefixable_types = form.get("prefixable_types", None) - portal_types = api.get_tool("portal_types") - # Content Type Validation + portal_types = api.get_tool("portal_types") if content_types is not None: content_types = [t.strip() for t in content_types.split(",")] content_types = filter(lambda ct: ct in portal_types, content_types) - # Content Type Validation + if unwanted_content_types is not None: unwanted_content_types = [t.strip() for t in unwanted_content_types.split(",")] @@ -90,14 +89,14 @@ def __call__(self): self.add_status_message("Please enter a valid Prefix.", "error") return self.template() - - prefixable_types = [t.strip() for t in prefixable_types.split(",")] + prefixable_types = [t.strip() for t + in prefixable_types.split(",")] prefixable_types = filter(lambda ct: ct in portal_types, prefixable_types) if prefix and not prefixable_types: self.add_status_message("Please enter valid Content Types to be" - " prefixified.", "error") + " created with the Prefix.", "error") return self.template() data = { diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index ee87176..121e9fd 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -308,7 +308,7 @@ def _do_obj_creation(self, row): .format(remote_path)) return None - local_path = self.translate_path_with_prefix(remote_path) + local_path = self.translate_path(remote_path) existing = self.portal.unrestrictedTraverse(local_path, None) if existing: local_uid = self.sh.find_unique(REMOTE_PATH, remote_path).get(LOCAL_UID, @@ -349,7 +349,7 @@ def _parents_created(self, remote_path): return True # Incoming path was remote path, translate it into local one - local_p_path = self.translate_path_with_prefix(p_path) + local_p_path = self.translate_path(p_path) # Check if the parent already exists. If yes, make sure it has # 'local_uid' value set in the soup table. diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index bc22cfe..460ef2b 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -54,15 +54,13 @@ def __init__(self, data): if not any([self.domain_name, self.url, self.username, self.password]): self.fail("Missing parameter in Sync Step: {}".format(data)) - def translate_path(self, path): - """ Translate the physical path to a local path - """ - portal_id = self.portal.getId() - remote_portal_id = path.split("/")[1] - return str(path.replace(remote_portal_id, portal_id)) - - def translate_path_with_prefix(self, remote_path): - """ + def translate_path(self, remote_path): + """ Translates a remote physical path into local path taking into account + the prefix. If prefix is not enabled, then just the Remote Site ID will + be replaced by the Local one. In case prefixes are enabled, then walk + through all parents and add prefixes if necessary. + :param remote_path: a path in a remote instance + :return string: the translated path """ if self.is_portal_path(remote_path): return api.get_path(self.portal) @@ -77,12 +75,14 @@ def translate_path_with_prefix(self, remote_path): if rec is None: raise SyncError("Missing Remote path in Soup table: {}".format( remote_path)) + + # Check if previously translated and saved if rec[LOCAL_PATH]: return str(rec[LOCAL_PATH]) # Get parent's local path remote_parent_path = utils.get_parent_path(remote_path) - parent_path = self.translate_path_with_prefix(remote_parent_path) + parent_path = self.translate_path(remote_parent_path) # Will check whether prefix needed by portal type portal_type = rec[PORTAL_TYPE] @@ -90,13 +90,13 @@ def translate_path_with_prefix(self, remote_path): res = "{0}/{1}{2}".format(parent_path, prefix, rem_id) res = res.replace(remote_portal_id, portal_id) + # Save the local path in the Souper to use in the future self.sh.update_by_remote_path(remote_path, LOCAL_PATH = res) return str(res) def get_prefix(self, portal_type): """ - - :param portal_type: + :param portal_type: content type to get the prefix for :return: """ if self.prefix and portal_type in self.prefixable_types: From 3830b01735715d448b261ab4f41716c3dbabe9c1 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 12:56:00 +0200 Subject: [PATCH 19/28] Raise SyncError if Remote Path is empty in a Souper Record. --- src/senaite/sync/importstep.py | 38 +++++++++++++++++----------------- src/senaite/sync/syncstep.py | 8 +++++-- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 121e9fd..c9b3581 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -269,24 +269,24 @@ def _handle_obj(self, row, handle_dependencies=True): :type row: dict """ r_uid = row.get(REMOTE_UID) - try: - if row.get("updated", "0") == "1": - return True - self._queue.append(r_uid) - obj = self._do_obj_creation(row) - if obj is None: - logger.error('Object creation failed: {}'.format(row)) - return - obj_data = self.get_json(r_uid, complete=True, - workflow=True) - if handle_dependencies: - self._create_dependencies(obj, obj_data) - self._update_object_with_data(obj, obj_data) - self.sh.mark_update(r_uid) - self._queue.remove(r_uid) - except Exception, e: - self._queue.remove(r_uid) - logger.error('Failed to handle {} : {} '.format(row, str(e))) + # try: + if row.get("updated", "0") == "1": + return True + self._queue.append(r_uid) + obj = self._do_obj_creation(row) + if obj is None: + logger.error('Object creation failed: {}'.format(row)) + return + obj_data = self.get_json(r_uid, complete=True, + workflow=True) + if handle_dependencies: + self._create_dependencies(obj, obj_data) + self._update_object_with_data(obj, obj_data) + self.sh.mark_update(r_uid) + self._queue.remove(r_uid) + # except Exception, e: + # self._queue.remove(r_uid) + # logger.error('Failed to handle {} : {} '.format(row, str(e))) return True @@ -307,7 +307,7 @@ def _do_obj_creation(self, row): logger.warning("Parent creation failed previously, skipping: {}" .format(remote_path)) return None - + import pdb; pdb.set_trace() local_path = self.translate_path(remote_path) existing = self.portal.unrestrictedTraverse(local_path, None) if existing: diff --git a/src/senaite/sync/syncstep.py b/src/senaite/sync/syncstep.py index 460ef2b..2361bd6 100644 --- a/src/senaite/sync/syncstep.py +++ b/src/senaite/sync/syncstep.py @@ -62,6 +62,10 @@ def translate_path(self, remote_path): :param remote_path: a path in a remote instance :return string: the translated path """ + if not remote_path or "/" not in remote_path: + raise SyncError("error", "Invalid remote path: '{}'" + .format(remote_path)) + if self.is_portal_path(remote_path): return api.get_path(self.portal) @@ -73,8 +77,8 @@ def translate_path(self, remote_path): rem_id = utils.get_id_from_path(remote_path) rec = self.sh.find_unique(REMOTE_PATH, remote_path) if rec is None: - raise SyncError("Missing Remote path in Soup table: {}".format( - remote_path)) + raise SyncError("error", "Missing Remote path in Soup table: {}" + .format(remote_path)) # Check if previously translated and saved if rec[LOCAL_PATH]: From 734bfcf711d2c28d014fd1dd41bc9285ab0bbfb3 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Fri, 4 May 2018 13:46:07 +0200 Subject: [PATCH 20/28] Front End. --- src/senaite/sync/browser/templates/sync.pt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/senaite/sync/browser/templates/sync.pt b/src/senaite/sync/browser/templates/sync.pt index 99d5c4b..1578c7a 100644 --- a/src/senaite/sync/browser/templates/sync.pt +++ b/src/senaite/sync/browser/templates/sync.pt @@ -44,9 +44,13 @@
    - Import settings: - Import registry: - Import users: + Prefix:

    + Settings will NOT, + Registry will NOT, + Users will NOT be imported.

    + Content types to be imported:

    + Content types to be skipped:

    + Content types that will contain prefix:


    Date: Fri, 4 May 2018 14:01:19 +0200 Subject: [PATCH 21/28] Polish. --- src/senaite/sync/browser/add.py | 6 +++--- src/senaite/sync/browser/templates/sync.pt | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index 6158cf9..01f7998 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -63,12 +63,12 @@ def __call__(self): # Content Type Validation portal_types = api.get_tool("portal_types") - if content_types is not None: + if content_types: content_types = [t.strip() for t in content_types.split(",")] content_types = filter(lambda ct: ct in portal_types, content_types) - if unwanted_content_types is not None: + if unwanted_content_types: unwanted_content_types = [t.strip() for t in unwanted_content_types.split(",")] unwanted_content_types = filter(lambda ct: ct in portal_types, @@ -84,7 +84,7 @@ def __call__(self): self.add_status_message("Long Prefix!", "warning") # Content Type Validation - if prefixable_types is not None: + if prefixable_types: if not prefix: self.add_status_message("Please enter a valid Prefix.", "error") diff --git a/src/senaite/sync/browser/templates/sync.pt b/src/senaite/sync/browser/templates/sync.pt index 1578c7a..c840359 100644 --- a/src/senaite/sync/browser/templates/sync.pt +++ b/src/senaite/sync/browser/templates/sync.pt @@ -43,14 +43,19 @@
    -
    +
    Prefix:

    Settings will NOT, Registry will NOT, Users will NOT be imported.

    - Content types to be imported:

    - Content types to be skipped:

    - Content types that will contain prefix:

    + + Content types to be imported:

    + Content types to be skipped:

    + Content types that will contain prefix:


    Date: Mon, 7 May 2018 12:10:54 +0200 Subject: [PATCH 22/28] 'Senaite.Lims' compatibility. --- src/senaite/sync/browser/templates/sync.pt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/senaite/sync/browser/templates/sync.pt b/src/senaite/sync/browser/templates/sync.pt index c840359..d6f7e3e 100644 --- a/src/senaite/sync/browser/templates/sync.pt +++ b/src/senaite/sync/browser/templates/sync.pt @@ -29,6 +29,7 @@ +
    @@ -36,7 +37,7 @@
    - + ( Items)
    @@ -69,10 +70,10 @@ name="clear_storage" i18n:attributes="value" value="Clear this Storage"/> -
    +
    - Run Complement Step + RUN COMPLEMENT STEP
    @@ -104,6 +105,7 @@
    +
    From 2f6dda44622e86927aa4aaea7979202d2dc8453d Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Mon, 7 May 2018 12:12:21 +0200 Subject: [PATCH 23/28] Changes.rst --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index bba1a58..4ba2a6c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,7 @@ Changelog **Added** +- #42 New Advanced Configuration Options - #34 Complement step for migration - #33 Recover step for failed objects in data import - #32 Log the estimated date of end From ac6210ba0d0aab03bdd440dbac2719c0c37a8cbd Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Mon, 7 May 2018 14:49:58 +0200 Subject: [PATCH 24/28] Remove forgotten pdb. --- src/senaite/sync/importstep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index c9b3581..9f9c29f 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -307,7 +307,7 @@ def _do_obj_creation(self, row): logger.warning("Parent creation failed previously, skipping: {}" .format(remote_path)) return None - import pdb; pdb.set_trace() + local_path = self.translate_path(remote_path) existing = self.portal.unrestrictedTraverse(local_path, None) if existing: From 9c8841bcb87f0311b3dd36f67db1cca1f20179a1 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Mon, 7 May 2018 15:10:06 +0200 Subject: [PATCH 25/28] Uncomment error catch. --- src/senaite/sync/importstep.py | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 9f9c29f..121e9fd 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -269,24 +269,24 @@ def _handle_obj(self, row, handle_dependencies=True): :type row: dict """ r_uid = row.get(REMOTE_UID) - # try: - if row.get("updated", "0") == "1": - return True - self._queue.append(r_uid) - obj = self._do_obj_creation(row) - if obj is None: - logger.error('Object creation failed: {}'.format(row)) - return - obj_data = self.get_json(r_uid, complete=True, - workflow=True) - if handle_dependencies: - self._create_dependencies(obj, obj_data) - self._update_object_with_data(obj, obj_data) - self.sh.mark_update(r_uid) - self._queue.remove(r_uid) - # except Exception, e: - # self._queue.remove(r_uid) - # logger.error('Failed to handle {} : {} '.format(row, str(e))) + try: + if row.get("updated", "0") == "1": + return True + self._queue.append(r_uid) + obj = self._do_obj_creation(row) + if obj is None: + logger.error('Object creation failed: {}'.format(row)) + return + obj_data = self.get_json(r_uid, complete=True, + workflow=True) + if handle_dependencies: + self._create_dependencies(obj, obj_data) + self._update_object_with_data(obj, obj_data) + self.sh.mark_update(r_uid) + self._queue.remove(r_uid) + except Exception, e: + self._queue.remove(r_uid) + logger.error('Failed to handle {} : {} '.format(row, str(e))) return True From bc77d58315d5877493086daa93c16db6be7ede57 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 10 May 2018 11:43:01 +0200 Subject: [PATCH 26/28] Review Changes. --- src/senaite/sync/browser/add.py | 52 +++++++++------------- src/senaite/sync/browser/templates/add.pt | 2 +- src/senaite/sync/browser/templates/sync.pt | 14 +++--- src/senaite/sync/browser/views.py | 33 ++++++++++---- src/senaite/sync/importstep.py | 1 + src/senaite/sync/utils.py | 18 ++++++++ 6 files changed, 72 insertions(+), 48 deletions(-) diff --git a/src/senaite/sync/browser/add.py b/src/senaite/sync/browser/add.py index 01f7998..7f633fc 100644 --- a/src/senaite/sync/browser/add.py +++ b/src/senaite/sync/browser/add.py @@ -9,6 +9,7 @@ from senaite.sync.browser.interfaces import ISync from senaite.sync.browser.views import Sync from senaite.sync.fetchstep import FetchStep +from senaite.sync import utils from zope.interface import implements SYNC_STORAGE = "senaite.sync" @@ -43,7 +44,7 @@ def __call__(self): url = form.get("url", "") if not url.startswith("http"): - url = "http://{}".format(url) + url = "https://{}".format(url) domain_name = form.get("domain_name", None) username = form.get("ac_name", None) password = form.get("ac_password", None) @@ -53,26 +54,17 @@ def __call__(self): self.add_status_message(message, "error") return self.template() - import_settings = True if form.get("import_settings") == 'on' else False - import_users = True if form.get("import_users") == 'on' else False - import_registry = True if form.get("import_registry") == 'on' else False - content_types = form.get("content_types", None) - unwanted_content_types = form.get("unwanted_content_types", None) - prefix = form.get("prefix", None) - prefixable_types = form.get("prefixable_types", None) - - # Content Type Validation - portal_types = api.get_tool("portal_types") - if content_types: - content_types = [t.strip() for t in content_types.split(",")] - content_types = filter(lambda ct: ct in portal_types, - content_types) + import_settings = (form.get("import_settings") == 'on') + import_users = (form.get("import_users") == 'on') + import_registry = (form.get("import_registry") == 'on') + content_types = utils.filter_content_types( + form.get("content_types")) + unwanted_content_types = utils.filter_content_types( + form.get("unwanted_content_types")) - if unwanted_content_types: - unwanted_content_types = [t.strip() for t - in unwanted_content_types.split(",")] - unwanted_content_types = filter(lambda ct: ct in portal_types, - unwanted_content_types) + prefix = form.get("prefix", None) + prefixable_types = utils.filter_content_types( + form.get("prefixable_types")) # Prefix Validation if prefix: @@ -80,24 +72,19 @@ def __call__(self): if not prefix: self.add_status_message("Invalid Prefix!", "error") return self.template() + if len(prefix) > 3: self.add_status_message("Long Prefix!", "warning") - # Content Type Validation - if prefixable_types: - if not prefix: + if not prefixable_types: + self.add_status_message("Please enter valid Content Types " + "to be created with the Prefix.", "error") + return self.template() + else: + if prefixable_types: self.add_status_message("Please enter a valid Prefix.", "error") return self.template() - prefixable_types = [t.strip() for t - in prefixable_types.split(",")] - prefixable_types = filter(lambda ct: ct in portal_types, - prefixable_types) - - if prefix and not prefixable_types: - self.add_status_message("Please enter valid Content Types to be" - " created with the Prefix.", "error") - return self.template() data = { "url": url, @@ -123,3 +110,4 @@ def __call__(self): # render the template return self.template() + diff --git a/src/senaite/sync/browser/templates/add.pt b/src/senaite/sync/browser/templates/add.pt index 05a0676..35784d4 100644 --- a/src/senaite/sync/browser/templates/add.pt +++ b/src/senaite/sync/browser/templates/add.pt @@ -251,7 +251,7 @@ type="submit" name="fetch" i18n:attributes="value" - value="Save & Fetch"/> + value="Save and Fetch"/>

    diff --git a/src/senaite/sync/browser/templates/sync.pt b/src/senaite/sync/browser/templates/sync.pt index d6f7e3e..0e4ee26 100644 --- a/src/senaite/sync/browser/templates/sync.pt +++ b/src/senaite/sync/browser/templates/sync.pt @@ -45,14 +45,14 @@
    - Prefix:

    - Settings will NOT, - Registry will NOT, - Users will NOT be imported.

    + Prefix:

    + Settings will NOT, + Registry will NOT, + Users will NOT be imported.

    Content types to be imported:

    Content types to be skipped:

    diff --git a/src/senaite/sync/browser/views.py b/src/senaite/sync/browser/views.py index 7a5d3d9..cddf79e 100644 --- a/src/senaite/sync/browser/views.py +++ b/src/senaite/sync/browser/views.py @@ -65,10 +65,14 @@ def __call__(self): url = storage["credentials"]["url"] username = storage["credentials"]["username"] password = storage["credentials"]["password"] - content_types = storage["configuration"].get("content_types", []) - unwanted_content_types = storage["configuration"].get("unwanted_content_types", []) - prefix = storage["configuration"].get("prefix", None) - prefixable_types = storage["configuration"].get("prefixable_types", []) + prefix = self.get_storage_config(domain_name, "prefix", None) + content_types = self.get_storage_config( + domain_name, "content_types", []) + unwanted_content_types = self.get_storage_config( + domain_name, "unwanted_content_types", []) + prefixable_types = self.get_storage_config( + domain_name, "prefixable_types", []) + data = { "url": url, "domain_name": domain_name, @@ -106,10 +110,14 @@ def __call__(self): url = storage["credentials"]["url"] username = storage["credentials"]["username"] password = storage["credentials"]["password"] - content_types = storage["configuration"].get("content_types", []) - unwanted_content_types = storage["configuration"].get("unwanted_content_types", []) - prefix = storage["configuration"].get("prefix", None) - prefixable_types = storage["configuration"].get("prefixable_types", []) + prefix = self.get_storage_config(domain_name, "prefix", None) + content_types = self.get_storage_config( + domain_name, "content_types", []) + unwanted_content_types = self.get_storage_config( + domain_name, "unwanted_content_types", []) + prefixable_types = self.get_storage_config( + domain_name, "prefixable_types", []) + data = { "url": url, "domain_name": domain_name, @@ -137,6 +145,15 @@ def __call__(self): # always render the template return self.template() + def get_storage_config(self, domain_name, config_name, default = None): + """ Get the advanced configuration setting for a given domain + :param config_name: advanced configuration section name + :param default: default value if configuration value is not set + :return: + """ + storage = self.get_storage(domain_name) + return storage["configuration"].get(config_name, default) + def add_status_message(self, message, level="info"): """Set a portal status message """ diff --git a/src/senaite/sync/importstep.py b/src/senaite/sync/importstep.py index 121e9fd..1a9a615 100644 --- a/src/senaite/sync/importstep.py +++ b/src/senaite/sync/importstep.py @@ -361,6 +361,7 @@ def _parents_created(self, remote_path): return False p_local_uid = p_row.get(LOCAL_UID, None) if not p_local_uid: + # Update parent's local path if it is not set already if hasattr(existing, "UID") and existing.UID(): p_local_uid = existing.UID() self.sh.update_by_remote_path(p_path, local_uid=p_local_uid) diff --git a/src/senaite/sync/utils.py b/src/senaite/sync/utils.py index 1c727b5..cf4c1ea 100644 --- a/src/senaite/sync/utils.py +++ b/src/senaite/sync/utils.py @@ -46,6 +46,24 @@ def has_valid_portal_type(item): return True +def filter_content_types(content_types): + """ + + :param content_types: + :return: + """ + ret = list() + if not content_types: + return ret + + # Get available portal types and make it all lowercase + portal_types = api.get_tool("portal_types") + + ret = [t.strip() for t in content_types.split(",") if t] + ret = filter(lambda ct: ct.lower() in portal_types, ret) + return ret + + def get_parent_path(path): """ Gets the parent path for a given object path. From 431886d369647e417f9b5d13bb45c413f98aeb71 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 10 May 2018 12:23:05 +0200 Subject: [PATCH 27/28] Polish. --- src/senaite/sync/browser/templates/sync.pt | 14 +++++++------- src/senaite/sync/utils.py | 7 +++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/senaite/sync/browser/templates/sync.pt b/src/senaite/sync/browser/templates/sync.pt index 0e4ee26..1d29ff9 100644 --- a/src/senaite/sync/browser/templates/sync.pt +++ b/src/senaite/sync/browser/templates/sync.pt @@ -45,14 +45,14 @@
    - Prefix:

    - Settings will NOT, - Registry will NOT, - Users will NOT be imported.

    + Prefix:

    + Settings will NOT, + Registry will NOT, + Users will NOT be imported.

    Content types to be imported:

    Content types to be skipped:

    diff --git a/src/senaite/sync/utils.py b/src/senaite/sync/utils.py index cf4c1ea..5374124 100644 --- a/src/senaite/sync/utils.py +++ b/src/senaite/sync/utils.py @@ -38,7 +38,7 @@ def has_valid_portal_type(item): if not isinstance(item, dict): return False - portal_types = api.get_tool("portal_types") + portal_types = api.get_tool("portal_types").listContentTypes() pt = item.get("portal_type", None) if pt not in portal_types: return False @@ -57,10 +57,9 @@ def filter_content_types(content_types): return ret # Get available portal types and make it all lowercase - portal_types = api.get_tool("portal_types") - + portal_types = api.get_tool("portal_types").listContentTypes() ret = [t.strip() for t in content_types.split(",") if t] - ret = filter(lambda ct: ct.lower() in portal_types, ret) + ret = filter(lambda ct, types=portal_types: ct in types, ret) return ret From 2c7e107b54913585ad7f1101b6e7e350b0101834 Mon Sep 17 00:00:00 2001 From: Nihadness <1992.nihad@gmail.com> Date: Thu, 10 May 2018 12:29:05 +0200 Subject: [PATCH 28/28] Review Change. --- src/senaite/sync/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/senaite/sync/utils.py b/src/senaite/sync/utils.py index 5374124..785f3e0 100644 --- a/src/senaite/sync/utils.py +++ b/src/senaite/sync/utils.py @@ -58,8 +58,11 @@ def filter_content_types(content_types): # Get available portal types and make it all lowercase portal_types = api.get_tool("portal_types").listContentTypes() + portal_types = [t.lower for t in portal_types] + ret = [t.strip() for t in content_types.split(",") if t] - ret = filter(lambda ct, types=portal_types: ct in types, ret) + ret = filter(lambda ct, types=portal_types: ct.lower() in types, ret) + ret = list(set(ret)) return ret