From 418e42299cdc20dd535809d6dd645fdea3730450 Mon Sep 17 00:00:00 2001 From: Ramon Bartl Date: Mon, 11 Jul 2022 21:13:00 +0200 Subject: [PATCH 1/2] Fix recatalog logging counter --- src/senaite/core/catalog/base_catalog.py | 29 ++++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/senaite/core/catalog/base_catalog.py b/src/senaite/core/catalog/base_catalog.py index 97ac0478d1..3e2a46e492 100644 --- a/src/senaite/core/catalog/base_catalog.py +++ b/src/senaite/core/catalog/base_catalog.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from threading import RLock + import transaction from AccessControl import ClassSecurityInfo from AccessControl.Permissions import \ @@ -19,6 +21,8 @@ CATALOG_ID = "senaite_catalog_base" CATALOG_TITLE = "Senaite Base Catalog" +progress_rlock = RLock() + INDEXES = [ # id, indexed attribute, type ("allowedRolesAndUsers", "", "KeywordIndex"), @@ -65,6 +69,7 @@ class BaseCatalog(CatalogTool): def __init__(self, id, title="", **kw): # CatalogTool does not take any parameters in __init__ ZCatalog.__init__(self, id, title=title, **kw) + self.progress_counter = 0 @property def mapped_catalog_types(self): @@ -104,6 +109,19 @@ def get_mapped_types(self): mapped_at_types = self.get_mapped_at_types() return mapped_catalog_types + mapped_at_types + def log_progress(self): + """Log reindex progress + """ + with progress_rlock: + self.progress_counter += 1 + + if self.progress_counter % 100 == 0: + logger.info("Progress: {} objects have been cataloged for {}." + .format(self.progress_counter, self.id)) + + if self.progress_counter % 10000 == 0: + transaction.savepoint(optimistic=True) + @security.protected(ManageZCatalogEntries) def clearFindAndRebuild(self): """Considers only mapped types when reindexing the whole catalog @@ -130,14 +148,14 @@ def indexObject(obj, path): # catalog, but does not take DX multiplexing into # consideration. self._reindexObject(obj, idxs=idxs) # bypass queue - self.counter += 1 + self.log_progress() elif api.is_dexterity_content(obj): # NOTE: Catalog multiplexing is only available for DX types # and stores the catalogs in a variable `_catalogs`. multiplex_catalogs = getattr(obj, "_catalogs", []) if self.id in multiplex_catalogs: self._reindexObject(obj, idxs=idxs) # bypass queue - self.counter += 1 + self.log_progress() else: return except TypeError: @@ -145,13 +163,10 @@ def indexObject(obj, path): # take different args, and will fail pass - if self.counter and self.counter % 100 == 0: - logger.info("Progress: {} objects have been cataloged for {}." - .format(self.counter, self.id)) - transaction.savepoint(optimistic=True) + # reset the progress counter + self.progress_counter = 0 logger.info("Cleaning and rebuilding catalog '%s'..." % self.id) - self.counter = 0 self.manage_catalogClear() portal = aq_parent(aq_inner(self)) portal.ZopeFindAndApply( From e90e002d090be9ca1f6490923bcc2101dadfdf46 Mon Sep 17 00:00:00 2001 From: Ramon Bartl Date: Mon, 11 Jul 2022 21:26:58 +0200 Subject: [PATCH 2/2] Changelog --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 52d7e32a01..cc443caaca 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,7 @@ Changelog 2.3.0 (unreleased) ------------------ +- #2048 Fix catalog logging counter duplicates - #2047 Make resultsinterpretation.pt to retrieve departments from viewlet - #2045 Fix instrument types instruments view - #2044 Skip Invoice for content exports