Slow Searches in Listing Views

CHANGES.rst

@@ -15,6 +15,7 @@ Changelog
 
 **Fixed**
 
+-  #771 Slow Searches in Listing Views
 
 **Security**
 

RELEASE_NOTES.rst

@@ -1,6 +1,15 @@
 Release notes
 =============
 
+Update from 1.2.4 to 1.2.5
+--------------------------
+
+- This update requires the execution of `bin/buildout`, because
+  Products.TextIndexNG3 has been added. It will help to search by wildcards in
+  TextIndexNG3 indexes instead of looking for the keyword inside wildcards.
+  For now, it is used only in AR listing catalog.
+  https://pypi.python.org/pypi/Products.TextIndexNG3/
+
 Update from 1.2.3 to 1.2.4
 --------------------------
 

bika/lims/browser/bika_listing.py

@@ -1291,7 +1291,7 @@ def search(self, searchterm="", ignorecase=True):
 
         # strip whitespaces off the searchterm
         searchterm = searchterm.strip()
-        # strip illegal characters off the searchterm
+        # Strip illegal characters of the searchterm
         searchterm = searchterm.strip(u"*.!$%&/()=-+:'`´^")
         logger.info(u"ListingView::search:searchterm='{}'".format(searchterm))
 
@@ -1302,19 +1302,66 @@ def search(self, searchterm="", ignorecase=True):
 
         # search the catalog
         catalog = api.get_tool(self.catalog)
-        brains = catalog(query)
+
+        # return the unfiltered catalog results if no searchterm
+        if not searchterm:
+            brains = catalog(query)
+
+        # check if there is ng3 index in the catalog to query by wildcards
+        elif "listing_searchable_text" in catalog.indexes():
+            # Always expand all categories if we have a searchterm
+            self.expand_all_categories = True
+            brains = self.ng3_index_search(catalog, query, searchterm)
+
+        else:
+            self.expand_all_categories = True
+            brains = self.metadata_search(catalog, query, searchterm, ignorecase)
 
         # Sort manually?
         if self.manual_sort_on is not None:
             brains = self.sort_brains(brains, sort_on=self.manual_sort_on)
 
-        # return the unfiltered catalog results
-        if not searchterm:
-            logger.info(u"ListingView::search: return {} results".format(len(brains)))
-            return brains
+        end = time.time()
+        logger.info(u"ListingView::search: Search for '{}' executed in "
+                    u"{:.2f}s ({} matches)"
+                    .format(searchterm, end - start, len(brains)))
+        return brains
 
-        # Always expand all categories if we have a searchterm
-        self.expand_all_categories = True
+    def ng3_index_search(self, catalog, query, searchterm):
+        """ Searches given catalog by query and also looks for a keyword in the
+        specific index called 'listing_searchable_text'
+        #REMEMBER TextIndexNG indexes are the only indexes that wildcards can be
+        used in the beginning of the string.
+        http://zope.readthedocs.io/en/latest/zope2book/SearchingZCatalog.html#textindexng
+        :param catalog: catalog to search
+        :param query:
+        :param searchterm: a keyword to look for in 'listing_searchable_text'
+        :return: brains matching the search result
+        """
+        logger.info(u"ListingView::search: Prepare NG3 index query for '{}'"
+                    .format(self.catalog))
+        # Remove quotation mark
+        searchterm = searchterm.replace('"', '')
+        # If the keyword is not encoded in searches, TextIndexNG3 encodes by
+        # default encoding which we cannot always trust
+        searchterm = searchterm.encode("utf-8")
+        query["listing_searchable_text"] = "*" + searchterm + "*"
+        return catalog(query)
+
+    def metadata_search(self, catalog, query, searchterm, ignorecase=True):
+        """ Retrieves all the brains from given catalog and returns the ones
+        with at least one metadata containing the search term
+        :param catalog: catalog to search
+        :param query:
+        :param searchterm:
+        :param ignorecase:
+        :return: brains matching search result
+        """
+        # create a catalog query
+        logger.info(u"ListingView::search: Prepare metadata query for '{}'"
+                    .format(self.catalog))
+
+        brains = catalog(query)
 
         # Build a regular expression for the given searchterm
         regex = self.make_regex_for(searchterm, ignorecase=ignorecase)
@@ -1332,12 +1379,7 @@ def match(brain):
             return False
 
         # Filtered brains by searchterm -> metadata match
-        out = filter(match, brains)
-
-        end = time.time()
-        logger.info(u"ListingView::search: Search for '{}' executed in {:.2f}s ({} matches)"
-                    .format(searchterm, end - start, len(out)))
-        return out
+        return filter(match, brains)
 
     def get_searchterm(self):
         """Get the user entered search value from the request

bika/lims/catalog/analysisrequest_catalog.py

@@ -40,6 +40,9 @@
     'getClientTitle': 'FieldIndex',
     'getPrioritySortkey': 'FieldIndex',
     'assigned_state': 'FieldIndex',
+    # Searchable Text Index by wildcards
+    # http://zope.readthedocs.io/en/latest/zope2book/SearchingZCatalog.html#textindexng
+    'listing_searchable_text': 'TextIndexNG3',
 }
 # Defining the columns for this catalog
 _columns_list = [

bika/lims/catalog/indexers/analysisrequest.py

@@ -6,8 +6,8 @@
 # Some rights reserved. See LICENSE.rst, CONTRIBUTORS.rst.
 
 from bika.lims import api
+from bika.lims import logger
 from bika.lims.interfaces import IAnalysisRequest
-from bika.lims.workflow import getCurrentState
 from plone.indexer import indexer
 
 
@@ -28,3 +28,41 @@ def assigned_state(instance):
             return 'unassigned'
 
     return 'assigned'
+
+
+@indexer(IAnalysisRequest)
+def listing_searchable_text(instance):
+    """ Indexes values of desired fields for searches in listing view. All the
+    field names added to 'plain_text_fields' will be available to search by
+    wildcards.
+    Please choose the searchable fields carefully and add only fields that
+    can be useful to search by. For example, there is no need to add 'SampleId'
+    since 'getId' of AR already contains that value. Nor 'ClientTitle' because
+    AR's are/can be filtered by client in Clients' 'AR Listing View'
+    :return: values of the fields defined as a string
+    """
+    entries = []
+    plain_text_fields = ('getId', 'getContactFullName', 'getSampleTypeTitle',
+                         'getSamplePointTitle', 'getCreatorFullName',
+                         'getProfilesTitle', 'getStorageLocationTitle',
+                         'getClientOrderNumber', 'getClientReference',
+                         'getClientSampleID', 'getTemplateTitle', )
+
+    # Concatenate plain text fields as they are
+    for field_name in plain_text_fields:
+        try:
+            value = api.safe_getattr(instance, field_name)
+        except:
+            logger.error("{} has no attribute called '{}' ".format(
+                            repr(instance), field_name))
+            continue
+
+        if not value:
+            continue
+        if isinstance(value, list):
+            value = " ".join(value)
+
+        entries.append(value)
+
+    # Concatenate all strings to one text blob
+    return " ".join(entries)

bika/lims/catalog/indexers/configure.zcml

@@ -5,5 +5,6 @@
   <adapter name="sortable_title" factory=".analysiscategory.sortable_title"/>
   <adapter name="sortable_title" factory=".baseanalysis.sortable_title"/>
   <adapter name="assigned_state" factory=".analysisrequest.assigned_state"/>
+  <adapter name="listing_searchable_text" factory=".analysisrequest.listing_searchable_text"/>
 
 </configure>

bika/lims/content/analysisrequest.py

@@ -2551,6 +2551,12 @@ def getSamplingRoundUID(self):
         else:
             return ''
 
+    def getStorageLocationTitle(self):
+        """ A method for AR listing catalog metadata
+        :return: Title of Storage Location
+        """
+        return self.getStorageLocation() and self.getStorageLocation().Title() or ''
+
     @security.public
     def getResultsRange(self):
         """Returns the valid result ranges for the analyses this Analysis

bika/lims/upgrade/v01_02_005.py

@@ -6,6 +6,7 @@
 # Some rights reserved. See LICENSE.rst, CONTRIBUTORS.rst.
 from bika.lims import api
 from bika.lims import logger
+from bika.lims.catalog.analysisrequest_catalog import CATALOG_ANALYSIS_REQUEST_LISTING
 from bika.lims.config import PROJECTNAME as product
 from bika.lims.upgrade import upgradestep
 from bika.lims.upgrade.utils import UpgradeUtils
@@ -28,7 +29,9 @@ def upgrade(tool):
     logger.info("Upgrading {0}: {1} -> {2}".format(product, ver_from, version))
 
     # -------- ADD YOUR STUFF HERE --------
-
+    ut.addIndex(CATALOG_ANALYSIS_REQUEST_LISTING, "listing_searchable_text",
+                "TextIndexNG3")
+    ut.refreshCatalogs()
     logger.info("{0} upgraded to version {1}".format(product, version))
 
     return True

setup.py

@@ -42,6 +42,7 @@
         'Products.DataGridField',
         'Products.AdvancedQuery',
         'Products.TinyMCE',
+        'Products.TextIndexNG3',
         'collective.monkeypatcher',
         'collective.js.jqueryui',
         'plone.app.z3cform',
@@ -61,6 +62,7 @@
         'plone.resource',
         'CairoSVG==1.0.20',
         'collective.taskqueue',
+        'zopyx.txng3.ext==3.4.0'
     ],
     extras_require={
         'test': [