Convert LDL/UDL fields to string (senaite#2103)

* Migrate LDL/UDL FixedPoint -> StringField

* Changelog updated

* Cleanup of the migration steps

* Fixed typo

* Re-added custom getters

* Cut off trailing zeros

Co-authored-by: Jordi Puiggené <[email protected]>

Author: ramonski

CHANGES.rst

@@ -5,8 +5,10 @@ Changelog
 2.3.0 (unreleased)
 ------------------
 
+- #2103 Convert LDL/UDL fields to string
 - #2101 Add help text for numeric result
-- #2097 Fix Attribute Error in Multi- Sample Add form when current user is linked to a client contact
+- #2097 Fix Attribute Error in Multi- Sample Add form when current user is a client contact
+- #2096 Convert uncertainty field to string
 - #2095 Fix rounded uncertainty value is stored in the database
 - #2094 Skip Auditlog catalog if disabled for DX types catalog multiplexer
 - #2090 Add support for dates before 1900

src/bika/lims/content/abstractbaseanalysis.py

@@ -136,16 +136,13 @@
     )
 )
 
-# TODO: Use a plain string field when converting to DX!
-#
 # If the value is below this limit, it means that the measurement lacks
 # accuracy and this will be shown in manage_results and also on the final
 # report.
-LowerDetectionLimit = FixedPointField(
+LowerDetectionLimit = StringField(
     "LowerDetectionLimit",
     schemata="Analysis",
     default="0.0",
-    precision=1000,  # avoid precision cut-off done by the field
     widget=DecimalWidget(
         label=_("Lower Detection Limit (LDL)"),
         description=_(
@@ -156,16 +153,13 @@
     )
 )
 
-# TODO: Use a plain string field when converting to DX!
-#
 # If the value is above this limit, it means that the measurement lacks
 # accuracy and this will be shown in manage_results and also on the final
 # report.
-UpperDetectionLimit = FixedPointField(
+UpperDetectionLimit = StringField(
     "UpperDetectionLimit",
     schemata="Analysis",
     default="1000000000.0",
-    precision=1000,  # avoid precision cut-off done by the field
     widget=DecimalWidget(
         label=_("Upper Detection Limit (UDL)"),
         description=_(
@@ -851,28 +845,24 @@ def getAnalysisCategories(self):
 
     @security.public
     def getLowerDetectionLimit(self):
-        """Get the lower detection limit without trailing zeros
+        """Get the lower detection limit
         """
         field = self.getField("LowerDetectionLimit")
         value = field.get(self)
-        # NOTE: This is a workaround to avoid the cut-off done by the field
-        #       if the value is lower than the precision
-        #       => we should use a string instead
-        # remove trailing zeros and possible trailing dot
-        value = value.rstrip("0").rstrip(".")
+        # cut off trailing zeros
+        if "." in value:
+            value = value.rstrip("0").rstrip(".")
         return value
 
     @security.public
     def getUpperDetectionLimit(self):
-        """Get the upper detection limit without trailing zeros
+        """Get the upper detection limit
         """
         field = self.getField("UpperDetectionLimit")
         value = field.get(self)
-        # NOTE: This is a workaround to avoid the cut-off done by the field
-        #       if the value is lower than the precision
-        #       => we should use a string instead
-        # remove trailing zeros and possible trailing dot
-        value = value.rstrip("0").rstrip(".")
+        # cut off trailing zeros
+        if "." in value:
+            value = value.rstrip("0").rstrip(".")
         return value
 
     @security.public

src/senaite/core/upgrade/v02_03_000.py

@@ -24,6 +24,7 @@
 from senaite.core.catalog import ANALYSIS_CATALOG
 from senaite.core.catalog import REPORT_CATALOG
 from senaite.core.catalog import SAMPLE_CATALOG
+from senaite.core.catalog import SETUP_CATALOG
 from senaite.core.catalog.report_catalog import ReportCatalog
 from senaite.core.config import PROJECTNAME as product
 from senaite.core.setuphandlers import _run_import_step
@@ -79,6 +80,7 @@ def upgrade(tool):
     fix_interface_interpretation_template(portal)
     fix_unassigned_samples(portal)
     move_arreports_to_report_catalog(portal)
+    migrate_analysis_services_fields(portal)
     migrate_analyses_fields(portal)
 
     logger.info("{0} upgraded to version {1}".format(product, version))
@@ -284,8 +286,35 @@ def move_arreports_to_report_catalog(portal):
     logger.info("Move ARReports to SENAITE Report Catalog [DONE]")
 
 
+def migrate_analysis_services_fields(portal):
+    """Migrate fields in AnalysisService objects
+    """
+    logger.info("Migrate Analysis Services Fields ...")
+    cat = api.get_tool(SETUP_CATALOG)
+    query = {"portal_type": ["AnalysisService"]}
+    brains = cat.search(query)
+    total = len(brains)
+
+    for num, brain in enumerate(brains):
+        if num and num % 100 == 0:
+            logger.info("Migrated {0}/{1} analyses fields".format(num, total))
+
+        obj = api.get_object(brain)
+
+        # Migrate UDL FixedPointField -> StringField
+        migrate_udl_field_to_string(obj)
+
+        # Migrate LDL FixedPointField -> StringField
+        migrate_ldl_field_to_string(obj)
+
+        # Flush the object from memory
+        obj._p_deactivate()  # noqa
+
+    logger.info("Migrate Analysis Services [DONE]")
+
+
 def migrate_analyses_fields(portal):
-    """return all analyses
+    """Migrate fields in Analysis/ReferenceAnalysis objects
     """
     logger.info("Migrate Analyses Fields ...")
     cat = api.get_tool(ANALYSIS_CATALOG)
@@ -297,35 +326,82 @@ def migrate_analyses_fields(portal):
         if num and num % 100 == 0:
             logger.info("Migrated {0}/{1} analyses fields".format(num, total))
 
-        analysis = api.get_object(brain)
+        obj = api.get_object(brain)
 
         # Migrate Uncertainty FixedPointField -> StringField
-        migrate_uncertainty_field_to_string(analysis)
+        migrate_uncertainty_field_to_string(obj)
+
+        # Migrate UDL FixedPointField -> StringField
+        migrate_udl_field_to_string(obj)
+
+        # Migrate LDL FixedPointField -> StringField
+        migrate_ldl_field_to_string(obj)
 
         # Flush the object from memory
-        analysis._p_deactivate()  # noqa
+        obj._p_deactivate()  # noqa
 
     logger.info("Migrate Analyses Fields [DONE]")
 
 
-def migrate_uncertainty_field_to_string(analysis):
+def migrate_udl_field_to_string(obj):
+    """Migrate the UDL field to string
+    """
+    field = obj.getField("UpperDetectionLimit")
+    value = field.get(obj)
+
+    # Leave any other value type unchanged
+    if isinstance(value, tuple):
+        migrated_value = fixed_point_value_to_string(value, 7)
+        logger.info("Migrating UDL field of %s: %s -> %s" % (
+            api.get_path(obj), value, migrated_value))
+        value = migrated_value
+
+    # set the new value
+    field.set(obj, value)
+
+
+def migrate_ldl_field_to_string(obj):
+    """Migrate the LDL field to string
+    """
+    field = obj.getField("LowerDetectionLimit")
+    value = field.get(obj)
+
+    # Leave any other value type unchanged
+    if isinstance(value, tuple):
+        migrated_value = fixed_point_value_to_string(value, 7)
+        logger.info("Migrating LDL field of %s: %s -> %s" % (
+            api.get_path(obj), value, migrated_value))
+        value = migrated_value
+
+    # set the new value
+    field.set(obj, value)
+
+
+def migrate_uncertainty_field_to_string(obj):
     """Migrate the uncertainty field to string
     """
-    field = analysis.getField("Uncertainty")
-    value = field.get(analysis)
+    field = obj.getField("Uncertainty")
+    value = field.get(obj)
 
     # Leave any other value type unchanged
     if isinstance(value, tuple):
-        value = fixed_point_value_to_string(value)
+        migrated_value = fixed_point_value_to_string(value, 10)
+        logger.info("Migrating Uncertainty field of %s: %s -> %s" % (
+            api.get_pat(obj), value, migrated_value))
+        value = migrated_value
 
     # set the new value
-    field.set(analysis, value)
+    field.set(obj, value)
 
 
-def fixed_point_value_to_string(value):
+def fixed_point_value_to_string(value, precision):
     """Code taken and modified from FixedPointField get method
+
+    IMPORTANT: The precision has to be the same as it was initially
+               defined in the field when the value was set!
+               Otherwise, values > 0, e.g. 0.0005 are converted wrong!
     """
-    template = "%%s%%d.%%0%dd" % 10
+    template = "%%s%%d.%%0%dd" % precision
     front, fra = value
     sign = ""
     # Numbers between -1 and 0 are store with a negative fraction.