mb/tidy-up

.github/workflows/run_edps.yaml

@@ -82,7 +82,6 @@ jobs:
           export SOF_DATA="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/data"
           export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/sof"
           edps -lw
-          edps -w metis.metis_lm_img_wkf -i $SOF_DATA -c
-          edps -w metis.metis_lm_img_wkf -i $SOF_DATA -lt
-          edps -w metis.metis_lm_img_wkf -i $SOF_DATA -m all| tee edps.stdout.txt
+          edps -w metis.metis_wkf -i $SOF_DATA -c
+          edps -w metis.metis_wkf -i $SOF_DATA | tee edps.stdout.txt
           ! grep "'FAILED'" edps.stdout.txt

metisp/pymetis/src/pymetis/base/impl.py

@@ -48,9 +48,9 @@ def __init__(self, recipe: 'MetisRecipe') -> None:
         self.inputset: PipelineInputSet = None
         self.frameset: cpl.ui.FrameSet | None = None
         self.header: cpl.core.PropertyList | None = None
-        self.products: Dict[str, PipelineProduct] = {}
+        self.products: [PipelineProduct] = {}
         self.product_frames = cpl.ui.FrameSet()
-        
+
     def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet:
         """
             The main function of the recipe implementation. It mirrors the signature of `Recipe.run`
@@ -64,14 +64,14 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
 
         try:
             self.frameset = frameset
-            self.import_settings(settings)                # Import and process the provided settings dict
-            self.inputset = self.InputSet(frameset)       # Create an appropriate InputSet object
+            self.import_settings(settings)                  # Import and process the provided settings dict
+            self.inputset = self.InputSet(frameset)         # Create an appropriate InputSet object
             self.inputset.print_debug()
-            self.inputset.validate()                      # Verify that they are valid (maybe with `schema` too?)
-            products = self.process_images()              # Do all the actual processing
-            self.save_products(products)                  # Save the output products
+            self.inputset.validate()                        # Verify that they are valid (maybe with `schema` too?)
+            self.products = self.process_images()           # Do all the actual processing
+            self._save_products()                           # Save the output products
 
-            return self.build_product_frameset(products)  # Return the output as a pycpl FrameSet
+            return self.build_product_frameset()            # Return the output as a pycpl FrameSet
         except cpl.core.DataNotFoundError as e:
             Msg.error(self.__class__.__qualname__, f"Data not found error: {e.message}")
             raise e
@@ -84,20 +84,20 @@ def import_settings(self, settings: Dict[str, Any]) -> None:
                 self.parameters[key].value = value
             except KeyError:
                 Msg.warning(self.__class__.__qualname__,
-                            f"Settings includes '{key}':{value} "
+                            f"Settings include '{key}' = {value} "
                             f"but class {self.__class__.__qualname__} "
                             f"has no parameter named {key}.")
 
     @abstractmethod
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         """
         The core method of the recipe implementation. It should contain all the processing logic.
         At its entry point the Input class must be already loaded and validated.
 
         All pixel manipulation should happen inside this function (or something it calls from within).
         Put explicitly,
-            - no pixel manipulation before entering `process_images`,
-            - and no pixel manipulation after exiting `process_images`.
+            - no pixel manipulation *before* entering `process_images`,
+            - and no pixel manipulation *after* exiting `process_images`.
 
         The basic workflow inside this function should be as follows:
 
@@ -118,22 +118,24 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         """
         return {}
 
-    def save_products(self, products: Dict[str, PipelineProduct]) -> None:
+    def _save_products(self) -> None:
         """
         Save and register the created products.
         """
-        for name, product in products.items():
-            Msg.debug(self.__class__.__qualname__,
-                      f"Saving product {name}")
+        assert self.products is not None, "Products have not been created yet!"
+
+        Msg.debug(self.__class__.__qualname__,
+                  f"Saving {len(self.products)} products: {self.products}")
+        for product in self.products:
             product.save()
 
-    def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui.FrameSet:
+    def build_product_frameset(self) -> cpl.ui.FrameSet:
         """
         Gather all the products and build a FrameSet from their frames so that it can be returned from `run`.
         """
         Msg.debug(self.__class__.__qualname__,
                   f"Building the product frameset")
-        return cpl.ui.FrameSet([product.as_frame() for product in products.values()])
+        return cpl.ui.FrameSet([product.as_frame() for product in self.products])
 
     def as_dict(self) -> dict[str, Any]:
         """
@@ -147,7 +149,7 @@ def as_dict(self) -> dict[str, Any]:
             'title': self.name,
             'inputset': self.inputset.as_dict(),
             'products': {
-                product.tag: product.as_dict() for product in self.products.values()
+                str(product.category): product.as_dict() for product in self.products
             }
         }
 

metisp/pymetis/src/pymetis/base/product.py

@@ -17,7 +17,7 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 """
 
-from abc import ABC, abstractmethod
+from abc import ABC
 
 import cpl
 from cpl.core import Msg
@@ -31,10 +31,10 @@ class PipelineProduct(ABC):
         one FITS file with associated headers and a frame
     """
 
-    tag: str = None
-    group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.PRODUCT        # ToDo: Is this a sensible default?
-    level: cpl.ui.Frame.FrameLevel = None
-    frame_type: cpl.ui.Frame.FrameType = None
+    _tag: str = None
+    _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.PRODUCT        # ToDo: Is this a sensible default?
+    _level: cpl.ui.Frame.FrameLevel = None
+    _frame_type: cpl.ui.Frame.FrameType = None
 
     def __init__(self,
                  recipe_impl: 'MetisRecipeImpl',
@@ -120,6 +120,22 @@ def save(self):
             header=self.header,
         )
 
+    @property
+    def tag(self):
+        return self._tag
+
+    @property
+    def group(self):
+        return self._group
+
+    @property
+    def level(self):
+        return self._level
+
+    @property
+    def frame_type(self):
+        return self._frame_type
+
     @property
     def category(self) -> str:
         """

metisp/pymetis/src/pymetis/base/recipe.py

@@ -42,6 +42,8 @@ class MetisRecipe(cpl.ui.PyRecipe):
                     "Bonus points if it is not visible from pyesorex.")
 
     parameters = cpl.ui.ParameterList([])           # By default, a recipe does not have any parameters.
+                                                    # ToDo: There is a pyesorex bug that prevents
+                                                    # this from being actually used.
     implementation_class: type["MetisRecipeImpl"]   # Dummy class, this must be overridden in the derived classes anyway
 
     def __init__(self):
@@ -61,4 +63,4 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
             It just calls the same method in the decoupled implementation.
         """
         self.implementation = self.dispatch_implementation_class(frameset)(self)
-        return self.implementation.run(frameset, settings)
+        return self.implementation.run(frameset, settings)

metisp/pymetis/src/pymetis/prefab/flat.py

@@ -52,9 +52,9 @@ def __init__(self, frameset: cpl.ui.FrameSet):
             #self.inputs |= {self.persistence, self.linearity, self.gain_map}
 
     class Product(PipelineProduct):
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _group = cpl.ui.Frame.FrameGroup.PRODUCT
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
         band: str = None
 
         @property
@@ -69,7 +69,7 @@ def output_file_name(self) -> str:
         def tag(self) -> str:
             return self.category
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         """
         Do the actual processing of the images.
         Here, it means loading the input images and a master dark,
@@ -94,7 +94,6 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
         combined_image = self.combine_images(self.inputset.load_raw_images(), method)
 
-        self.products = {
-            self.name.upper(): self.Product(self, header, combined_image),
-        }
-        return self.products
+        product = self.Product(self, header, combined_image)
+
+        return [product]

metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py

@@ -62,9 +62,9 @@ class ProductCombined(PipelineProduct):
         """
         Final product: combined, background-subtracted images of the WCU source
         """
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _group = cpl.ui.Frame.FrameGroup.PRODUCT
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
         @property
         def category(self) -> str:
@@ -83,9 +83,9 @@ class ProductBackground(PipelineProduct):
         """
         Intermediate product: the instrumental background (WCU OFF)
         """
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
-        level = cpl.ui.Frame.FrameLevel.INTERMEDIATE
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _group = cpl.ui.Frame.FrameGroup.PRODUCT
+        _level = cpl.ui.Frame.FrameLevel.INTERMEDIATE
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
         @property
         def category(self) -> str:
@@ -100,7 +100,7 @@ def tag(self) -> str:
             return rf"{self.category}"
 
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         """do something"""
 
         background_hdr = cpl.core.PropertyList()
@@ -114,13 +114,10 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         raw_images.subtract_image(background_img)
         combined_img = self.combine_images(raw_images, "median")
 
-        self.products = {
-            rf"{self.target}_COMBINED":
+        return [
             self.ProductCombined(self, combined_hdr, combined_img),
-            rf"{self.target}_BACKGROUND":
             self.ProductBackground(self, background_hdr, background_img),
-        }
-        return self.products
+        ]
 
     def load_images(self, frameset: cpl.ui.FrameSet) -> cpl.core.ImageList:
         """Load an imagelist from a FrameSet

metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_calibrate.py

@@ -53,24 +53,22 @@ def __init__(self, frameset: cpl.ui.FrameSet):
             self.inputs |= {self.sci_reduced, self.telluric, self.fluxcal}
 
     class ProductSciCubeCalibrated(PipelineProduct):
-        tag = rf"IFU_SCI_CUBE_CALIBRATED"
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _tag = rf"IFU_SCI_CUBE_CALIBRATED"
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
 
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         # self.correct_telluric()
         # self.apply_fluxcal()
 
         header = self._create_dummy_header()
         image = self._create_dummy_image()
 
-        self.products = {
-            product.category: product(self, header, image)
-            for product in [self.ProductSciCubeCalibrated]
-        }
-        return self.products
+        product_scc = self.ProductSciCubeCalibrated(self, header, image)
+
+        return [product_scc]
 
 
 class MetisIfuCalibrate(MetisRecipe):

metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_distortion.py

@@ -47,18 +47,16 @@ def __init__(self, frameset: cpl.ui.FrameSet):
 
 
     class ProductIfuDistortionTable(PipelineProduct):
-        category = rf"IFU_DISTORTION_TABLE"
-        tag = category
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.TABLE
+        _tag = r"IFU_DISTORTION_TABLE"
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.TABLE
 
     class ProductIfuDistortionReduced(PipelineProduct):
-        category = rf"IFU_DIST_REDUCED"
-        tag = category
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _tag = r"IFU_DIST_REDUCED"
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         raw_images = cpl.core.ImageList()
 
         for idx, frame in enumerate(self.inputset.raw.frameset):
@@ -72,11 +70,10 @@ def process_images(self) -> Dict[str, PipelineProduct]:
 
         combined_image = self.combine_images(raw_images, "average")
 
-        self.products = {
-            product.category: product(self, self.header, combined_image)
-            for product in [self.ProductIfuDistortionTable, self.ProductIfuDistortionReduced]
-        }
-        return self.products
+        product_distortion = self.ProductIfuDistortionTable(self, self.header, combined_image)
+        product_distortion_reduced = self.ProductIfuDistortionReduced(self, self.header, combined_image)
+
+        return [product_distortion, product_distortion_reduced]
 
 
 class MetisIfuDistortion(MetisRecipe):

metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_postprocess.py

@@ -41,8 +41,8 @@ def __init__(self, frameset: cpl.ui.FrameSet):
             self.inputs |= {self.sci_cube_calibrated}
 
     class ProductSciCoadd(PipelineProduct):
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
         @property
         def tag(self) -> str:
@@ -57,18 +57,15 @@ def resample_cubes(self):
     def coadd_cubes(self):
         pass
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         self.determine_output_grid()
         self.resample_cubes()
         self.coadd_cubes()
 
-        header = cpl.core.PropertyList()
+        header = self._create_dummy_header()
         image = cpl.core.Image.load(self.inputset.sci_cube_calibrated.frame.file) # ToDo actual processing
 
-        self.products = {
-            r'IFU_SCI_COADD': self.ProductSciCoadd(self, header, image), # ToDo is just a dummy for now
-        }
-        return self.products
+        return [self.ProductSciCoadd(self, header, image)] # ToDo is just a dummy for now
 
 
 class MetisIfuPostprocess(MetisRecipe):