Cy/issue 0033

.github/workflows/run_edps.yaml

@@ -71,5 +71,5 @@ jobs:
           export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/sof"
           edps -lw
           edps -w metis.metis_lm_img_wkf -i $SOF_DATA -c
-          edps -w metis.metis_lm_img_wkf -i $SOF_DATA | tee edps.stdout.txt
+          edps -w metis.metis_lm_img_wkf -i $SOF_DATA -m science| tee edps.stdout.txt
           ! grep "'FAILED'" edps.stdout.txt

.gitignore

@@ -1,3 +1,6 @@
+# Skip __init__.py setting
+__init__.py
+
 # Skip vscode setting
 .vscode
 

metisp/README.md

@@ -83,6 +83,12 @@ Running one specific recipe
 
 ```
 
+Running Meta-target
+```
+ edps -w metis.metis_wkf -i $SOF_DATA -m science 
+```
+
+
 Getting report in a better way
 ```
  edps -w metis.metis_lm_img_wkf -i $SOF_DATA -t metis_det_dark -od
@@ -93,9 +99,12 @@ Getting report in a better way
 
 Making plots
 ```
-edps -w metis.metis_lm_img_wkf -i /home/chyan/METIS_Simulations/ESO/output -g > test.dot
+edps -w metis.metis_lm_img_wkf -i $SOF_DATA -g > test.dot
 dot -T png test.dot > mygraph.png
 ```
+The gerated plotting code can plot using online tool as well
+[GraphvizOnline](https://dreampuf.github.io/GraphvizOnline/)
+
 
 ## Note for developers
 When you're using the Python Debugger (pdb) and an error occurs, pdb will automatically enter post-mortem debugging mode, allowing you to inspect the state of the program at the point where the error occurred. Here's how you can find out where the error happened:

metisp/pymetis/src/pymetis/base/impl.py

@@ -50,7 +50,7 @@ def __init__(self, recipe: 'MetisRecipe') -> None:
         self.header: cpl.core.PropertyList | None = None
         self.products: Dict[str, PipelineProduct] = {}
         self.product_frames = cpl.ui.FrameSet()
-
+        
     def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet:
         """
             The main function of the recipe implementation. It mirrors the signature of `Recipe.run`

metisp/pymetis/src/pymetis/inputs/multiple.py

@@ -53,6 +53,20 @@ def __init__(self,
 
         self.extract_tag_parameters(tag_matches)
 
+    @classmethod
+    def get_target_name(cls, frameset: cpl.ui.FrameSet):  
+        """
+        Extracts the 'target' name from the input string based on the '_tags' regex.
+
+        :param inputString: The string to be matched against the pattern.
+        :return: The target name if a match is found, otherwise None.
+        """
+        for frame in frameset:
+            if match := cls._tags.match(frame.tag):
+                return match.group("target")
+            else: 
+                return None
+
     def extract_tag_parameters(self, matches: [dict[str, str]]):
         if len(matches) == 0:
             return

metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py

@@ -25,60 +25,80 @@
 
 from pymetis.base import MetisRecipeImpl
 from pymetis.base.recipe import MetisRecipe
-from pymetis.base.product import PipelineProduct
-from pymetis.inputs import PipelineInputSet, SinglePipelineInput
+from pymetis.base.product import PipelineProduct, TargetSpecificProduct
+from pymetis.inputs import RawInput, SinglePipelineInput
+from pymetis.prefab.rawimage import RawImageProcessor
 
 
-class MetisLmImgBackgroundImpl(MetisRecipeImpl):
-    class InputSet(PipelineInputSet):
-        class LmBasicReducedInput(SinglePipelineInput):
-            _tags: re.Pattern = re.compile(r"LM_(?P<target>SCI|STD)_BASIC_REDUCED")
+class MetisLmImgBackgroundImpl(RawImageProcessor):
+
+    class InputSet(RawImageProcessor.InputSet):
+        class RawInput(RawInput):
+            _tags = re.compile(r"LM_(?P<target>SCI|STD)_BASIC_REDUCED")
+
+        class SkyInput(RawInput):
+            _tags = re.compile(r"LM_(?P<target>SKY)_BASIC_REDUCED")
 
         def __init__(self, frameset: cpl.ui.FrameSet):
             super().__init__(frameset)
-            self.basic_reduced = self.LmBasicReducedInput(frameset)
-
+            self.basic_reduced = self.RawInput(frameset)
+            self.sky_reduced = self.SkyInput(frameset)
+
             # We need to register the inputs (just to be able to do `for x in self.inputs:`)
-            self.inputs |= {self.basic_reduced}
-
-    class ProductBkg(PipelineProduct):
-        tag: str = "LM_{target}_BKG"
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
+            self.inputs |= {self.basic_reduced, self.sky_reduced}
+
+    class ProductBkg(TargetSpecificProduct):
+        @property
+        def category(self):
+            return f"LM_{self.target:s}_BKG"
+        #category = rf"LM_{self.target}_BKG"
+        tag = category
         level = cpl.ui.Frame.FrameLevel.FINAL
         frame_type = cpl.ui.Frame.FrameType.IMAGE
 
-    class ProductBkgSubtracted(PipelineProduct):
-        tag: str = "LM_{target}_BKG_SUBTRACTED"
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
+    class ProductBkgSubtracted(TargetSpecificProduct):
+        @property
+        def category(self):
+            return f"LM_{self.target:s}_BKG_SUBTRACTED"
+        tag = category
         level = cpl.ui.Frame.FrameLevel.FINAL
         frame_type = cpl.ui.Frame.FrameType.IMAGE
-
-    class ProductObjectCat(PipelineProduct):
-        tag: str = "LM_{target}_OBJECT_CAT"
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
+
+    class ProductObjectCat(TargetSpecificProduct):
+        @property
+        def category(self):
+            return rf"LM_{self.target:s}_OBJECT_CAT"
+        tag = category
         level = cpl.ui.Frame.FrameLevel.FINAL
         frame_type = cpl.ui.Frame.FrameType.TABLE
 
     def process_images(self) -> Dict[str, PipelineProduct]:
-        Msg.info(self.__class__.__qualname__, f"Starting processing image attribute.")
+        raw_images = cpl.core.ImageList()
+
+        for idx, frame in enumerate(self.inputset.raw.frameset):
+            Msg.info(self.name, f"Loading raw image {frame.file}")
 
-        header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
-        image_bkg = cpl.core.Image() # ToDo implementation missing
-        image_bkg_subtracted = cpl.core.Image() # ToDo implementation missing
-        table_object_cat = cpl.core.Table()
+            if idx == 0:
+                self.header = cpl.core.PropertyList.load(frame.file, 0)
 
+            raw_image = cpl.core.Image.load(frame.file, extension=0)
+            raw_images.append(raw_image)
+
+        combined_image = self.combine_images(raw_images, "average")
+        #import pdb ; pdb.set_trace()
+
+        #dir(self.InputSet)
+        #print(self.inputset.RawInput.get_target_name())
+        self.target = self.inputset.RawInput.get_target_name(self.inputset.raw.frameset)
+
         self.products = {
-            self.ProductBkg.tag:
-                self.ProductBkg(self, header, image_bkg),
-            self.ProductBkgSubtracted.tag:
-                self.ProductBkgSubtracted(self, header, image_bkg_subtracted),
-            self.ProductObjectCat.tag:
-                self.ProductObjectCat(self, header, table_object_cat),
+            product.category: product(self, self.header, combined_image, target=self.target)
+            for product in [self.ProductBkg, self.ProductBkgSubtracted, self.ProductObjectCat]
         }
-
         return self.products
 
 
+
 class MetisLmImgBackground(MetisRecipe):
     _name = "metis_lm_img_background"
     _version = "0.1"
@@ -88,5 +108,14 @@ class MetisLmImgBackground(MetisRecipe):
     _synopsis = "Basic reduction of raw exposures from the LM-band imager"
     _description = ""
 
-    parameters = cpl.ui.ParameterList([])
+    parameters = cpl.ui.ParameterList([
+        cpl.ui.ParameterEnum(
+            name="metis_lm_img_background.stacking.method",
+            context="metis_lm_img_background",
+            description="Name of the method used to combine the input images",
+            default="add",
+            alternatives=("add", "average", "median"),
+        )
+    ])
+
     implementation_class = MetisLmImgBackgroundImpl

metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py

@@ -31,6 +31,7 @@
 
 
 class MetisLmImgBasicReduceImpl(DarkImageProcessor):
+
     class InputSet(DarkImageProcessor.InputSet):
         """
         The first step of writing a recipe is to define an InputSet: the singleton class
@@ -58,7 +59,7 @@ class InputSet(DarkImageProcessor.InputSet):
         # RawImageProcessor.InputSet. It already knows that it wants a RawInput and MasterDarkInput class,
         # but does not know about the tags yet. So here we define tags for the raw input:
         class RawInput(RawInput):
-            _tags = re.compile(r"LM_IMAGE_(?P<target>SCI|STD)_RAW")
+            _tags = re.compile(r"LM_IMAGE_(?P<target>SCI|SKY|STD)_RAW")
 
         # Now we need a master dark. Since nothing is changed and the tag is always the same,
         # we just point to the provided MasterDarkInput.
@@ -96,7 +97,7 @@ class Product(TargetSpecificProduct):
 
         @property
         def category(self) -> str:
-            return rf"LM_{self.target:s}_REDUCED"
+            return rf"LM_{self.target:s}_BASIC_REDUCED"
 
         @property
         def output_file_name(self):
@@ -165,8 +166,10 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         images = self.prepare_images(self.inputset.raw.frameset, flat, bias)
         combined_image = self.combine_images(images, self.parameters["metis_lm_img_basic_reduce.stacking.method"].value)
         header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
-
-        self.target = "SCI" # hardcoded for now
+
+
+        self.target = self.inputset.RawInput.get_target_name(self.inputset.raw.frameset)
+        #self.target = 'SCI'
         self.products = {
             fr'OBJECT_REDUCED_{self.detector}':
                 self.Product(self, header, combined_image, target=self.target),

metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_cal_distortion.py

@@ -0,0 +1,125 @@
+"""
+This file is part of the METIS Pipeline.
+Copyright (C) 2024 European Southern Observatory
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+"""
+
+import re
+
+import cpl
+from cpl.core import Msg
+from typing import Dict
+
+from pymetis.inputs.common import RawInput, LinearityInput, BadpixMapInput, PersistenceMapInput, GainMapInput
+from pymetis.base.recipe import MetisRecipe
+from pymetis.base.product import PipelineProduct
+from pymetis.inputs import RawInput, SinglePipelineInput
+from pymetis.prefab.rawimage import RawImageProcessor
+
+
+class MetisLmImgCalDistortionImpl(RawImageProcessor):
+    class InputSet(RawImageProcessor.InputSet):
+        class RawInput(RawInput):
+            _tags = re.compile(r"LM_WCU_OFF_RAW")
+
+        class DistortionInput(SinglePipelineInput):
+            _tags = re.compile(r"LM_DISTORTION_RAW")
+            _title = "Distortion map"
+            _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
+
+        class PinholeTableInput(SinglePipelineInput):
+            _tags = re.compile(r"PINHOLE_TABLE")
+            _title = "pinhole table"
+            _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
+
+        def __init__(self, frameset: cpl.ui.FrameSet):
+            super().__init__(frameset)
+            self.pinhole_table = SinglePipelineInput(frameset,
+                                                     tags=re.compile(r"PINHOLE_TABLE"),
+                                                     title="pinhole table",
+                                                     group=cpl.ui.Frame.FrameGroup.CALIB)
+
+            self.distortion = self.DistortionInput(frameset, required=False) 
+            self.linearity = LinearityInput(frameset, required=False) # But should be
+            self.badpix_map = BadpixMapInput(frameset, required=False)
+            self.persistence_map = PersistenceMapInput(frameset, required=False) # But should be
+            self.gain_map = GainMapInput(frameset, required=False) # But should be
+
+            self.inputs |= {self.pinhole_table, self.linearity, self.distortion,
+                            self.badpix_map, self.persistence_map, self.gain_map}
+
+
+    class ProductLmDistortionTable(PipelineProduct):
+        category = rf"ILM_DISTORTION_TABLE"
+        tag = category
+        level = cpl.ui.Frame.FrameLevel.FINAL
+        frame_type = cpl.ui.Frame.FrameType.TABLE
+
+    class ProductLmDistortionMap(PipelineProduct):
+        category = rf"LM_DIST_MAP"
+        tag = category
+        level = cpl.ui.Frame.FrameLevel.FINAL
+        frame_type = cpl.ui.Frame.FrameType.IMAGE
+
+    class ProductLmDistortionReduced(PipelineProduct):
+        category = rf"LM_DIST_REDUCED"
+        tag = category
+        level = cpl.ui.Frame.FrameLevel.FINAL
+        frame_type = cpl.ui.Frame.FrameType.IMAGE
+
+    def process_images(self) -> Dict[str, PipelineProduct]:
+        raw_images = cpl.core.ImageList()
+
+        for idx, frame in enumerate(self.inputset.raw.frameset):
+            Msg.info(self.name, f"Loading raw image {frame.file}")
+
+            if idx == 0:
+                self.header = cpl.core.PropertyList.load(frame.file, 0)
+
+            raw_image = cpl.core.Image.load(frame.file, extension=1)
+            raw_images.append(raw_image)
+
+        combined_image = self.combine_images(raw_images, "average")
+
+        self.products = {
+            product.category: product(self, self.header, combined_image)
+            for product in [self.ProductLmDistortionTable, self.ProductLmDistortionMap,
+                            self.ProductLmDistortionReduced]
+        }
+        return self.products
+
+
+class MetisLmImgCalDistortion(MetisRecipe):
+    _name = "metis_lm_img_cal_distortion"
+    _version = "0.1"
+    _author = "Chi-Hung Yan"
+    _email = "[email protected]"
+    _synopsis = "Determine optical distortion coefficients for the LM imager."
+    _description = (
+        "Currently just a skeleton prototype."
+    )
+
+    parameters = cpl.ui.ParameterList([
+        cpl.ui.ParameterEnum(
+            name="metis_lm_img_cal_distortion.stacking.method",
+            context="metis_lm_img_cal_distortion",
+            description="Name of the method used to combine the input images",
+            default="average",
+            alternatives=("add", "average", "median", "sigclip"),
+        ),
+    ])
+
+    implementation_class = MetisLmImgCalDistortionImpl