diff --git a/.github/workflows/run_edps.yaml b/.github/workflows/run_edps.yaml index c742767e..942a2dfb 100644 --- a/.github/workflows/run_edps.yaml +++ b/.github/workflows/run_edps.yaml @@ -71,5 +71,5 @@ jobs: export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/sof" edps -lw edps -w metis.metis_lm_img_wkf -i $SOF_DATA -c - edps -w metis.metis_lm_img_wkf -i $SOF_DATA | tee edps.stdout.txt + edps -w metis.metis_lm_img_wkf -i $SOF_DATA -m science| tee edps.stdout.txt ! grep "'FAILED'" edps.stdout.txt diff --git a/.gitignore b/.gitignore index eb8261ef..4be73c39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Skip __init__.py setting +__init__.py + # Skip vscode setting .vscode diff --git a/metisp/README.md b/metisp/README.md index c47633bf..7b514194 100644 --- a/metisp/README.md +++ b/metisp/README.md @@ -83,6 +83,12 @@ Running one specific recipe ``` +Running Meta-target +``` + edps -w metis.metis_wkf -i $SOF_DATA -m science +``` + + Getting report in a better way ``` edps -w metis.metis_lm_img_wkf -i $SOF_DATA -t metis_det_dark -od @@ -93,9 +99,12 @@ Getting report in a better way Making plots ``` -edps -w metis.metis_lm_img_wkf -i /home/chyan/METIS_Simulations/ESO/output -g > test.dot +edps -w metis.metis_lm_img_wkf -i $SOF_DATA -g > test.dot dot -T png test.dot > mygraph.png ``` +The gerated plotting code can plot using online tool as well +[GraphvizOnline](https://dreampuf.github.io/GraphvizOnline/) + ## Note for developers When you're using the Python Debugger (pdb) and an error occurs, pdb will automatically enter post-mortem debugging mode, allowing you to inspect the state of the program at the point where the error occurred. Here's how you can find out where the error happened: diff --git a/metisp/pymetis/src/pymetis/base/impl.py b/metisp/pymetis/src/pymetis/base/impl.py index 238abbab..d487f478 100644 --- a/metisp/pymetis/src/pymetis/base/impl.py +++ b/metisp/pymetis/src/pymetis/base/impl.py @@ -50,7 +50,7 @@ def __init__(self, recipe: 'MetisRecipe') -> None: self.header: cpl.core.PropertyList | None = None self.products: Dict[str, PipelineProduct] = {} self.product_frames = cpl.ui.FrameSet() - + def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet: """ The main function of the recipe implementation. It mirrors the signature of `Recipe.run` diff --git a/metisp/pymetis/src/pymetis/inputs/multiple.py b/metisp/pymetis/src/pymetis/inputs/multiple.py index 0bff0db9..e60fe158 100644 --- a/metisp/pymetis/src/pymetis/inputs/multiple.py +++ b/metisp/pymetis/src/pymetis/inputs/multiple.py @@ -53,6 +53,20 @@ def __init__(self, self.extract_tag_parameters(tag_matches) + @classmethod + def get_target_name(cls, frameset: cpl.ui.FrameSet): + """ + Extracts the 'target' name from the input string based on the '_tags' regex. + + :param inputString: The string to be matched against the pattern. + :return: The target name if a match is found, otherwise None. + """ + for frame in frameset: + if match := cls._tags.match(frame.tag): + return match.group("target") + else: + return None + def extract_tag_parameters(self, matches: [dict[str, str]]): if len(matches) == 0: return diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py index c74eb9d1..bae32995 100644 --- a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py @@ -25,60 +25,80 @@ from pymetis.base import MetisRecipeImpl from pymetis.base.recipe import MetisRecipe -from pymetis.base.product import PipelineProduct -from pymetis.inputs import PipelineInputSet, SinglePipelineInput +from pymetis.base.product import PipelineProduct, TargetSpecificProduct +from pymetis.inputs import RawInput, SinglePipelineInput +from pymetis.prefab.rawimage import RawImageProcessor -class MetisLmImgBackgroundImpl(MetisRecipeImpl): - class InputSet(PipelineInputSet): - class LmBasicReducedInput(SinglePipelineInput): - _tags: re.Pattern = re.compile(r"LM_(?PSCI|STD)_BASIC_REDUCED") +class MetisLmImgBackgroundImpl(RawImageProcessor): + + class InputSet(RawImageProcessor.InputSet): + class RawInput(RawInput): + _tags = re.compile(r"LM_(?PSCI|STD)_BASIC_REDUCED") + + class SkyInput(RawInput): + _tags = re.compile(r"LM_(?PSKY)_BASIC_REDUCED") def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) - self.basic_reduced = self.LmBasicReducedInput(frameset) - + self.basic_reduced = self.RawInput(frameset) + self.sky_reduced = self.SkyInput(frameset) + # We need to register the inputs (just to be able to do `for x in self.inputs:`) - self.inputs |= {self.basic_reduced} - - class ProductBkg(PipelineProduct): - tag: str = "LM_{target}_BKG" - group = cpl.ui.Frame.FrameGroup.PRODUCT + self.inputs |= {self.basic_reduced, self.sky_reduced} + + class ProductBkg(TargetSpecificProduct): + @property + def category(self): + return f"LM_{self.target:s}_BKG" + #category = rf"LM_{self.target}_BKG" + tag = category level = cpl.ui.Frame.FrameLevel.FINAL frame_type = cpl.ui.Frame.FrameType.IMAGE - class ProductBkgSubtracted(PipelineProduct): - tag: str = "LM_{target}_BKG_SUBTRACTED" - group = cpl.ui.Frame.FrameGroup.PRODUCT + class ProductBkgSubtracted(TargetSpecificProduct): + @property + def category(self): + return f"LM_{self.target:s}_BKG_SUBTRACTED" + tag = category level = cpl.ui.Frame.FrameLevel.FINAL frame_type = cpl.ui.Frame.FrameType.IMAGE - - class ProductObjectCat(PipelineProduct): - tag: str = "LM_{target}_OBJECT_CAT" - group = cpl.ui.Frame.FrameGroup.PRODUCT + + class ProductObjectCat(TargetSpecificProduct): + @property + def category(self): + return rf"LM_{self.target:s}_OBJECT_CAT" + tag = category level = cpl.ui.Frame.FrameLevel.FINAL frame_type = cpl.ui.Frame.FrameType.TABLE def process_images(self) -> Dict[str, PipelineProduct]: - Msg.info(self.__class__.__qualname__, f"Starting processing image attribute.") + raw_images = cpl.core.ImageList() + + for idx, frame in enumerate(self.inputset.raw.frameset): + Msg.info(self.name, f"Loading raw image {frame.file}") - header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0) - image_bkg = cpl.core.Image() # ToDo implementation missing - image_bkg_subtracted = cpl.core.Image() # ToDo implementation missing - table_object_cat = cpl.core.Table() + if idx == 0: + self.header = cpl.core.PropertyList.load(frame.file, 0) + raw_image = cpl.core.Image.load(frame.file, extension=0) + raw_images.append(raw_image) + + combined_image = self.combine_images(raw_images, "average") + #import pdb ; pdb.set_trace() + + #dir(self.InputSet) + #print(self.inputset.RawInput.get_target_name()) + self.target = self.inputset.RawInput.get_target_name(self.inputset.raw.frameset) + self.products = { - self.ProductBkg.tag: - self.ProductBkg(self, header, image_bkg), - self.ProductBkgSubtracted.tag: - self.ProductBkgSubtracted(self, header, image_bkg_subtracted), - self.ProductObjectCat.tag: - self.ProductObjectCat(self, header, table_object_cat), + product.category: product(self, self.header, combined_image, target=self.target) + for product in [self.ProductBkg, self.ProductBkgSubtracted, self.ProductObjectCat] } - return self.products + class MetisLmImgBackground(MetisRecipe): _name = "metis_lm_img_background" _version = "0.1" @@ -88,5 +108,14 @@ class MetisLmImgBackground(MetisRecipe): _synopsis = "Basic reduction of raw exposures from the LM-band imager" _description = "" - parameters = cpl.ui.ParameterList([]) + parameters = cpl.ui.ParameterList([ + cpl.ui.ParameterEnum( + name="metis_lm_img_background.stacking.method", + context="metis_lm_img_background", + description="Name of the method used to combine the input images", + default="add", + alternatives=("add", "average", "median"), + ) + ]) + implementation_class = MetisLmImgBackgroundImpl diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py index 41bafc57..3c5b224d 100644 --- a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py @@ -31,6 +31,7 @@ class MetisLmImgBasicReduceImpl(DarkImageProcessor): + class InputSet(DarkImageProcessor.InputSet): """ The first step of writing a recipe is to define an InputSet: the singleton class @@ -58,7 +59,7 @@ class InputSet(DarkImageProcessor.InputSet): # RawImageProcessor.InputSet. It already knows that it wants a RawInput and MasterDarkInput class, # but does not know about the tags yet. So here we define tags for the raw input: class RawInput(RawInput): - _tags = re.compile(r"LM_IMAGE_(?PSCI|STD)_RAW") + _tags = re.compile(r"LM_IMAGE_(?PSCI|SKY|STD)_RAW") # Now we need a master dark. Since nothing is changed and the tag is always the same, # we just point to the provided MasterDarkInput. @@ -96,7 +97,7 @@ class Product(TargetSpecificProduct): @property def category(self) -> str: - return rf"LM_{self.target:s}_REDUCED" + return rf"LM_{self.target:s}_BASIC_REDUCED" @property def output_file_name(self): @@ -165,8 +166,10 @@ def process_images(self) -> Dict[str, PipelineProduct]: images = self.prepare_images(self.inputset.raw.frameset, flat, bias) combined_image = self.combine_images(images, self.parameters["metis_lm_img_basic_reduce.stacking.method"].value) header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0) - - self.target = "SCI" # hardcoded for now + + + self.target = self.inputset.RawInput.get_target_name(self.inputset.raw.frameset) + #self.target = 'SCI' self.products = { fr'OBJECT_REDUCED_{self.detector}': self.Product(self, header, combined_image, target=self.target), diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_cal_distortion.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_cal_distortion.py new file mode 100644 index 00000000..b0c6025b --- /dev/null +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_cal_distortion.py @@ -0,0 +1,125 @@ +""" +This file is part of the METIS Pipeline. +Copyright (C) 2024 European Southern Observatory + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import re + +import cpl +from cpl.core import Msg +from typing import Dict + +from pymetis.inputs.common import RawInput, LinearityInput, BadpixMapInput, PersistenceMapInput, GainMapInput +from pymetis.base.recipe import MetisRecipe +from pymetis.base.product import PipelineProduct +from pymetis.inputs import RawInput, SinglePipelineInput +from pymetis.prefab.rawimage import RawImageProcessor + + +class MetisLmImgCalDistortionImpl(RawImageProcessor): + class InputSet(RawImageProcessor.InputSet): + class RawInput(RawInput): + _tags = re.compile(r"LM_WCU_OFF_RAW") + + class DistortionInput(SinglePipelineInput): + _tags = re.compile(r"LM_DISTORTION_RAW") + _title = "Distortion map" + _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB + + class PinholeTableInput(SinglePipelineInput): + _tags = re.compile(r"PINHOLE_TABLE") + _title = "pinhole table" + _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB + + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + self.pinhole_table = SinglePipelineInput(frameset, + tags=re.compile(r"PINHOLE_TABLE"), + title="pinhole table", + group=cpl.ui.Frame.FrameGroup.CALIB) + + self.distortion = self.DistortionInput(frameset, required=False) + self.linearity = LinearityInput(frameset, required=False) # But should be + self.badpix_map = BadpixMapInput(frameset, required=False) + self.persistence_map = PersistenceMapInput(frameset, required=False) # But should be + self.gain_map = GainMapInput(frameset, required=False) # But should be + + self.inputs |= {self.pinhole_table, self.linearity, self.distortion, + self.badpix_map, self.persistence_map, self.gain_map} + + + class ProductLmDistortionTable(PipelineProduct): + category = rf"ILM_DISTORTION_TABLE" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.TABLE + + class ProductLmDistortionMap(PipelineProduct): + category = rf"LM_DIST_MAP" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.IMAGE + + class ProductLmDistortionReduced(PipelineProduct): + category = rf"LM_DIST_REDUCED" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.IMAGE + + def process_images(self) -> Dict[str, PipelineProduct]: + raw_images = cpl.core.ImageList() + + for idx, frame in enumerate(self.inputset.raw.frameset): + Msg.info(self.name, f"Loading raw image {frame.file}") + + if idx == 0: + self.header = cpl.core.PropertyList.load(frame.file, 0) + + raw_image = cpl.core.Image.load(frame.file, extension=1) + raw_images.append(raw_image) + + combined_image = self.combine_images(raw_images, "average") + + self.products = { + product.category: product(self, self.header, combined_image) + for product in [self.ProductLmDistortionTable, self.ProductLmDistortionMap, + self.ProductLmDistortionReduced] + } + return self.products + + +class MetisLmImgCalDistortion(MetisRecipe): + _name = "metis_lm_img_cal_distortion" + _version = "0.1" + _author = "Chi-Hung Yan" + _email = "chyan@asiaa.sinica.edu.tw" + _synopsis = "Determine optical distortion coefficients for the LM imager." + _description = ( + "Currently just a skeleton prototype." + ) + + parameters = cpl.ui.ParameterList([ + cpl.ui.ParameterEnum( + name="metis_lm_img_cal_distortion.stacking.method", + context="metis_lm_img_cal_distortion", + description="Name of the method used to combine the input images", + default="average", + alternatives=("add", "average", "median", "sigclip"), + ), + ]) + + implementation_class = MetisLmImgCalDistortionImpl \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_calibrate.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_calibrate.py new file mode 100644 index 00000000..ee9259b9 --- /dev/null +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_calibrate.py @@ -0,0 +1,111 @@ +""" +This file is part of the METIS Pipeline. +Copyright (C) 2024 European Southern Observatory + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import re + +import cpl +from cpl.core import Msg +from typing import Dict + +from pymetis.inputs.common import RawInput, LinearityInput, BadpixMapInput, PersistenceMapInput, GainMapInput +from pymetis.base.recipe import MetisRecipe +from pymetis.base.product import PipelineProduct +from pymetis.inputs import RawInput, SinglePipelineInput +from pymetis.prefab.rawimage import RawImageProcessor + + +class MetisLmImgCalibrateImpl(RawImageProcessor): + class InputSet(RawImageProcessor.InputSet): + class RawInput(RawInput): + _tags = re.compile(r"LM_SCI_BKG_SUBTRACTED") + + class PinholeTableInput(SinglePipelineInput): + _tags = re.compile(r"FLUXCAL_TAB") + _title = "flux table" + _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB + + class PinholeTableInput(SinglePipelineInput): + _tags = re.compile(r"ILM_DISTORTION_TABLE") + _title = "distortion table" + _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB + + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + self.flux_table = SinglePipelineInput(frameset, + tags=re.compile(r"FLUXCAL_TAB"), + title="pinhole table", + group=cpl.ui.Frame.FrameGroup.CALIB) + + self.distortion_table = SinglePipelineInput(frameset, + tags=re.compile(r"ILM_DISTORTION_TABLE"), + title="distortion table", + group=cpl.ui.Frame.FrameGroup.CALIB) + + + self.inputs |= {self.flux_table, self.distortion_table} + + class ProductLmSciCalibrated(PipelineProduct): + category = rf"LM_SCI_CALIBRATED" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.IMAGE + + + def process_images(self) -> Dict[str, PipelineProduct]: + raw_images = cpl.core.ImageList() + + for idx, frame in enumerate(self.inputset.raw.frameset): + Msg.info(self.name, f"Loading raw image {frame.file}") + + if idx == 0: + self.header = cpl.core.PropertyList.load(frame.file, 0) + + raw_image = cpl.core.Image.load(frame.file, extension=0) + raw_images.append(raw_image) + + combined_image = self.combine_images(raw_images, "average") + + self.products = { + product.category: product(self, self.header, combined_image) + for product in [self.ProductLmSciCalibrated] + } + return self.products + + +class MetisLmImgCalibrate(MetisRecipe): + _name = "metis_lm_img_calibrate" + _version = "0.1" + _author = "Chi-Hung Yan" + _email = "chyan@asiaa.sinica.edu.tw" + _synopsis = "Determine optical distortion coefficients for the LM imager." + _description = ( + "Currently just a skeleton prototype." + ) + + parameters = cpl.ui.ParameterList([ + cpl.ui.ParameterEnum( + name="metis_lm_img_calibrate.stacking.method", + context="metis_lm_img_calibrate", + description="Name of the method used to combine the input images", + default="average", + alternatives=("add", "average", "median", "sigclip"), + ), + ]) + + implementation_class = MetisLmImgCalibrateImpl \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_sci_postprocess.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_sci_postprocess.py new file mode 100644 index 00000000..9c958db0 --- /dev/null +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_sci_postprocess.py @@ -0,0 +1,91 @@ +""" +This file is part of the METIS Pipeline. +Copyright (C) 2024 European Southern Observatory + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import re + +import cpl +from cpl.core import Msg +from typing import Dict + +from pymetis.base.recipe import MetisRecipe +from pymetis.base.product import PipelineProduct +from pymetis.inputs import RawInput, SinglePipelineInput +from pymetis.prefab.rawimage import RawImageProcessor + + +class MetisLmImgSciPostProcessImpl(RawImageProcessor): + class InputSet(RawImageProcessor.InputSet): + class RawInput(RawInput): + _tags = re.compile(r"LM_SCI_CALIBRATED") + + + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + + #self.inputs += [self.fluxstd_table] + + + class ProductLmImgSciCoadd(PipelineProduct): + category = rf"LM_SCI_COADD" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.IMAGE + + def process_images(self) -> Dict[str, PipelineProduct]: + raw_images = cpl.core.ImageList() + + for idx, frame in enumerate(self.inputset.raw.frameset): + Msg.info(self.name, f"Loading raw image {frame.file}") + + if idx == 0: + self.header = cpl.core.PropertyList.load(frame.file, 0) + + raw_image = cpl.core.Image.load(frame.file, extension=0) + raw_images.append(raw_image) + + combined_image = self.combine_images(raw_images, "average") + + self.products = { + product.category: product(self, self.header, combined_image) + for product in [self.ProductLmImgSciCoadd] + } + return self.products + + +class MetisLmImgSciPostProcess(MetisRecipe): + _name = "metis_lm_img_sci_postprocess" + _version = "0.1" + _author = "Chi-Hung Yan" + _email = "chyan@asiaa.sinica.edu.tw" + _synopsis = "Coadd reduced images" + _description = ( + "Currently just a skeleton prototype." + ) + + parameters = cpl.ui.ParameterList([ + cpl.ui.ParameterEnum( + name="metis_lm_img_sci_postprocess.stacking.method", + context="metis_lm_img_sci_postprocess", + description="Name of the method used to combine the input images", + default="average", + alternatives=("add", "average", "median", "sigclip"), + ), + ]) + #import pdb ; pdb.set_trace() + implementation_class = MetisLmImgSciPostProcessImpl \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_std_process.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_std_process.py new file mode 100644 index 00000000..1ddc7413 --- /dev/null +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_std_process.py @@ -0,0 +1,104 @@ +""" +This file is part of the METIS Pipeline. +Copyright (C) 2024 European Southern Observatory + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import re + +import cpl +from cpl.core import Msg +from typing import Dict + +from pymetis.base.recipe import MetisRecipe +from pymetis.base.product import PipelineProduct +from pymetis.inputs import RawInput, SinglePipelineInput +from pymetis.prefab.rawimage import RawImageProcessor + + +class MetisLmImgsStdProcessImpl(RawImageProcessor): + class InputSet(RawImageProcessor.InputSet): + class RawInput(RawInput): + _tags = re.compile(r"LM_STD_BKG_SUBTRACTED") + + class FluxTableInput(SinglePipelineInput): + _tags = re.compile(r"FLUXSTD_CATALOG") + _title = "flux standard star catalogue table" + _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB + + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + self.fluxstd_table = SinglePipelineInput(frameset, + tags=re.compile(r"FLUXSTD_CATALOG"), + title="flux standard star catalogue table", + group=cpl.ui.Frame.FrameGroup.CALIB) + self.inputs |= {self.fluxstd_table} + + #import pdb ; pdb.set_trace() + class ProductLmImgFluxCalTable(PipelineProduct): + category = rf"FLUXCAL_TAB" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.TABLE + + class ProductLmImgStdCombined(PipelineProduct): + category = rf"LM_STD_COMBINED" + tag = category + level = cpl.ui.Frame.FrameLevel.FINAL + frame_type = cpl.ui.Frame.FrameType.IMAGE + + def process_images(self) -> Dict[str, PipelineProduct]: + raw_images = cpl.core.ImageList() + + for idx, frame in enumerate(self.inputset.raw.frameset): + Msg.info(self.name, f"Loading raw image {frame.file}") + + if idx == 0: + self.header = cpl.core.PropertyList.load(frame.file, 0) + + raw_image = cpl.core.Image.load(frame.file, extension=0) + raw_images.append(raw_image) + + combined_image = self.combine_images(raw_images, "average") + + self.products = { + product.category: product(self, self.header, combined_image) + for product in [self.ProductLmImgFluxCalTable, self.ProductLmImgStdCombined] + } + return self.products + + +class MetisLmImgStdProcess(MetisRecipe): + _name = "metis_lm_img_std_process" + _version = "0.1" + _author = "Chi-Hung Yan" + _email = "chyan@asiaa.sinica.edu.tw" + _synopsis = "Determine conversion factor between detector counts and physical source flux" + _description = ( + "Currently just a skeleton prototype." + ) + + parameters = cpl.ui.ParameterList([ + cpl.ui.ParameterEnum( + name="metis_lm_img_std_process.stacking.method", + context="metis_lm_img_std_process", + description="Name of the method used to combine the input images", + default="average", + alternatives=("add", "average", "median", "sigclip"), + ), + ]) + + implementation_class = MetisLmImgsStdProcessImpl \ No newline at end of file diff --git a/metisp/pyrecipes/metis_recipes.py b/metisp/pyrecipes/metis_recipes.py index d7dcd3bc..3f90172d 100644 --- a/metisp/pyrecipes/metis_recipes.py +++ b/metisp/pyrecipes/metis_recipes.py @@ -21,6 +21,11 @@ from pymetis.recipes.metis_det_dark import MetisDetDark from pymetis.recipes.img.metis_lm_img_basic_reduce import MetisLmImgBasicReduce from pymetis.recipes.img.metis_lm_img_flat import MetisLmImgFlat +from pymetis.recipes.img.metis_lm_img_background import MetisLmImgBackground +from pymetis.recipes.img.metis_lm_img_std_process import MetisLmImgStdProcess +from pymetis.recipes.img.metis_lm_img_cal_distortion import MetisLmImgCalDistortion +from pymetis.recipes.img.metis_lm_img_calibrate import MetisLmImgCalibrate +from pymetis.recipes.img.metis_lm_img_sci_postprocess import MetisLmImgSciPostProcess from pymetis.recipes.img.metis_n_img_flat import MetisNImgFlat from pymetis.recipes.ifu.metis_ifu_distortion import MetisIfuDistortion from pymetis.recipes.ifu.metis_ifu_calibrate import MetisIfuCalibrate @@ -37,8 +42,13 @@ MetisDetLinGain, MetisDetDark, MetisLmImgBasicReduce, + MetisLmImgBackground, + MetisLmImgStdProcess, MetisLmImgFlat, MetisNImgFlat, + MetisLmImgCalDistortion, + MetisLmImgCalibrate, + MetisLmImgSciPostProcess, MetisIfuDistortion, MetisIfuCalibrate, MetisIfuPostprocess, diff --git a/metisp/workflows/metis/metis_classification.py b/metisp/workflows/metis/metis_classification.py index ea1bfe91..32ee0ed1 100644 --- a/metisp/workflows/metis/metis_classification.py +++ b/metisp/workflows/metis/metis_classification.py @@ -2,7 +2,7 @@ from . import metis_keywords as metis_kwd # Detector linearity calibration classification -detlin_class = classification_rule("DETLIN_2RG_RAW", +detlin_2rg_raw_class = classification_rule("DETLIN_2RG_RAW", {metis_kwd.instrume: "METIS", metis_kwd.dpr_catg: "CALIB", metis_kwd.dpr_type: "DETLIN", @@ -17,6 +17,21 @@ metis_kwd.drp_tech: "IMAGE,LM", }) +lm_distortion_raw_class = classification_rule("LM_DISTORTION_RAW", + {metis_kwd.instrume: "METIS", + metis_kwd.dpr_catg: "CALIB", + metis_kwd.dpr_type: "DISTORTION", + metis_kwd.drp_tech: "IMAGE,LM", + }) + +lm_wcu_off_raw_class = classification_rule("LM_WCU_OFF_RAW", + {metis_kwd.instrume: "METIS", + metis_kwd.dpr_catg: "CALIB", + metis_kwd.dpr_type: "DARK,WCUOFF", + metis_kwd.drp_tech: "IMAGE,LM", + }) + + # Lamp flat calibration classification lm_lampflat_class = classification_rule("LM_FLAT_LAMP_RAW", {metis_kwd.instrume: "METIS", @@ -39,4 +54,28 @@ metis_kwd.dpr_catg: "SCIENCE", metis_kwd.dpr_type: "OBJECT", metis_kwd.drp_tech: "IMAGE,LM", - }) \ No newline at end of file + }) + +raw_sky_class = classification_rule("LM_IMAGE_SKY_RAW", + {metis_kwd.instrume: "METIS", + metis_kwd.dpr_catg: "SCIENCE", + metis_kwd.dpr_type: "SKY", + metis_kwd.drp_tech: "IMAGE,LM", + }) + +raw_std_class = classification_rule("LM_IMAGE_STD_RAW", + {metis_kwd.instrume: "METIS", + metis_kwd.dpr_catg: "CALIB", + metis_kwd.dpr_type: "STD", + metis_kwd.drp_tech: "IMAGE,LM", + }) + +# Flux standard catalog classification +fluxstd_catalog_class = classification_rule("FLUXSTD_CATALOG", + {metis_kwd.pro_catg: "FLUXSTD_CATALOG", + }) + +# Pinhole table classification +pinhole_table_class = classification_rule("PINHOLE_TABLE", + {metis_kwd.pro_catg: "PINHOLE_TABLE", + }) diff --git a/metisp/workflows/metis/metis_datasources.py b/metisp/workflows/metis/metis_datasources.py index 44c369b9..cb2b578d 100644 --- a/metisp/workflows/metis/metis_datasources.py +++ b/metisp/workflows/metis/metis_datasources.py @@ -19,8 +19,8 @@ # --- Data sources --- -detlin_raw = (data_source() - .with_classification_rule(detlin_class) +detlin_2rg_raw = (data_source() + .with_classification_rule(detlin_2rg_raw_class) .with_match_keywords(["instrume"]) .build()) @@ -34,7 +34,37 @@ .with_match_keywords(["instrume"]) .build()) +lm_distortion_raw = (data_source() + .with_classification_rule(lm_distortion_raw_class) + .with_match_keywords(["instrume"]) + .build()) + +lm_wcu_off_raw = (data_source() + .with_classification_rule(lm_wcu_off_raw_class) + .with_match_keywords(["instrume"]) + .build()) + lm_raw_science = (data_source() .with_classification_rule(raw_science_class) .with_match_keywords(["instrume"]) .build()) + +lm_raw_sky = (data_source() + .with_classification_rule(raw_sky_class) + .with_match_keywords(["instrume"]) + .build()) + +lm_raw_std = (data_source() + .with_classification_rule(raw_std_class) + .with_match_keywords(["instrume"]) + .build()) + +fluxstd_catalog = (data_source() + .with_classification_rule(fluxstd_catalog_class) + .with_match_keywords(["instrume"]) + .build()) + +pinehole_tab = (data_source() + .with_classification_rule(pinhole_table_class) + .with_match_keywords(["instrume"]) + .build()) \ No newline at end of file diff --git a/metisp/workflows/metis/metis_lm_img_wkf.py b/metisp/workflows/metis/metis_lm_img_wkf.py index bb1bcb2e..9fe7eec7 100644 --- a/metisp/workflows/metis/metis_lm_img_wkf.py +++ b/metisp/workflows/metis/metis_lm_img_wkf.py @@ -10,7 +10,7 @@ .build()) lingain_task = (task('metis_det_detlin') - .with_main_input(detlin_raw) + .with_main_input(detlin_2rg_raw) .with_associated_input(dark_task) .with_recipe("metis_det_lingain") .build()) @@ -21,7 +21,15 @@ .with_recipe("metis_lm_img_flat") .build()) -basic_reduction = (task('metis_lm_img_basic_reduce') +distortion_task = (task('metis_lm_img_cal_distortion') + .with_main_input(lm_distortion_raw) + .with_associated_input(lm_wcu_off_raw) + .with_associated_input(pinehole_tab) + .with_associated_input(lingain_task) + .with_recipe('metis_lm_img_cal_distortion') + .build()) + +basic_reduction_sci = (task('metis_lm_img_basic_reduce_sci') .with_recipe('metis_lm_img_basic_reduce') .with_main_input(lm_raw_science) .with_associated_input(lingain_task) @@ -30,3 +38,56 @@ .with_meta_targets([SCIENCE]) .build()) +basic_reduction_sky = (task('metis_lm_img_basic_reduce_sky') + .with_recipe('metis_lm_img_basic_reduce') + .with_main_input(lm_raw_sky) + .with_associated_input(lingain_task) + .with_associated_input(dark_task) + .with_associated_input(flat_task) + .with_meta_targets([SCIENCE]) + .build()) + +basic_reduction_std = (task('metis_lm_img_basic_reduce_std') + .with_recipe('metis_lm_img_basic_reduce') + .with_main_input(lm_raw_std) + .with_associated_input(lingain_task) + .with_associated_input(dark_task) + .with_associated_input(flat_task) + .with_meta_targets([SCIENCE]) + .build()) + +background_sci_task = (task('metis_lm_img_background_sci') + .with_recipe('metis_lm_img_background') + .with_main_input(basic_reduction_sci) + .with_associated_input(basic_reduction_sky) + .with_meta_targets([SCIENCE]) + .build()) + +background_std_task = (task('metis_lm_img_background_std') + .with_recipe('metis_lm_img_background') + .with_main_input(basic_reduction_std) + .with_associated_input(basic_reduction_sky) + .with_meta_targets([SCIENCE]) + .build()) + +standard_flux_task = (task('metis_lm_img_standard_flux') + .with_recipe('metis_lm_img_std_process') + .with_main_input(background_std_task) + .with_associated_input(fluxstd_catalog) + .with_meta_targets([SCIENCE]) + .build()) + +img_calib = (task('metis_lm_img_calib') + .with_recipe('metis_lm_img_calibrate') + .with_main_input(background_sci_task) + .with_associated_input(standard_flux_task) + .with_associated_input(distortion_task) + .with_meta_targets([SCIENCE]) + .build()) + +img_coadd = (task('metis_lm_img_coadd') + .with_recipe('metis_lm_img_sci_postprocess') + .with_main_input(img_calib) + .with_meta_targets([SCIENCE]) + .build()) +# QC1 \ No newline at end of file