Integrate preprocess scripts

hexrd/cli/main.py

@@ -16,6 +16,7 @@
 from hexrd.cli import find_orientations
 from hexrd.cli import fit_grains
 from hexrd.cli import pickle23
+from hexrd.cli import preprocess
 
 
 try:
@@ -66,6 +67,7 @@ def main():
     find_orientations.configure_parser(sub_parsers)
     fit_grains.configure_parser(sub_parsers)
     pickle23.configure_parser(sub_parsers)
+    preprocess.configure_parser(sub_parsers)
 
     try:
         import argcomplete

hexrd/cli/preprocess.py

@@ -0,0 +1,101 @@
+import dataclasses
+from hexrd.preprocess.profiles import HexrdPPScript_Arguments
+from hexrd.preprocess.preprocessors import preprocess
+from dataclasses import fields
+
+import argparse
+
+_description = 'Preprocess detector images'
+_help = "Preprocess data from detector and attach metadata"
+
+
+def configure_parser(sub_parsers):
+    parser = sub_parsers.add_parser(
+        'preprocess', description=_description, help=_help
+    )
+
+    subparsers = parser.add_subparsers(
+        dest="profile", required=True, help="Select detector profile"
+    )
+
+    for fmt in HexrdPPScript_Arguments.known_formats():
+        klass = HexrdPPScript_Arguments.create_args(fmt)
+        add_profile_subparser(subparsers, fmt, klass)
+
+    parser.set_defaults(func=execute)
+
+
+def execute(args, parser):
+    kwargs, extra = _remove_non_dataclass_args(vars(args))
+
+    if extra["generate_default_config"]:
+        s = HexrdPPScript_Arguments.create_default_config(extra["profile"])
+        print(s)
+    else:
+        if extra["config"] is not None:
+            args_object = HexrdPPScript_Arguments.load_from_config(
+                extra["config"].read()
+            )
+        else:
+            args_object = HexrdPPScript_Arguments.create_args(
+                extra["profile"], **kwargs
+            )
+        preprocess(args_object)
+
+
+def add_profile_subparser(subparsers, name, klass):
+    """Create a subparser with the options related to detector `name` using
+    `klass` to retrieve default values"""
+
+    subparser = subparsers.add_parser(
+        name,
+        help=f"{name} help",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    help_messages = getattr(klass, "help_messages")
+    short_switches = getattr(klass, "short_switches")
+
+    for field in fields(klass):
+        switches = [f"--{field.name}"]
+        if field.name in short_switches:
+            switch = short_switches[field.name]
+            switches.insert(0, f"-{switch}")
+        default_value = None
+        if field.default_factory != dataclasses.MISSING:
+            default_value = field.default_factory()
+        else:
+            default_value = field.default
+
+        subparser.add_argument(
+            *switches,
+            type=field.type,
+            default=default_value,
+            help=help_messages[field.name],
+        )
+
+    subparser.add_argument(
+        "--generate-default-config",
+        action="store_true",
+        help="Generate config file with default values",
+    )
+    subparser.add_argument(
+        "--config",
+        type=argparse.FileType("r"),
+        required=False,
+        help="Read arguments from .pp config file",
+    )
+
+
+def _remove_non_dataclass_args(args_dict):
+    """Remove args that do not belong to any dataclass. These are standard args
+    we manually inserted and now remove to allow the rest of the arguments to
+    initialize dataclass"""
+
+    extra = {}
+    for key in ["profile", "config", "generate_default_config"]:
+        v = args_dict.get(key, None)
+        extra[key] = v
+        if v is not None:
+            del args_dict[key]
+    return args_dict, extra

hexrd/preprocess/argument_classes_factory.py

@@ -0,0 +1,25 @@
+class ArgumentClassesFactory:
+    """A factory to collect all Argument classes"""
+
+    _creators = {}
+
+    @classmethod
+    def register(cls, klass):
+        cls._creators[klass.profile_name] = klass
+
+    @classmethod
+    def get_registered(cls):
+        return cls._creators.keys()
+
+    @classmethod
+    def get_args(cls, profile_name):
+        creator = cls._creators.get(profile_name)
+        if not creator:
+            raise ValueError(format)
+        return creator
+
+
+def autoregister(cls):
+    """decorator that registers cls with ArgumentClassesFactory"""
+    ArgumentClassesFactory().register(cls)
+    return cls

hexrd/preprocess/preprocessors.py

@@ -0,0 +1,205 @@
+from hexrd.preprocess.profiles import Eiger_Arguments, Dexelas_Arguments
+from hexrd import imageseries
+from hexrd.imageseries.process import ProcessedImageSeries
+import os
+import time
+
+
+class PP_Base(object):
+    PROCFMT = None
+    RAWFMT = None
+
+    def __init__(
+        self, fname, omw, panel_opts, frame_start=0, style="npz", **kwargs
+    ):
+        self.fname = fname
+        self.omwedges = omw
+        self.panel_opts = panel_opts
+        self.frame_start = frame_start
+        self.raw = imageseries.open(self.fname, format=self.RAWFMT)
+        self.use_frame_list = self.nframes != len(self.raw)
+        self.style = style
+        print(
+            f"On Init:\n\t{self.fname}, {self.nframes} frames,"
+            f"{self.omwedges.nframes} omw, {len(self.raw)} total"
+        )
+
+    @property
+    def oplist(self):
+        return self.panel_opts
+
+    @property
+    def framelist(self):
+        return range(self.frame_start, self.nframes + self.frame_start)
+
+    @property
+    def nframes(self):
+        return self.omwedges.nframes
+
+    @property
+    def omegas(self):
+        return self.omwedges.omegas
+
+    def processed(self):
+        kw = {}
+        if self.use_frame_list:
+            kw = dict(frame_list=self.framelist)
+        return ProcessedImageSeries(self.raw, self.oplist, **kw)
+
+    def _attach_metadata(self, metadata):
+        metadata["omega"] = self.omegas
+
+    def save_processed(self, name, threshold, output_dir=None):
+        if output_dir is None:
+            output_dir = os.getcwd()
+        else:
+            os.mkdir(output_dir)
+
+        # add omegas
+        pims = self.processed()
+        self._attach_metadata(pims.metadata)
+        cache = f"{name}-cachefile." + f"{self.style}"
+        imageseries.write(
+            pims,
+            "dummy",
+            self.PROCFMT,
+            style=self.style,
+            threshold=threshold,
+            cache_file=cache,
+        )
+
+
+class PP_Eiger(PP_Base):
+    """PP_Eiger"""
+
+    PROCFMT = "frame-cache"
+    RAWFMT = "eiger-stream-v1"
+
+    def __init__(self, fname, omw, panel_opts, frame_start=0, style="npz"):
+        super().__init__(
+            fname=fname,
+            omw=omw,
+            panel_opts=panel_opts,
+            frame_start=frame_start,
+            style=style,
+        )
+
+
+class PP_Dexela(PP_Base):
+    """PP_Dexela"""
+
+    PROCFMT = "frame-cache"
+    RAWFMT = "hdf5"
+
+    RAWPATH = "/imageseries"
+    DARKPCTILE = 50
+
+    def __init__(
+        self,
+        fname,
+        omw,
+        panel_opts,
+        panel_id,
+        frame_start=0,
+        style="npz",
+        raw_format="hdf5",
+        dark=None,
+    ):
+        super().__init__(
+            fname=fname,
+            omw=omw,
+            panel_opts=panel_opts,
+            frame_start=frame_start,
+            style=style,
+        )
+
+        self._panel_id = panel_id
+        if raw_format.lower() == "hdf5":
+            self.raw = imageseries.open(
+                self.fname, self.RAWFMT, path=self.RAWPATH
+            )
+        else:
+            self.raw = imageseries.open(self.fname, raw_format.lower())
+        self._dark = dark
+
+        self.use_frame_list = self.nframes != len(
+            self.raw
+        )  # Framelist fix, DCP 6/18/18
+
+    def _attach_metadata(self, metadata):
+        super()._attach_metadata(metadata)
+        metadata["panel_id"] = self.panel_id
+
+    @property
+    def panel_id(self):
+        return self._panel_id
+
+    @property
+    def dark(self, nframes=100):
+        """build and return dark image"""
+        if self._dark is None:
+            usenframes = min(nframes, self.nframes)
+            print(
+                "building dark images using %s frames (may take a while)..."
+                % usenframes
+            )
+            start = time.time()
+            #            self._dark = imageseries.stats.percentile(
+            #                    self.raw, self.DARKPCTILE, nframes=usenframes
+            #            )
+            self._dark = imageseries.stats.median(
+                self.raw, nframes=usenframes
+            )  # changed to median by DCP 11/18/17
+            elapsed = time.time() - start
+            print(
+                "done building background (dark) image: "
+                + "elapsed time is %f seconds" % elapsed
+            )
+
+        return self._dark
+
+
+def preprocess(args):
+    omw = imageseries.omega.OmegaWedges(args.num_frames)
+    omw.addwedge(args.ome_start, args.ome_end, args.num_frames)
+
+    if type(args) == Eiger_Arguments:
+        pp = PP_Eiger(
+            fname=args.file_name,
+            omw=omw,
+            panel_opts=args.panel_opts,
+            frame_start=args.start_frame,
+            style=args.style,
+        )
+        pp.save_processed(args.output, args.threshold)
+    elif type(args) == Dexelas_Arguments:
+        pp = PP_Dexela(
+            fname=args.file_name,
+            omw=omw,
+            panel_opts=args.panel_opts,
+            frame_start=args.start_frame,
+            style=args.style,
+        )
+
+        for file_name in pp.file_name:
+            for key in args.panel_keys:
+                if key.lower() in file_name:
+                    pp = PP_Dexela(
+                        fname=args.file_name,
+                        omw=omw,
+                        panel_opts=args.panel_opts[key],
+                        panel_id=key,
+                        frame_start=args.start_frame,
+                        style=args.style,
+                    )
+
+                    output_name = (
+                        args.samp_name
+                        + "_"
+                        + str(args.scan_number)
+                        + "_"
+                        + file_name.split("/")[-1].split(".")[0]
+                    )
+                    pp.save_processed(output_name, args.threshold)
+    else:
+        raise AttributeError(f"Unknown argument type: {type(args)}")