From a85622d48a546cf97a737b7302779dd1301bc8c8 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 16 Feb 2023 16:30:15 +0000 Subject: [PATCH 01/11] something odd about how images are being passed --- setup.py | 4 +- src/deepsparse/open_pif_paf/README_temp.md | 10 + src/deepsparse/open_pif_paf/pipelines.py | 29 +- src/deepsparse/open_pif_paf/schemas.py | 13 +- src/deepsparse/open_pif_paf/validation.py | 420 +++++++++++++++++++++ src/deepsparse/yolact/utils/utils.py | 8 - 6 files changed, 457 insertions(+), 27 deletions(-) create mode 100644 src/deepsparse/open_pif_paf/README_temp.md create mode 100644 src/deepsparse/open_pif_paf/validation.py diff --git a/setup.py b/setup.py index 0194eb856c..66ecf77417 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,7 @@ def _parse_requirements_file(file_path): "protobuf>=3.12.2,<=3.20.1", "click>=7.1.2,!=8.0.0", # latest version < 8.0 + blocked version with reported bug ] -_nm_deps = [f"{'sparsezoo' if is_release else 'sparsezoo-nightly'}~={version_base}"] +_nm_deps = []#[f"{'sparsezoo' if is_release else 'sparsezoo-nightly'}~={version_base}"] _dev_deps = [ "beautifulsoup4>=4.9.3", "black==22.12.0", @@ -133,7 +133,7 @@ def _parse_requirements_file(file_path): "opencv-python<=4.6.0.66", ] _openpifpaf_integration_deps = [ - "openpifpaf==0.13.6", + "openpifpaf==0.13.11", "opencv-python<=4.6.0.66", ] # haystack dependencies are installed from a requirements file to avoid diff --git a/src/deepsparse/open_pif_paf/README_temp.md b/src/deepsparse/open_pif_paf/README_temp.md new file mode 100644 index 0000000000..7362a0bd83 --- /dev/null +++ b/src/deepsparse/open_pif_paf/README_temp.md @@ -0,0 +1,10 @@ +For training and evaluation, you need to download the dataset. + +mkdir data-crowdpose +cd data-crowdpose +# download links here: https://github.com/Jeff-sjtu/CrowdPose +unzip annotations.zip +unzip images.zip +Now you can use the standard openpifpaf.train and openpifpaf.eval commands as documented in Training with --dataset=crowdpose. + +install pycocotools \ No newline at end of file diff --git a/src/deepsparse/open_pif_paf/pipelines.py b/src/deepsparse/open_pif_paf/pipelines.py index ddaa9f1fc3..6994f1a4b3 100644 --- a/src/deepsparse/open_pif_paf/pipelines.py +++ b/src/deepsparse/open_pif_paf/pipelines.py @@ -22,7 +22,7 @@ import cv2 import torch -from deepsparse.open_pif_paf.schemas import OpenPifPafInput, OpenPifPafOutput +from deepsparse.open_pif_paf.schemas import OpenPifPafInput, OpenPifPafOutput, OpenPifPafFields from deepsparse.pipeline import Pipeline from deepsparse.yolact.utils import preprocess_array from openpifpaf import decoder, network @@ -57,12 +57,10 @@ class OpenPifPafPipeline(Pipeline): """ def __init__( - self, *, image_size: Union[int, Tuple[int, int]] = (384, 384), **kwargs + self, *, output_fields = False, **kwargs ): super().__init__(**kwargs) - self._image_size = ( - image_size if isinstance(image_size, Tuple) else (image_size, image_size) - ) + self.output_fields = output_fields # necessary openpifpaf dependencies for now model_cpu, _ = network.Factory().factory(head_metas=None) self.processor = decoder.factory(model_cpu.head_metas) @@ -79,7 +77,7 @@ def output_schema(self) -> Type[OpenPifPafOutput]: """ :return: pydantic model class that outputs to this pipeline must comply to """ - return OpenPifPafOutput + return OpenPifPafOutput if not self.output_fields else OpenPifPafFields def setup_onnx_file_path(self) -> str: """ @@ -93,16 +91,13 @@ class properties into an inference ready onnx file to be compiled by the def process_inputs(self, inputs: OpenPifPafInput) -> List[numpy.ndarray]: - images = inputs.images - - if not isinstance(images, list): - images = [images] - - image_batch = list(self.executor.map(self._preprocess_image, images)) - - image_batch = numpy.concatenate(image_batch, axis=0) + image = inputs.images + image = image.astype(numpy.float32) + #image = image.transpose(0, 2, 3, 1) + image /= 255 + image = numpy.ascontiguousarray(image) - return [image_batch] + return [image] def process_engine_outputs( self, fields: List[numpy.ndarray], **kwargs @@ -114,6 +109,8 @@ def process_engine_outputs( :return: Outputs of engine post-processed into an object in the `output_schema` format of this pipeline """ + if self.output_fields: + return OpenPifPafFields(fields=fields) data_batch, skeletons_batch, scores_batch, keypoints_batch = [], [], [], [] @@ -137,4 +134,4 @@ def _preprocess_image(self, image) -> numpy.ndarray: if isinstance(image, str): image = cv2.imread(image) - return preprocess_array(image, input_image_size=self._image_size) + return preprocess_array(image) diff --git a/src/deepsparse/open_pif_paf/schemas.py b/src/deepsparse/open_pif_paf/schemas.py index 2c384807b5..b150bf689f 100644 --- a/src/deepsparse/open_pif_paf/schemas.py +++ b/src/deepsparse/open_pif_paf/schemas.py @@ -13,7 +13,7 @@ # limitations under the License. from typing import List, Tuple - +import numpy from pydantic import BaseModel, Field from deepsparse.pipelines.computer_vision import ComputerVisionSchema @@ -22,6 +22,7 @@ __all__ = [ "OpenPifPafInput", "OpenPifPafOutput", + "OpenPifPafFields", ] @@ -32,6 +33,16 @@ class OpenPifPafInput(ComputerVisionSchema): pass +class OpenPifPafFields(BaseModel): + """ + # TODO + """ + + fields: List[numpy.ndarray] = Field(description="") + + class Config: + arbitrary_types_allowed = True + class OpenPifPafOutput(BaseModel): """ diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py new file mode 100644 index 0000000000..dedeba1899 --- /dev/null +++ b/src/deepsparse/open_pif_paf/validation.py @@ -0,0 +1,420 @@ +"""Evaluation on COCO data.""" +from openpifpaf.eval import cli, Evaluator, Predictor, network, count_ops, __version__, LOG, show +from openpifpaf import datasets, decoder, visualizer, transforms +from openpifpaf.decoder import CifCaf +import typing as t +import json +from collections import defaultdict +import sys +import os +import time +import PIL.Image +from openpifpaf.decoder.multi import Multi +import argparse + +import PIL +import torch + +from deepsparse import Pipeline + +class DummyPool(): + @staticmethod + def starmap(f, iterable): + return [f(*i) for i in iterable] + +def remove_normalization(data_loader: t.Iterable[t.Any]) -> t.Iterable[t.Any]: + dataset = data_loader.dataset + if not hasattr(dataset, "preprocess"): + raise AttributeError("") + + preprocess = dataset.preprocess + assert len(preprocess.preprocess_list) == 6 + + image_preprocess = preprocess.preprocess_list[5] + assert len(image_preprocess.preprocess_list) == 3 + + assert image_preprocess.preprocess_list[2].image_transform.__class__.__name__ == "Normalize" + + preprocess.preprocess_list[5].preprocess_list = image_preprocess.preprocess_list[:2] + + + return data_loader + +class DeepSparseDecoder(decoder.Decoder): + def batch(self, pipeline, model, image_batch, *, device=None, gt_anns_batch=None): + """From image batch straight to annotations batch.""" + start_nn = time.perf_counter() + fields_batch = self.fields_batch(model, image_batch, device=device) + self.last_nn_time = time.perf_counter() - start_nn + + if gt_anns_batch is None: + gt_anns_batch = [None for _ in fields_batch] + + if not isinstance(self.worker_pool, DummyPool): + # remove debug_images to save time during pickle + image_batch = [None for _ in fields_batch] + gt_anns_batch = [None for _ in fields_batch] + + LOG.debug('parallel execution with worker %s', self.worker_pool) + start_decoder = time.perf_counter() + result = self.worker_pool.starmap( + self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch)) + self.last_decoder_time = time.perf_counter() - start_decoder + + LOG.debug('time: nn = %.1fms, dec = %.1fms', + self.last_nn_time * 1000.0, + self.last_decoder_time * 1000.0) + return result + +def fields_batch_deepsparse_to_torch(fields_batch,device): + result = [] + fields = fields_batch.fields + for idx, (cif, caf) in enumerate(zip(*fields)): + result.append([torch.from_numpy(cif), torch.from_numpy(caf)]) + return result + +import numpy +class DeepSparseCifCaf(CifCaf): + def __init__(self, head_metas): + cif_metas, caf_metas = head_metas + super().__init__([cif_metas], [caf_metas]) + + def batch(self, pipeline, model, image_batch, *, device=None, gt_anns_batch=None): + """From image batch straight to annotations batch.""" + start_nn = time.perf_counter() + fields_batch = self.fields_batch(model, image_batch, device=device) + image_batch = image_batch.numpy() * 255 + image_batch = image_batch[:,:,:480, :640] + fields_batch_deepsparse= pipeline(images=image_batch.astype(numpy.uint8)) + fields_batch = fields_batch_deepsparse_to_torch(fields_batch_deepsparse, device=device) + + self.last_nn_time = time.perf_counter() - start_nn + + if gt_anns_batch is None: + gt_anns_batch = [None for _ in fields_batch] + + if not isinstance(self.worker_pool, DummyPool): + # remove debug_images to save time during pickle + image_batch = [None for _ in fields_batch] + gt_anns_batch = [None for _ in fields_batch] + + LOG.debug('parallel execution with worker %s', self.worker_pool) + start_decoder = time.perf_counter() + result = self.worker_pool.starmap( + self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch)) + self.last_decoder_time = time.perf_counter() - start_decoder + + LOG.debug('time: nn = %.1fms, dec = %.1fms', + self.last_nn_time * 1000.0, + self.last_decoder_time * 1000.0) + return result + +class DeepSparsePredictor: + """Convenience class to predict from various inputs with a common configuration.""" + + batch_size = 1 #: batch size + device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') #: device + fast_rescaling = True #: fast rescaling + loader_workers = None #: loader workers + long_edge = None #: long edge + + def __init__(self, checkpoint=None, head_metas=None, *, + json_data=False, + visualize_image=False, + visualize_processed_image=False): + if checkpoint is not None: + network.Factory.checkpoint = checkpoint + self.json_data = json_data + self.visualize_image = visualize_image + self.visualize_processed_image = visualize_processed_image + + self.model_cpu, _ = network.Factory().factory(head_metas=head_metas) + self.model = self.model_cpu.to(self.device) + if self.device.type == 'cuda' and torch.cuda.device_count() > 1: + LOG.info('Using multiple GPUs: %d', torch.cuda.device_count()) + self.model = torch.nn.DataParallel(self.model) + self.model.base_net = self.model_cpu.base_net + self.model.head_nets = self.model_cpu.head_nets + + self.preprocess = self._preprocess_factory() + LOG.debug('head names = %s', [meta.name for meta in head_metas]) + self.processor = DeepSparseCifCaf(self.model_cpu.head_metas) + + self.last_decoder_time = 0.0 + self.last_nn_time = 0.0 + self.total_nn_time = 0.0 + self.total_decoder_time = 0.0 + self.total_images = 0 + + LOG.info('neural network device: %s (CUDA available: %s, count: %d)', + self.device, torch.cuda.is_available(), torch.cuda.device_count()) + + @classmethod + def cli(cls, parser: argparse.ArgumentParser, *, + skip_batch_size=False, skip_loader_workers=False): + """Add command line arguments. + + When using this class together with datasets (e.g. in eval), + skip the cli arguments for batch size and loader workers as those + will be provided via the datasets module. + """ + group = parser.add_argument_group('Predictor') + + if not skip_batch_size: + group.add_argument('--batch-size', default=cls.batch_size, type=int, + help='processing batch size') + + if not skip_loader_workers: + group.add_argument('--loader-workers', default=cls.loader_workers, type=int, + help='number of workers for data loading') + + group.add_argument('--long-edge', default=cls.long_edge, type=int, + help='rescale the long side of the image (aspect ratio maintained)') + group.add_argument('--precise-rescaling', dest='fast_rescaling', + default=True, action='store_false', + help='use more exact image rescaling (requires scipy)') + + @classmethod + def configure(cls, args: argparse.Namespace): + """Configure from command line parser.""" + cls.batch_size = args.batch_size + cls.device = args.device + cls.fast_rescaling = args.fast_rescaling + cls.loader_workers = args.loader_workers + cls.long_edge = args.long_edge + + def _preprocess_factory(self): + rescale_t = None + if self.long_edge: + rescale_t = transforms.RescaleAbsolute(self.long_edge, fast=self.fast_rescaling) + + pad_t = None + if self.batch_size > 1: + assert self.long_edge, '--long-edge must be provided for batch size > 1' + pad_t = transforms.CenterPad(self.long_edge) + else: + pad_t = transforms.CenterPadTight(16) + + return transforms.Compose([ + transforms.NormalizeAnnotations(), + rescale_t, + pad_t, + transforms.EVAL_TRANSFORM, + ]) + + def dataset(self, data): + """Predict from a dataset.""" + loader_workers = self.loader_workers + if loader_workers is None: + loader_workers = self.batch_size if len(data) > 1 else 0 + + dataloader = torch.utils.data.DataLoader( + data, batch_size=self.batch_size, shuffle=False, + pin_memory=self.device.type != 'cpu', + num_workers=loader_workers, + collate_fn=datasets.collate_images_anns_meta) + + yield from self.dataloader(dataloader) + + def enumerated_dataloader(self, pipeline, enumerated_dataloader): + """Predict from an enumerated dataloader.""" + for batch_i, item in enumerated_dataloader: + if len(item) == 3: + processed_image_batch, gt_anns_batch, meta_batch = item + image_batch = [None for _ in processed_image_batch] + elif len(item) == 4: + image_batch, processed_image_batch, gt_anns_batch, meta_batch = item + if self.visualize_processed_image: + visualizer.Base.processed_image(processed_image_batch[0]) + + pred_batch = self.processor.batch(pipeline, self.model, processed_image_batch, device=self.device) + self.last_decoder_time = self.processor.last_decoder_time + self.last_nn_time = self.processor.last_nn_time + self.total_decoder_time += self.processor.last_decoder_time + self.total_nn_time += self.processor.last_nn_time + self.total_images += len(processed_image_batch) + + # un-batch + for image, pred, gt_anns, meta in \ + zip(image_batch, pred_batch, gt_anns_batch, meta_batch): + LOG.info('batch %d: %s', batch_i, meta.get('file_name', 'no-file-name')) + + # load the original image if necessary + if self.visualize_image: + visualizer.Base.image(image, meta=meta) + + pred = [ann.inverse_transform(meta) for ann in pred] + gt_anns = [ann.inverse_transform(meta) for ann in gt_anns] + + if self.json_data: + pred = [ann.json_data() for ann in pred] + + yield pred, gt_anns, meta + + def dataloader(self, dataloader): + """Predict from a dataloader.""" + yield from self.enumerated_dataloader(enumerate(dataloader)) + + def image(self, file_name): + """Predict from an image file name.""" + return next(iter(self.images([file_name]))) + + def images(self, file_names, **kwargs): + """Predict from image file names.""" + data = datasets.ImageList( + file_names, preprocess=self.preprocess, with_raw_image=True) + yield from self.dataset(data, **kwargs) + + def pil_image(self, image): + """Predict from a Pillow image.""" + return next(iter(self.pil_images([image]))) + + def pil_images(self, pil_images, **kwargs): + """Predict from Pillow images.""" + data = datasets.PilImageList( + pil_images, preprocess=self.preprocess, with_raw_image=True) + yield from self.dataset(data, **kwargs) + + def numpy_image(self, image): + """Predict from a numpy image.""" + return next(iter(self.numpy_images([image]))) + + def numpy_images(self, numpy_images, **kwargs): + """Predict from numpy images.""" + data = datasets.NumpyImageList( + numpy_images, preprocess=self.preprocess, with_raw_image=True) + yield from self.dataset(data, **kwargs) + + def image_file(self, file_pointer): + """Predict from an opened image file pointer.""" + pil_image = PIL.Image.open(file_pointer).convert('RGB') + return self.pil_image(pil_image) + + +class DeepSparseEvaluator(Evaluator): + def __init__(self, pipeline: Pipeline, dataset_name: str, **kwargs): + self.pipeline = pipeline + super().__init__(dataset_name = dataset_name,**kwargs) + self.data_loader = remove_normalization(self.data_loader) + + def accumulate(self, predictor, metrics): + prediction_loader = predictor.enumerated_dataloader(self.pipeline, enumerate(self.data_loader)) + if self.loader_warmup: + LOG.info('Data loader warmup (%.1fs) ...', self.loader_warmup) + time.sleep(self.loader_warmup) + LOG.info('Done.') + + total_start = time.perf_counter() + loop_start = time.perf_counter() + + for image_i, (pred, gt_anns, image_meta) in enumerate(prediction_loader): + LOG.info('image %d / %d, last loop: %.3fs, images per second=%.1f', + image_i, len(self.data_loader), time.perf_counter() - loop_start, + image_i / max(1, (time.perf_counter() - total_start))) + loop_start = time.perf_counter() + + for metric in metrics: + metric.accumulate(pred, image_meta, ground_truth=gt_anns) + + if self.show_final_image: + # show ground truth and predictions on original image + annotation_painter = show.AnnotationPainter() + with open(image_meta['local_file_path'], 'rb') as f: + cpu_image = PIL.Image.open(f).convert('RGB') + + with show.image_canvas(cpu_image) as ax: + if self.show_final_ground_truth: + annotation_painter.annotations(ax, gt_anns, color='grey') + annotation_painter.annotations(ax, pred) + + if self.n_images is not None and image_i >= self.n_images - 1: + break + if image_i == 100: + break + + total_time = time.perf_counter() - total_start + return total_time + + def evaluate(self, output: t.Optional[str]): + # generate a default output filename + if output is None: + assert self.args is not None + output = self.default_output_name(self.args) + + #if self.skip_existing: + # stats_file = output + '.stats.json' + # if os.path.exists(stats_file): + # print('Output file {} exists already. Exiting.'.format(stats_file)) + # return + # print('{} not found. Processing: {}'.format(stats_file, network.Factory.checkpoint)) + + predictor = DeepSparsePredictor(head_metas=self.datamodule.head_metas) + metrics = self.datamodule.metrics() + + total_time = self.accumulate(predictor, metrics) + + # model stats + counted_ops = list(count_ops(predictor.model_cpu)) + file_size = -1 # TODO + + # write + additional_data = { + 'args': sys.argv, + 'version': __version__, + 'dataset': self.dataset_name, + 'total_time': total_time, + 'checkpoint': network.Factory.checkpoint, + 'count_ops': counted_ops, + 'file_size': file_size, + 'n_images': predictor.total_images, + 'decoder_time': predictor.total_decoder_time, + 'nn_time': predictor.total_nn_time, + } + + metric_stats = defaultdict(list) + for metric in metrics: + if self.write_predictions: + metric.write_predictions(output, additional_data=additional_data) + + this_metric_stats = metric.stats() + assert (len(this_metric_stats.get('text_labels', [])) + == len(this_metric_stats.get('stats', []))) + + for k, v in this_metric_stats.items(): + metric_stats[k] = metric_stats[k] + v + + stats = dict(**metric_stats, **additional_data) + + # write stats file + with open(output + '.stats.json', 'w') as f: + json.dump(stats, f) + + LOG.info('stats:\n%s', json.dumps(stats, indent=4)) + LOG.info( + 'time per image: decoder = %.0fms, nn = %.0fms, total = %.0fms', + 1000 * stats['decoder_time'] / stats['n_images'], + 1000 * stats['nn_time'] / stats['n_images'], + 1000 * stats['total_time'] / stats['n_images'], + ) + + + +def main(): + args = cli() + args.dataset = "cocokp" + args.output = "funny" + args.decoder = ["cifcaf"] + args.checkpoint = "shufflenetv2k16" + pipeline = Pipeline.create(task="open_pif_paf", model_path="openpifpaf-resnet50.onnx", output_fields=True) + #evaluator = Evaluator(dataset_name=args.dataset) + evaluator = DeepSparseEvaluator(pipeline = pipeline, dataset_name=args.dataset) + + if args.watch: + assert args.output is None + evaluator.watch(args.checkpoint, args.watch) + else: + evaluator.evaluate(args.output) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/src/deepsparse/yolact/utils/utils.py b/src/deepsparse/yolact/utils/utils.py index 30e5e82266..ef6af7eebc 100644 --- a/src/deepsparse/yolact/utils/utils.py +++ b/src/deepsparse/yolact/utils/utils.py @@ -50,14 +50,6 @@ def preprocess_array( """ image = image.astype(numpy.float32) image = _assert_channels_last(image) - if image.ndim == 4 and image.shape[:2] != input_image_size: - image = numpy.stack([cv2.resize(img, input_image_size) for img in image]) - - else: - if image.shape[:2] != input_image_size: - image = cv2.resize(image, input_image_size) - image = numpy.expand_dims(image, 0) - image = image.transpose(0, 3, 1, 2) image /= 255 image = numpy.ascontiguousarray(image) From 908f2009d529bd3e47c2356fed5a77d09b744223 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Mon, 20 Feb 2023 16:21:55 +0000 Subject: [PATCH 02/11] getting the same mAP! --- src/deepsparse/open_pif_paf/__init__.py | 1 + src/deepsparse/open_pif_paf/pipelines.py | 2 +- .../open_pif_paf/utils/validation/__init__.py | 1 + .../utils/validation/deepsparse_decoder.py | 47 ++ .../utils/validation/deepsparse_evaluator.py | 100 +++++ .../utils/validation/deepsparse_predictor.py | 20 + .../open_pif_paf/utils/validation/helpers.py | 15 + src/deepsparse/open_pif_paf/validation.py | 405 +----------------- 8 files changed, 188 insertions(+), 403 deletions(-) create mode 100644 src/deepsparse/open_pif_paf/utils/validation/__init__.py create mode 100644 src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py create mode 100644 src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py create mode 100644 src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py create mode 100644 src/deepsparse/open_pif_paf/utils/validation/helpers.py diff --git a/src/deepsparse/open_pif_paf/__init__.py b/src/deepsparse/open_pif_paf/__init__.py index 0c44f887a4..fb5016974f 100644 --- a/src/deepsparse/open_pif_paf/__init__.py +++ b/src/deepsparse/open_pif_paf/__init__.py @@ -11,3 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .utils import * \ No newline at end of file diff --git a/src/deepsparse/open_pif_paf/pipelines.py b/src/deepsparse/open_pif_paf/pipelines.py index 6994f1a4b3..a7c84df9af 100644 --- a/src/deepsparse/open_pif_paf/pipelines.py +++ b/src/deepsparse/open_pif_paf/pipelines.py @@ -94,7 +94,7 @@ def process_inputs(self, inputs: OpenPifPafInput) -> List[numpy.ndarray]: image = inputs.images image = image.astype(numpy.float32) #image = image.transpose(0, 2, 3, 1) - image /= 255 + #image /= 255 image = numpy.ascontiguousarray(image) return [image] diff --git a/src/deepsparse/open_pif_paf/utils/validation/__init__.py b/src/deepsparse/open_pif_paf/utils/validation/__init__.py new file mode 100644 index 0000000000..e69af2abe6 --- /dev/null +++ b/src/deepsparse/open_pif_paf/utils/validation/__init__.py @@ -0,0 +1 @@ +from .deepsparse_evaluator import * diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py new file mode 100644 index 0000000000..0f72264c9f --- /dev/null +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py @@ -0,0 +1,47 @@ +import time +import torch +import logging +from openpifpaf.decoder import CifCaf +from openpifpaf.decoder.decoder import DummyPool +from typing import List +from deepsparse import Pipeline + +from deepsparse.open_pif_paf.utils.validation.helpers import deepsparse_fields_to_torch + +LOG = logging.getLogger(__name__) + +class DeepSparseCifCaf(CifCaf): + def __init__(self, pipeline: Pipeline, head_metas: List[None]): + self.pipeline = pipeline + cif_metas, caf_metas = head_metas + super().__init__([cif_metas], [caf_metas]) + + # adapted from OPENPIFPAF GITHUB: + # https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/decoder/decoder.py + # the appropriate edits are marked with # deepsparse edit: + def batch(self, model, image_batch, *, device=None, gt_anns_batch=None): + """From image batch straight to annotations batch.""" + start_nn = time.perf_counter() + fields_batch = self.fields_batch(model, image_batch, device=device) + fields_batch = deepsparse_fields_to_torch(self.pipeline(images=image_batch.numpy())) + self.last_nn_time = time.perf_counter() - start_nn + + if gt_anns_batch is None: + gt_anns_batch = [None for _ in fields_batch] + + if not isinstance(self.worker_pool, DummyPool): + # remove debug_images to save time during pickle + image_batch = [None for _ in fields_batch] + gt_anns_batch = [None for _ in fields_batch] + + LOG.debug('parallel execution with worker %s', self.worker_pool) + start_decoder = time.perf_counter() + result = self.worker_pool.starmap( + self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch)) + self.last_decoder_time = time.perf_counter() - start_decoder + + LOG.debug('time: nn = %.1fms, dec = %.1fms', + self.last_nn_time * 1000.0, + self.last_decoder_time * 1000.0) + return result + diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py new file mode 100644 index 0000000000..865d6da0d5 --- /dev/null +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py @@ -0,0 +1,100 @@ +from collections import defaultdict +import json +import logging +import os +import sys +import typing as t +from openpifpaf.eval import Evaluator, count_ops +from openpifpaf import network, __version__ + +from deepsparse.open_pif_paf.utils.validation.helpers import apply_deepsparse_preprocessing +from deepsparse.open_pif_paf.utils.validation.deepsparse_predictor import DeepSparsePredictor + + +from deepsparse import Pipeline + + +LOG = logging.getLogger(__name__) + +__all__ = ["DeepSparseEvaluator"] + +# adapted from OPENPIFPAF GITHUB: +# https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/eval.py +# the appropriate edits are marked with # deepsparse edit: +class DeepSparseEvaluator(Evaluator): + # deepsparse edit: allow for passing in a pipeline + def __init__(self, pipeline: Pipeline, img_size: int, **kwargs): + self.pipeline = pipeline + super().__init__(**kwargs) + # deepsparse edit: required to enforce square images + apply_deepsparse_preprocessing(self.data_loader, img_size) + + def evaluate(self, output: t.Optional[str]): + # generate a default output filename + if output is None: + assert self.args is not None + output = self.default_output_name(self.args) + + # skip existing? + if self.skip_epoch0: + assert network.Factory.checkpoint is not None + if network.Factory.checkpoint.endswith('.epoch000'): + print('Not evaluating epoch 0.') + return + if self.skip_existing: + stats_file = output + '.stats.json' + if os.path.exists(stats_file): + print('Output file {} exists already. Exiting.'.format(stats_file)) + return + print('{} not found. Processing: {}'.format(stats_file, network.Factory.checkpoint)) + + # deepsparse edit: allow for passing in a pipeline + predictor = DeepSparsePredictor(pipeline = self.pipeline, head_metas=self.datamodule.head_metas) + metrics = self.datamodule.metrics() + + total_time = self.accumulate(predictor, metrics) + + # model stats + # deepsparse edit: compute stats for ONNX file not torch model + counted_ops = list(count_ops(predictor.model_cpu)) + file_size = -1.0 # TODO get file size + + # write + additional_data = { + 'args': sys.argv, + 'version': __version__, + 'dataset': self.dataset_name, + 'total_time': total_time, + 'checkpoint': network.Factory.checkpoint, + 'count_ops': counted_ops, + 'file_size': file_size, + 'n_images': predictor.total_images, + 'decoder_time': predictor.total_decoder_time, + 'nn_time': predictor.total_nn_time, + } + + metric_stats = defaultdict(list) + for metric in metrics: + if self.write_predictions: + metric.write_predictions(output, additional_data=additional_data) + + this_metric_stats = metric.stats() + assert (len(this_metric_stats.get('text_labels', [])) + == len(this_metric_stats.get('stats', []))) + + for k, v in this_metric_stats.items(): + metric_stats[k] = metric_stats[k] + v + + stats = dict(**metric_stats, **additional_data) + + # write stats file + with open(output + '.stats.json', 'w') as f: + json.dump(stats, f) + + LOG.info('stats:\n%s', json.dumps(stats, indent=4)) + LOG.info( + 'time per image: decoder = %.0fms, nn = %.0fms, total = %.0fms', + 1000 * stats['decoder_time'] / stats['n_images'], + 1000 * stats['nn_time'] / stats['n_images'], + 1000 * stats['total_time'] / stats['n_images'], + ) diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py new file mode 100644 index 0000000000..b4dd7bc89e --- /dev/null +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py @@ -0,0 +1,20 @@ +import logging +from openpifpaf import Predictor +from deepsparse import Pipeline +from deepsparse.open_pif_paf.utils.validation.deepsparse_decoder import DeepSparseCifCaf + +LOG = logging.getLogger(__name__) + +# adapted from OPENPIFPAF GITHUB: +# https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/predictor.py +# the appropriate edits are marked with # deepsparse edit: +class DeepSparsePredictor(Predictor): + """Convenience class to predict from various inputs with a common configuration.""" + # deepsparse edit: allow for passing in a pipeline + def __init__(self, pipeline: Pipeline, **kwargs): + super().__init__(**kwargs) + # deepsparse edit: allow for passing in a pipeline and fix the processor + # to CifCaf processor + self.processor = DeepSparseCifCaf(pipeline = pipeline, head_metas = self.model_cpu.head_metas) + + diff --git a/src/deepsparse/open_pif_paf/utils/validation/helpers.py b/src/deepsparse/open_pif_paf/utils/validation/helpers.py new file mode 100644 index 0000000000..69f826be6f --- /dev/null +++ b/src/deepsparse/open_pif_paf/utils/validation/helpers.py @@ -0,0 +1,15 @@ +import torch +from openpifpaf import transforms +import numpy + +__all__ = ["apply_deepsparse_preprocessing", "deepsparse_fields_to_torch"] + +def deepsparse_fields_to_torch(fields_batch, device='cpu'): + result = [] + fields = fields_batch.fields + for idx, (cif, caf) in enumerate(zip(*fields)): + result.append([torch.from_numpy(cif).to(device), torch.from_numpy(caf).to(device)]) + return + +def apply_deepsparse_preprocessing(data_loader: torch.utils.data.DataLoader, img_size: int) -> torch.utils.data.DataLoader: + data_loader.dataset.preprocess.preprocess_list[2] = transforms.CenterPad(img_size) \ No newline at end of file diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py index dedeba1899..998e25068f 100644 --- a/src/deepsparse/open_pif_paf/validation.py +++ b/src/deepsparse/open_pif_paf/validation.py @@ -1,404 +1,7 @@ """Evaluation on COCO data.""" -from openpifpaf.eval import cli, Evaluator, Predictor, network, count_ops, __version__, LOG, show -from openpifpaf import datasets, decoder, visualizer, transforms -from openpifpaf.decoder import CifCaf -import typing as t -import json -from collections import defaultdict -import sys -import os -import time -import PIL.Image -from openpifpaf.decoder.multi import Multi -import argparse - -import PIL -import torch - +from openpifpaf.eval import cli +from deepsparse.open_pif_paf.utils.validation.deepsparse_evaluator import DeepSparseEvaluator from deepsparse import Pipeline - -class DummyPool(): - @staticmethod - def starmap(f, iterable): - return [f(*i) for i in iterable] - -def remove_normalization(data_loader: t.Iterable[t.Any]) -> t.Iterable[t.Any]: - dataset = data_loader.dataset - if not hasattr(dataset, "preprocess"): - raise AttributeError("") - - preprocess = dataset.preprocess - assert len(preprocess.preprocess_list) == 6 - - image_preprocess = preprocess.preprocess_list[5] - assert len(image_preprocess.preprocess_list) == 3 - - assert image_preprocess.preprocess_list[2].image_transform.__class__.__name__ == "Normalize" - - preprocess.preprocess_list[5].preprocess_list = image_preprocess.preprocess_list[:2] - - - return data_loader - -class DeepSparseDecoder(decoder.Decoder): - def batch(self, pipeline, model, image_batch, *, device=None, gt_anns_batch=None): - """From image batch straight to annotations batch.""" - start_nn = time.perf_counter() - fields_batch = self.fields_batch(model, image_batch, device=device) - self.last_nn_time = time.perf_counter() - start_nn - - if gt_anns_batch is None: - gt_anns_batch = [None for _ in fields_batch] - - if not isinstance(self.worker_pool, DummyPool): - # remove debug_images to save time during pickle - image_batch = [None for _ in fields_batch] - gt_anns_batch = [None for _ in fields_batch] - - LOG.debug('parallel execution with worker %s', self.worker_pool) - start_decoder = time.perf_counter() - result = self.worker_pool.starmap( - self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch)) - self.last_decoder_time = time.perf_counter() - start_decoder - - LOG.debug('time: nn = %.1fms, dec = %.1fms', - self.last_nn_time * 1000.0, - self.last_decoder_time * 1000.0) - return result - -def fields_batch_deepsparse_to_torch(fields_batch,device): - result = [] - fields = fields_batch.fields - for idx, (cif, caf) in enumerate(zip(*fields)): - result.append([torch.from_numpy(cif), torch.from_numpy(caf)]) - return result - -import numpy -class DeepSparseCifCaf(CifCaf): - def __init__(self, head_metas): - cif_metas, caf_metas = head_metas - super().__init__([cif_metas], [caf_metas]) - - def batch(self, pipeline, model, image_batch, *, device=None, gt_anns_batch=None): - """From image batch straight to annotations batch.""" - start_nn = time.perf_counter() - fields_batch = self.fields_batch(model, image_batch, device=device) - image_batch = image_batch.numpy() * 255 - image_batch = image_batch[:,:,:480, :640] - fields_batch_deepsparse= pipeline(images=image_batch.astype(numpy.uint8)) - fields_batch = fields_batch_deepsparse_to_torch(fields_batch_deepsparse, device=device) - - self.last_nn_time = time.perf_counter() - start_nn - - if gt_anns_batch is None: - gt_anns_batch = [None for _ in fields_batch] - - if not isinstance(self.worker_pool, DummyPool): - # remove debug_images to save time during pickle - image_batch = [None for _ in fields_batch] - gt_anns_batch = [None for _ in fields_batch] - - LOG.debug('parallel execution with worker %s', self.worker_pool) - start_decoder = time.perf_counter() - result = self.worker_pool.starmap( - self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch)) - self.last_decoder_time = time.perf_counter() - start_decoder - - LOG.debug('time: nn = %.1fms, dec = %.1fms', - self.last_nn_time * 1000.0, - self.last_decoder_time * 1000.0) - return result - -class DeepSparsePredictor: - """Convenience class to predict from various inputs with a common configuration.""" - - batch_size = 1 #: batch size - device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') #: device - fast_rescaling = True #: fast rescaling - loader_workers = None #: loader workers - long_edge = None #: long edge - - def __init__(self, checkpoint=None, head_metas=None, *, - json_data=False, - visualize_image=False, - visualize_processed_image=False): - if checkpoint is not None: - network.Factory.checkpoint = checkpoint - self.json_data = json_data - self.visualize_image = visualize_image - self.visualize_processed_image = visualize_processed_image - - self.model_cpu, _ = network.Factory().factory(head_metas=head_metas) - self.model = self.model_cpu.to(self.device) - if self.device.type == 'cuda' and torch.cuda.device_count() > 1: - LOG.info('Using multiple GPUs: %d', torch.cuda.device_count()) - self.model = torch.nn.DataParallel(self.model) - self.model.base_net = self.model_cpu.base_net - self.model.head_nets = self.model_cpu.head_nets - - self.preprocess = self._preprocess_factory() - LOG.debug('head names = %s', [meta.name for meta in head_metas]) - self.processor = DeepSparseCifCaf(self.model_cpu.head_metas) - - self.last_decoder_time = 0.0 - self.last_nn_time = 0.0 - self.total_nn_time = 0.0 - self.total_decoder_time = 0.0 - self.total_images = 0 - - LOG.info('neural network device: %s (CUDA available: %s, count: %d)', - self.device, torch.cuda.is_available(), torch.cuda.device_count()) - - @classmethod - def cli(cls, parser: argparse.ArgumentParser, *, - skip_batch_size=False, skip_loader_workers=False): - """Add command line arguments. - - When using this class together with datasets (e.g. in eval), - skip the cli arguments for batch size and loader workers as those - will be provided via the datasets module. - """ - group = parser.add_argument_group('Predictor') - - if not skip_batch_size: - group.add_argument('--batch-size', default=cls.batch_size, type=int, - help='processing batch size') - - if not skip_loader_workers: - group.add_argument('--loader-workers', default=cls.loader_workers, type=int, - help='number of workers for data loading') - - group.add_argument('--long-edge', default=cls.long_edge, type=int, - help='rescale the long side of the image (aspect ratio maintained)') - group.add_argument('--precise-rescaling', dest='fast_rescaling', - default=True, action='store_false', - help='use more exact image rescaling (requires scipy)') - - @classmethod - def configure(cls, args: argparse.Namespace): - """Configure from command line parser.""" - cls.batch_size = args.batch_size - cls.device = args.device - cls.fast_rescaling = args.fast_rescaling - cls.loader_workers = args.loader_workers - cls.long_edge = args.long_edge - - def _preprocess_factory(self): - rescale_t = None - if self.long_edge: - rescale_t = transforms.RescaleAbsolute(self.long_edge, fast=self.fast_rescaling) - - pad_t = None - if self.batch_size > 1: - assert self.long_edge, '--long-edge must be provided for batch size > 1' - pad_t = transforms.CenterPad(self.long_edge) - else: - pad_t = transforms.CenterPadTight(16) - - return transforms.Compose([ - transforms.NormalizeAnnotations(), - rescale_t, - pad_t, - transforms.EVAL_TRANSFORM, - ]) - - def dataset(self, data): - """Predict from a dataset.""" - loader_workers = self.loader_workers - if loader_workers is None: - loader_workers = self.batch_size if len(data) > 1 else 0 - - dataloader = torch.utils.data.DataLoader( - data, batch_size=self.batch_size, shuffle=False, - pin_memory=self.device.type != 'cpu', - num_workers=loader_workers, - collate_fn=datasets.collate_images_anns_meta) - - yield from self.dataloader(dataloader) - - def enumerated_dataloader(self, pipeline, enumerated_dataloader): - """Predict from an enumerated dataloader.""" - for batch_i, item in enumerated_dataloader: - if len(item) == 3: - processed_image_batch, gt_anns_batch, meta_batch = item - image_batch = [None for _ in processed_image_batch] - elif len(item) == 4: - image_batch, processed_image_batch, gt_anns_batch, meta_batch = item - if self.visualize_processed_image: - visualizer.Base.processed_image(processed_image_batch[0]) - - pred_batch = self.processor.batch(pipeline, self.model, processed_image_batch, device=self.device) - self.last_decoder_time = self.processor.last_decoder_time - self.last_nn_time = self.processor.last_nn_time - self.total_decoder_time += self.processor.last_decoder_time - self.total_nn_time += self.processor.last_nn_time - self.total_images += len(processed_image_batch) - - # un-batch - for image, pred, gt_anns, meta in \ - zip(image_batch, pred_batch, gt_anns_batch, meta_batch): - LOG.info('batch %d: %s', batch_i, meta.get('file_name', 'no-file-name')) - - # load the original image if necessary - if self.visualize_image: - visualizer.Base.image(image, meta=meta) - - pred = [ann.inverse_transform(meta) for ann in pred] - gt_anns = [ann.inverse_transform(meta) for ann in gt_anns] - - if self.json_data: - pred = [ann.json_data() for ann in pred] - - yield pred, gt_anns, meta - - def dataloader(self, dataloader): - """Predict from a dataloader.""" - yield from self.enumerated_dataloader(enumerate(dataloader)) - - def image(self, file_name): - """Predict from an image file name.""" - return next(iter(self.images([file_name]))) - - def images(self, file_names, **kwargs): - """Predict from image file names.""" - data = datasets.ImageList( - file_names, preprocess=self.preprocess, with_raw_image=True) - yield from self.dataset(data, **kwargs) - - def pil_image(self, image): - """Predict from a Pillow image.""" - return next(iter(self.pil_images([image]))) - - def pil_images(self, pil_images, **kwargs): - """Predict from Pillow images.""" - data = datasets.PilImageList( - pil_images, preprocess=self.preprocess, with_raw_image=True) - yield from self.dataset(data, **kwargs) - - def numpy_image(self, image): - """Predict from a numpy image.""" - return next(iter(self.numpy_images([image]))) - - def numpy_images(self, numpy_images, **kwargs): - """Predict from numpy images.""" - data = datasets.NumpyImageList( - numpy_images, preprocess=self.preprocess, with_raw_image=True) - yield from self.dataset(data, **kwargs) - - def image_file(self, file_pointer): - """Predict from an opened image file pointer.""" - pil_image = PIL.Image.open(file_pointer).convert('RGB') - return self.pil_image(pil_image) - - -class DeepSparseEvaluator(Evaluator): - def __init__(self, pipeline: Pipeline, dataset_name: str, **kwargs): - self.pipeline = pipeline - super().__init__(dataset_name = dataset_name,**kwargs) - self.data_loader = remove_normalization(self.data_loader) - - def accumulate(self, predictor, metrics): - prediction_loader = predictor.enumerated_dataloader(self.pipeline, enumerate(self.data_loader)) - if self.loader_warmup: - LOG.info('Data loader warmup (%.1fs) ...', self.loader_warmup) - time.sleep(self.loader_warmup) - LOG.info('Done.') - - total_start = time.perf_counter() - loop_start = time.perf_counter() - - for image_i, (pred, gt_anns, image_meta) in enumerate(prediction_loader): - LOG.info('image %d / %d, last loop: %.3fs, images per second=%.1f', - image_i, len(self.data_loader), time.perf_counter() - loop_start, - image_i / max(1, (time.perf_counter() - total_start))) - loop_start = time.perf_counter() - - for metric in metrics: - metric.accumulate(pred, image_meta, ground_truth=gt_anns) - - if self.show_final_image: - # show ground truth and predictions on original image - annotation_painter = show.AnnotationPainter() - with open(image_meta['local_file_path'], 'rb') as f: - cpu_image = PIL.Image.open(f).convert('RGB') - - with show.image_canvas(cpu_image) as ax: - if self.show_final_ground_truth: - annotation_painter.annotations(ax, gt_anns, color='grey') - annotation_painter.annotations(ax, pred) - - if self.n_images is not None and image_i >= self.n_images - 1: - break - if image_i == 100: - break - - total_time = time.perf_counter() - total_start - return total_time - - def evaluate(self, output: t.Optional[str]): - # generate a default output filename - if output is None: - assert self.args is not None - output = self.default_output_name(self.args) - - #if self.skip_existing: - # stats_file = output + '.stats.json' - # if os.path.exists(stats_file): - # print('Output file {} exists already. Exiting.'.format(stats_file)) - # return - # print('{} not found. Processing: {}'.format(stats_file, network.Factory.checkpoint)) - - predictor = DeepSparsePredictor(head_metas=self.datamodule.head_metas) - metrics = self.datamodule.metrics() - - total_time = self.accumulate(predictor, metrics) - - # model stats - counted_ops = list(count_ops(predictor.model_cpu)) - file_size = -1 # TODO - - # write - additional_data = { - 'args': sys.argv, - 'version': __version__, - 'dataset': self.dataset_name, - 'total_time': total_time, - 'checkpoint': network.Factory.checkpoint, - 'count_ops': counted_ops, - 'file_size': file_size, - 'n_images': predictor.total_images, - 'decoder_time': predictor.total_decoder_time, - 'nn_time': predictor.total_nn_time, - } - - metric_stats = defaultdict(list) - for metric in metrics: - if self.write_predictions: - metric.write_predictions(output, additional_data=additional_data) - - this_metric_stats = metric.stats() - assert (len(this_metric_stats.get('text_labels', [])) - == len(this_metric_stats.get('stats', []))) - - for k, v in this_metric_stats.items(): - metric_stats[k] = metric_stats[k] + v - - stats = dict(**metric_stats, **additional_data) - - # write stats file - with open(output + '.stats.json', 'w') as f: - json.dump(stats, f) - - LOG.info('stats:\n%s', json.dumps(stats, indent=4)) - LOG.info( - 'time per image: decoder = %.0fms, nn = %.0fms, total = %.0fms', - 1000 * stats['decoder_time'] / stats['n_images'], - 1000 * stats['nn_time'] / stats['n_images'], - 1000 * stats['total_time'] / stats['n_images'], - ) - - - def main(): args = cli() args.dataset = "cocokp" @@ -406,9 +9,7 @@ def main(): args.decoder = ["cifcaf"] args.checkpoint = "shufflenetv2k16" pipeline = Pipeline.create(task="open_pif_paf", model_path="openpifpaf-resnet50.onnx", output_fields=True) - #evaluator = Evaluator(dataset_name=args.dataset) - evaluator = DeepSparseEvaluator(pipeline = pipeline, dataset_name=args.dataset) - + evaluator = DeepSparseEvaluator(pipeline = pipeline, dataset_name=args.dataset, skip_epoch0=False, img_size= args.coco_eval_long_edge) if args.watch: assert args.output is None evaluator.watch(args.checkpoint, args.watch) From c175205c81207fcf6e94af8190a896a61a9b52c0 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Mon, 20 Feb 2023 17:54:37 +0000 Subject: [PATCH 03/11] second round of refactoring --- setup.py | 4 +- src/deepsparse/open_pif_paf/__init__.py | 2 +- src/deepsparse/open_pif_paf/pipelines.py | 43 ++++-- src/deepsparse/open_pif_paf/schemas.py | 15 +- .../open_pif_paf/utils/validation/__init__.py | 14 ++ .../utils/validation/deepsparse_decoder.py | 49 ++++-- .../utils/validation/deepsparse_evaluator.py | 88 ++++++----- .../utils/validation/deepsparse_predictor.py | 31 +++- .../open_pif_paf/utils/validation/helpers.py | 63 ++++++-- src/deepsparse/open_pif_paf/validation.py | 144 ++++++++++++++++-- src/deepsparse/yolact/utils/utils.py | 14 +- 11 files changed, 377 insertions(+), 90 deletions(-) diff --git a/setup.py b/setup.py index 66ecf77417..086520221c 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,9 @@ def _parse_requirements_file(file_path): "protobuf>=3.12.2,<=3.20.1", "click>=7.1.2,!=8.0.0", # latest version < 8.0 + blocked version with reported bug ] -_nm_deps = []#[f"{'sparsezoo' if is_release else 'sparsezoo-nightly'}~={version_base}"] +_nm_deps = ( + [] +) # [f"{'sparsezoo' if is_release else 'sparsezoo-nightly'}~={version_base}"] _dev_deps = [ "beautifulsoup4>=4.9.3", "black==22.12.0", diff --git a/src/deepsparse/open_pif_paf/__init__.py b/src/deepsparse/open_pif_paf/__init__.py index fb5016974f..8256228a57 100644 --- a/src/deepsparse/open_pif_paf/__init__.py +++ b/src/deepsparse/open_pif_paf/__init__.py @@ -11,4 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .utils import * \ No newline at end of file +from .utils import * diff --git a/src/deepsparse/open_pif_paf/pipelines.py b/src/deepsparse/open_pif_paf/pipelines.py index a7c84df9af..eb934e7f11 100644 --- a/src/deepsparse/open_pif_paf/pipelines.py +++ b/src/deepsparse/open_pif_paf/pipelines.py @@ -22,7 +22,11 @@ import cv2 import torch -from deepsparse.open_pif_paf.schemas import OpenPifPafInput, OpenPifPafOutput, OpenPifPafFields +from deepsparse.open_pif_paf.schemas import ( + OpenPifPafFields, + OpenPifPafInput, + OpenPifPafOutput, +) from deepsparse.pipeline import Pipeline from deepsparse.yolact.utils import preprocess_array from openpifpaf import decoder, network @@ -57,10 +61,19 @@ class OpenPifPafPipeline(Pipeline): """ def __init__( - self, *, output_fields = False, **kwargs + self, + *, + image_size: Union[int, Tuple[int, int]] = (384, 384), + return_cifcaf_fields: bool = False, + **kwargs, ): super().__init__(**kwargs) - self.output_fields = output_fields + self._image_size = ( + image_size if isinstance(image_size, Tuple) else (image_size, image_size) + ) + # whether to return the cif and caf fields or the + # complete decoded output + self.return_cifcaf_fields = return_cifcaf_fields # necessary openpifpaf dependencies for now model_cpu, _ = network.Factory().factory(head_metas=None) self.processor = decoder.factory(model_cpu.head_metas) @@ -77,7 +90,7 @@ def output_schema(self) -> Type[OpenPifPafOutput]: """ :return: pydantic model class that outputs to this pipeline must comply to """ - return OpenPifPafOutput if not self.output_fields else OpenPifPafFields + return OpenPifPafOutput if not self.return_cifcaf_fields else OpenPifPafFields def setup_onnx_file_path(self) -> str: """ @@ -91,13 +104,14 @@ class properties into an inference ready onnx file to be compiled by the def process_inputs(self, inputs: OpenPifPafInput) -> List[numpy.ndarray]: - image = inputs.images - image = image.astype(numpy.float32) - #image = image.transpose(0, 2, 3, 1) - #image /= 255 - image = numpy.ascontiguousarray(image) + images = inputs.images + if not isinstance(images, list): + images = [images] + + image_batch = list(self.executor.map(self._preprocess_image, images)) + image_batch = numpy.concatenate(image_batch, axis=0) - return [image] + return [image_batch] def process_engine_outputs( self, fields: List[numpy.ndarray], **kwargs @@ -109,8 +123,11 @@ def process_engine_outputs( :return: Outputs of engine post-processed into an object in the `output_schema` format of this pipeline """ - if self.output_fields: - return OpenPifPafFields(fields=fields) + if self.return_cifcaf_fields: + batch_fields = [] + for cif, caf in zip(*fields): + batch_fields.append([cif, caf]) + return OpenPifPafFields(fields=batch_fields) data_batch, skeletons_batch, scores_batch, keypoints_batch = [], [], [], [] @@ -134,4 +151,4 @@ def _preprocess_image(self, image) -> numpy.ndarray: if isinstance(image, str): image = cv2.imread(image) - return preprocess_array(image) + return preprocess_array(image, input_image_size=self._image_size) diff --git a/src/deepsparse/open_pif_paf/schemas.py b/src/deepsparse/open_pif_paf/schemas.py index b150bf689f..740d89fa6e 100644 --- a/src/deepsparse/open_pif_paf/schemas.py +++ b/src/deepsparse/open_pif_paf/schemas.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import List, Tuple + import numpy from pydantic import BaseModel, Field @@ -33,12 +34,22 @@ class OpenPifPafInput(ComputerVisionSchema): pass + class OpenPifPafFields(BaseModel): """ - # TODO + Open Pif Paf is composed of two stages: + - Computing Cif/Caf fields using a parametrized model + - Applying a matching algorithm to obtain the final pose + predictions + In some cases (e.g. for validation), it may be useful to + obtain the Cif/Caf fields as output. """ - fields: List[numpy.ndarray] = Field(description="") + fields: List[List[numpy.ndarray]] = Field( + description="Cif/Caf fields returned by the network. " + "The outer list is the batch dimension, while the second " + "list contains two numpy arrays: Cif and Caf field values" + ) class Config: arbitrary_types_allowed = True diff --git a/src/deepsparse/open_pif_paf/utils/validation/__init__.py b/src/deepsparse/open_pif_paf/utils/validation/__init__.py index e69af2abe6..fec8d68416 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/__init__.py +++ b/src/deepsparse/open_pif_paf/utils/validation/__init__.py @@ -1 +1,15 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .deepsparse_evaluator import * diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py index 0f72264c9f..dbc7984c9c 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py @@ -1,17 +1,36 @@ -import time -import torch +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging +import time +from typing import List, Union + +from deepsparse import Pipeline +from deepsparse.open_pif_paf.utils.validation.helpers import deepsparse_fields_to_torch from openpifpaf.decoder import CifCaf from openpifpaf.decoder.decoder import DummyPool -from typing import List -from deepsparse import Pipeline -from deepsparse.open_pif_paf.utils.validation.helpers import deepsparse_fields_to_torch LOG = logging.getLogger(__name__) + class DeepSparseCifCaf(CifCaf): - def __init__(self, pipeline: Pipeline, head_metas: List[None]): + def __init__( + self, + head_metas: List[Union["Cif", "Caf"]], # noqa: F821 + pipeline: Pipeline, + ): self.pipeline = pipeline cif_metas, caf_metas = head_metas super().__init__([cif_metas], [caf_metas]) @@ -23,7 +42,9 @@ def batch(self, model, image_batch, *, device=None, gt_anns_batch=None): """From image batch straight to annotations batch.""" start_nn = time.perf_counter() fields_batch = self.fields_batch(model, image_batch, device=device) - fields_batch = deepsparse_fields_to_torch(self.pipeline(images=image_batch.numpy())) + fields_batch = deepsparse_fields_to_torch( + self.pipeline(images=image_batch.numpy()) + ) self.last_nn_time = time.perf_counter() - start_nn if gt_anns_batch is None: @@ -34,14 +55,16 @@ def batch(self, model, image_batch, *, device=None, gt_anns_batch=None): image_batch = [None for _ in fields_batch] gt_anns_batch = [None for _ in fields_batch] - LOG.debug('parallel execution with worker %s', self.worker_pool) + LOG.debug("parallel execution with worker %s", self.worker_pool) start_decoder = time.perf_counter() result = self.worker_pool.starmap( - self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch)) + self._mappable_annotations, zip(fields_batch, image_batch, gt_anns_batch) + ) self.last_decoder_time = time.perf_counter() - start_decoder - LOG.debug('time: nn = %.1fms, dec = %.1fms', - self.last_nn_time * 1000.0, - self.last_decoder_time * 1000.0) + LOG.debug( + "time: nn = %.1fms, dec = %.1fms", + self.last_nn_time * 1000.0, + self.last_decoder_time * 1000.0, + ) return result - diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py index 865d6da0d5..2fa86c7a58 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_evaluator.py @@ -1,23 +1,40 @@ -from collections import defaultdict +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import logging import os import sys import typing as t -from openpifpaf.eval import Evaluator, count_ops -from openpifpaf import network, __version__ - -from deepsparse.open_pif_paf.utils.validation.helpers import apply_deepsparse_preprocessing -from deepsparse.open_pif_paf.utils.validation.deepsparse_predictor import DeepSparsePredictor - +from collections import defaultdict from deepsparse import Pipeline +from deepsparse.open_pif_paf.utils.validation.deepsparse_predictor import ( + DeepSparsePredictor, +) +from deepsparse.open_pif_paf.utils.validation.helpers import ( + apply_deepsparse_preprocessing, +) +from openpifpaf import __version__, network +from openpifpaf.eval import Evaluator LOG = logging.getLogger(__name__) __all__ = ["DeepSparseEvaluator"] + # adapted from OPENPIFPAF GITHUB: # https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/eval.py # the appropriate edits are marked with # deepsparse edit: @@ -38,39 +55,41 @@ def evaluate(self, output: t.Optional[str]): # skip existing? if self.skip_epoch0: assert network.Factory.checkpoint is not None - if network.Factory.checkpoint.endswith('.epoch000'): - print('Not evaluating epoch 0.') + if network.Factory.checkpoint.endswith(".epoch000"): + print("Not evaluating epoch 0.") return if self.skip_existing: - stats_file = output + '.stats.json' + stats_file = output + ".stats.json" if os.path.exists(stats_file): - print('Output file {} exists already. Exiting.'.format(stats_file)) + print("Output file {} exists already. Exiting.".format(stats_file)) return - print('{} not found. Processing: {}'.format(stats_file, network.Factory.checkpoint)) + print( + "{} not found. Processing: {}".format( + stats_file, network.Factory.checkpoint + ) + ) # deepsparse edit: allow for passing in a pipeline - predictor = DeepSparsePredictor(pipeline = self.pipeline, head_metas=self.datamodule.head_metas) + predictor = DeepSparsePredictor( + pipeline=self.pipeline, head_metas=self.datamodule.head_metas + ) metrics = self.datamodule.metrics() total_time = self.accumulate(predictor, metrics) # model stats - # deepsparse edit: compute stats for ONNX file not torch model - counted_ops = list(count_ops(predictor.model_cpu)) - file_size = -1.0 # TODO get file size + # deepsparse edit: removed model stats that are + # only applicable to torch models # write additional_data = { - 'args': sys.argv, - 'version': __version__, - 'dataset': self.dataset_name, - 'total_time': total_time, - 'checkpoint': network.Factory.checkpoint, - 'count_ops': counted_ops, - 'file_size': file_size, - 'n_images': predictor.total_images, - 'decoder_time': predictor.total_decoder_time, - 'nn_time': predictor.total_nn_time, + "args": sys.argv, + "version": __version__, + "dataset": self.dataset_name, + "total_time": total_time, + "n_images": predictor.total_images, + "decoder_time": predictor.total_decoder_time, + "nn_time": predictor.total_nn_time, } metric_stats = defaultdict(list) @@ -79,8 +98,9 @@ def evaluate(self, output: t.Optional[str]): metric.write_predictions(output, additional_data=additional_data) this_metric_stats = metric.stats() - assert (len(this_metric_stats.get('text_labels', [])) - == len(this_metric_stats.get('stats', []))) + assert len(this_metric_stats.get("text_labels", [])) == len( + this_metric_stats.get("stats", []) + ) for k, v in this_metric_stats.items(): metric_stats[k] = metric_stats[k] + v @@ -88,13 +108,13 @@ def evaluate(self, output: t.Optional[str]): stats = dict(**metric_stats, **additional_data) # write stats file - with open(output + '.stats.json', 'w') as f: + with open(output + ".stats.json", "w") as f: json.dump(stats, f) - LOG.info('stats:\n%s', json.dumps(stats, indent=4)) + LOG.info("stats:\n%s", json.dumps(stats, indent=4)) LOG.info( - 'time per image: decoder = %.0fms, nn = %.0fms, total = %.0fms', - 1000 * stats['decoder_time'] / stats['n_images'], - 1000 * stats['nn_time'] / stats['n_images'], - 1000 * stats['total_time'] / stats['n_images'], + "time per image: decoder = %.0fms, nn = %.0fms, total = %.0fms", + 1000 * stats["decoder_time"] / stats["n_images"], + 1000 * stats["nn_time"] / stats["n_images"], + 1000 * stats["total_time"] / stats["n_images"], ) diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py index b4dd7bc89e..644b1d9898 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py @@ -1,20 +1,41 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from openpifpaf import Predictor + from deepsparse import Pipeline from deepsparse.open_pif_paf.utils.validation.deepsparse_decoder import DeepSparseCifCaf +from openpifpaf import Predictor + LOG = logging.getLogger(__name__) + # adapted from OPENPIFPAF GITHUB: # https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/predictor.py # the appropriate edits are marked with # deepsparse edit: class DeepSparsePredictor(Predictor): - """Convenience class to predict from various inputs with a common configuration.""" + """ + Convenience class to predict from various + inputs with a common configuration. + """ + # deepsparse edit: allow for passing in a pipeline def __init__(self, pipeline: Pipeline, **kwargs): super().__init__(**kwargs) # deepsparse edit: allow for passing in a pipeline and fix the processor # to CifCaf processor - self.processor = DeepSparseCifCaf(pipeline = pipeline, head_metas = self.model_cpu.head_metas) - - + self.processor = DeepSparseCifCaf( + pipeline=pipeline, head_metas=self.model_cpu.head_metas + ) diff --git a/src/deepsparse/open_pif_paf/utils/validation/helpers.py b/src/deepsparse/open_pif_paf/utils/validation/helpers.py index 69f826be6f..a5b2dc6777 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/helpers.py +++ b/src/deepsparse/open_pif_paf/utils/validation/helpers.py @@ -1,15 +1,60 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + import torch +from deepsparse.open_pif_paf.schemas import OpenPifPafFields from openpifpaf import transforms -import numpy + __all__ = ["apply_deepsparse_preprocessing", "deepsparse_fields_to_torch"] -def deepsparse_fields_to_torch(fields_batch, device='cpu'): - result = [] - fields = fields_batch.fields - for idx, (cif, caf) in enumerate(zip(*fields)): - result.append([torch.from_numpy(cif).to(device), torch.from_numpy(caf).to(device)]) - return -def apply_deepsparse_preprocessing(data_loader: torch.utils.data.DataLoader, img_size: int) -> torch.utils.data.DataLoader: - data_loader.dataset.preprocess.preprocess_list[2] = transforms.CenterPad(img_size) \ No newline at end of file +def deepsparse_fields_to_torch( + fields_batch: OpenPifPafFields, device="cpu" +) -> List[List[torch.Tensor]]: + """ + Convert a batch of fields from the deepsparse + openpifpaf fields schema to torch tensors + + :param fields_batch: the batch of fields to convert + :param device: the device to move the tensors to + :return: a list of lists of torch tensors. The first + list is the batch dimension, the second list + contains two tensors: Cif and Caf field values + """ + return [ + [ + torch.from_numpy(array).to(device) + for field in fields_batch.fields + for array in field + ] + ] + + +def apply_deepsparse_preprocessing( + data_loader: torch.utils.data.DataLoader, img_size: int +) -> torch.utils.data.DataLoader: + """ + Replace the CenterPadTight transform in the data loader + with a CenterPad transform to ensure that the images + from the data loader are (B, 3, D, D) where D is + the img_size. This function changes `data_loader` + in place + + :param data_loader: the data loader to modify + :param img_size: the image size to pad to + """ + data_loader.dataset.preprocess.preprocess_list[2] = transforms.CenterPad(img_size) diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py index 998e25068f..4c07cae810 100644 --- a/src/deepsparse/open_pif_paf/validation.py +++ b/src/deepsparse/open_pif_paf/validation.py @@ -1,15 +1,139 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Evaluation on COCO data.""" -from openpifpaf.eval import cli -from deepsparse.open_pif_paf.utils.validation.deepsparse_evaluator import DeepSparseEvaluator +from typing import Optional + +import click + from deepsparse import Pipeline -def main(): +from deepsparse.open_pif_paf.utils.validation.deepsparse_evaluator import ( + DeepSparseEvaluator, +) +from openpifpaf.eval import cli + + +DEEPSPARSE_ENGINE = "deepsparse" +ORT_ENGINE = "onnxruntime" +SUPPORTED_DATASET_CONFIGS = ["cocokp"] + + +@click.command( + context_settings=( + dict(token_normalize_func=lambda x: x.replace("-", "_"), show_default=True) + ) +) +@click.option( + "--model-path", + # required=True, + default="openpifpaf-resnet50.onnx", + help="Path to the OpenPifPaf onnx model or" "SparseZoo stub to be evaluated.", +) +@click.option( + "--dataset", + type=str, + default="cocokp", + show_default=True, + help="Dataset name supported by the openpifpaf framework. ", +) +@click.option( + "--num-cores", + type=int, + default=None, + show_default=True, + help="Number of CPU cores to run deepsparse with, default is all available", +) +@click.option( + "--batch-size", + type=int, + default=1, + show_default=True, + help="Validation batch size", +) +@click.option( + "--image_size", + type=int, + default=641, + show_default=True, + help="Image size to use for evaluation. Will " + "be used to resize images to the same size " + "(B, C, image_size, image_size)", +) +@click.option( + "--name-validation-run", + type=str, + default="openpifpaf_validation", + show_default=True, + help="Name of the validation run, used for" "creating a file to store the results", +) +@click.option( + "--engine-type", + default=DEEPSPARSE_ENGINE, + type=click.Choice([DEEPSPARSE_ENGINE, ORT_ENGINE]), + show_default=True, + help="engine type to use, valid choices: ['deepsparse', 'onnxruntime']", +) +@click.option( + "--device", + default="cuda", + type=str, + show_default=True, + help="Use 'device=cpu' or pass valid CUDA device(s) if available, " + "i.e. 'device=0' or 'device=0,1,2,3' for Multi-GPU", +) +def main( + model_path: str, + dataset: str, + num_cores: Optional[int], + batch_size: int, + image_size: int, + name_validation_run: str, + engine_type: str, + device: str, +): + + if dataset not in SUPPORTED_DATASET_CONFIGS: + raise ValueError( + f"Dataset {dataset} is not supported. " + f"Supported datasets are {SUPPORTED_DATASET_CONFIGS}" + ) args = cli() - args.dataset = "cocokp" - args.output = "funny" - args.decoder = ["cifcaf"] + args.dataset = dataset + args.batch_size = batch_size + args.output = name_validation_run args.checkpoint = "shufflenetv2k16" - pipeline = Pipeline.create(task="open_pif_paf", model_path="openpifpaf-resnet50.onnx", output_fields=True) - evaluator = DeepSparseEvaluator(pipeline = pipeline, dataset_name=args.dataset, skip_epoch0=False, img_size= args.coco_eval_long_edge) + args.device = device + + if dataset == "cocokp": + # eval for coco keypoints dataset + args.coco_eval_long_edge = image_size + + pipeline = Pipeline.create( + task="open_pif_paf", + model_path=model_path, + engine_type=engine_type, + num_cores=num_cores, + image_size=args.coco_eval_long_edge, + return_cifcaf_fields=True, + ) + + evaluator = DeepSparseEvaluator( + pipeline=pipeline, + dataset_name=args.dataset, + skip_epoch0=False, + img_size=args.coco_eval_long_edge, + ) if args.watch: assert args.output is None evaluator.watch(args.checkpoint, args.watch) @@ -17,5 +141,5 @@ def main(): evaluator.evaluate(args.output) -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/src/deepsparse/yolact/utils/utils.py b/src/deepsparse/yolact/utils/utils.py index ef6af7eebc..1a09d6e46d 100644 --- a/src/deepsparse/yolact/utils/utils.py +++ b/src/deepsparse/yolact/utils/utils.py @@ -48,10 +48,20 @@ def preprocess_array( :return: preprocessed numpy array (B, C, D, D); where (D,D) is image size expected by the network. It is a contiguous array with RGB channel order. """ - image = image.astype(numpy.float32) + uint8_image = image[0].dtype == numpy.uint8 image = _assert_channels_last(image) + if image.ndim == 4 and image.shape[:2] != input_image_size: + image = numpy.stack([cv2.resize(img, input_image_size) for img in image]) + + else: + if image.shape[:2] != input_image_size: + image = cv2.resize(image, input_image_size) + image = numpy.expand_dims(image, 0) + image = image.transpose(0, 3, 1, 2) - image /= 255 + if uint8_image: + image = image.astype(numpy.float32) + image /= 255 image = numpy.ascontiguousarray(image) return image From 30a5add3fe7797542382910f27eee7a094b70d13 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 21 Feb 2023 13:02:21 +0000 Subject: [PATCH 04/11] ready to break it down into smaller PRs --- setup.py | 5 ++--- src/deepsparse/open_pif_paf/__init__.py | 1 + src/deepsparse/open_pif_paf/pipelines.py | 2 ++ src/deepsparse/open_pif_paf/utils/__init__.py | 1 + .../open_pif_paf/utils/validation/__init__.py | 2 +- .../utils/validation/deepsparse_decoder.py | 4 +++- src/deepsparse/open_pif_paf/validation.py | 11 +++++------ 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 9379289919..650047af06 100644 --- a/setup.py +++ b/setup.py @@ -93,9 +93,7 @@ def _parse_requirements_file(file_path): "protobuf>=3.12.2,<=3.20.1", "click>=7.1.2,!=8.0.0", # latest version < 8.0 + blocked version with reported bug ] -_nm_deps = ( - [] -) # [f"{'sparsezoo' if is_release else 'sparsezoo-nightly'}~={version_base}"] +_nm_deps = [f"{'sparsezoo' if is_release else 'sparsezoo-nightly'}~={version_base}"] _dev_deps = [ "beautifulsoup4>=4.9.3", "black==22.12.0", @@ -279,6 +277,7 @@ def _setup_entry_points() -> Dict: "deepsparse.yolov8.annotate=deepsparse.yolov8.annotate:main", "deepsparse.yolov8.eval=deepsparse.yolov8.validation:main", "deepsparse.pose_estimation.annotate=deepsparse.open_pif_paf.annotate:main", + "deepsparse.pose_estimation.eval=deepsparse.open_pif_paf.validation:main", "deepsparse.image_classification.annotate=deepsparse.image_classification.annotate:main", # noqa E501 "deepsparse.instance_segmentation.annotate=deepsparse.yolact.annotate:main", f"deepsparse.image_classification.eval={ic_eval}", diff --git a/src/deepsparse/open_pif_paf/__init__.py b/src/deepsparse/open_pif_paf/__init__.py index 8256228a57..8d3ec2e88e 100644 --- a/src/deepsparse/open_pif_paf/__init__.py +++ b/src/deepsparse/open_pif_paf/__init__.py @@ -11,4 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# flake8: noqa from .utils import * diff --git a/src/deepsparse/open_pif_paf/pipelines.py b/src/deepsparse/open_pif_paf/pipelines.py index eb934e7f11..12c6842a60 100644 --- a/src/deepsparse/open_pif_paf/pipelines.py +++ b/src/deepsparse/open_pif_paf/pipelines.py @@ -105,10 +105,12 @@ class properties into an inference ready onnx file to be compiled by the def process_inputs(self, inputs: OpenPifPafInput) -> List[numpy.ndarray]: images = inputs.images + if not isinstance(images, list): images = [images] image_batch = list(self.executor.map(self._preprocess_image, images)) + image_batch = numpy.concatenate(image_batch, axis=0) return [image_batch] diff --git a/src/deepsparse/open_pif_paf/utils/__init__.py b/src/deepsparse/open_pif_paf/utils/__init__.py index be2130c93b..0873a8ab4d 100644 --- a/src/deepsparse/open_pif_paf/utils/__init__.py +++ b/src/deepsparse/open_pif_paf/utils/__init__.py @@ -13,3 +13,4 @@ # limitations under the License. # flake8: noqa from .annotate import * +from .validation import * diff --git a/src/deepsparse/open_pif_paf/utils/validation/__init__.py b/src/deepsparse/open_pif_paf/utils/validation/__init__.py index fec8d68416..0805886a7b 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/__init__.py +++ b/src/deepsparse/open_pif_paf/utils/validation/__init__.py @@ -11,5 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# flake8: noqa from .deepsparse_evaluator import * diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py index dbc7984c9c..c32c690c03 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py @@ -41,7 +41,9 @@ def __init__( def batch(self, model, image_batch, *, device=None, gt_anns_batch=None): """From image batch straight to annotations batch.""" start_nn = time.perf_counter() - fields_batch = self.fields_batch(model, image_batch, device=device) + # deepsparse edit: inference using deepsparse pipeline + # instead of torch model + # fields_batch = self.fields_batch(model, image_batch, device=device) fields_batch = deepsparse_fields_to_torch( self.pipeline(images=image_batch.numpy()) ) diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py index 4c07cae810..b9df9591ab 100644 --- a/src/deepsparse/open_pif_paf/validation.py +++ b/src/deepsparse/open_pif_paf/validation.py @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Evaluation on COCO data.""" from typing import Optional import click from deepsparse import Pipeline -from deepsparse.open_pif_paf.utils.validation.deepsparse_evaluator import ( - DeepSparseEvaluator, -) +from deepsparse.open_pif_paf.utils.validation import DeepSparseEvaluator from openpifpaf.eval import cli @@ -124,7 +121,7 @@ def main( model_path=model_path, engine_type=engine_type, num_cores=num_cores, - image_size=args.coco_eval_long_edge, + image_size=image_size, return_cifcaf_fields=True, ) @@ -132,9 +129,11 @@ def main( pipeline=pipeline, dataset_name=args.dataset, skip_epoch0=False, - img_size=args.coco_eval_long_edge, + img_size=image_size, ) if args.watch: + # this pathway has not been tested + # and is not supported assert args.output is None evaluator.watch(args.checkpoint, args.watch) else: From 4b70e45c6554040cf7b20c78df8a7ad7fbfa3c3d Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 21 Feb 2023 13:05:50 +0000 Subject: [PATCH 05/11] initial commit --- src/deepsparse/open_pif_paf/pipelines.py | 22 +++++++++++++++++++--- src/deepsparse/open_pif_paf/schemas.py | 22 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/deepsparse/open_pif_paf/pipelines.py b/src/deepsparse/open_pif_paf/pipelines.py index ddaa9f1fc3..12c6842a60 100644 --- a/src/deepsparse/open_pif_paf/pipelines.py +++ b/src/deepsparse/open_pif_paf/pipelines.py @@ -22,7 +22,11 @@ import cv2 import torch -from deepsparse.open_pif_paf.schemas import OpenPifPafInput, OpenPifPafOutput +from deepsparse.open_pif_paf.schemas import ( + OpenPifPafFields, + OpenPifPafInput, + OpenPifPafOutput, +) from deepsparse.pipeline import Pipeline from deepsparse.yolact.utils import preprocess_array from openpifpaf import decoder, network @@ -57,12 +61,19 @@ class OpenPifPafPipeline(Pipeline): """ def __init__( - self, *, image_size: Union[int, Tuple[int, int]] = (384, 384), **kwargs + self, + *, + image_size: Union[int, Tuple[int, int]] = (384, 384), + return_cifcaf_fields: bool = False, + **kwargs, ): super().__init__(**kwargs) self._image_size = ( image_size if isinstance(image_size, Tuple) else (image_size, image_size) ) + # whether to return the cif and caf fields or the + # complete decoded output + self.return_cifcaf_fields = return_cifcaf_fields # necessary openpifpaf dependencies for now model_cpu, _ = network.Factory().factory(head_metas=None) self.processor = decoder.factory(model_cpu.head_metas) @@ -79,7 +90,7 @@ def output_schema(self) -> Type[OpenPifPafOutput]: """ :return: pydantic model class that outputs to this pipeline must comply to """ - return OpenPifPafOutput + return OpenPifPafOutput if not self.return_cifcaf_fields else OpenPifPafFields def setup_onnx_file_path(self) -> str: """ @@ -114,6 +125,11 @@ def process_engine_outputs( :return: Outputs of engine post-processed into an object in the `output_schema` format of this pipeline """ + if self.return_cifcaf_fields: + batch_fields = [] + for cif, caf in zip(*fields): + batch_fields.append([cif, caf]) + return OpenPifPafFields(fields=batch_fields) data_batch, skeletons_batch, scores_batch, keypoints_batch = [], [], [], [] diff --git a/src/deepsparse/open_pif_paf/schemas.py b/src/deepsparse/open_pif_paf/schemas.py index 2c384807b5..740d89fa6e 100644 --- a/src/deepsparse/open_pif_paf/schemas.py +++ b/src/deepsparse/open_pif_paf/schemas.py @@ -14,6 +14,7 @@ from typing import List, Tuple +import numpy from pydantic import BaseModel, Field from deepsparse.pipelines.computer_vision import ComputerVisionSchema @@ -22,6 +23,7 @@ __all__ = [ "OpenPifPafInput", "OpenPifPafOutput", + "OpenPifPafFields", ] @@ -33,6 +35,26 @@ class OpenPifPafInput(ComputerVisionSchema): pass +class OpenPifPafFields(BaseModel): + """ + Open Pif Paf is composed of two stages: + - Computing Cif/Caf fields using a parametrized model + - Applying a matching algorithm to obtain the final pose + predictions + In some cases (e.g. for validation), it may be useful to + obtain the Cif/Caf fields as output. + """ + + fields: List[List[numpy.ndarray]] = Field( + description="Cif/Caf fields returned by the network. " + "The outer list is the batch dimension, while the second " + "list contains two numpy arrays: Cif and Caf field values" + ) + + class Config: + arbitrary_types_allowed = True + + class OpenPifPafOutput(BaseModel): """ Output model for Open Pif Paf From e5102f70d20eab95656c63fb93a50e375848c70e Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 21 Feb 2023 14:28:45 +0000 Subject: [PATCH 06/11] ready for testing --- setup.py | 1 + src/deepsparse/open_pif_paf/README.md | 36 ++++++++++++++++++++-- src/deepsparse/open_pif_paf/README_temp.md | 10 ------ src/deepsparse/yolact/utils/utils.py | 16 +++++----- 4 files changed, 43 insertions(+), 20 deletions(-) delete mode 100644 src/deepsparse/open_pif_paf/README_temp.md diff --git a/setup.py b/setup.py index 650047af06..a5d55b901b 100644 --- a/setup.py +++ b/setup.py @@ -135,6 +135,7 @@ def _parse_requirements_file(file_path): _openpifpaf_integration_deps = [ "openpifpaf==0.13.11", "opencv-python<=4.6.0.66", + "pycocotools >=2.0.6", ] _yolov8_integration_deps = _yolo_integration_deps + ["ultralytics==8.0.30"] diff --git a/src/deepsparse/open_pif_paf/README.md b/src/deepsparse/open_pif_paf/README.md index 456f92ad4d..ebf0d3f94f 100644 --- a/src/deepsparse/open_pif_paf/README.md +++ b/src/deepsparse/open_pif_paf/README.md @@ -10,14 +10,44 @@ DeepSparse pipeline for OpenPifPaf ```python from deepsparse import Pipeline -model_path: str = ... # path to open_pif_paf model -pipeline = Pipeline.create(task="open_pif_paf", batch_size=1, model_path=model_path) +model_path: str = ... # path to open_pif_paf model (SparseZoo stub or onnx model) +pipeline = Pipeline.create(task="open_pif_paf", model_path=model_path) predictions = pipeline(images=['dancers.jpg']) # predictions have attributes `data', 'keypoints', 'scores', 'skeletons' predictions[0].scores >> scores=[0.8542259724243828, 0.7930507659912109] ``` -# predictions have attributes `data', 'keypoints', 'scores', 'skeletons' +### Output CifCaf fields +Alternatively, instead of returning the detected poses, it is possible to return the intermediate output - the CifCaf fields. +This is the representation returned directly by the neural network, but not yet processed by the matching algorithm + +```python +... +pipeline = Pipeline.create(task="open_pif_paf", model_path=model_path, return_cifcaf_fields=True) +predictions = pipeline(images=['dancers.jpg']) +predictions.fields +``` + +## Validation script: +This paragraph describes how to run validation of the ONNX model/SparseZoo stub + +### Dataset +For evaluation, you need to download the dataset. The [Open Pif Paf documentation](https://openpifpaf.github.io/) describes +thoroughly how to prepare different datasets for validation. This is the example for `crowdpose` dataset: + +```bash +mkdir data-crowdpose +cd data-crowdpose +# download links here: https://github.com/Jeff-sjtu/CrowdPose +unzip annotations.zip +unzip images.zip +# Now you can use the standard openpifpaf.train and openpifpaf.eval +# commands as documented in Training with --dataset=crowdpose. +``` + +### Validation command +Once the dataset has been downloaded, run the command: +... ## The necessity of external OpenPifPaf helper function image diff --git a/src/deepsparse/open_pif_paf/README_temp.md b/src/deepsparse/open_pif_paf/README_temp.md deleted file mode 100644 index 7362a0bd83..0000000000 --- a/src/deepsparse/open_pif_paf/README_temp.md +++ /dev/null @@ -1,10 +0,0 @@ -For training and evaluation, you need to download the dataset. - -mkdir data-crowdpose -cd data-crowdpose -# download links here: https://github.com/Jeff-sjtu/CrowdPose -unzip annotations.zip -unzip images.zip -Now you can use the standard openpifpaf.train and openpifpaf.eval commands as documented in Training with --dataset=crowdpose. - -install pycocotools \ No newline at end of file diff --git a/src/deepsparse/yolact/utils/utils.py b/src/deepsparse/yolact/utils/utils.py index 1a09d6e46d..c3a78e9293 100644 --- a/src/deepsparse/yolact/utils/utils.py +++ b/src/deepsparse/yolact/utils/utils.py @@ -48,23 +48,25 @@ def preprocess_array( :return: preprocessed numpy array (B, C, D, D); where (D,D) is image size expected by the network. It is a contiguous array with RGB channel order. """ - uint8_image = image[0].dtype == numpy.uint8 + + is_uint8 = image.dtype == numpy.uint8 + + # put channel last to be compatible with cv2.resize image = _assert_channels_last(image) + # resize image to expected size (if needed) if image.ndim == 4 and image.shape[:2] != input_image_size: image = numpy.stack([cv2.resize(img, input_image_size) for img in image]) - else: if image.shape[:2] != input_image_size: image = cv2.resize(image, input_image_size) image = numpy.expand_dims(image, 0) - + # put channel "first" image = image.transpose(0, 3, 1, 2) - if uint8_image: + # if uint8 image, convert to float32 and normalize + if is_uint8: image = image.astype(numpy.float32) image /= 255 - image = numpy.ascontiguousarray(image) - - return image + return numpy.ascontiguousarray(image) def jaccard( From f20027cf650860c12e19b85ab1e3f9797d20fce3 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 21 Feb 2023 14:54:31 +0000 Subject: [PATCH 07/11] remove torch model from the ported code --- .../utils/validation/deepsparse_decoder.py | 5 +++-- .../utils/validation/deepsparse_predictor.py | 4 +++- src/deepsparse/open_pif_paf/validation.py | 13 +------------ 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py index c32c690c03..f19743d0ab 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py @@ -38,12 +38,13 @@ def __init__( # adapted from OPENPIFPAF GITHUB: # https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/decoder/decoder.py # the appropriate edits are marked with # deepsparse edit: - def batch(self, model, image_batch, *, device=None, gt_anns_batch=None): + + # deepsparse edit: removed model argument (not needed, substituted with '_') + def batch(self, _, image_batch, *, device=None, gt_anns_batch=None): """From image batch straight to annotations batch.""" start_nn = time.perf_counter() # deepsparse edit: inference using deepsparse pipeline # instead of torch model - # fields_batch = self.fields_batch(model, image_batch, device=device) fields_batch = deepsparse_fields_to_torch( self.pipeline(images=image_batch.numpy()) ) diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py index 644b1d9898..de8b421594 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py @@ -35,7 +35,9 @@ class DeepSparsePredictor(Predictor): def __init__(self, pipeline: Pipeline, **kwargs): super().__init__(**kwargs) # deepsparse edit: allow for passing in a pipeline and fix the processor - # to CifCaf processor + # to CifCaf processor. Note: we are creating here a default torch model + # but we only use it to get its head metas. This is required to + # initialize the DeepSparseCifCaf processor. self.processor = DeepSparseCifCaf( pipeline=pipeline, head_metas=self.model_cpu.head_metas ) diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py index b9df9591ab..9cb5e96d56 100644 --- a/src/deepsparse/open_pif_paf/validation.py +++ b/src/deepsparse/open_pif_paf/validation.py @@ -33,8 +33,7 @@ ) @click.option( "--model-path", - # required=True, - default="openpifpaf-resnet50.onnx", + required=True, help="Path to the OpenPifPaf onnx model or" "SparseZoo stub to be evaluated.", ) @click.option( @@ -51,13 +50,6 @@ show_default=True, help="Number of CPU cores to run deepsparse with, default is all available", ) -@click.option( - "--batch-size", - type=int, - default=1, - show_default=True, - help="Validation batch size", -) @click.option( "--image_size", type=int, @@ -93,7 +85,6 @@ def main( model_path: str, dataset: str, num_cores: Optional[int], - batch_size: int, image_size: int, name_validation_run: str, engine_type: str, @@ -107,9 +98,7 @@ def main( ) args = cli() args.dataset = dataset - args.batch_size = batch_size args.output = name_validation_run - args.checkpoint = "shufflenetv2k16" args.device = device if dataset == "cocokp": From 868095c331b2547bfabcc694d552d2e09497cbcd Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 21 Feb 2023 15:32:36 +0000 Subject: [PATCH 08/11] solving some issues with logging --- src/deepsparse/open_pif_paf/README.md | 11 ++- .../open_pif_paf/utils/validation/__init__.py | 1 + .../open_pif_paf/utils/validation/helpers.py | 82 ++++++++++++++++++- src/deepsparse/open_pif_paf/validation.py | 3 +- 4 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/deepsparse/open_pif_paf/README.md b/src/deepsparse/open_pif_paf/README.md index ebf0d3f94f..111f80ed4b 100644 --- a/src/deepsparse/open_pif_paf/README.md +++ b/src/deepsparse/open_pif_paf/README.md @@ -44,10 +44,19 @@ unzip images.zip # Now you can use the standard openpifpaf.train and openpifpaf.eval # commands as documented in Training with --dataset=crowdpose. ``` +### Create an ONNX model: + +```bash +python3 -m openpifpaf.export_onnx --input-width 641 --input-height 641 +``` ### Validation command Once the dataset has been downloaded, run the command: -... +```bash +deepsparse.pose_estimation.eval --model-path openpifpaf-resnet50.onnx --dataset cocokp --image_size 641 +``` + +### Expected output: ## The necessity of external OpenPifPaf helper function image diff --git a/src/deepsparse/open_pif_paf/utils/validation/__init__.py b/src/deepsparse/open_pif_paf/utils/validation/__init__.py index 0805886a7b..30c5963274 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/__init__.py +++ b/src/deepsparse/open_pif_paf/utils/validation/__init__.py @@ -13,3 +13,4 @@ # limitations under the License. # flake8: noqa from .deepsparse_evaluator import * +from .helpers import * diff --git a/src/deepsparse/open_pif_paf/utils/validation/helpers.py b/src/deepsparse/open_pif_paf/utils/validation/helpers.py index a5b2dc6777..5d31a8bd18 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/helpers.py +++ b/src/deepsparse/open_pif_paf/utils/validation/helpers.py @@ -12,14 +12,92 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse +import logging from typing import List import torch from deepsparse.open_pif_paf.schemas import OpenPifPafFields -from openpifpaf import transforms +from openpifpaf import ( + Predictor, + __version__, + datasets, + decoder, + logger, + network, + show, + transforms, + visualizer, +) +from openpifpaf.eval import CustomFormatter, Evaluator -__all__ = ["apply_deepsparse_preprocessing", "deepsparse_fields_to_torch"] +LOG = logging.getLogger(__name__) + +__all__ = ["cli", "apply_deepsparse_preprocessing", "deepsparse_fields_to_torch"] + + +# adapted from OPENPIFPAF GITHUB: +# https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/eval.py +# the appropriate edits are marked with # deepsparse edit: +def cli(): + parser = argparse.ArgumentParser( + prog="python3 -m openpifpaf.eval", + usage="%(prog)s [options]", + description=__doc__, + formatter_class=CustomFormatter, + ) + parser.add_argument( + "--version", + action="version", + version="OpenPifPaf {version}".format(version=__version__), + ) + + datasets.cli(parser) + decoder.cli(parser) + logger.cli(parser) + network.Factory.cli(parser) + Predictor.cli(parser, skip_batch_size=True, skip_loader_workers=True) + show.cli(parser) + visualizer.cli(parser) + Evaluator.cli(parser) + + parser.add_argument("--disable-cuda", action="store_true", help="disable CUDA") + parser.add_argument( + "--output", default=None, help="output filename without file extension" + ) + parser.add_argument( + "--watch", + default=False, + const=60, + nargs="?", + type=int, + help=( + "Watch a directory for new checkpoint files. " + "Optionally specify the number of seconds between checks." + ), + ) + + # deepsparse edit: replace the parse_args call with parse_known_args + args, unknown = parser.parse_known_args() + + # add args.device + args.device = torch.device("cpu") + args.pin_memory = False + if not args.disable_cuda and torch.cuda.is_available(): + args.device = torch.device("cuda") + args.pin_memory = True + LOG.debug("neural network device: %s", args.device) + + datasets.configure(args) + decoder.configure(args) + network.Factory.configure(args) + Predictor.configure(args) + show.configure(args) + visualizer.configure(args) + Evaluator.configure(args) + + return args def deepsparse_fields_to_torch( diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py index 9cb5e96d56..4dbb52b307 100644 --- a/src/deepsparse/open_pif_paf/validation.py +++ b/src/deepsparse/open_pif_paf/validation.py @@ -17,8 +17,7 @@ import click from deepsparse import Pipeline -from deepsparse.open_pif_paf.utils.validation import DeepSparseEvaluator -from openpifpaf.eval import cli +from deepsparse.open_pif_paf.utils.validation import DeepSparseEvaluator, cli DEEPSPARSE_ENGINE = "deepsparse" From 801da7e965a2366e1de517cdce7409506e512cb6 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 21 Feb 2023 16:00:29 +0000 Subject: [PATCH 09/11] ready for review --- setup.py | 1 + src/deepsparse/open_pif_paf/README.md | 17 ++++ .../open_pif_paf/utils/validation/__init__.py | 2 +- .../open_pif_paf/utils/validation/cli.py | 94 +++++++++++++++++++ .../utils/validation/deepsparse_decoder.py | 2 + .../utils/validation/deepsparse_predictor.py | 2 + .../open_pif_paf/utils/validation/helpers.py | 80 +--------------- src/deepsparse/open_pif_paf/validation.py | 3 + 8 files changed, 123 insertions(+), 78 deletions(-) create mode 100644 src/deepsparse/open_pif_paf/utils/validation/cli.py diff --git a/setup.py b/setup.py index a5d55b901b..2899e7a7e7 100644 --- a/setup.py +++ b/setup.py @@ -136,6 +136,7 @@ def _parse_requirements_file(file_path): "openpifpaf==0.13.11", "opencv-python<=4.6.0.66", "pycocotools >=2.0.6", + "scipy==1.10.1", ] _yolov8_integration_deps = _yolo_integration_deps + ["ultralytics==8.0.30"] diff --git a/src/deepsparse/open_pif_paf/README.md b/src/deepsparse/open_pif_paf/README.md index 111f80ed4b..bd6fb25728 100644 --- a/src/deepsparse/open_pif_paf/README.md +++ b/src/deepsparse/open_pif_paf/README.md @@ -56,6 +56,23 @@ Once the dataset has been downloaded, run the command: deepsparse.pose_estimation.eval --model-path openpifpaf-resnet50.onnx --dataset cocokp --image_size 641 ``` +This should result in the evaluation output similar to this: +```bash +... + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.502 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.732 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.523 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.429 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.605 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.534 + Average Recall (AR) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.744 + Average Recall (AR) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.554 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.457 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.643 +... +```` + + ### Expected output: ## The necessity of external OpenPifPaf helper function diff --git a/src/deepsparse/open_pif_paf/utils/validation/__init__.py b/src/deepsparse/open_pif_paf/utils/validation/__init__.py index 30c5963274..5280e7e84b 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/__init__.py +++ b/src/deepsparse/open_pif_paf/utils/validation/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. # flake8: noqa +from .cli import * from .deepsparse_evaluator import * -from .helpers import * diff --git a/src/deepsparse/open_pif_paf/utils/validation/cli.py b/src/deepsparse/open_pif_paf/utils/validation/cli.py new file mode 100644 index 0000000000..6efb85c073 --- /dev/null +++ b/src/deepsparse/open_pif_paf/utils/validation/cli.py @@ -0,0 +1,94 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging + +import torch +from deepsparse.open_pif_paf.utils.validation.deepsparse_evaluator import ( + DeepSparseEvaluator, +) +from deepsparse.open_pif_paf.utils.validation.deepsparse_predictor import ( + DeepSparsePredictor, +) +from openpifpaf import __version__, datasets, decoder, logger, network, show, visualizer +from openpifpaf.eval import CustomFormatter + + +LOG = logging.getLogger(__name__) + +__all__ = ["cli"] + + +# adapted from OPENPIFPAF GITHUB: +# https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/eval.py +# the appropriate edits are marked with # deepsparse edit: +def cli(): + parser = argparse.ArgumentParser( + prog="python3 -m openpifpaf.eval", + usage="%(prog)s [options]", + description=__doc__, + formatter_class=CustomFormatter, + ) + parser.add_argument( + "--version", + action="version", + version="OpenPifPaf {version}".format(version=__version__), + ) + + datasets.cli(parser) + decoder.cli(parser) + logger.cli(parser) + network.Factory.cli(parser) + DeepSparsePredictor.cli(parser, skip_batch_size=True, skip_loader_workers=True) + show.cli(parser) + visualizer.cli(parser) + DeepSparseEvaluator.cli(parser) + + parser.add_argument("--disable-cuda", action="store_true", help="disable CUDA") + parser.add_argument( + "--output", default=None, help="output filename without file extension" + ) + parser.add_argument( + "--watch", + default=False, + const=60, + nargs="?", + type=int, + help=( + "Watch a directory for new checkpoint files. " + "Optionally specify the number of seconds between checks." + ), + ) + + # deepsparse edit: replace the parse_args call with parse_known_args + args, unknown = parser.parse_known_args() + + # add args.device + args.device = torch.device("cpu") + args.pin_memory = False + if not args.disable_cuda and torch.cuda.is_available(): + args.device = torch.device("cuda") + args.pin_memory = True + LOG.debug("neural network device: %s", args.device) + + datasets.configure(args) + decoder.configure(args) + network.Factory.configure(args) + DeepSparsePredictor.configure(args) + show.configure(args) + visualizer.configure(args) + DeepSparseEvaluator.configure(args) + + return args diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py index f19743d0ab..8e2a9af102 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_decoder.py @@ -24,6 +24,8 @@ LOG = logging.getLogger(__name__) +__all__ = ["DeepSparseCifCaf"] + class DeepSparseCifCaf(CifCaf): def __init__( diff --git a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py index de8b421594..b22af1dc96 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py +++ b/src/deepsparse/open_pif_paf/utils/validation/deepsparse_predictor.py @@ -21,6 +21,8 @@ LOG = logging.getLogger(__name__) +__all__ = ["DeepSparsePredictor"] + # adapted from OPENPIFPAF GITHUB: # https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/predictor.py diff --git a/src/deepsparse/open_pif_paf/utils/validation/helpers.py b/src/deepsparse/open_pif_paf/utils/validation/helpers.py index 5d31a8bd18..36c43c4996 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/helpers.py +++ b/src/deepsparse/open_pif_paf/utils/validation/helpers.py @@ -12,92 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse + import logging from typing import List import torch from deepsparse.open_pif_paf.schemas import OpenPifPafFields -from openpifpaf import ( - Predictor, - __version__, - datasets, - decoder, - logger, - network, - show, - transforms, - visualizer, -) -from openpifpaf.eval import CustomFormatter, Evaluator +from openpifpaf import transforms LOG = logging.getLogger(__name__) -__all__ = ["cli", "apply_deepsparse_preprocessing", "deepsparse_fields_to_torch"] - - -# adapted from OPENPIFPAF GITHUB: -# https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/eval.py -# the appropriate edits are marked with # deepsparse edit: -def cli(): - parser = argparse.ArgumentParser( - prog="python3 -m openpifpaf.eval", - usage="%(prog)s [options]", - description=__doc__, - formatter_class=CustomFormatter, - ) - parser.add_argument( - "--version", - action="version", - version="OpenPifPaf {version}".format(version=__version__), - ) - - datasets.cli(parser) - decoder.cli(parser) - logger.cli(parser) - network.Factory.cli(parser) - Predictor.cli(parser, skip_batch_size=True, skip_loader_workers=True) - show.cli(parser) - visualizer.cli(parser) - Evaluator.cli(parser) - - parser.add_argument("--disable-cuda", action="store_true", help="disable CUDA") - parser.add_argument( - "--output", default=None, help="output filename without file extension" - ) - parser.add_argument( - "--watch", - default=False, - const=60, - nargs="?", - type=int, - help=( - "Watch a directory for new checkpoint files. " - "Optionally specify the number of seconds between checks." - ), - ) - - # deepsparse edit: replace the parse_args call with parse_known_args - args, unknown = parser.parse_known_args() - - # add args.device - args.device = torch.device("cpu") - args.pin_memory = False - if not args.disable_cuda and torch.cuda.is_available(): - args.device = torch.device("cuda") - args.pin_memory = True - LOG.debug("neural network device: %s", args.device) - - datasets.configure(args) - decoder.configure(args) - network.Factory.configure(args) - Predictor.configure(args) - show.configure(args) - visualizer.configure(args) - Evaluator.configure(args) - - return args +__all__ = ["apply_deepsparse_preprocessing", "deepsparse_fields_to_torch"] def deepsparse_fields_to_torch( diff --git a/src/deepsparse/open_pif_paf/validation.py b/src/deepsparse/open_pif_paf/validation.py index 4dbb52b307..9ca4ae70f5 100644 --- a/src/deepsparse/open_pif_paf/validation.py +++ b/src/deepsparse/open_pif_paf/validation.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from typing import Optional import click @@ -24,6 +25,8 @@ ORT_ENGINE = "onnxruntime" SUPPORTED_DATASET_CONFIGS = ["cocokp"] +logging.basicConfig(level=logging.INFO) + @click.command( context_settings=( From 54e2a2c17aeb37e5d9b218c313dce595dfd58267 Mon Sep 17 00:00:00 2001 From: dbogunowicz <97082108+dbogunowicz@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:05:10 +0100 Subject: [PATCH 10/11] Update src/deepsparse/open_pif_paf/utils/validation/cli.py --- src/deepsparse/open_pif_paf/utils/validation/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deepsparse/open_pif_paf/utils/validation/cli.py b/src/deepsparse/open_pif_paf/utils/validation/cli.py index 6efb85c073..b92c839d06 100644 --- a/src/deepsparse/open_pif_paf/utils/validation/cli.py +++ b/src/deepsparse/open_pif_paf/utils/validation/cli.py @@ -73,7 +73,7 @@ def cli(): ) # deepsparse edit: replace the parse_args call with parse_known_args - args, unknown = parser.parse_known_args() + args, _ = parser.parse_known_args() # add args.device args.device = torch.device("cpu") From 4bea820df2237f5133c4aaef58b2608a84720aac Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 28 Feb 2023 10:05:30 +0000 Subject: [PATCH 11/11] prohibit openpifpaf fields in the server --- src/deepsparse/open_pif_paf/pipelines.py | 11 ++++++++- src/deepsparse/open_pif_paf/schemas.py | 29 ++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/deepsparse/open_pif_paf/pipelines.py b/src/deepsparse/open_pif_paf/pipelines.py index 12c6842a60..fe9bb1b30a 100644 --- a/src/deepsparse/open_pif_paf/pipelines.py +++ b/src/deepsparse/open_pif_paf/pipelines.py @@ -111,9 +111,18 @@ def process_inputs(self, inputs: OpenPifPafInput) -> List[numpy.ndarray]: image_batch = list(self.executor.map(self._preprocess_image, images)) + original_image_shapes = None + if image_batch and isinstance(image_batch[0], tuple): + # splits image batch is of format: + # [(preprocesses_img, original_image_shape), ...] into separate lists + image_batch, original_image_shapes = list(map(list, zip(*image_batch))) + image_batch = numpy.concatenate(image_batch, axis=0) - return [image_batch] + postprocessing_kwargs = dict( + original_image_shapes=original_image_shapes, + ) + return [image_batch], postprocessing_kwargs def process_engine_outputs( self, fields: List[numpy.ndarray], **kwargs diff --git a/src/deepsparse/open_pif_paf/schemas.py b/src/deepsparse/open_pif_paf/schemas.py index 740d89fa6e..5628bb2b22 100644 --- a/src/deepsparse/open_pif_paf/schemas.py +++ b/src/deepsparse/open_pif_paf/schemas.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple +from typing import Iterable, List, TextIO, Tuple import numpy +from PIL import Image from pydantic import BaseModel, Field from deepsparse.pipelines.computer_vision import ComputerVisionSchema @@ -48,9 +49,33 @@ class OpenPifPafFields(BaseModel): fields: List[List[numpy.ndarray]] = Field( description="Cif/Caf fields returned by the network. " "The outer list is the batch dimension, while the second " - "list contains two numpy arrays: Cif and Caf field values" + "list contains two numpy arrays: Cif and Caf field values. " ) + @classmethod + def from_files( + cls, files: Iterable[TextIO], *args, from_server: bool = False, **kwargs + ) -> "OpenPifPafFields": + """ + :param files: Iterable of file pointers to create OpenPifPafFields from + :param kwargs: extra keyword args to pass to OpenPifPafFields constructor + :return: OpenPifPafFields constructed from files + """ + if "images" in kwargs: + raise ValueError( + f"argument 'images' cannot be specified in {cls.__name__} when " + "constructing from file(s)" + ) + if from_server: + raise ValueError( + "Cannot construct OpenPifPafFields from server. This will create" + "numpy arrays that are not serializable." + ) + + files_numpy = [numpy.array(Image.open(file)) for file in files] + input_schema = cls(*args, images=files_numpy, **kwargs) + return input_schema + class Config: arbitrary_types_allowed = True