From 0ed02e71d5d82a624e622716de148907a66294a3 Mon Sep 17 00:00:00 2001 From: Julia Werner Date: Tue, 14 May 2024 11:26:47 +0000 Subject: [PATCH 1/2] Fixed correct train val test split, by getting validation set as proportion... --- hannah/datasets/vision/kvasir.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hannah/datasets/vision/kvasir.py b/hannah/datasets/vision/kvasir.py index 4aac843b..55bfcea6 100644 --- a/hannah/datasets/vision/kvasir.py +++ b/hannah/datasets/vision/kvasir.py @@ -192,8 +192,9 @@ def process_official_split(df: pd.DataFrame): train_images = split0_paths train_labels = split0_labels - val_images = split1_paths - val_labels = split1_labels + train_images, val_images, train_labels, val_labels = train_test_split( + train_images, train_labels, test_size=0.25, random_state=1 + ) test_images = split1_paths test_labels = split1_labels From cd3dfc67fd7d158fea37321c04bf67cad8fcca44 Mon Sep 17 00:00:00 2001 From: Moritz Reiber Date: Wed, 15 May 2024 13:43:07 +0000 Subject: [PATCH 2/2] Merge ICCAD24 changes into main --- .../augmentation/cifar_augment.yaml | 20 + experiments/residual_choice/config.yaml | 45 ++ .../experiment/ae_nas_cifar10.yaml | 32 + .../experiment/ae_nas_cifar10_1000k.yaml | 32 + .../experiment/ae_nas_cifar10_1000k_256m.yaml | 32 + .../experiment/ae_nas_cifar10_150k.yaml | 33 + .../ae_nas_cifar10_150k_noconstraints.yaml | 35 + .../experiment/ae_nas_cifar10_250k.yaml | 31 + .../experiment/ae_nas_cifar10_30k.yaml | 33 + .../ae_nas_cifar10_30k_noconstraints.yaml | 35 + .../experiment/ae_nas_cifar10_500k_256m.yaml | 32 + .../ae_nas_cifar10_500k_nobounds.yaml | 28 + .../experiment/random_nas_cifar10_30k.yaml | 33 + .../experiment/train_best_model.yaml | 23 + .../experiment/train_best_model_1000_256.yaml | 23 + .../experiment/train_best_model_550_256.yaml | 23 + hannah/conf/model/capsule_net.yaml | 67 -- hannah/conf/nas/aging_evolution_nas.yaml | 4 +- hannah/conf/nas/predictor/gcn.yaml | 4 +- hannah/models/capsule_net/__init__.py | 1 - hannah/models/capsule_net/expressions.py | 84 --- hannah/models/capsule_net/models.py | 492 -------------- hannah/models/capsule_net/operators.py | 285 -------- hannah/models/capsule_net/utils.py | 24 - hannah/models/embedded_vision_net/blocks.py | 8 +- .../models/embedded_vision_net/operators.py | 5 + hannah/nas/constraints/random_walk.py | 1 + hannah/nas/functional_operators/op.py | 11 + hannah/nas/parameters/parameters.py | 6 +- hannah/nas/parameters/parametrize.py | 8 +- .../features/dataset.py | 35 +- .../model_trainer/simple_model_trainer.py | 2 +- hannah/nas/search/sampler/mutator.py | 12 +- hannah/nas/search/search.py | 21 +- hannah/nas/test/test_symbolic_metrics.py | 137 ---- test/test_capsule_net.py | 84 --- test/test_capsule_net_constraints.py | 79 --- test/test_symbolic_check.py | 73 -- test/test_symbolic_metrics.py | 623 ++++++++++++++++++ 39 files changed, 1185 insertions(+), 1371 deletions(-) create mode 100644 experiments/residual_choice/augmentation/cifar_augment.yaml create mode 100644 experiments/residual_choice/config.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_1000k.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_1000k_256m.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_150k.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_150k_noconstraints.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_250k.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_30k.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_30k_noconstraints.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_500k_256m.yaml create mode 100644 experiments/residual_choice/experiment/ae_nas_cifar10_500k_nobounds.yaml create mode 100644 experiments/residual_choice/experiment/random_nas_cifar10_30k.yaml create mode 100644 experiments/residual_choice/experiment/train_best_model.yaml create mode 100644 experiments/residual_choice/experiment/train_best_model_1000_256.yaml create mode 100644 experiments/residual_choice/experiment/train_best_model_550_256.yaml delete mode 100644 hannah/conf/model/capsule_net.yaml delete mode 100644 hannah/models/capsule_net/__init__.py delete mode 100644 hannah/models/capsule_net/expressions.py delete mode 100644 hannah/models/capsule_net/models.py delete mode 100644 hannah/models/capsule_net/operators.py delete mode 100644 hannah/models/capsule_net/utils.py delete mode 100644 hannah/nas/test/test_symbolic_metrics.py delete mode 100644 test/test_capsule_net.py delete mode 100644 test/test_capsule_net_constraints.py delete mode 100644 test/test_symbolic_check.py create mode 100644 test/test_symbolic_metrics.py diff --git a/experiments/residual_choice/augmentation/cifar_augment.yaml b/experiments/residual_choice/augmentation/cifar_augment.yaml new file mode 100644 index 00000000..24d4f70c --- /dev/null +++ b/experiments/residual_choice/augmentation/cifar_augment.yaml @@ -0,0 +1,20 @@ +batch_augment: + pipeline: null + transforms: + #RandomVerticalFlip: + # p: 0.5 + RandomHorizontalFlip: + p: 0.5 + RandomAffine: + degrees: [-15, 15] + translate: [0.1, 0.1] + scale: [0.9, 1.1] + shear: [-5, 5] + p: 0.5 + RandomCrop: + size: [32,32] + padding: 4 + RandomErasing: + p: 0.5 + #scale: [0.!, 0.3] + #value: [0.4914, 0.4822, 0.4465] diff --git a/experiments/residual_choice/config.yaml b/experiments/residual_choice/config.yaml new file mode 100644 index 00000000..c418df73 --- /dev/null +++ b/experiments/residual_choice/config.yaml @@ -0,0 +1,45 @@ +## +## Copyright (c) 2022 University of Tübingen. +## +## This file is part of hannah. +## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +defaults: + - base_config + - experiment: optional + - override dataset: cifar10 # Dataset configuration name + - override features: identity # Feature extractor configuration name (use identity for vision datasets) + #- override model: timm_mobilenetv3_small_075 # Neural network name (for now timm_resnet50 or timm_efficientnet_lite1) + - override scheduler: 1cycle # learning rate scheduler config name + - override optimizer: adamw # Optimizer config name + - override normalizer: null # Feature normalizer (used for quantized neural networks) + - override module: image_classifier # Lightning module config for the training loop (image classifier for image classification tasks) + - _self_ + + +dataset: + data_folder: ${oc.env:HANNAH_DATA_FOLDER,${hydra:runtime.cwd}/../../datasets/} + +module: + batch_size: 128 + num_workers: 0 + +trainer: + max_epochs: 10 + +scheduler: + max_lr: 0.001 + +fx_mac_summary: True diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10.yaml new file mode 100644 index 00000000..38beb63c --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 550000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 550000 + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w550000_m128000000_noinitconstraint" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_1000k.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_1000k.yaml new file mode 100644 index 00000000..9addb206 --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_1000k.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 1000000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 1000000 + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w1000000_m128000000_noinitconstraint" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_1000k_256m.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_1000k_256m.yaml new file mode 100644 index 00000000..e2002746 --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_1000k_256m.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 1000000 + - name: macs + upper: 256000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 256000000 + total_weights: 1000000 + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w1000000_m256000000" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_150k.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_150k.yaml new file mode 100644 index 00000000..96f47f0e --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_150k.yaml @@ -0,0 +1,33 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 150000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 150000 + + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w150000_m128000000" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_150k_noconstraints.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_150k_noconstraints.yaml new file mode 100644 index 00000000..f5176140 --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_150k_noconstraints.yaml @@ -0,0 +1,35 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + #upper: 150000 + - name: macs + #upper: 128000000 + + + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 150000 + + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w150000_m128000000_noconstraints_v2" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_250k.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_250k.yaml new file mode 100644 index 00000000..ac918525 --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_250k.yaml @@ -0,0 +1,31 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 250000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 250000 + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w250000_m128000000_presample" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_30k.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_30k.yaml new file mode 100644 index 00000000..0bb04e04 --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_30k.yaml @@ -0,0 +1,33 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 30000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 30000 + + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w30000_m128000000" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_30k_noconstraints.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_30k_noconstraints.yaml new file mode 100644 index 00000000..3f51ef7b --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_30k_noconstraints.yaml @@ -0,0 +1,35 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + #upper: 150000 + - name: macs + #upper: 128000000 + + + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 128000000 + total_weights: 30000 + + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w30000_m128000000_noconstraints" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_500k_256m.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_500k_256m.yaml new file mode 100644 index 00000000..aa790bb2 --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_500k_256m.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 550000 + - name: macs + upper: 256000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + total_macs: 256000000 + total_weights: 550000 + + + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w550000_m256000000" diff --git a/experiments/residual_choice/experiment/ae_nas_cifar10_500k_nobounds.yaml b/experiments/residual_choice/experiment/ae_nas_cifar10_500k_nobounds.yaml new file mode 100644 index 00000000..a8cbcd6e --- /dev/null +++ b/experiments/residual_choice/experiment/ae_nas_cifar10_500k_nobounds.yaml @@ -0,0 +1,28 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 550000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + bounds: + val_error: 0.1 + +seed: [1234] + +experiment_id: "ae_nas_cifar10_w550000_m128000000_nobounds" diff --git a/experiments/residual_choice/experiment/random_nas_cifar10_30k.yaml b/experiments/residual_choice/experiment/random_nas_cifar10_30k.yaml new file mode 100644 index 00000000..3725e973 --- /dev/null +++ b/experiments/residual_choice/experiment/random_nas_cifar10_30k.yaml @@ -0,0 +1,33 @@ +# @package _global_ +defaults: + - override /nas: random_nas + - override /model: embedded_vision_net + - override /dataset: cifar10 + - override /nas/constraint_model: random_walk + + +model: + num_classes: 10 + + constraints: + - name: weights + upper: 30000 + - name: macs + upper: 128000000 + +nas: + budget: 600 + n_jobs: 1 + num_selected_candidates: 20 + total_candidates: 50 + # bounds: + # val_error: 0.1 + # total_macs: 128000000 + # total_weights: 150000 + + + + +seed: [1234] + +experiment_id: "random_nas_cifar10_w30000_m128000000" diff --git a/experiments/residual_choice/experiment/train_best_model.yaml b/experiments/residual_choice/experiment/train_best_model.yaml new file mode 100644 index 00000000..94f9e8a4 --- /dev/null +++ b/experiments/residual_choice/experiment/train_best_model.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /model: embedded_vision_net_model + - override /augmentation: cifar_augment + +model: + param_path: /local/reiber/hannah/experiments/residual_choice/parameters.pkl + task_name: 1000w_128m_presample + index: 553 + input_shape: [1, 3, 32, 32] + labels: 10 + +module: + batch_size: 64 + num_workers: 8 + +trainer: + max_epochs: 600 + +scheduler: + max_lr: 0.001 + +experiment_id: best_model_1000w128m_augment_adam0001_b64_600e \ No newline at end of file diff --git a/experiments/residual_choice/experiment/train_best_model_1000_256.yaml b/experiments/residual_choice/experiment/train_best_model_1000_256.yaml new file mode 100644 index 00000000..1582dd18 --- /dev/null +++ b/experiments/residual_choice/experiment/train_best_model_1000_256.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /model: embedded_vision_net_model + - override /augmentation: cifar_augment + +model: + param_path: /local/reiber/hannah/experiments/residual_choice/parameters.pkl + task_name: 1000w_256m + index: 407 + input_shape: [1, 3, 32, 32] + labels: 10 + +module: + batch_size: 64 + num_workers: 8 + +trainer: + max_epochs: 600 + +scheduler: + max_lr: 0.001 + +experiment_id: best_model_1000w256m_augment_adam0001_b64_600e \ No newline at end of file diff --git a/experiments/residual_choice/experiment/train_best_model_550_256.yaml b/experiments/residual_choice/experiment/train_best_model_550_256.yaml new file mode 100644 index 00000000..48221f5f --- /dev/null +++ b/experiments/residual_choice/experiment/train_best_model_550_256.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /model: embedded_vision_net_model + - override /augmentation: cifar_augment + +model: + param_path: /local/reiber/hannah/experiments/residual_choice/parameters.pkl + task_name: 550w_256m + index: 371 + input_shape: [1, 3, 32, 32] + labels: 10 + +module: + batch_size: 64 + num_workers: 8 + +trainer: + max_epochs: 600 + +scheduler: + max_lr: 0.001 + +experiment_id: best_model_550w256m_augment_adam0001_b64_600e \ No newline at end of file diff --git a/hannah/conf/model/capsule_net.yaml b/hannah/conf/model/capsule_net.yaml deleted file mode 100644 index 699e112d..00000000 --- a/hannah/conf/model/capsule_net.yaml +++ /dev/null @@ -1,67 +0,0 @@ -_target_: hannah.models.capsule_net.SearchSpace -name: capsule_net -params: - min_reduction: 4 - max_layer: 18 - num_blocks: - min: 3 - max: 10 - - block: - depth: - min: 1 - max: 3 - patterns: - choices: ["convolution", "expand_reduce", "reduce_expand", "pooling"] - out_channels: - min: 16 - max: 64 - step: 16 - stride: - choices: [1, 2] - convolution: - kernel_size: - choices: [3, 5, 7, 9] - stride: - choices: [1, 2] - out_channels: - min: 16 - max: 64 - step: 16 - expand_reduce: - ratio: - min: 1.5 - max: 6 - reduce_expand: - ratio: - min: 6 - max: 10 - pooling: - kernel_size: - choices: [3, 5, 7] - stride: - choices: [2] - type: - choices: ['max', 'avg'] - activation: - choices: ["relu"] - residual: - activation: - choices: ["relu"] - - stem: - convolution: - kernel_size: - choices: [3] - stride: - choices: [1, 2] - out_channels: - min: 8 - max: 32 - step: 8 - activation: - choices: ['relu'] - head: [] - -labels: 10 - diff --git a/hannah/conf/nas/aging_evolution_nas.yaml b/hannah/conf/nas/aging_evolution_nas.yaml index 12f932e4..25372389 100644 --- a/hannah/conf/nas/aging_evolution_nas.yaml +++ b/hannah/conf/nas/aging_evolution_nas.yaml @@ -33,5 +33,5 @@ total_candidates: 50 num_selected_candidates: 20 bounds: val_error: 0.1 - total_macs: 128000000 - total_weights: 500000 + # total_macs: 128000000 + # total_weights: 500000 diff --git a/hannah/conf/nas/predictor/gcn.yaml b/hannah/conf/nas/predictor/gcn.yaml index ebd51575..bb73ef17 100644 --- a/hannah/conf/nas/predictor/gcn.yaml +++ b/hannah/conf/nas/predictor/gcn.yaml @@ -17,10 +17,8 @@ ## limitations under the License. ## - - gcn: _target_: hannah.nas.performance_prediction.simple.GCNPredictor model: _target_: hannah.nas.performance_prediction.gcn.predictor.GaussianProcessPredictor - input_feature_size: 31 + input_feature_size: 35 diff --git a/hannah/models/capsule_net/__init__.py b/hannah/models/capsule_net/__init__.py deleted file mode 100644 index 448f9a06..00000000 --- a/hannah/models/capsule_net/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .models import * # noqa \ No newline at end of file diff --git a/hannah/models/capsule_net/expressions.py b/hannah/models/capsule_net/expressions.py deleted file mode 100644 index 137a779c..00000000 --- a/hannah/models/capsule_net/expressions.py +++ /dev/null @@ -1,84 +0,0 @@ -from hannah.nas.expressions.logic import And, If -from hannah.nas.expressions.arithmetic import Ceil - - -def padding_expression(kernel_size, stride, dilation = 1): - """Symbolically calculate padding such that for a given kernel_size, stride and dilation - the padding is such that the output dimension is kept the same(stride=1) or halved(stride=2). - Note: If the input dimension is 1 and stride = 2, the calculated padding will result in - an output with also dimension 1. - - Parameters - ---------- - kernel_size : Union[int, Expression] - stride : Union[int, Expression] - dilation : Union[int, Expression], optional - _description_, by default 1 - - Returns - ------- - Expression - """ - # r = 1 - (kernel_size % 2) - p = (dilation * (kernel_size - 1) - stride + 1) / 2 - return Ceil(p) - - -def expr_product(expressions: list): - res = None - for expr in expressions: - if res: - res = res * expr - else: - res = expr - return res - - -def expr_sum(expressions: list): - res = None - for expr in expressions: - if res: - res = res + expr - else: - res = expr - return res - - -# FIXME: replace in search space with depth_aware_sum -def num_layer_constraint(depth_expr, num_blocks): - num = num_blocks - ct = 1 - res = None - for expr in depth_expr: - if res: - res = res + (expr * If(num >= ct, 1, 0)) - else: - res = expr - ct += 1 - return res - -def depth_aware_downsampling(strides, params): - res = strides.pop(-1) # get stride of stem - for stride in strides: - block_num = int(stride.id.split(".")[1]) - pattern_num = int(stride.id.split(".")[3]) - block_depth = params[f"block.{block_num}.depth"] - num_blocks = params["num_blocks"] - - if res: - res = res * If(num_blocks > block_num, If(block_depth > pattern_num, stride, 1), 1) - else: - res = If(num_blocks > block_num, If(block_depth > pattern_num, stride, 1), 1) - return res - - -def depth_aware_sum(param_list, depth_param): - ct = 1 - res = None - for expr in param_list: - if res: - res = res + (expr * If(depth_param >= ct, 1, 0)) - else: - res = expr - ct += 1 - return res diff --git a/hannah/models/capsule_net/models.py b/hannah/models/capsule_net/models.py deleted file mode 100644 index 49084efc..00000000 --- a/hannah/models/capsule_net/models.py +++ /dev/null @@ -1,492 +0,0 @@ -from omegaconf import OmegaConf -import torch -import torch.nn as nn -from hannah.nas.expressions.logic import And -from hannah.nas.expressions.metrics import conv2d_macs, conv2d_weights -from hannah.nas.expressions.shapes import conv2d_shape -from hannah.nas.parameters.lazy import Lazy -from hannah.nas.parameters.parametrize import parametrize -from hannah.nas.parameters.iterators import RangeIterator -from hannah.nas.parameters.parameters import IntScalarParameter, CategoricalParameter -from hannah.nas.expressions.arithmetic import Ceil -from hannah.nas.expressions.choice import Choice -from hannah.nas.expressions.types import Int - -from hannah.models.capsule_net.utils import handle_parameter -from hannah.models.capsule_net.expressions import depth_aware_downsampling, depth_aware_sum, expr_sum, num_layer_constraint -from hannah.models.capsule_net.operators import ( - Convolution, - DepthwiseConvolution, - Linear, - PointwiseConvolution, - Relu, - Identity, - BatchNorm, - Pooling, -) - - -conv2d = Lazy(nn.Conv2d, shape_func=conv2d_shape) - -@parametrize -class Activation(nn.Module): - def __init__(self, params, id) -> None: - super().__init__() - self.params = params - self.id = id - - # FIXME: Introduce choice here - relu_act = Relu(self.id) - identity_act = Identity(self.id) - - self.mods = nn.ModuleDict({"relu": relu_act, "identity": identity_act}) - # self.mods = {"relu": relu_act, "identity": identity_act} - self.choice = handle_parameter(self, params, "choice") - self.active_module = Choice(self.mods, self.choice) - - def initialize(self): - self.active_module.evaluate().initialize() - - def forward(self, x): - return self.active_module.evaluate()(x) - - -@parametrize -class ReducedComplexitySpatialOperator(nn.Module): - def __init__(self) -> None: - super().__init__() - - -@parametrize -class ExpandReduce(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.params = params - self.inputs = inputs - input_shape = inputs[0] - self.id = id - self.in_channels = input_shape[1] - - # FIXME: Share parameters over patterns - self.out_channels = handle_parameter(self, self.params.convolution.out_channels, "out_channels") - self.expand_ratio = handle_parameter(self, self.params.expand_reduce.ratio, "expand_ratio") - - self.expanded_channels = Int(self.expand_ratio * self.in_channels) - self.expansion = PointwiseConvolution(self.expanded_channels, self.id + '.expand', inputs) - self.bn0 = BatchNorm(None, self.id, [self.expansion.shape]) - self.act0 = Activation(params.activation, self.id + ".activation0") - - # FIXME: Replace with ReducedComplexitySpatialOperator - self.spatial_correlations = DepthwiseConvolution(params.convolution, self.id, [self.expansion.shape]) - self.bn1 = BatchNorm(None, self.id, [self.spatial_correlations.shape]) - self.act1 = Activation(params.activation, self.id + ".activation1") - self.reduction = PointwiseConvolution(self.out_channels, self.id + '.reduce', [self.spatial_correlations.shape]) - - def initialize(self): - self.expansion.initialize() - self.bn0.initialize() - self.act0.initialize() - self.spatial_correlations.initialize() - self.bn1.initialize() - self.act1.initialize() - self.reduction.initialize() - - def forward(self, x): - out = self.expansion(x) - out = self.bn0(out) - out = self.act0(out) - out = self.spatial_correlations(out) - out = self.bn1(out) - out = self.act1(out) - out = self.reduction(out) - return out - - @property - def shape(self): - return self.reduction.shape - - @property - def macs(self): - return self.expansion.macs + self.spatial_correlations.macs + self.reduction.macs - - @property - def weights(self): - return self.expansion.weights + self.spatial_correlations.weights + self.reduction.weights - - -@parametrize -class ReduceExpand(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.params = params - self.id = id - self.inputs = inputs - self.input_shape = inputs[0] - - self.in_channels = self.input_shape[1] - - # FIXME: Share parameters over patterns - self.out_channels = handle_parameter(self, self.params.convolution.out_channels, f"out_channels") - self.reduce_ratio = handle_parameter(self, self.params.reduce_expand.ratio, f"reduce_ratio") - - self.reduced_channels = Int(self.reduce_ratio * self.in_channels) - self.reduction = PointwiseConvolution(self.reduced_channels, self.id + '.expand', inputs) - self.bn0 = BatchNorm(None, self.id, [self.reduction.shape]) - self.act0 = Activation(params.activation, self.id + ".activation0") - - # FIXME: convolution out channels? - self.conv = Convolution(params.convolution, self.id, [self.reduction.shape]) - self.bn1 = BatchNorm(None, self.id, [self.conv.shape]) - self.act1 = Activation(params.activation, self.id + ".activation1") - self.expansion = PointwiseConvolution(self.out_channels, self.id + '.expand', [self.conv.shape]) - - def initialize(self): - self.reduction.initialize() - self.bn0.initialize() - self.act0.initialize() - self.conv.initialize() - self.bn1.initialize() - self.act1.initialize() - self.expansion.initialize() - - def forward(self, x): - out = self.reduction(x) - out = self.bn0(out) - out = self.act0(out) - out = self.conv(out) - out = self.bn1(out) - out = self.act1(out) - out = self.expansion(out) - return out - - @property - def shape(self): - return self.expansion.shape - - @property - def macs(self): - return self.reduction.macs + self.conv.macs + self.expansion.macs - - @property - def weights(self): - return self.reduction.weights + self.conv.weights + self.expansion.weights - - -@parametrize -class Residual(nn.Module): - def __init__(self, params, input_shape, output_shape, id) -> None: - super().__init__() - self.id = id - self.params = params - self.input_shape = input_shape - self.output_shape = output_shape - self.in_channels = input_shape[1] - self.out_channels = output_shape[1] - self.in_fmap = input_shape[2] - self.out_fmap = output_shape[2] - self.stride = Int(Ceil(self.in_fmap / self.out_fmap)) - - # FIXME: Add potential alternative downsampling (pooling) - # FIXME: Move lazy conv to operators - self.conv = conv2d(self.id + ".conv", - inputs=[input_shape], - in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=1, - stride=self.stride, - padding=0) - self.bn = BatchNorm(None, self.id, [self.conv.shape]) - self.activation = Activation(params.activation, self.id + ".activation") - - def initialize(self): - self.downsample_conv = self.conv.instantiate() - self.bn.initialize() - self.activation.initialize() - - def forward(self, x): - out = self.downsample_conv(x) - out = self.bn(out) - out = self.activation(out) - return out - - - @property - def shape(self): - return self.conv.shape - - @property - def macs(self): - return conv2d_macs(self.input_shape, self.shape, self.conv.kwargs) - - @property - def weights(self): - return conv2d_weights(self.input_shape, self.shape, self.conv.kwargs) - - - -@parametrize -class Pattern(nn.Module): - def __init__(self, params, input_shape, id) -> None: - super().__init__() - self.params = params - self.input_shape = input_shape - self.id = id - - # shared parameters - self.stride = handle_parameter(self, params.stride, "stride") - self.out_channels = handle_parameter(self, params.out_channels, "out_channels") - - conv_params = OmegaConf.create({'convolution': {'stride': self.stride, 'out_channels': self.out_channels}, - 'pooling': {'stride': self.stride}}, flags={"allow_objects": True}) - self.params = OmegaConf.merge(self.params, conv_params) - - # FIXME: Check whether the self.add_params is necessary (probably yes, for child parameters to be registered when calling search_space.parameterization()) - convolution = self.add_param(f"{self.id}.convolution", Convolution(self.params.convolution, f"{self.id}.convolution", [self.input_shape])) - expand_reduce = self.add_param(f"{self.id}.expand_reduce", ExpandReduce(self.params, f"{self.id}.expand_reduce", [self.input_shape])) - reduce_expand = self.add_param(f"{self.id}.reduce_expand", ReduceExpand(self.params, f"{self.id}.reduce_expand", [self.input_shape])) - pooling = self.add_param(f"{self.id}.pooling", Pooling(self.params.pooling, f"{self.id}.pooling", [self.input_shape])) - - self.mods = nn.ModuleDict({'convolution': convolution, - 'expand_reduce': expand_reduce, - 'reduce_expand': reduce_expand, - 'pooling': pooling}) - - self.choice = self.add_param(f"{self.id}.choice", CategoricalParameter(choices=params.choices)) - self.active_module = Choice(self.mods, self.choice) - self.bn = BatchNorm(None, self.id, [self.shape]) - # FIXME: Make activation dependent on last activation in active module - self.activation = Activation(params.activation, self.id + ".activation") - - def initialize(self): - self.active_module.evaluate().initialize() # FIXME: See whether this works or whether we have to initialize all modules - self.activation.initialize() - - def forward(self, x): - out = self.active_module.evaluate()(x) - out = self.activation(out) - return out - - @property - def shape(self): - return[self.active_module.get('shape')[0], - self.active_module.get('shape')[1], - self.active_module.get('shape')[2], - self.active_module.get('shape')[3]] - - @property - def macs(self): - return self.active_module.get('macs') - - @property - def weights(self): - return self.active_module.get('weights') - - - -@parametrize -class Block(nn.Module): - def __init__(self, params, input_shape, id) -> None: - super().__init__() - self.id = id - self.input_shape = input_shape - self.depth = handle_parameter(self, params.depth, 'depth') - self.mods = nn.ModuleList() - self.params = params - - next_input = self.input_shape - for i in RangeIterator(self.depth, instance=False): - mod = self.add_param(f"{self.id}.pattern.{i}", Pattern(params.patterns, next_input, f"{self.id}.pattern.{i}")) - self.mods.append(mod) - next_input = mod.shape - - self.last_mod = Choice(self.mods, self.depth - 1) - self.residual = self.add_param(f"{self.id}.residual", Residual(params=params.residual, - input_shape=input_shape, - output_shape=self.shape, - id=f"{self.id}.residual")) - - def initialize(self): - for d in RangeIterator(self.depth, instance=False): - self.mods[d].initialize() - self.residual.initialize() - - def forward(self, x): - out = x - for d in RangeIterator(self.depth, instance=True): - out = self.mods[d](out) - res_out = self.residual(x) - out = torch.add(out, res_out) - return out - - @property - def shape(self): - # shape = self.last_mod.get('shape') - shape = [self.last_mod.get("shape")[0], - self.last_mod.get("shape")[1], - self.last_mod.get("shape")[2], - self.last_mod.get("shape")[3]] - return shape # FIXME: validate whether this works - - @property - def macs(self): - mac_list = [] - for d in RangeIterator(self.depth, instance=False): - mac_list.append(self.mods[d].macs) - - return depth_aware_sum(mac_list, self.depth) + self.residual.macs - - @property - def weights(self): - weight_list = [] - for d in RangeIterator(self.depth, instance=False): - weight_list.append(self.mods[d].weights) - - return depth_aware_sum(weight_list, self.depth) + self.residual.weights - - -@parametrize -class Stem(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.params = params - self.id = id - self.inputs = inputs - self.input_shape = self.inputs[0] - - self.conv = self.add_param(f"{self.id}.convolution", Convolution(params.convolution, self.id, self.inputs)) - self.bn = BatchNorm(None, self.id, [self.conv.shape]) - self.act = Activation(params.activation, self.id) - - def initialize(self): - self.conv.initialize() - self.bn.initialize() - self.act.initialize() - - def forward(self, x): - out = self.conv(x) - out = self.bn(out) - out = self.act(out) - return out - - @property - def shape(self): - return self.conv.shape - - @property - def macs(self): - return self.conv.macs - - @property - def weights(self): - return self.conv.weights - - -@parametrize -class Neck(nn.Module): - def __init__(self, params, input_shape) -> None: - super().__init__() - self.params = params - self.input_shape = input_shape - - @property - def shape(self): - pass # FIXME: - - @property - def macs(self): - pass - - @property - def weights(self): - pass - - -@parametrize -class SearchSpace(nn.Module): - def __init__(self, name, params, input_shape, labels) -> None: - super().__init__() - self.name = name - self.input_shape = input_shape - self.labels = labels - self.params = params - self.num_blocks = IntScalarParameter(self.params.num_blocks.min, self.params.num_blocks.max) - self.add_param("num_blocks", self.num_blocks) - - self.blocks = nn.ModuleList() - # FIXME: Subsampling - self.stem = self.add_param("stem", Stem(params.stem, "stem", [self.input_shape])) - next_input = self.stem.shape - - for n in RangeIterator(self.num_blocks, instance=False): - block = self.add_param(f"block.{n}", Block(params.block, next_input, f"block.{n}")) - self.blocks.append(block) - - next_input = block.shape - - last_block = Choice(self.blocks, self.num_blocks - 1) - - # self.neck = self.add_param("neck", Neck(params.neck, next_input)) - # NOTE: One can apparently not just write last_block.get("shape") and then index on the receiving end - self.head = self.add_param("head", Linear(params.head, - [last_block.get("shape")[0], - last_block.get("shape")[1], - last_block.get("shape")[2], - last_block.get("shape")[3]], - self.labels)) - - - strides = [] - block_depths = [] - for n, p in self.parametrization(flatten=True).items(): - if 'stride' in n: - strides.append(p) - elif 'depth' in n: - block_depths.append(p) - self.downsampling = depth_aware_downsampling(strides, self.parametrization(flatten=True)) - - self.cond(And(self.downsampling <= self.input_shape[2], self.downsampling >= (self.input_shape[2] / self.params.min_reduction))) - block_depths.sort(key=lambda x: x.id) - num_blocks = self.parametrization(flatten=True)['num_blocks'] - self.num_layers = num_layer_constraint(block_depths, num_blocks) - # self.cond(self.num_layers <= self.params.max_layer) - # self.cond(self.macs <= 1000000000) - print() - - def initialize(self): - self.stem.initialize() - for n in RangeIterator(self.num_blocks, instance=False): - self.blocks[n].initialize() - # self.neck.initialize() - self.head.initialize() - - def forward(self, x): - out = self.stem(x) - for n in RangeIterator(self.num_blocks, instance=True): - out = self.blocks[n](out) - # out = self.neck(out) - out = self.head(out) - return out - - @property - def shape(self): - # FIXME: Doesnt work right now - return self.head.shape - - @property - def macs(self): - mac_list = [] - for d in RangeIterator(self.num_blocks, instance=False): - mac_list.append(self.blocks[d].macs) - - blocks = depth_aware_sum(mac_list, self.num_blocks) - return self.stem.macs + blocks + self.head.macs - - @property - def weights(self): - weight_list = [] - for d in RangeIterator(self.num_blocks, instance=False): - weight_list.append(self.blocks[d].weights) - - blocks = depth_aware_sum(weight_list, self.num_blocks) - return self.stem.weights + blocks + self.head.weights - diff --git a/hannah/models/capsule_net/operators.py b/hannah/models/capsule_net/operators.py deleted file mode 100644 index 584ecce6..00000000 --- a/hannah/models/capsule_net/operators.py +++ /dev/null @@ -1,285 +0,0 @@ -import torch -import torch.nn as nn -from hannah.models.capsule_net.expressions import padding_expression -from hannah.models.capsule_net.utils import handle_parameter -from hannah.nas.expressions.shapes import conv2d_shape, identity_shape, linear_shape, pool_shape -from hannah.nas.parameters.lazy import Lazy -from hannah.nas.parameters.parametrize import parametrize -from hannah.nas.expressions.arithmetic import Ceil -from hannah.nas.expressions.choice import Choice -from hannah.nas.expressions.metrics import conv2d_macs, conv2d_weights, linear_macs, linear_weights - - -conv2d = Lazy(nn.Conv2d, shape_func=conv2d_shape) -linear = Lazy(nn.Linear, shape_func=linear_shape) -batch_norm = Lazy(nn.BatchNorm2d, shape_func=identity_shape) -relu = Lazy(nn.ReLU) -tensor = Lazy(torch.Tensor, shape_func=identity_shape) -identity = Lazy(nn.Identity) -avg_pooling = Lazy(nn.AvgPool2d, shape_func=pool_shape) -max_pooling = Lazy(nn.MaxPool2d, shape_func=pool_shape) - - - -@parametrize -class Relu(nn.Module): - def __init__(self, id) -> None: - super().__init__() - self.id = id - self.act = relu(self.id + '.relu') - - def initialize(self): - self.act_mod = self.act.instantiate() - - def forward(self, x): - return self.act_mod(x) - - -@parametrize -class Identity(nn.Module): - def __init__(self, id) -> None: - super().__init__() - self.id = id - self.act = identity(self.id + '.identity') - - def initialize(self): - self.act_mod = self.act.instantiate() - - def forward(self, x): - return self.act_mod(x) - - -@parametrize -class BatchNorm(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.params = params - self.id = id - self.inputs = inputs - self.input_shape = inputs[0] - self.channels = self.input_shape[1] - - self.bn = batch_norm(self.id + '.batch_norm', num_features=self.channels, inputs=inputs) - - def initialize(self): - self.tbn = self.bn.instantiate() - - def forward(self, x): - return self.tbn(x) - - -@parametrize -class Subsampling(nn.Module): - def __init__(self) -> None: - super().__init__() - - -# FIXME: Expand padding calculations to allow other windows and strides -@parametrize -class Pooling(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.params = params - self.id = id - self.inputs = inputs - - self.kernel_size = handle_parameter(self, params.kernel_size, f"{self.id}.kernel_size") - self.stride = handle_parameter(self, params.stride, f"{self.id}.stride") - - self.avg = avg_pooling(self.id + ".avg_pool", - inputs=inputs, - kernel_size=self.kernel_size, - stride=self.stride, - padding=padding_expression(self.kernel_size, self.stride)) - - self.max = max_pooling(self.id + ".max_pool", - inputs=inputs, - kernel_size=self.kernel_size, - stride=self.stride, - padding=padding_expression(self.kernel_size, self.stride)) - - self.mods = {'max': self.max, - 'avg': self.avg} - self.choice = handle_parameter(self, params.type, f"{self.id}.type") - self.active_module = Choice(self.mods, self.choice) - - def initialize(self): - self.tpool = self.active_module.evaluate().instantiate() - - def forward(self, x): - return self.tpool(x) - - @property - def shape(self): - return [self.active_module.get('shape')[0], - self.active_module.get('shape')[1], - self.active_module.get('shape')[2], - self.active_module.get('shape')[3]] - - @property - def macs(self): - return 0 # FIXME: - - @property - def weights(self): - return 0 - - - - -@parametrize -class Convolution(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.id = id - self.inputs = inputs - input_shape = inputs[0] - self.in_channels = input_shape[1] - self.out_channels = handle_parameter(self, params.out_channels, f"{self.id}.out_channels") - self.kernel_size = handle_parameter(self, params.kernel_size, name=f"{self.id}.kernel_size") - self.stride = handle_parameter(self, params.stride, f"{self.id}.stride") - - self.conv = conv2d(self.id + ".conv", - inputs=inputs, - in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=self.kernel_size, - stride=self.stride, - padding=padding_expression(self.kernel_size, self.stride)) - - def initialize(self): - self.tconv = self.conv.instantiate() - - def forward(self, x): - out = self.tconv(x) - return out - - @property - def shape(self): - return self.conv.shape - - @property - def macs(self): - return conv2d_macs(self.inputs[0], self.shape, self.conv.kwargs) - - @property - def weights(self): - return conv2d_weights(self.inputs[0], self.shape, self.conv.kwargs) - - -@parametrize -class DepthwiseConvolution(nn.Module): - def __init__(self, params, id, inputs) -> None: - super().__init__() - self.params = params - self.id = id - self.inputs = inputs - - input_shape = inputs[0] - self.in_channels = input_shape[1] - self.kernel_size = handle_parameter(self, params.kernel_size, f"{self.id}.kernel_size") - self.stride = handle_parameter(self, params.stride, f"{self.id}.stride") - - self.conv = conv2d(self.id + ".depthwise", - inputs=inputs, - in_channels=self.in_channels, - out_channels=self.in_channels, - kernel_size=self.kernel_size, - stride=self.stride, - groups=self.in_channels, - padding=padding_expression(self.kernel_size, self.stride)) - - def initialize(self): - self.tconv = self.conv.instantiate() - - def forward(self, x): - return self.tconv(x) - - @property - def shape(self): - return self.conv.shape - - @property - def macs(self): - return conv2d_macs(self.inputs[0], self.shape, self.conv.kwargs) - - @property - def weights(self): - return conv2d_weights(self.inputs[0], self.shape, self.conv.kwargs) - - -@parametrize -class PointwiseConvolution(nn.Module): - def __init__(self, out_channels, id, inputs) -> None: - super().__init__() - self.out_channels = out_channels - self.id = id - self.inputs = inputs - input_shape = inputs[0] - self.in_channels = input_shape[1] - - self.conv = conv2d(self.id + ".pointwise", - inputs=inputs, - in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=1, - stride=1, - padding=0) - - def initialize(self): - self.tconv = self.conv.instantiate() - - def forward(self, x): - return self.tconv(x) - - @property - def shape(self): - return self.conv.shape - - @property - def macs(self): - return conv2d_macs(self.inputs[0], self.shape, self.conv.kwargs) - - @property - def weights(self): - return conv2d_weights(self.inputs[0], self.shape, self.conv.kwargs) - - -@parametrize -class Linear(nn.Module): - def __init__(self, params, input, labels) -> None: - super().__init__() - self.params = params - self.input = input - self.labels = labels - - in_features = self.input[1] * self.input[2] * self.input[3] - self._linear = self.add_param('linear', - linear("linear", - inputs=[self.input], - in_features=in_features, - out_features=self.labels)) - - def initialize(self): - self.linear = self._linear.instantiate() - - def forward(self, x): - out = x.view(x.shape[0], -1) - out = self.linear(out) - return out - - @property - def shape(self): - return self._linear.shape - - @property - def shape(self): - return self._linear.shape - - @property - def macs(self): - return linear_macs(self.input, self.shape, self._linear.kwargs) - - @property - def weights(self): - return linear_weights(self.input, self.shape, self._linear.kwargs) diff --git a/hannah/models/capsule_net/utils.py b/hannah/models/capsule_net/utils.py deleted file mode 100644 index 29c6a522..00000000 --- a/hannah/models/capsule_net/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -from hannah.nas.parameters.parameters import FloatScalarParameter, IntScalarParameter, CategoricalParameter, Parameter -from omegaconf import DictConfig - - - -def handle_parameter(mod, param, name=None): - if isinstance(param, Parameter): - res = param - elif isinstance(param, DictConfig): - assert name, "For parameter creation, name has to be specified" - if 'min' in param: - if isinstance(param.min, float) or isinstance(param.max, float): - scalar_parameter = FloatScalarParameter - else: - scalar_parameter = IntScalarParameter - if 'step' in param: - res = mod.add_param(name, scalar_parameter(min=param.min, max=param.max, step_size=param.step, name=name)) - else: - res = mod.add_param(name, scalar_parameter(min=param.min, max=param.max, name=name)) - elif 'choices' in param: - res = mod.add_param(name, CategoricalParameter(choices=param.choices, name=name)) - else: - raise Exception(f"Given parameter of type {type(param)} can not be handled: {param}") - return res diff --git a/hannah/models/embedded_vision_net/blocks.py b/hannah/models/embedded_vision_net/blocks.py index 280401a5..58640c36 100644 --- a/hannah/models/embedded_vision_net/blocks.py +++ b/hannah/models/embedded_vision_net/blocks.py @@ -3,7 +3,7 @@ from hannah.nas.expressions.arithmetic import Ceil from hannah.nas.expressions.types import Int from hannah.nas.functional_operators.op import scope -from hannah.models.embedded_vision_net.operators import adaptive_avg_pooling, add, conv2d, conv_relu, depthwise_conv2d, dynamic_depth, grouped_conv2d, interleave_channels, pointwise_conv2d, linear, relu, batch_norm, choice, identity, max_pool, avg_pool +from hannah.models.embedded_vision_net.operators import adaptive_avg_pooling, add, conv2d, conv_relu, depthwise_conv2d, dynamic_depth, grouped_conv2d, interleave_channels, pointwise_conv2d, linear, relu, batch_norm, choice, identity, max_pool, avg_pool, alternative_paths from hannah.nas.parameters.parameters import CategoricalParameter, IntScalarParameter @@ -138,8 +138,10 @@ def block(input, depth, stride, out_channels, kernel_size, expand_ratio, reduce_ out = dynamic_depth(*exits, switch=depth) res = residual(input, out.shape()) - out = add(out, res) - + res_add = add(out, res) + residual_choice = IntScalarParameter(0, 1, name="residual_choice") + out = alternative_paths(out, res_add, switch=residual_choice) + # out = res_add return out diff --git a/hannah/models/embedded_vision_net/operators.py b/hannah/models/embedded_vision_net/operators.py index 85894731..c7315d6c 100644 --- a/hannah/models/embedded_vision_net/operators.py +++ b/hannah/models/embedded_vision_net/operators.py @@ -107,6 +107,11 @@ def choice(input, *choices, switch=None): return ChoiceOp(*choices, switch=switch)(input) +def alternative_paths(*exits, switch): + return ChoiceOp(*exits, switch=switch)() + + +# ALIAS for alternative paths def dynamic_depth(*exits, switch): return ChoiceOp(*exits, switch=switch)() diff --git a/hannah/nas/constraints/random_walk.py b/hannah/nas/constraints/random_walk.py index f30a95d9..99f47b4a 100644 --- a/hannah/nas/constraints/random_walk.py +++ b/hannah/nas/constraints/random_walk.py @@ -155,6 +155,7 @@ def right_direction(self, current, new, direction): return False def solve(self, module, parameters, fix_vars=[]): + print("Start constraint solving") mod = deepcopy(module) # FIXME copying is inefficient # mod = module self.solution = deepcopy(parameters) diff --git a/hannah/nas/functional_operators/op.py b/hannah/nas/functional_operators/op.py index 59fc2da6..489ed80f 100644 --- a/hannah/nas/functional_operators/op.py +++ b/hannah/nas/functional_operators/op.py @@ -75,6 +75,17 @@ def get_unique_id(): return _id +_id = 0 + + +def get_unique_id(): + global _id + _id += 1 + return _id + + +# FIXME: Traverses nodes to often -> massively increases time when building +# search spaces def get_highest_scope_counter(start_nodes, scope): ct = -1 for start_node in start_nodes: diff --git a/hannah/nas/parameters/parameters.py b/hannah/nas/parameters/parameters.py index e3814ffa..d9672d7f 100644 --- a/hannah/nas/parameters/parameters.py +++ b/hannah/nas/parameters/parameters.py @@ -117,7 +117,7 @@ def __init__( def evaluate_field(self, field_str): field = getattr(self, field_str) if isinstance(field, Parameter): - return field.instantiate() + return int(field.instantiate()) elif isinstance(field, int): return field else: @@ -134,7 +134,7 @@ def sample(self): min, max = self.get_bounds() values = np.arange(min, max + 1, self.step_size) # self.current_value = self.rng.integers(min, max+1) - self.current_value = self.rng.choice(values) + self.current_value = int(self.rng.choice(values)) return self.current_value def instantiate(self): @@ -207,7 +207,7 @@ def sample(self): idx = int(self.rng.choice(range(len(self.choices)))) self.current_value = self.choices[idx] if is_parametrized(self.current_value): - self.current_value = self.current_value.sample() + self.current_value = self.current_value.sample() # FIXME: This doesnt seem right? It should probably just call sample but not assign to SELF current value return self.current_value def instantiate(self): diff --git a/hannah/nas/parameters/parametrize.py b/hannah/nas/parameters/parametrize.py index 14862116..6572ff76 100644 --- a/hannah/nas/parameters/parametrize.py +++ b/hannah/nas/parameters/parametrize.py @@ -207,13 +207,13 @@ def get_parameters( self, scope: Optional[str] = None, include_empty=False, flatten=False ): params = {} - visited = [] + visited = [self] queue = [] queue.extend(self._PARAMETERS.values()) while queue: current = queue.pop(-1) - visited.append(current.id) + # visited.append(current.id) if current.id is None: name = current.name else: @@ -226,8 +226,10 @@ def get_parameters( param.id = param.name if isinstance(param, Parameter) and param.id not in visited: param.id = current.id + '.' + param.name - if param.id not in visited: + # if param not in visited: + if not any([param is v for v in visited]): queue.append(param) + visited.append(param) params = hierarchical_parameter_dict(params, include_empty, flatten) return params diff --git a/hannah/nas/performance_prediction/features/dataset.py b/hannah/nas/performance_prediction/features/dataset.py index 270498e1..a8e944ed 100644 --- a/hannah/nas/performance_prediction/features/dataset.py +++ b/hannah/nas/performance_prediction/features/dataset.py @@ -34,22 +34,17 @@ # FIXME: Find better way -COLUMNS = ['output_quant_bits', 'output_shape_0', 'output_shape_1', - 'output_shape_2', 'output_shape_3', 'attrs_in_channels', - 'attrs_out_channels', 'attrs_kernel_size_0', 'attrs_kernel_size_1', - 'attrs_stride_0', 'attrs_stride_1', 'attrs_dilation_0', - 'attrs_dilation_1', 'attrs_groups', 'attrs_padding_0', - 'attrs_padding_1', 'weight_quant_bits', 'weight_shape_0', - 'weight_shape_1', 'weight_shape_2', 'weight_shape_3', 'bias_quant_bits', - 'bias_shape_0', 'attrs_in_features', 'attrs_out_features', 'type_0', - 'type_add', 'type_batch_norm', 'type_conv', 'type_linear', - 'type_placeholder', 'type_pooling', 'type_relu', 'output_quant_dtype_0', - 'output_quant_dtype_float', 'output_quant_method_0', - 'output_quant_method_none', 'weight_quant_dtype_0', - 'weight_quant_dtype_float', 'weight_quant_method_0', - 'weight_quant_method_none', 'bias_quant_dtype_0', - 'bias_quant_dtype_float', 'bias_quant_method_0', - 'bias_quant_method_none'] +COLUMNS = ['attrs_dilation', 'attrs_groups', 'attrs_in_channels', + 'attrs_in_features', 'attrs_kernel_size', 'attrs_out_channels', + 'attrs_out_features', 'attrs_padding', 'attrs_stride', 'bias_nan', + 'output_quant_bits', 'output_quant_dtype_float', + 'output_quant_dtype_nan', 'output_quant_method_nan', + 'output_quant_method_none', 'output_quant_nan', 'output_shape_0', + 'output_shape_1', 'output_shape_2', 'output_shape_3', 'type_add', + 'type_batch_norm', 'type_conv', 'type_flatten', 'type_linear', + 'type_nan', 'type_placeholder', 'type_pooling', 'type_relu', + 'type_tensor', 'weight_quant_nan', 'weight_shape_0', 'weight_shape_1', + 'weight_shape_2', 'weight_shape_3'] class NASGraphDataset(DGLDataset): def __init__(self, result_folder: str): @@ -155,11 +150,11 @@ def get_features(nx_graph): dataframes.append(df) df = pd.concat(dataframes) # df.dropna(axis = 0, how = 'all', inplace = True) + df = pd.get_dummies(df, dummy_na=True) df = df.fillna(0) - df = pd.get_dummies(df) - # for col in COLUMNS: - # if col not in df.columns: - # df[col] = 0 + for col in COLUMNS: + if col not in df.columns: + df[col] = 0 df = df.reindex(sorted(df.columns), axis=1) # Sort to have consistency return df diff --git a/hannah/nas/search/model_trainer/simple_model_trainer.py b/hannah/nas/search/model_trainer/simple_model_trainer.py index 019b802c..f8120de5 100644 --- a/hannah/nas/search/model_trainer/simple_model_trainer.py +++ b/hannah/nas/search/model_trainer/simple_model_trainer.py @@ -96,7 +96,7 @@ def run_training(self, model, num, global_num, config): msglogger.critical("Training failed with exception") msglogger.critical(str(e)) print(traceback.format_exc()) - # sys.exit(1) + sys.exit(1) res = {} for monitor in opt_monitor: diff --git a/hannah/nas/search/sampler/mutator.py b/hannah/nas/search/sampler/mutator.py index cff5846d..04e1168e 100644 --- a/hannah/nas/search/sampler/mutator.py +++ b/hannah/nas/search/sampler/mutator.py @@ -35,6 +35,8 @@ def mutate_parameter(self, parameter): return self.mutate_int_scalar(parameter) elif isinstance(parameter, FloatScalarParameter): return self.mutate_float_scalar(parameter) + else: + return self.mutate_generic(parameter) def mutate_choice(self, parameter): mutations = self.get_choice_mutations() @@ -44,13 +46,18 @@ def mutate_choice(self, parameter): def mutate_int_scalar(self, parameter): mutations = self.get_int_mutations() chosen_mutation = self.rng.choice(mutations) - return chosen_mutation(parameter) + return int(chosen_mutation(parameter)) def mutate_float_scalar(self, parameter): mutations = self.get_float_mutations() chosen_mutation = self.rng.choice(mutations) return chosen_mutation(parameter) + def mutate_generic(self, parameter): + # we assume each parameter has at least a sample() method + # FIXME: Gather custom mutations + return parameter.sample() + # gather the relevant mutations def get_choice_mutations(self): return [self.random_choice, self.increase_choice, self.decrease_choice] @@ -64,7 +71,8 @@ def get_float_mutations(self): #################################### # The individual mutations def random_choice(self, parameter): - return parameter.rng.choice(parameter.choices) + idx = int(parameter.rng.choice(range(len(parameter.choices)))) + return parameter.choices[idx] def increase_choice(self, parameter): index = parameter.choices.index(parameter.current_value) diff --git a/hannah/nas/search/search.py b/hannah/nas/search/search.py index 72bc7c94..6e8b6760 100644 --- a/hannah/nas/search/search.py +++ b/hannah/nas/search/search.py @@ -20,6 +20,7 @@ import copy import logging import os +import sys import traceback from abc import ABC, abstractmethod from typing import Any, Mapping, Optional @@ -34,6 +35,7 @@ from hannah.nas.functional_operators.op import Tensor from hannah.nas.graph_conversion import model_to_graph from hannah.nas.performance_prediction.simple import MACPredictor +from hannah.nas.performance_prediction.protocol import FitablePredictor from hannah.nas.search.sampler.aging_evolution import FitnessFunction from hannah.nas.search.utils import WorklistItem, save_config_to_file from hannah.utils.utils import common_callbacks @@ -144,7 +146,7 @@ def before_search(self): for name, config in predictor_config.items(): predictor = instantiate(config) - if os.path.exists("performance_data"): + if isinstance(predictor, FitablePredictor) and os.path.exists("performance_data"): predictor.load("performance_data") self.predictors[name] = predictor @@ -161,7 +163,7 @@ def init_candidates(self): self.candidates = [] if remaining_candidates > 0: self.candidates = self.sample_candidates( - remaining_candidates, remaining_candidates, presample=self.presample + remaining_candidates, remaining_candidates, presample=self.presample, constrain=False ) def search(self): @@ -175,9 +177,10 @@ def search(self): self.worklist = [] if len(self.candidates) == 0: - for name, predictor in self.predictors: + for name, predictor in self.predictors.items(): try: - predictor.update(self.new_points, self.example_input_array) + if isinstance(predictor, FitablePredictor): + predictor.update(self.new_points, self.example_input_array) except Exception as e: # FIXME: Find reason for NaN in embeddings msglogger.error("Updating predictor failed:") @@ -232,12 +235,12 @@ def after_search(self): # self.extract_best_model() def sample_candidates( - self, num_total, num_candidates=None, sort_key="val_error", presample=False + self, num_total, num_candidates=None, sort_key="val_error", presample=False, constrain=False ): candidates = [] skip_ct = 0 while len(candidates) < num_total: - parameters = self.sample() + parameters = self.sample(constrain) model = self.build_model(parameters) estimated_metrics, satisfied_bounds = self.estimate_metrics( copy.deepcopy(model) @@ -315,8 +318,8 @@ def train_model(self, model): trainer = instantiate(self.config.trainer, callbacks=self.callbacks) trainer.fit(model) - def sample(self): - if self.constraint_model: + def sample(self, constrain=True): + if self.constraint_model and constrain: while True: try: parameters, keys = self.sampler.next_parameters() @@ -334,6 +337,8 @@ def sample(self): except Exception as e: print("Error occured while sampling: ") print(str(e)) + # print(traceback.format_exc()) + # sys.exit(1) else: parameters, keys = self.sampler.next_parameters() return parameters diff --git a/hannah/nas/test/test_symbolic_metrics.py b/hannah/nas/test/test_symbolic_metrics.py deleted file mode 100644 index b5408f27..00000000 --- a/hannah/nas/test/test_symbolic_metrics.py +++ /dev/null @@ -1,137 +0,0 @@ -import torch -import torch.nn as nn -from hannah.models.capsule_net.expressions import padding_expression -from hannah.nas.expressions.shapes import conv2d_shape, linear_shape -from hannah.nas.expressions.metrics import conv2d_macs, conv2d_weights, linear_macs, linear_weights -from hannah.nas.parameters.lazy import Lazy -from hannah.nas.parameters.parameters import CategoricalParameter, IntScalarParameter - -from hannah.nas.parameters.parametrize import parametrize -from hannah.callbacks.summaries import walk_model - -conv2d = Lazy(nn.Conv2d, shape_func=conv2d_shape) -linear = Lazy(nn.Linear, shape_func=linear_shape) - - -@parametrize -class Convolution(nn.Module): - def __init__(self, inputs) -> None: - super().__init__() - self.id = 'convolution' - self.inputs = inputs - input_shape = inputs[0] - self.in_channels = input_shape[1] - self.out_channels = IntScalarParameter(4, 64, 4) - self.kernel_size = CategoricalParameter([1, 3, 5]) - self.stride = CategoricalParameter([1, 2]) - - self.conv = conv2d(self.id + ".conv", - inputs=inputs, - in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=self.kernel_size, - stride=self.stride, - padding=padding_expression(self.kernel_size, self.stride)) - - def initialize(self): - self.tconv = self.conv.instantiate() - - def forward(self, x): - out = self.tconv(x) - return out - - @property - def shape(self): - return self.conv.shape - - @property - def macs(self): - return conv2d_macs(self.inputs[0], self.shape, self.conv.kwargs) - - @property - def weights(self): - return conv2d_weights(self.inputs[0], self.shape, self.conv.kwargs) - - -@parametrize -class Linear(nn.Module): - def __init__(self, input, labels) -> None: - super().__init__() - self.input = input - self.labels = labels - - in_features = self.input[1] * self.input[2] * self.input[3] - self._linear = linear("linear", - inputs=[self.input], - in_features=in_features, - out_features=self.labels) - - def initialize(self): - self.linear = self._linear.instantiate() - - def forward(self, x): - out = x.view(x.shape[0], -1) - out = self.linear(out) - return out - - @property - def shape(self): - return self._linear.shape - - @property - def macs(self): - return linear_macs(self.input, self.shape, self._linear.kwargs) - - @property - def weights(self): - return linear_weights(self.input, self.shape, self._linear.kwargs) - - -def test_conv_metrics(): - input_tensor = torch.ones(1, 3, 32, 32) - conv = Convolution([input_tensor.shape]) - conv.initialize() - conv.sample() - out = conv(input_tensor) - summary = walk_model(conv, input_tensor) - - mac_summary = summary['MACs'].item() - mac_symbolic = conv.macs.evaluate() - - assert mac_summary == mac_symbolic - - weight_summary = summary['Weights volume'].item() - weights_symbolic = conv.weights.evaluate() - - assert weight_summary == weights_symbolic - print() - - -def test_linear_metrics(): - input_tensor = torch.ones(1, 3, 32, 32) - conv = Convolution([input_tensor.shape]) - fc = Linear(conv.shape, 10) - conv.initialize() - fc.initialize() - conv.sample() - fc.sample() - conv_out = conv(input_tensor) - out = fc(conv_out) - - summary = walk_model(fc, conv_out) - - mac_summary = summary['MACs'].item() - mac_symbolic = fc.macs.evaluate() - - assert mac_summary == mac_symbolic - - weight_summary = summary['Weights volume'].item() - weights_symbolic = fc.weights.evaluate() - - assert weight_summary == weights_symbolic - print() - - -if __name__ == '__main__': - test_conv_metrics() - test_linear_metrics() \ No newline at end of file diff --git a/test/test_capsule_net.py b/test/test_capsule_net.py deleted file mode 100644 index 8c10be94..00000000 --- a/test/test_capsule_net.py +++ /dev/null @@ -1,84 +0,0 @@ -import os -from pathlib import Path -import numpy as np -from omegaconf import OmegaConf -import torch -import time - -import yaml - -import pandas as pd - -from hannah.models.capsule_net.models import SearchSpace -from hannah.nas.parameters.parameters import CategoricalParameter -from hannah.nas.constraints.constraint_model import ConstraintModel -from z3 import Z3_reset_memory - - -def diff(search_space, solver): - for solv in solver.solver: - solv.check() - mod = solv.model() - par = {} - sol = {} - for name, p in search_space.parametrization(flatten=True).items(): - if name in solver.vars[solv]: - par[name] = int(p.current_value) - sol[name] = mod[solver.vars[solv][name]].as_long() - - print(f"{name}: {par[name]} - {sol[name]}") - par = np.array(list(par.values())) - sol = np.array(list(sol.values())) - return np.linalg.norm(par-sol) - - -def test_capsule_net(): - cwd = os.getcwd() - config_path = Path(cwd + "/hannah/conf/model/capsule_net.yaml") - input_shape = [3, 3, 336, 336] - with config_path.open("r") as config_file: - config = yaml.unsafe_load(config_file) - config = OmegaConf.create(config) - search_space = SearchSpace("resnet", params=config.params, input_shape=input_shape, labels=config.labels) - search_space.sample() - - - solver = ConstraintModel(method='naive') - solver.build_model(search_space._conditions) - - df = pd.DataFrame(columns=["run", "method", "time", "diff"]) - # for i in range(1): - # search_space.sample() - # t0 = time.perf_counter() - # solver.soft_constrain_current_parametrization(search_space, key="stride", method='linear') - # t1 = time.perf_counter() - # td = t1 - t0 - # data = {"run": i, "method": 'linear', 'time': td, 'diff': diff(search_space, solver)} - # df = df.append(data, ignore_index=True) - # print("Run {} Method: {} Time: {:.4f}".format(i, 'linear', td)) - - # t0 = time.perf_counter() - # solver.soft_constrain_current_parametrization(search_space) - # t1 = time.perf_counter() - # td = t1 - t0 - - # data = {"run": i, "method": 'naive', 'time': td, 'diff': diff(search_space, solver)} - # df = df.append(data, ignore_index=True) - # print("Run {} Method: {} Time: {:.4f}".format(i, 'naive', td)) - - solver.soft_constrain_current_parametrization(search_space) - print(diff(search_space, solver)) - solver.insert_model_values_to_module(search_space) - # search_space.check() - - x = torch.randn(input_shape) - search_space.initialize() - out = search_space(x) - assert out.shape == (3, 10) - - print() - -if __name__ == '__main__': - test_capsule_net() - - diff --git a/test/test_capsule_net_constraints.py b/test/test_capsule_net_constraints.py deleted file mode 100644 index 9ae332a8..00000000 --- a/test/test_capsule_net_constraints.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -from pathlib import Path -import numpy as np -from omegaconf import OmegaConf -import torch -import time - -import yaml - -import pandas as pd - -from hannah.models.capsule_net.models import SearchSpace -from hannah.nas.parameters.parameters import CategoricalParameter -from hannah.nas.constraints.constraint_model import ConstraintModel -from hannah.callbacks.summaries import walk_model - -# from z3 import Z3_global_param_set -# Z3_global_param_set("combined_solver.solver2_timeout", '5000') - - -def diff(search_space, solver): - for solv in solver.solver: - solv.check() - mod = solv.model() - par = {} - sol = {} - for name, p in search_space.parametrization(flatten=True).items(): - if name in solver.vars[solv]: - par[name] = int(p.current_value) - sol[name] = mod[solver.vars[solv][name]].as_long() - - print(f"{name}: {par[name]} - {sol[name]}") - par = np.array(list(par.values())) - sol = np.array(list(sol.values())) - return np.linalg.norm(par-sol) - - -def test_symbolic_macs(): - cwd = os.getcwd() - config_path = Path(cwd + "/hannah/conf/model/capsule_net.yaml") - input_shape = [3, 3, 32, 32] - with config_path.open("r") as config_file: - config = yaml.unsafe_load(config_file) - config = OmegaConf.create(config) - search_space = SearchSpace("resnet", params=config.params, input_shape=input_shape, labels=config.labels) - search_space.sample() - search_space.initialize() - input_tensor = torch.ones(input_shape) - - - t0 = time.perf_counter() - summary = walk_model(search_space, dummy_input=input_tensor) - t1 = time.perf_counter() - print("walk_model: {:.4f}s.".format(t1 - t0)) - - solver = ConstraintModel(method='linear') - # solver.mac_constraint(search_space) - solver.build_model(search_space._conditions) - fixed_vars = {n: var for n, var in search_space.parametrization(flatten=True).items()} - # for n, v in fixed_vars.items(): - # solver.fix_var(solver.solver[0], v) - t0 = time.perf_counter() - print("Start check") - # solver.solver[0].check() - solver.soft_constrain_current_parametrization(search_space) - search_space = solver.insert_model_values_to_module(search_space) - t1 = time.perf_counter() - print("Model constrained in {:.4f}s".format(t1 - t0)) - assert search_space.downsampling.evaluate() <= input_shape[2] - assert search_space.downsampling.evaluate() >= input_shape[2] / 4 - # new_macs = search_space.macs.evaluate() - # new_macs = search_space.easy_macs.evaluate() - # print("MACs after constraining: {:.4E}".format(new_macs)) - # assert new_macs <= 1000000000 - print() - - -if __name__ == '__main__': - test_symbolic_macs() \ No newline at end of file diff --git a/test/test_symbolic_check.py b/test/test_symbolic_check.py deleted file mode 100644 index c78850c1..00000000 --- a/test/test_symbolic_check.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -from pathlib import Path -import numpy as np -from omegaconf import OmegaConf -import torch -import time - -import yaml - -import pandas as pd - -from hannah.models.capsule_net.models import SearchSpace -from hannah.nas.parameters.parameters import CategoricalParameter -from hannah.nas.constraints.constraint_model import ConstraintModel -from hannah.callbacks.summaries import walk_model - - -def test_symbolic_macs(): - cwd = os.getcwd() - config_path = Path(cwd + "/hannah/conf/model/capsule_net.yaml") - input_shape = [3, 3, 32, 32] - with config_path.open("r") as config_file: - config = yaml.unsafe_load(config_file) - config = OmegaConf.create(config) - search_space = SearchSpace("resnet", params=config.params, input_shape=input_shape, labels=config.labels) - search_space.sample() - search_space.initialize() - input_tensor = torch.ones(input_shape) - - summary = walk_model(search_space, dummy_input=input_tensor) - macs = search_space.macs.evaluate() - # print("MACs before constraining: {:.4E}".format(macs)) - - # assert summary['MACs'].sum() == macs - solver = ConstraintModel(method='naive') - t0 = time.perf_counter() - solver.soft_constrain_current_parametrization(search_space) - solver.insert_model_values_to_module(search_space) - t1 = time.perf_counter() - # print("Model constrained in {:.4f}s".format(t1 - t0)) - new_macs = search_space.macs.evaluate() - # print("MACs after constraining: {:.4E}".format(new_macs)) - - times = [] - macs = [] - tries = [] - # print("MACs before constraining: {:.4E}".format(macs)) - budget = 100 - - # for i in range(100): - # t0 = time.perf_counter() - # ct = 0 - # while True: - # search_space.sample() - # try: - # search_space.check() - # break - # except Exception: - # ct += 1 - # t1 = time.perf_counter() - # times.append(t1 - t0) - # tries.append(ct) - # print("Model constrained in {:.4f}s".format(t1 - t0)) - # new_macs = search_space.macs.evaluate() - # print("MACs after constraining: {:.4E}".format(new_macs)) - # macs.append(new_macs) - # df = pd.DataFrame({'times': times, 'tries': tries, 'macs': macs}) - # df.to_csv("./random_pre_constrainting.csv") - # print() - - -if __name__ == '__main__': - test_symbolic_macs() diff --git a/test/test_symbolic_metrics.py b/test/test_symbolic_metrics.py new file mode 100644 index 00000000..4f45ea0f --- /dev/null +++ b/test/test_symbolic_metrics.py @@ -0,0 +1,623 @@ +from omegaconf import OmegaConf +import torch +from hannah.models.embedded_vision_net.models import search_space +from hannah.nas.functional_operators.executor import BasicExecutor +from hannah.nas.functional_operators.op import Tensor +from hannah.callbacks.summaries import FxMACSummaryCallback +from hannah.nas.parameters.parametrize import set_parametrization + + +PARAMS = {"ChoiceOp_0.num_blocks": 1, + "block_9.ChoiceOp_0.residual_choice": 1, + "block_9.ChoiceOp_0.depth": 0, + "block_9.pattern_2.ChoiceOp_0.choice": 4, + "block_9.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_9.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_9.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_9.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_9.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_9.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_9.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_9.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_9.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_9.pattern_2.Conv2d_0.weight.kernel_size": 3, + "block_9.pattern_2.Conv2d_0.weight.out_channels": 48, + "block_9.pattern_1.ChoiceOp_0.choice": 1, + "block_9.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_9.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_9.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_9.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_9.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_9.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_9.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_9.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_9.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_9.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_9.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_9.pattern_1.Conv2d_0.weight.kernel_size": 9, + "block_9.pattern_1.Conv2d_0.weight.out_channels": 288, + "block_9.pattern_0.ChoiceOp_0.choice": 3, + "block_9.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_9.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_9.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_9.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_9.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_9.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_9.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_9.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_9.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_9.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_9.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_9.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_9.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_9.pattern_0.Conv2d_0.stride": 1, + "block_9.pattern_0.Conv2d_0.weight.kernel_size": 9, + "block_9.pattern_0.Conv2d_0.weight.out_channels": 384, + "block_8.ChoiceOp_0.residual_choice": 0, + "block_8.ChoiceOp_0.depth": 1, + "block_8.pattern_2.ChoiceOp_0.choice": 1, + "block_8.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_8.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_8.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_8.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_8.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_8.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_8.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_8.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_8.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_8.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 6, + "block_8.pattern_2.Conv2d_0.weight.kernel_size": 3, + "block_8.pattern_2.Conv2d_0.weight.out_channels": 496, + "block_8.pattern_1.ChoiceOp_0.choice": 0, + "block_8.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_8.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_8.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_8.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_8.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_8.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_8.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_8.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_8.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_8.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_8.pattern_1.Conv2d_0.weight.kernel_size": 3, + "block_8.pattern_1.Conv2d_0.weight.out_channels": 272, + "block_8.pattern_0.ChoiceOp_0.choice": 4, + "block_8.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_8.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_8.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_8.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_8.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_8.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_8.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_8.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_8.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_8.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_8.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_8.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 5, + "block_8.pattern_0.Conv2d_0.stride": 2, + "block_8.pattern_0.Conv2d_0.weight.kernel_size": 9, + "block_8.pattern_0.Conv2d_0.weight.out_channels": 232, + "block_7.ChoiceOp_0.residual_choice": 0, + "block_7.ChoiceOp_0.depth": 2, + "block_7.pattern_2.ChoiceOp_0.choice": 0, + "block_7.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_7.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_7.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_7.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_7.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_7.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_7.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_7.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_7.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_7.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_7.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_7.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_7.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_7.pattern_2.Conv2d_0.weight.kernel_size": 3, + "block_7.pattern_2.Conv2d_0.weight.out_channels": 248, + "block_7.pattern_1.ChoiceOp_0.choice": 3, + "block_7.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_7.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_7.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_7.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_7.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_7.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_7.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_7.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_7.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_7.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_7.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_7.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_7.pattern_1.Conv2d_0.weight.kernel_size": 5, + "block_7.pattern_1.Conv2d_0.weight.out_channels": 104, + "block_7.pattern_0.ChoiceOp_0.choice": 3, + "block_7.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_7.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_7.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_7.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_7.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_7.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_7.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_7.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_7.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_7.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_7.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_7.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_7.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_7.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 3, + "block_7.pattern_0.Conv2d_0.stride": 2, + "block_7.pattern_0.Conv2d_0.weight.kernel_size": 5, + "block_7.pattern_0.Conv2d_0.weight.out_channels": 336, + "block_6.ChoiceOp_0.residual_choice": 0, + "block_6.ChoiceOp_0.depth": 0, + "block_6.pattern_2.ChoiceOp_0.choice": 3, + "block_6.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_6.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_6.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_6.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_6.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_6.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_6.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_6.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_6.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_6.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_6.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_6.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 1, + "block_6.pattern_2.Conv2d_0.weight.kernel_size": 9, + "block_6.pattern_2.Conv2d_0.weight.out_channels": 160, + "block_6.pattern_1.ChoiceOp_0.choice": 2, + "block_6.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_6.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_6.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_6.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_6.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_6.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_6.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_6.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_6.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_6.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_6.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_6.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_6.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_6.pattern_1.Conv2d_0.weight.kernel_size": 9, + "block_6.pattern_1.Conv2d_0.weight.out_channels": 384, + "block_6.pattern_0.ChoiceOp_0.choice": 1, + "block_6.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_6.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_6.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_6.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_6.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_6.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_6.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_6.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_6.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_6.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_6.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_6.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_6.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 3, + "block_6.pattern_0.Conv2d_0.stride": 1, + "block_6.pattern_0.Conv2d_0.weight.kernel_size": 3, + "block_6.pattern_0.Conv2d_0.weight.out_channels": 40, + "block_5.ChoiceOp_0.residual_choice": 1, + "block_5.ChoiceOp_0.depth": 1, + "block_5.pattern_2.ChoiceOp_0.choice": 1, + "block_5.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_5.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_5.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_5.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_5.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_5.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_5.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_5.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_5.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_5.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_5.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_5.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_5.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_5.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_5.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_5.pattern_2.Conv2d_0.weight.kernel_size": 7, + "block_5.pattern_2.Conv2d_0.weight.out_channels": 272, + "block_5.pattern_1.ChoiceOp_0.choice": 1, + "block_5.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_5.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_5.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_5.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_5.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_5.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_5.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_5.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_5.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_5.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_5.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_5.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_5.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_5.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_5.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 3, + "block_5.pattern_1.Conv2d_0.weight.kernel_size": 9, + "block_5.pattern_1.Conv2d_0.weight.out_channels": 400, + "block_5.pattern_0.ChoiceOp_0.choice": 0, + "block_5.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_5.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_5.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_5.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_5.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_5.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_5.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_5.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_5.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_5.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 4, + "block_5.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_5.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_5.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_5.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_5.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 3, + "block_5.pattern_0.Conv2d_0.stride": 1, + "block_5.pattern_0.Conv2d_0.weight.kernel_size": 7, + "block_5.pattern_0.Conv2d_0.weight.out_channels": 32, + "block_4.ChoiceOp_0.residual_choice": 0, + "block_4.ChoiceOp_0.depth": 1, + "block_4.pattern_2.ChoiceOp_0.choice": 3, + "block_4.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_4.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_4.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_4.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_4.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_4.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_4.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_4.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_4.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_4.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_4.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_4.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_4.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_4.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_4.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 3, + "block_4.pattern_2.Conv2d_0.weight.kernel_size": 9, + "block_4.pattern_2.Conv2d_0.weight.out_channels": 136, + "block_4.pattern_1.ChoiceOp_0.choice": 1, + "block_4.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_4.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_4.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_4.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_4.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_4.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_4.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_4.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_4.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_4.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_4.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_4.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_4.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_4.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_4.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_4.pattern_1.Conv2d_0.weight.kernel_size": 7, + "block_4.pattern_1.Conv2d_0.weight.out_channels": 208, + "block_4.pattern_0.ChoiceOp_0.choice": 2, + "block_4.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_4.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_4.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_4.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_4.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_4.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_4.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_4.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_4.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_4.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_4.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_4.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_4.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_4.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_4.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_4.pattern_0.Conv2d_0.stride": 2, + "block_4.pattern_0.Conv2d_0.weight.kernel_size": 3, + "block_4.pattern_0.Conv2d_0.weight.out_channels": 320, + "block_3.ChoiceOp_0.residual_choice": 0, + "block_3.ChoiceOp_0.depth": 0, + "block_3.pattern_2.ChoiceOp_0.choice": 4, + "block_3.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_3.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_3.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_3.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_3.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_3.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_3.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_3.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_3.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 16, + "block_3.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 5, + "block_3.pattern_2.Conv2d_0.weight.kernel_size": 3, + "block_3.pattern_2.Conv2d_0.weight.out_channels": 80, + "block_3.pattern_1.ChoiceOp_0.choice": 3, + "block_3.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_3.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_3.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_3.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_3.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_3.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_3.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_3.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_3.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 3, + "block_3.pattern_1.Conv2d_0.weight.kernel_size": 9, + "block_3.pattern_1.Conv2d_0.weight.out_channels": 272, + "block_3.pattern_0.ChoiceOp_0.choice": 2, + "block_3.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_3.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_3.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_3.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_3.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_3.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_3.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_3.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_3.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_3.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_3.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_3.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_3.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_3.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_3.pattern_0.Conv2d_0.stride": 1, + "block_3.pattern_0.Conv2d_0.weight.kernel_size": 5, + "block_3.pattern_0.Conv2d_0.weight.out_channels": 248, + "block_2.ChoiceOp_0.residual_choice": 1, + "block_2.ChoiceOp_0.depth": 0, + "block_2.pattern_2.ChoiceOp_0.choice": 4, + "block_2.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_2.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_2.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_2.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_2.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_2.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_2.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_2.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_2.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_2.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 1, + "block_2.pattern_2.Conv2d_0.weight.kernel_size": 3, + "block_2.pattern_2.Conv2d_0.weight.out_channels": 280, + "block_2.pattern_1.ChoiceOp_0.choice": 4, + "block_2.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_2.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_2.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_2.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_2.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_2.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_2.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_2.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_2.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_2.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_2.pattern_1.Conv2d_0.weight.kernel_size": 7, + "block_2.pattern_1.Conv2d_0.weight.out_channels": 432, + "block_2.pattern_0.ChoiceOp_0.choice": 4, + "block_2.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_2.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_2.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_2.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_2.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_2.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_2.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_2.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_2.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_2.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_2.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_2.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_2.pattern_0.Conv2d_0.stride": 1, + "block_2.pattern_0.Conv2d_0.weight.kernel_size": 7, + "block_2.pattern_0.Conv2d_0.weight.out_channels": 88, + "block_1.ChoiceOp_0.residual_choice": 1, + "block_1.ChoiceOp_0.depth": 0, + "block_1.pattern_2.ChoiceOp_0.choice": 4, + "block_1.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_1.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_1.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_1.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_1.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_1.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_1.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_1.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_1.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_1.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_1.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_1.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_1.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_1.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_1.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 4, + "block_1.pattern_2.Conv2d_0.weight.kernel_size": 3, + "block_1.pattern_2.Conv2d_0.weight.out_channels": 248, + "block_1.pattern_1.ChoiceOp_0.choice": 3, + "block_1.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_1.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_1.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_1.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_1.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_1.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_1.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_1.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_1.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_1.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 3, + "block_1.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_1.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_1.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_1.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_1.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 1, + "block_1.pattern_1.Conv2d_0.weight.kernel_size": 5, + "block_1.pattern_1.Conv2d_0.weight.out_channels": 488, + "block_1.pattern_0.ChoiceOp_0.choice": 1, + "block_1.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_1.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_1.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_1.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_1.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_1.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_1.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_1.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 1, + "block_1.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_1.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_1.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_1.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_1.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_1.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_1.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 2, + "block_1.pattern_0.Conv2d_0.stride": 1, + "block_1.pattern_0.Conv2d_0.weight.kernel_size": 5, + "block_1.pattern_0.Conv2d_0.weight.out_channels": 120, + "block_0.ChoiceOp_0.residual_choice": 1, + "stem_0.Conv2d_0.stride": 2, + "stem_0.Conv2d_0.weight.kernel_size": 3, + "stem_0.Conv2d_0.weight.out_channels": 60, + "block_0.ChoiceOp_0.depth": 0, + "block_0.pattern_2.ChoiceOp_0.choice": 3, + "block_0.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_0.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_0.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_0.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_0.pattern_2.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_0.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_0.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_0.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_0.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 8, + "block_0.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_0.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_0.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_0.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_0.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_0.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 1, + "block_0.pattern_2.Conv2d_0.weight.kernel_size": 7, + "block_0.pattern_2.Conv2d_0.weight.out_channels": 352, + "block_0.pattern_1.ChoiceOp_0.choice": 4, + "block_0.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 1, + "block_0.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_0.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 0, + "block_0.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_0.pattern_1.pooling_0.ChoiceOp_0.pool_mode": 0, + "block_0.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 1, + "block_0.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_0.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_0.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_0.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_0.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 0, + "block_0.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_0.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 1, + "block_0.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_0.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 5, + "block_0.pattern_1.Conv2d_0.weight.kernel_size": 9, + "block_0.pattern_1.Conv2d_0.weight.out_channels": 392, + "block_0.pattern_0.ChoiceOp_0.choice": 3, + "block_0.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice": 0, + "block_0.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 1, + "block_0.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice": 1, + "block_0.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 3, + "block_0.pattern_0.pooling_0.ChoiceOp_0.pool_mode": 1, + "block_0.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice": 0, + "block_0.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 2, + "block_0.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice": 0, + "block_0.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 10, + "block_0.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio": 2, + "block_0.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice": 1, + "block_0.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups": 4, + "block_0.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice": 0, + "block_0.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups": 30, + "block_0.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio": 1, + "block_0.pattern_0.Conv2d_0.stride": 1, + "block_0.pattern_0.Conv2d_0.weight.kernel_size": 9, + "block_0.pattern_0.Conv2d_0.weight.out_channels": 72} + + +def test_symbolic_metrics(): + # params = {'ChoiceOp_0.num_blocks': 4, 'block_9.ChoiceOp_0.residual_choice': 0, 'block_9.ChoiceOp_0.depth': 2, 'block_9.pattern_2.ChoiceOp_0.choice': 3, 'block_9.pattern_2.sandglass_block_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_2.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 8, 'block_9.pattern_2.sandglass_block_0.reduction_0.ChoiceOp_0.choice': 1, 'block_9.pattern_2.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 4, 'block_9.pattern_2.pooling_0.ChoiceOp_0.pool_mode': 0, 'block_9.pattern_2.reduce_expand_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_2.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 2, 'block_9.pattern_2.reduce_expand_0.reduction_0.ChoiceOp_0.choice': 0, 'block_9.pattern_2.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_2.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio': 4, 'block_9.pattern_2.expand_reduce_0.reduction_0.ChoiceOp_0.choice': 0, 'block_9.pattern_2.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_2.expand_reduce_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_2.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 8, 'block_9.pattern_2.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio': 6, 'block_9.pattern_2.Conv2d_0.weight.kernel_size': 3, 'block_9.pattern_2.Conv2d_0.weight.out_channels': 304, 'block_9.pattern_1.ChoiceOp_0.choice': 0, 'block_9.pattern_1.sandglass_block_0.expansion_0.ChoiceOp_0.choice': 1, 'block_9.pattern_1.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 8, 'block_9.pattern_1.sandglass_block_0.reduction_0.ChoiceOp_0.choice': 1, 'block_9.pattern_1.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_1.pooling_0.ChoiceOp_0.pool_mode': 1, 'block_9.pattern_1.reduce_expand_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_1.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_1.reduce_expand_0.reduction_0.ChoiceOp_0.choice': 1, 'block_9.pattern_1.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_1.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio': 2, 'block_9.pattern_1.expand_reduce_0.reduction_0.ChoiceOp_0.choice': 1, 'block_9.pattern_1.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 16, 'block_9.pattern_1.expand_reduce_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_1.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 2, 'block_9.pattern_1.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio': 6, 'block_9.pattern_1.Conv2d_0.weight.kernel_size': 7, 'block_9.pattern_1.Conv2d_0.weight.out_channels': 288, 'block_9.pattern_0.ChoiceOp_0.choice': 2, 'block_9.pattern_0.sandglass_block_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_0.sandglass_block_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 8, 'block_9.pattern_0.sandglass_block_0.reduction_0.ChoiceOp_0.choice': 1, 'block_9.pattern_0.sandglass_block_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_0.pooling_0.ChoiceOp_0.pool_mode': 1, 'block_9.pattern_0.reduce_expand_0.expansion_0.ChoiceOp_0.choice': 0, 'block_9.pattern_0.reduce_expand_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 2, 'block_9.pattern_0.reduce_expand_0.reduction_0.ChoiceOp_0.choice': 0, 'block_9.pattern_0.reduce_expand_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 1, 'block_9.pattern_0.reduce_expand_0.reduction_0.Conv2d_0.weight.reduce_ratio': 2, 'block_9.pattern_0.expand_reduce_0.reduction_0.ChoiceOp_0.choice': 1, 'block_9.pattern_0.expand_reduce_0.reduction_0.grouped_pointwise_0.InterleaveChannels_0.groups': 16, 'block_9.pattern_0.expand_reduce_0.expansion_0.ChoiceOp_0.choice': 1, 'block_9.pattern_0.expand_reduce_0.expansion_0.grouped_pointwise_0.InterleaveChannels_0.groups': 8, 'block_9.pattern_0.expand_reduce_0.expansion_0.Conv2d_0.weight.expand_ratio': 3, 'block_9.pattern_0.Conv2d_0.stride': 2, 'block_9.pattern_0.Conv2d_0.weight.kernel_size': 9, 'block_9.pattern_0.Conv2d_0.weight.out_channels': 288, 'block_8.ChoiceOp_0.residual_choice': 1, ...} + class DummyModule: + """ Dummy module with relevant fields to demonstrate usage of + MacSummaryCallback without the need for an actual ImageClassifierModule + """ + def __init__(self, model, device, example_feature_array) -> None: + self.model = model + self.device = device + self.example_feature_array = example_feature_array + + input_shape = [1, 3, 32, 32] + input = Tensor(name="input", shape=input_shape, axis=("N", "C", "H", "W")) + cons = OmegaConf.create([{"name": "weights"}, {"name": "macs"}]) + space = search_space(name="evn", input=input, num_classes=10, constraints=cons) + set_parametrization(PARAMS, space.parametrization()) + + model = BasicExecutor(space) + model.initialize() + input_tensor = torch.ones(input_shape) + + module = DummyModule(model, "cpu", input_tensor) + metrics_predictor = FxMACSummaryCallback() + summary = metrics_predictor._do_summary(module) + + mac_summary = summary['total_macs'].item() + mac_symbolic = space.macs.evaluate() + + assert mac_summary == mac_symbolic + + weight_summary = summary['total_weights'].item() + weights_symbolic = space.weights.evaluate() + + assert weight_summary == weights_symbolic + + +if __name__ == '__main__': + test_symbolic_metrics()