From 18aa16baf9498bf6abbc42afdf03a7113f059946 Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Fri, 8 Nov 2024 08:56:57 -0500 Subject: [PATCH 1/7] adapt constant layers in pytorch to work with Constant class introduced with QONNX parser --- hls4ml/converters/pytorch/core.py | 16 +++++++++ hls4ml/converters/pytorch_to_hls.py | 51 +++++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/hls4ml/converters/pytorch/core.py b/hls4ml/converters/pytorch/core.py index 2c05b7501f..994446d56c 100644 --- a/hls4ml/converters/pytorch/core.py +++ b/hls4ml/converters/pytorch/core.py @@ -1,5 +1,21 @@ from hls4ml.converters.pytorch_to_hls import pytorch_handler +import numpy as np +@pytorch_handler('Constant') +#def parse_constant_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config): +def parse_constant_layer(operation, layer_name, node): + assert 'Constant' in operation + + layer = {} + + layer['class_name'] = 'Constant' + layer['name'] = layer_name + + constant = np.array(node._args) + layer['value'] = constant + output_shape = constant.shape + + return layer, output_shape @pytorch_handler('Linear') def parse_linear_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config): diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index 79ca1fa5c6..c9508e2d23 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -1,4 +1,5 @@ import torch +import numpy as np from hls4ml.model import ModelGraph @@ -159,6 +160,31 @@ def parse_pytorch_model(config, verbose=True): n_inputs = 0 + print(traced_model.graph) + + # check for constant nodes + merge_layers = ['add','mul','sub','fmin','fmax'] + i=0 # count number of consts and use it in the name + for node in traced_model.graph.nodes: + if node.name in merge_layers: + for arg in node.args: + if np.isscalar(arg): + # add an input node with the constant value + new_node = traced_model.graph.placeholder( + name='const_'+str(i), + type_expr=torch.Tensor, + default_value=arg + ) + node.prepend(new_node) + node.update_arg(1,new_node) + i += 1 + + print(traced_model.graph) + #import pdb; breakpoint() + traced_model.graph.lint() + + + for node in traced_model.graph.nodes: if node.op == 'call_module': # modules that are part of a torch.nn.Sequential with name 'name' have target names 'name.x', @@ -249,13 +275,26 @@ def parse_pytorch_model(config, verbose=True): input_layer = {} input_layer['name'] = node.name - input_layer['class_name'] = 'InputLayer' - input_layer['input_shape'] = list(input_shapes[n_inputs][1:]) - layer_list.insert(n_inputs, input_layer) - output_shapes[input_layer['name']] = list(input_shapes[n_inputs]) - input_layers.append(input_layer['name']) - n_inputs += 1 + if 'const' in node.name: + pytorch_class = "Constant" + layer, output_shape = layer_handlers[pytorch_class](pytorch_class, node.name, node) + + layer_list.append(layer) + + assert output_shape is not None + output_shapes[layer['name']] = output_shape + + else: + + input_layer['class_name'] = 'InputLayer' + input_layer['input_shape'] = list(input_shapes[n_inputs][1:]) + layer_list.insert(n_inputs, input_layer) + + output_shapes[input_layer['name']] = list(input_shapes[n_inputs]) + + input_layers.append(input_layer['name']) + n_inputs += 1 layer_counter += 1 From c50c09e1e13e42dec39350130a0c80a9427d2c93 Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Tue, 12 Nov 2024 10:45:13 -0500 Subject: [PATCH 2/7] apply pre-commit --- hls4ml/converters/pytorch/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hls4ml/converters/pytorch/core.py b/hls4ml/converters/pytorch/core.py index 994446d56c..29fbf69394 100644 --- a/hls4ml/converters/pytorch/core.py +++ b/hls4ml/converters/pytorch/core.py @@ -1,8 +1,10 @@ -from hls4ml.converters.pytorch_to_hls import pytorch_handler import numpy as np +from hls4ml.converters.pytorch_to_hls import pytorch_handler + + @pytorch_handler('Constant') -#def parse_constant_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config): +# def parse_constant_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config): def parse_constant_layer(operation, layer_name, node): assert 'Constant' in operation @@ -17,6 +19,7 @@ def parse_constant_layer(operation, layer_name, node): return layer, output_shape + @pytorch_handler('Linear') def parse_linear_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config): assert 'Linear' in operation From 3413cb5f143d5b1efb068f56e874328049b8cd15 Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Tue, 12 Nov 2024 10:49:07 -0500 Subject: [PATCH 3/7] apply pre-commit --- hls4ml/converters/pytorch_to_hls.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index c9508e2d23..a8b8c2f4a2 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -1,5 +1,5 @@ -import torch import numpy as np +import torch from hls4ml.model import ModelGraph @@ -162,29 +162,25 @@ def parse_pytorch_model(config, verbose=True): print(traced_model.graph) - # check for constant nodes - merge_layers = ['add','mul','sub','fmin','fmax'] - i=0 # count number of consts and use it in the name + # check for constant nodes + merge_layers = ['add', 'mul', 'sub', 'fmin', 'fmax'] + i = 0 # count number of consts and use it in the name for node in traced_model.graph.nodes: if node.name in merge_layers: for arg in node.args: if np.isscalar(arg): # add an input node with the constant value new_node = traced_model.graph.placeholder( - name='const_'+str(i), - type_expr=torch.Tensor, - default_value=arg + name='const_' + str(i), type_expr=torch.Tensor, default_value=arg ) node.prepend(new_node) - node.update_arg(1,new_node) + node.update_arg(1, new_node) i += 1 print(traced_model.graph) - #import pdb; breakpoint() + # import pdb; breakpoint() traced_model.graph.lint() - - for node in traced_model.graph.nodes: if node.op == 'call_module': # modules that are part of a torch.nn.Sequential with name 'name' have target names 'name.x', @@ -278,12 +274,12 @@ def parse_pytorch_model(config, verbose=True): if 'const' in node.name: pytorch_class = "Constant" - layer, output_shape = layer_handlers[pytorch_class](pytorch_class, node.name, node) - + layer, output_shape = layer_handlers[pytorch_class](pytorch_class, node.name, node) + layer_list.append(layer) - + assert output_shape is not None - output_shapes[layer['name']] = output_shape + output_shapes[layer['name']] = output_shape else: @@ -292,7 +288,7 @@ def parse_pytorch_model(config, verbose=True): layer_list.insert(n_inputs, input_layer) output_shapes[input_layer['name']] = list(input_shapes[n_inputs]) - + input_layers.append(input_layer['name']) n_inputs += 1 From b39fedaa298b133861ed0c9ea6d0e2ff63063237 Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Tue, 12 Nov 2024 10:51:53 -0500 Subject: [PATCH 4/7] clean print statements and commented code --- hls4ml/converters/pytorch/core.py | 1 - hls4ml/converters/pytorch_to_hls.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/hls4ml/converters/pytorch/core.py b/hls4ml/converters/pytorch/core.py index 29fbf69394..7cc20fc282 100644 --- a/hls4ml/converters/pytorch/core.py +++ b/hls4ml/converters/pytorch/core.py @@ -4,7 +4,6 @@ @pytorch_handler('Constant') -# def parse_constant_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config): def parse_constant_layer(operation, layer_name, node): assert 'Constant' in operation diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index a8b8c2f4a2..06f7b95364 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -160,8 +160,6 @@ def parse_pytorch_model(config, verbose=True): n_inputs = 0 - print(traced_model.graph) - # check for constant nodes merge_layers = ['add', 'mul', 'sub', 'fmin', 'fmax'] i = 0 # count number of consts and use it in the name @@ -177,8 +175,6 @@ def parse_pytorch_model(config, verbose=True): node.update_arg(1, new_node) i += 1 - print(traced_model.graph) - # import pdb; breakpoint() traced_model.graph.lint() for node in traced_model.graph.nodes: From 92d65d9c0cf18b0eacd4942612267d5f692a32fc Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Tue, 12 Nov 2024 11:09:21 -0500 Subject: [PATCH 5/7] adhere to quote style --- hls4ml/converters/pytorch_to_hls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index 06f7b95364..8af2830365 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -269,7 +269,7 @@ def parse_pytorch_model(config, verbose=True): input_layer['name'] = node.name if 'const' in node.name: - pytorch_class = "Constant" + pytorch_class = 'Constant' layer, output_shape = layer_handlers[pytorch_class](pytorch_class, node.name, node) layer_list.append(layer) From 68b4c4fec42a7a115573ffa2a1a5cc439e052c0a Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Tue, 12 Nov 2024 11:46:59 -0500 Subject: [PATCH 6/7] fix name matching for multiple instances of mul etc --- hls4ml/converters/pytorch/core.py | 1 + hls4ml/converters/pytorch_to_hls.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hls4ml/converters/pytorch/core.py b/hls4ml/converters/pytorch/core.py index 7cc20fc282..57c42f401f 100644 --- a/hls4ml/converters/pytorch/core.py +++ b/hls4ml/converters/pytorch/core.py @@ -8,6 +8,7 @@ def parse_constant_layer(operation, layer_name, node): assert 'Constant' in operation layer = {} + layer['inputs'] = [] layer['class_name'] = 'Constant' layer['name'] = layer_name diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index 8af2830365..f3615200d0 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -164,7 +164,7 @@ def parse_pytorch_model(config, verbose=True): merge_layers = ['add', 'mul', 'sub', 'fmin', 'fmax'] i = 0 # count number of consts and use it in the name for node in traced_model.graph.nodes: - if node.name in merge_layers: + if node.name.split("_")[0] in merge_layers: for arg in node.args: if np.isscalar(arg): # add an input node with the constant value From 21f517934bbf1f26b6409fc4a29dbd9bc57cc150 Mon Sep 17 00:00:00 2001 From: Jan-Frederik Schulte Date: Fri, 10 Jan 2025 08:35:51 -0500 Subject: [PATCH 7/7] style fix --- hls4ml/converters/pytorch_to_hls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index f3615200d0..871026bc49 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -164,7 +164,7 @@ def parse_pytorch_model(config, verbose=True): merge_layers = ['add', 'mul', 'sub', 'fmin', 'fmax'] i = 0 # count number of consts and use it in the name for node in traced_model.graph.nodes: - if node.name.split("_")[0] in merge_layers: + if node.name.split('_')[0] in merge_layers: for arg in node.args: if np.isscalar(arg): # add an input node with the constant value