From ad8b153e77f82ad683b22edadb61ce4fc00bac96 Mon Sep 17 00:00:00 2001 From: Jorge Marques Date: Thu, 30 May 2024 17:57:09 -0300 Subject: [PATCH] Add HDL interface and lib parser w/ Makefile out Add HDL interface_ip.tcl and library *_ip.tcl and *_hw.tcl parser. Add HDL library Makefile writer. Signed-off-by: Jorge Marques --- adi_doctools/cli/hdl_gen.py | 83 ++++++++++-- adi_doctools/directive/hdl.py | 2 +- adi_doctools/parser/hdl.py | 233 ++++++++++++++++++++++++++++++++-- adi_doctools/typings/hdl.py | 40 ++++++ adi_doctools/writer/hdl.py | 62 ++++++++- tests/asset/core_ip.tcl | 33 +++++ tests/asset/interfaces_ip.tcl | 28 ++++ tests/test_hdl_interfaces.py | 64 ++++++++++ tests/test_hdl_library.py | 29 +++++ tests/test_hdl_vendor.py | 1 + 10 files changed, 549 insertions(+), 26 deletions(-) create mode 100644 adi_doctools/typings/hdl.py create mode 100644 tests/asset/core_ip.tcl create mode 100644 tests/asset/interfaces_ip.tcl create mode 100644 tests/test_hdl_interfaces.py create mode 100644 tests/test_hdl_library.py diff --git a/adi_doctools/cli/hdl_gen.py b/adi_doctools/cli/hdl_gen.py index b2fd307..c918a7b 100644 --- a/adi_doctools/cli/hdl_gen.py +++ b/adi_doctools/cli/hdl_gen.py @@ -1,15 +1,19 @@ import click import subprocess import re -from os import path, walk +from os import path, walk, pardir from glob import glob +from ..typings.hdl import vendors, Library, Carrier from ..parser.hdl import parse_hdl_regmap from ..parser.hdl import resolve_hdl_regmap from ..parser.hdl import expand_hdl_regmap from ..parser.hdl import parse_hdl_vendor +from ..parser.hdl import parse_hdl_library +from ..parser.hdl import resolve_hdl_library +from ..parser.hdl import parse_hdl_interfaces from ..writer.hdl import write_hdl_regmap - +from ..writer.hdl import write_hdl_library_makefile log = { 'hdl_f': "{} not found, are you sure this is the HDL repo?", @@ -61,23 +65,78 @@ def dir_assert(file, msg): has_tb = False if dir_assert(tbdir, log['hdl_tb']) else True # Generate HDL carrier dictionary - carrier = {} + carrier = Carrier() path_ = path.join(hdldir, 'projects', 'scripts') - glob_ = path.join(path_, "adi_project_*.tcl") - filenames = glob(glob_) - for file_ in filenames: - m = re.search("adi_project_(\\w+)\\.tcl", file_) - if not bool(m): - continue - - vendor_name = m.group(1) - carrier[vendor_name], msg = parse_hdl_vendor(file_) + for v in vendors: + file_ = path.join(path_, f"adi_project_{v}.tcl") + carrier[v], msg = parse_hdl_vendor(file_) for m in msg: click.echo(f"{file_}: {m}") # TODO do something with the parsed carriers, like get/validate library and # project dicts + # Generate HDL Library dictionary + types = ['*_ip.tcl', '*_hw.tcl'] + files = {} + library = {} + interfaces_ip_files = [] + for v in vendors: + files[v] = [] + for typ, v in zip(types, vendors): + glob_ = path.join(hdldir, 'library', '**', typ) + files[v].extend(glob(glob_, recursive=True)) + + for v in files: + for f in files[v]: + if 'interfaces_ip.tcl' in f: + files[v].remove(f) + interfaces_ip_files.append(f) + + # Generate the HDL interfaces dictionary + interfaces_ip = {} + for f in interfaces_ip_files: + interfaces_ip[path.dirname(f)], msg = parse_hdl_interfaces(f) + for m in msg: + click.echo(f"{f}: {m}") + + intf_key_file = {} + for f in interfaces_ip: + for k in interfaces_ip[f]: + intf_key_file[k['name']] = f + + for typ, v in zip(types, files): + for f in files[v]: + key = path.dirname(f) + lib = path.basename(key) + lib2 = path.basename(f)[:-len(typ)+1] + if lib != lib2: + click.echo(f"{f}: path basename '{lib}' does not match " + f"filename '{lib2}'") + lib_, msg = parse_hdl_library(f, lib2) + for m in msg: + click.echo(f"{f}: {m}") + if lib_: + if key not in library: + library[key] = Library( + name=lib2, + vendor={}, + generic={} + ) + library[key]['vendor'][v] = lib_ + + for key in library: + resolve_hdl_library(library[key], intf_key_file, hdldir, key) + + for key in library: + write_hdl_library_makefile(key, hdldir, library[key]) + + # Generate HDL Project dictionary + files = [] + glob_ = path.join(hdldir, 'projects', '**', 'system_project.tcl') + files.extend(glob(glob_, recursive=True)) + # TODO parse HDL Project, write Project Makefile + # Generate HDL Register Map dictionary rm = {} regdir = path.join(hdldir, 'docs', 'regmap') diff --git a/adi_doctools/directive/hdl.py b/adi_doctools/directive/hdl.py index 41fc080..1e76988 100644 --- a/adi_doctools/directive/hdl.py +++ b/adi_doctools/directive/hdl.py @@ -226,7 +226,7 @@ def tables(self, subnode, obj, key): # Underscore with word-breaking 0 width space default = default.replace("_", "_\u200B") # Ensure `` parsed as pre in formulas - for a in ['+', '-', '/', '*', '^']: + for a in ['+', '-', '/', '*', '^', '~', '!']: default = default.replace(a + "``", a + " ``") if type(bits) is tuple: diff --git a/adi_doctools/parser/hdl.py b/adi_doctools/parser/hdl.py index ff1dcdc..8431ec6 100644 --- a/adi_doctools/parser/hdl.py +++ b/adi_doctools/parser/hdl.py @@ -1,10 +1,12 @@ -from typing import List, Tuple, Dict +from typing import TypedDict, Optional, List, Tuple, Dict import re import contextlib from lxml import etree from os import path +from ..typings.hdl import Intf, IntfPort +from ..typings.hdl import LibraryVendor from ..directive.string import string_hdl @@ -165,7 +167,7 @@ def get_where(desc: str, warn: List, reg: str, fi=None) -> Tuple[any]: field_where = None field_import = False field_loc = data[fi + 1] - field_loc = field_loc.split(" ") + field_loc = field_loc.split() field_bits = field_loc[0].replace("[", "").replace("]", "") if field_bits != 'n': delimiters = ["+", "-", "*", "/"] @@ -233,7 +235,7 @@ def get_where(desc: str, warn: List, reg: str, fi=None) -> Tuple[any]: default_str = re.findall("[A-Z0-9_]+", default_str) for str_part in default_str: try: - default_tmp = int(str_part) + int(str_part) except Exception: reg_params.append(re.sub('\[[0-9:]+\]', ' ', str_part)) # TODO: Check if parameter exist in the parameters dict from the parsed pkg.ttcl (when it gets implemented) @@ -707,9 +709,10 @@ def parse_hdl_build_status(file: str) -> Tuple[List, int, List[str]]: try: s = 'build number' build_number = int(data[0][data[0].find(s)+len(s)+1:]) - except: + except Exception as e: build_number = -1 - warning.append(f"Couldn't get the build number from the first line of {file}.") + warning.append("Couldn't get the build number from the first line " + f"of '{file}', exception: {e}.") project = [] for i in range(5, len(data) - 2): @@ -723,11 +726,13 @@ def parse_hdl_build_status(file: str) -> Tuple[List, int, List[str]]: return (project, build_number, warning) -def parse_hdl_vendor(file: str, owners: List = []) -> Tuple[Tuple[str], List[str]]: +def parse_hdl_vendor( + file: str +) -> Tuple[Tuple[str], List[str]]: """ Obtain the carrier from the project vendor file. """ - carrier = set() + obj = set() if not path.isfile(file): return ((), ["File doesn't exist!"]) @@ -740,7 +745,217 @@ def parse_hdl_vendor(file: str, owners: List = []) -> Tuple[Tuple[str], List[str m = re.search("if \\[regexp \"_(\\w+)\" \\$project_name", line) if not bool(m): continue + obj.add(m.group(1)) - carrier.add(m.group(1)) + return (tuple(obj), []) - return (tuple(carrier), []) + +def parse_hdl_library( + file: str, + key: str, +) -> Tuple[Optional[LibraryVendor], List[str]]: + """ + Obtain the library dependencies and interfaces from the library file. + Vendor agnostic, even though we use: + * adi_ip_files: xilinx + * ad_ip_file: intel + Would be better to switch to the same method in the future. + """ + # TODO get parameters + warning = [] + + if not path.isfile(file): + warning.append("File doesn't exist!") + return (None, warning) + + with open(file, "r") as f: + data = f.readlines() + + # Check library name against key + for i, line in enumerate(data): + for m in ['adi_ip_create', 'ad_ip_create']: + if line.startswith(m): + line_ = line.split() + if key != line_[1]: + warning.append(f"'{m}' IP name '{line_[1]}' does not " + f"match name '{key}', line {i}") + break + + # Obtain the file dependencies + deps = set() + i = -1 + for i, line in enumerate(data): + # Xilinx + if (line.startswith('adi_ip_files')) and key in line: + break + # Altera + if (line.startswith('ad_ip_files')) and key in line: + break + # Without wrapper (improper) + if (line.startswith('add_files')): + break + if i != 0: + i += 1 + while (i != len(data) and len(data[i]) != 1): + dep = data[i] + for char in ['\n', '"', ']', '\\']: + dep = dep.replace(char, '') + dep = dep.strip() + if len(dep) > 0: + deps.add(dep) + if data[i][-2] != '\\': + break + i += 1 + # Add itself as a dependency + deps.add(path.basename(file)) + + def in_method_match(expr, start): + """ + Try to match group inside a tcl method accross multiple lines. + """ + v = set() + for i in range(0, len(data)): + if data[i].startswith(start): + while (i != len(data) and len(data[i]) != 1): + m = re.search(expr, data[i]) + i += 1 + if m is False or m is None: + continue + v_ = m.group(1) + v.add(v_) + if data[i-1][-2] != '\\': + break + return v + + # Obtain the library dependencies + lib_deps = in_method_match("analog\\.com:\\$VIVADO_IP_LIBRARY:(\\w+)", + "adi_ip_add_core_dependencies") + + # Obtain the interface dependencies + intf = in_method_match("analog\\.com:(?:interface|user):(\\w+)", + "adi_add_bus") + # Remove _rtl suffixed + intf = [e for e in intf if not e.endswith('_rtl')] + + lib = LibraryVendor( + path=file, + dependencies=tuple(deps), + library_dependencies=tuple(sorted(lib_deps)), + interfaces=tuple(intf) + ) + return (lib, warning) + + +def resolve_hdl_library( + library: str, + intf_lut: str, + root_path: str, + path_: str +) -> List[str]: + """ + Resolve a library by extracting generic dependencies, resolving paths + and checking interfaces + """ + warning = [] + + # Filter generic deps + deps = {} + for v in library['vendor']: + deps[v] = set(library['vendor'][v]['dependencies']) + deps['generic'] = set.intersection(*[deps[v] for v in deps]) + + # Expand $ad_hdl_dir and make it relative + for v in deps: + for k in deps[v]: + if k.startswith('$ad_hdl_dir/'): + deps[v].add(path.relpath(path.join(root_path, k[12:]), path_)) + deps[v].remove(k) + + for v in library['vendor']: + library['vendor'][v]['dependencies'] = sorted(deps[v] - deps['generic']) + library['generic']['dependencies'] = sorted(deps['generic']) + + # Find interfaces_ip.tcl source for intf db and obtain + # Effectively, Xilinx + for v in library['vendor']: + deps_intf = set() + interface_deps = set() + for intf in library['vendor'][v]['interfaces']: + if intf not in intf_lut: + warning.append(f"Interface {intf} does not exist in any " + "interfaces_ip.tcl file.") + else: + base = path.relpath(path.join(intf_lut[intf], intf), path_) + deps_intf.add(base + '.xml') + deps_intf.add(base + "_rtl.xml") + # XILINX_INTERFACE_DEPS are relative to the library folder + p_ = path.join(root_path, 'library') + interface_deps.add(path.relpath(intf_lut[intf], p_)) + library['vendor'][v]['interfaces'] = tuple(deps_intf) + library['vendor'][v]['interfaces_tcl'] = tuple(interface_deps) + + return warning + + +def parse_hdl_interfaces( + file: str, +) -> Tuple[Tuple[Intf], List[str]]: + """ + Obtain the interfaces from the interfaces file. + """ + warning = [] + obj = [] + + if not path.isfile(file): + warning.append("File doesn't exist!") + return ((), warning) + + with open(file, "r") as f: + data = f.readlines() + + intf = None + descr = None + for i, line in enumerate(data): + line = line.strip() + if len(line) > 0 and line[0] == '#': + descr = line[1:].strip() + if line.startswith('adi_if_define'): + intf = line.split()[1].replace('"', '') + obj.append(Intf( + description=descr, + name=intf, + ports=[] + )) + descr = None + + if line.startswith('adi_if_ports'): + if len(obj) == 0: + warning.append(f"'adi_if_ports' at line {i+1} " + "without precending adi_if_ports") + continue + ports = line.split() + try: + if len(ports) < 4: + raise Exception(f"too few arguments, got {len(ports)}") + if ports[1] not in ['input', 'output']: + raise Exception(f"unknown direction '{ports[1]}'") + direction = ports[3] + width = int(ports[2]) + name = ports[3] + domain = 'none' if len(ports) < 5 else ports[4] + domain = None if domain == 'none' else domain + default = None if len(ports) < 6 else int(ports[5]) + + obj[-1]['ports'].append(IntfPort( + direction=direction, + width=width, + name=name, + domain=domain, + default=default, + )) + except Exception as e: + warning.append(f"Malformed 'adi_if_ports' at line {i+1}, " + f"exception: {e}") + for o in obj: + o['ports'] = tuple(o['ports']) + return (tuple(obj), warning) diff --git a/adi_doctools/typings/hdl.py b/adi_doctools/typings/hdl.py new file mode 100644 index 0000000..34df2ad --- /dev/null +++ b/adi_doctools/typings/hdl.py @@ -0,0 +1,40 @@ +from typing import TypedDict, Optional, Tuple, Dict + +vendors = ("xilinx", "intel") + + +class IntfPort(TypedDict): + direction: str + width: int + name: str + domain: Optional[str] + default: Optional[int] + + +class Intf(TypedDict): + description: Optional[str] + name: str + ports: Tuple[IntfPort] + + +class LibraryGeneric(TypedDict): + dependencies: Tuple[str] + + +class LibraryVendor(TypedDict): + path: str + dependencies: Tuple[str] + library_dependencies: Tuple[str] + interfaces: Tuple[str] + interfaces_tcl: Tuple[str] + + +class Library(TypedDict): + name: str + vendor: Dict[str, LibraryVendor] + generic: LibraryGeneric + + +class Carrier(TypedDict): + xilinx: Tuple[str] + intel: Tuple[str] diff --git a/adi_doctools/writer/hdl.py b/adi_doctools/writer/hdl.py index 1a47ecd..bb2fd2a 100644 --- a/adi_doctools/writer/hdl.py +++ b/adi_doctools/writer/hdl.py @@ -1,9 +1,10 @@ -from typing import Dict +from typing import Dict, Tuple from datetime import datetime from os import path from ..__init__ import __version__ +from ..typings.hdl import vendors, Library svpkg_fn_new0 = """ function new( @@ -143,7 +144,7 @@ def svpkg_reg_decl(f, regmap: Dict): row += ")" row += f" {reg['name']}_R;\n" f.write(row) - + def svpkg_reg_inst(f, regmap: Dict): for reg in regmap['regmap']: @@ -165,13 +166,17 @@ def svpkg_reg_inst(f, regmap: Dict): row += f", {addr});\n" f.write(row) - + def svpkg_footer(f): f.write(" endclass;\n") f.write("endpackage;\n") -def write_hdl_regmap(path_: str, regmap: Dict, key: str): +def write_hdl_regmap( + path_: str, + regmap: Dict, + key: str +) -> None: fname = f"adi_regmap_{key}_pkg.sv" file = path.join(path_, fname) f = open(file, "w") @@ -191,3 +196,52 @@ def write_hdl_regmap(path_: str, regmap: Dict, key: str): svpkg_footer(f) f.close() + + +def write_hdl_library_makefile( + path_: str, + root_path: str, + library: Library +) -> Tuple[str]: + fname = "Makefile" + file = path.join(path_, fname) + f = open(file, "w") + f.write(f"""\ +#################################################################################### +## Copyright (c) 2018 - {datetime.now().year} Analog Devices, Inc. +### SPDX short identifier: BSD-1-Clause +## Auto-generated v{__version__}, do not modify! +#################################################################################### +""") + f.write("\n") + f.write(f"LIBRARY_NAME := {library['name']}\n") + f.write("\n") + for d in library['generic']['dependencies']: + f.write(f"GENERIC_DEPS += {d}\n") + f.write("\n") + for v in library['vendor']: + r = library['vendor'][v] + for d in r['dependencies']: + f.write(f"{v.upper()}_DEPS += {d}\n") + if len(r['dependencies']) > 0: + f.write("\n") + + for x in r['interfaces']: + f.write(f"{v.upper()}_DEPS += {x}\n") + if len(r['interfaces']) > 0: + f.write("\n") + + for d in r['library_dependencies']: + f.write(f"{v.upper()}_LIB_DEPS += {d}\n") + if len(r['library_dependencies']) > 0: + f.write("\n") + + for x in r['interfaces_tcl']: + f.write(f"{v.upper()}_INTERFACE_DEPS += {x}\n") + if len(r['interfaces_tcl']) > 0: + f.write("\n") + + p_ = path.join(root_path, 'library', 'scripts', 'library.mk') + p_ = path.relpath(p_, path_) + f.write(f"include {p_}\n") + f.close() diff --git a/tests/asset/core_ip.tcl b/tests/asset/core_ip.tcl new file mode 100644 index 0000000..288d868 --- /dev/null +++ b/tests/asset/core_ip.tcl @@ -0,0 +1,33 @@ +# ... + +adi_ip_files core [list \ + $special/file/path.v \ + "core.v" \ + "core_sub.v" \ +] + +## Interface definitions + +adi_add_bus "bus_0" "master" \ + "analog.com:interface:core_bus_0_rtl:1.0" \ + "analog.com:interface:core_bus_0:1.0" \ + { + {"bus_0_a" "a"} \ + {"bus_0_b" "b"} \ + } +adi_add_bus_clock "clk" "bus_0" "reset_n" + +adi_add_bus "bus_1" "slave" \ + "analog.com:interface:core_bus_1_rtl:1.0" \ + "analog.com:interface:core_bus_1:1.0" \ + { + {"bus_1_a" "a"} \ + {"bus_1_b" "b"} \ + } +adi_add_bus_clock "clk" "bus_1" "reset_n" + +adi_ip_add_core_dependencies [list \ + analog.com:$VIVADO_IP_LIBRARY:util_a:1.0 \ + analog.com:$VIVADO_IP_LIBRARY:util_b:1.0 \ + analog.com:$VIVADO_IP_LIBRARY:util_c:1.0 \ +] diff --git a/tests/asset/interfaces_ip.tcl b/tests/asset/interfaces_ip.tcl new file mode 100644 index 0000000..525c12f --- /dev/null +++ b/tests/asset/interfaces_ip.tcl @@ -0,0 +1,28 @@ + + +adi_if_define intf_0 +adi_if_ports output 1 a +adi_if_ports output 2 b +adi_if_ports output 3 c +adi_if_ports output 4 d +adi_if_ports input 5 e +adi_if_ports input 6 f + +# Interface description 1 + +adi_if_define intf_1 +adi_if_ports input 1 a +adi_if_ports output 2 b reset 1 +adi_if_ports output 3 c none 0 +adi_if_ports output -1 d + +adi_if_define intf_2 + +adi_if_ports output 1 a reset +adi_if_ports output 1 b clock + +# Interface description 2 + +adi_if_define intf_3 +adi_if_ports output 1 a reset +adi_if_ports output 1 b clock diff --git a/tests/test_hdl_interfaces.py b/tests/test_hdl_interfaces.py new file mode 100644 index 0000000..cbc063a --- /dev/null +++ b/tests/test_hdl_interfaces.py @@ -0,0 +1,64 @@ +import logging +from os import path + +from adi_doctools.parser.hdl import parse_hdl_interfaces +from adi_doctools.typings.hdl import Intf + +logger = logging.getLogger(__name__) + +intf_0 = Intf( + description=None, + name="intf_0" +) + +intf_1 = Intf( + description="Interface description 1", + name="intf_1" +) + +intf_2 = Intf( + description=None, + name="intf_2" +) + +intf_3 = Intf( + description="Interface description 2", + name="intf_3" +) + +intfs = [intf_0, intf_1, intf_2, intf_3] + +logger = logging.getLogger(__name__) + + +def test_hdl_interfaces(tmp_path): + + def log_assert(msg): + for m in msg: + logger.warning(m) + + assert len(msg) == 0 + + def log_info(obj): + import json + logger.info(json.dumps(obj, indent=4)) + + file = path.join("asset", "interfaces_ip.tcl") + + interfaces, msg = parse_hdl_interfaces(file) + log_info(interfaces) + log_assert(msg) + + assert len(interfaces) == 4 + + for a, b in zip(intfs, interfaces): + for key in ["description", "name"]: + assert a[key] == b[key] + + assert len(interfaces[0]['ports']) == 6 + i = 0 + for a in range(0, len(interfaces[0]['ports'])): + assert interfaces[0]['ports'][i]['width'] == i + 1 + + assert interfaces[1]['ports'][1]['domain'] == "reset" + assert interfaces[1]['ports'][2]['domain'] is None diff --git a/tests/test_hdl_library.py b/tests/test_hdl_library.py new file mode 100644 index 0000000..ba0e22d --- /dev/null +++ b/tests/test_hdl_library.py @@ -0,0 +1,29 @@ +import logging +from os import path + +from adi_doctools.parser.hdl import parse_hdl_library + +logger = logging.getLogger(__name__) + + +def test_hdl_library(tmp_path): + + def log_assert(msg): + for m in msg: + logger.warning(m) + + assert len(msg) == 0 + + def log_info(obj): + import json + logger.info(json.dumps(obj, indent=4)) + + file = path.join("asset", "core_ip.tcl") + + obj, msg = parse_hdl_library(file, "core") + log_info(obj) + log_assert(msg) + + assert len(obj["dependencies"]) == 4 + assert len(obj["interfaces"]) == 2 + assert len(obj["library_dependencies"]) == 3 diff --git a/tests/test_hdl_vendor.py b/tests/test_hdl_vendor.py index abbd2f0..668445a 100644 --- a/tests/test_hdl_vendor.py +++ b/tests/test_hdl_vendor.py @@ -17,6 +17,7 @@ def log_assert(msg): file = path.join('asset', "adi_project_vendor.tcl") carrier, msg = parse_hdl_vendor(file) + logger.info(carrier) log_assert(msg) assert sorted(carrier) == sorted(('dev_0', 'dev_1', 'dev_2'))