diff --git a/.ci.yml b/.ci.yml index f7bed684..19f1a291 100644 --- a/.ci.yml +++ b/.ci.yml @@ -79,9 +79,10 @@ package_cores: image: debian:bookworm script: - ./.github/scripts/ci.sh package_cores + - tar -cf build/cores_export.tar -C build/export . artifacts: paths: - - core_repo/** + - build/cores_export.tar pyright_check: variables: diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh index e96c93da..3f8e873e 100755 --- a/.github/scripts/ci.sh +++ b/.github/scripts/ci.sh @@ -142,7 +142,6 @@ generate_examples() { done } - generate_docs() { install_common_system_packages install_topwrap_system_deps @@ -156,20 +155,13 @@ generate_docs() { end_command_group } - package_cores() { install_common_system_packages install_topwrap_system_deps - - begin_command_group "Install Topwrap with parsing dependencies" - log_cmd pip install ".[topwrap-parse]" - end_command_group + install_nox begin_command_group "Package cores for release" - log_cmd mkdir core_repo - log_cmd pushd core_repo - log_cmd python ../.github/scripts/package_cores.py - log_cmd popd + log_cmd nox -s package_cores end_command_group } diff --git a/.github/scripts/package_cores.py b/.github/scripts/package_cores.py old mode 100755 new mode 100644 index 217acac0..edb84cbf --- a/.github/scripts/package_cores.py +++ b/.github/scripts/package_cores.py @@ -1,14 +1,28 @@ #!/usr/bin/env python3 # Copyright (c) 2024 Antmicro # SPDX-License-Identifier: Apache-2.0 + +"""This module contains a `package_fusesoc` function and helpers. The goal of `package_fusesoc` is to +download, parse and package cores from fusesoc library to use as external repos for topwrap projects""" + +import logging +import os +import re +import shutil +import subprocess from dataclasses import dataclass from pathlib import Path from typing import List +import click + +from topwrap.cli import parse_main from topwrap.repo.file_handlers import VerilogFileHandler from topwrap.repo.files import HttpGetFile from topwrap.repo.user_repo import UserRepo +logger = logging.getLogger(__name__) + @dataclass class RemoteRepo: @@ -28,7 +42,7 @@ class RemoteRepo: ] -def package_cores(): +def package_repos(): """Generates reusable cores package for usage in Topwrap project.""" core_repo = UserRepo() @@ -43,5 +57,171 @@ def package_cores(): core_repo.save(repo.name) +@click.command() +@click.option("--log-level", required=False, default="INFO", help="Log level") +def package_cores(log_level: str): + """Downloads, parses and packages fusesoc cores library to topwrap repos""" + if log_level in ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]: + logger.setLevel(log_level) + else: + logger.setLevel("INFO") + logger.warning(f"incorrect log level {log_level}, using INFO instead") + + # creating workspace directory, making sure it is empty + Path("build").mkdir(exist_ok=True) + shutil.rmtree("build/fusesoc_workspace", ignore_errors=True) + shutil.rmtree("build/export", ignore_errors=True) + Path("build/fusesoc_workspace").mkdir(exist_ok=True) + + os.chdir("build/fusesoc_workspace") # root/build/workspace + subprocess.run(["fusesoc", "config", "cache_root", "cache_dir"], check=True) + subprocess.run( + ["fusesoc", "library", "add", "fusesoc-cores", "https://github.com/fusesoc/fusesoc-cores"], + check=True, + ) + + os.chdir( + "fusesoc_libraries/fusesoc-cores" + ) # root/build/fusesoc_workspace/fusesoc_libraries/fusesoc-cores + cores_str = [str(x) for x in Path(".").glob("*") if x.is_dir()] + os.chdir("../..") # root/build/fusesoc_workspace + + IGNORE_LIST = [ + ".git", + ] + FILE_IGNORE_LIST = [ + "tests", + ] + FILE_IGNORE_PATTERN_LIST = [ + re.compile(raw_regex) + for raw_regex in [ + ".*_tb", + ".*/test_.*", + ".*_test", + ".*/tb_.*", + ".*/tst_.*", + ".*bench/.*", + ".*testbench.*", + ] + ] + + cores_downloaded: List[str] = [] + cores_failed = [] + for core in cores_str: + if core in IGNORE_LIST: + continue + try: + subprocess.run(["fusesoc", "fetch", core], check=True) + cores_downloaded.append(core) + except Exception as e: + cores_failed.append(core) + logger.warning(f"failed to fetch {core} due to {e}") + logger.warning(f"failed to download {len(cores_failed)} cores: {cores_failed}") + logger.warning(f"downloaded {len(cores_downloaded)} cores: {cores_downloaded}") + + # root/build/fusesoc_workspace/scratchpad - all hdl files + Path("scratchpad").mkdir(exist_ok=True) + # root/build/fusesoc_workspace/build - intermediate build dir + Path("build").mkdir(exist_ok=True) + + os.chdir("cache_dir") # root/build/fusesoc_workspace/cache_dir + + for c in [x for x in Path(".").rglob("*") if x.suffix in [".v", ".vh", ".vhd", ".vhdl"]]: + shutil.copy(str(c), "../scratchpad/") + + error_counter = 0 + pass_counter = 0 + error_parses: List[str] = [] + full_good: List[str] = [] + + # iterating over fusesoc cores, note that fusesoc core is a topwrap repo, not topwrap core + for core in cores_downloaded: + core_build_dir = "../../build/" + core + # finding the path of a fusesoc core - they have a suffix with version + for path in [x for x in Path(".").glob(core + "*")]: + os.chdir( + path + ) # root/build/fusesoc_workspace/cache_dir/[core] - intermediate build dir for fusesoc core (repo) + core_path_list = [ + x for x in Path(".").rglob("*") if x.suffix in [".v", ".vh", ".vhd", ".vhdl"] + ] # looking for hdl files + if len(core_path_list) > 0: + # root/build/fusesoc_workspace/build/[core] and workspace/build/[core]/cores + Path(core_build_dir).mkdir(exist_ok=True) + Path(core_build_dir + "/cores").mkdir(exist_ok=True) + err_ini = error_counter + # iterating over hdl files, cores in topwrap terminology + for core_path in core_path_list: + if any( + compiled_reg.match(str(core_path)) + for compiled_reg in FILE_IGNORE_PATTERN_LIST + ): + continue + if str(core_path.stem) in FILE_IGNORE_LIST: + continue + component_build_dir = core_build_dir + "/cores/" + str(core_path.stem) + # root/build/fusesoc_workspace/build/[core]/cores/[component] - intermediate build dir for topwrap core in a repo + Path(component_build_dir).mkdir(exist_ok=True) + logger.info(f"parsing {os.getcwd()} / {str(core_path)}") + # parsing core, first normally, then in scratchpad to resolve import errors + try: + with click.Context(parse_main) as ctx: + ctx.invoke( + parse_main, + use_yosys=False, + iface_deduce=False, + iface=(), + log_level=log_level, + files=[str(core_path)], + dest_dir=component_build_dir, + ) + # root/build/fusesoc_workspace/build/[core]/cores/[component]/srcs - path for hdl file describing [component] + Path(component_build_dir + "/srcs").mkdir(exist_ok=True) + shutil.copy(str(core_path), component_build_dir + "/srcs") + pass_counter += 1 + except Exception: + try: + with click.Context(parse_main) as ctx: + ctx.invoke( + parse_main, + use_yosys=False, + iface_deduce=False, + iface=(), + log_level=log_level, + files=[str(Path("../../scratchpad/" + core_path.name))], + dest_dir=component_build_dir, + ) + # root/build/fusesoc_workspaces/build/[core]/cores/[component]/srcs - path for hdl file describing [component] + Path(component_build_dir + "/srcs").mkdir(exist_ok=True) + shutil.copy(str(core_path), component_build_dir + "/srcs") + pass_counter += 1 + except Exception as e2: + logger.warning(f"failed to parse due to {e2}") + error_counter += 1 + error_parses.append(core + "/" + str(core_path)) + subprocess.run(["ls"], check=True) + subprocess.run(["pwd"], check=True) + # check if whole fusesoc core parsed correctly + if error_counter == err_ini: + full_good.append(core) + os.chdir("..") # root/build/fusesoc_workspace/cache_dir + for succ_core in full_good: + shutil.copytree( + Path("../build") / Path(succ_core), + Path("../../export") / Path(succ_core), + dirs_exist_ok=True, + ) + os.chdir("../..") # root/build + logger.warning(f"parses failed: {error_counter} out of {error_counter+pass_counter}") + logger.warning(error_parses) + logger.info(f"fully well parsed cores are {full_good} - a total of {len(full_good)}") + + os.chdir("export") + logger.info("packaging cores") + package_repos() + + if __name__ == "__main__": package_cores() + os.chdir("export") + package_repos() diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 7f838da4..f7b90d88 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -206,9 +206,10 @@ jobs: - name: Pack cores into a Topwrap repository run: | ./.github/scripts/ci.sh package_cores + tar -cf build/cores_export.tar -C build/export . - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: core_repo - path: core_repo/** + name: export_cores + path: build/cores_export.tar diff --git a/noxfile.py b/noxfile.py index b79985b1..75c7d1df 100644 --- a/noxfile.py +++ b/noxfile.py @@ -271,3 +271,13 @@ def print_table( for errtype, num in errorfiles.items(): if num - errorfiles_main[errtype] > 0: raise CommandFailed() + + +@nox.session +def package_cores(session: nox.Session) -> None: + session.install("-e", ".[topwrap-parse]") + session.install("fusesoc") + if len(session.posargs) > 0: + session.run("python", ".github/scripts/package_cores.py", "--log-level", session.posargs[0]) + else: + session.run("python", ".github/scripts/package_cores.py") diff --git a/tests/data/data_build/clog2/clog2_design.yaml b/tests/data/data_build/clog2/clog2_design.yaml new file mode 100644 index 00000000..82628170 --- /dev/null +++ b/tests/data/data_build/clog2/clog2_design.yaml @@ -0,0 +1,21 @@ +# Copyright (c) 2023-2024 Antmicro +# SPDX-License-Identifier: Apache-2.0 + +ips: + core1: + file: clog2_tester.yaml +design: + name: top + ports: + core1: + i_clk: PORT_CLK + i_waddr: PORT_VEC_IN + o_waddr: PORT_VEC_OUT + +external: + ports: + in: + - PORT_CLK + - PORT_VEC_IN + out: + - PORT_VEC_OUT diff --git a/tests/data/data_build/clog2/clog2_design2.yaml b/tests/data/data_build/clog2/clog2_design2.yaml new file mode 100644 index 00000000..51e5f4b3 --- /dev/null +++ b/tests/data/data_build/clog2/clog2_design2.yaml @@ -0,0 +1,21 @@ +# Copyright (c) 2023-2024 Antmicro +# SPDX-License-Identifier: Apache-2.0 + +ips: + core1: + file: clog2_tester2.yaml +design: + name: top + ports: + core1: + i_clk: PORT_CLK + i_waddr: PORT_VEC_IN + o_waddr: PORT_VEC_OUT + +external: + ports: + in: + - PORT_CLK + - PORT_VEC_IN + out: + - PORT_VEC_OUT diff --git a/tests/data/data_build/clog2/clog2_tester.v b/tests/data/data_build/clog2/clog2_tester.v new file mode 100644 index 00000000..3a4b9a97 --- /dev/null +++ b/tests/data/data_build/clog2/clog2_tester.v @@ -0,0 +1,14 @@ +module clog2_tester + #(parameter w=1, + parameter p4=4, + parameter depth=32*(32+p4)/w) + (input wire i_clk, + input wire [$clog2(depth)-1:0] i_waddr, + output wire [$clog2(depth)-1:0] o_waddr + ); + + always @(posedge i_clk) begin + o_waddr <= i_waddr; + end + +endmodule diff --git a/tests/data/data_build/clog2/clog2_tester.yaml b/tests/data/data_build/clog2/clog2_tester.yaml new file mode 100644 index 00000000..1d2f8f4c --- /dev/null +++ b/tests/data/data_build/clog2/clog2_tester.yaml @@ -0,0 +1,15 @@ +name: clog2_tester +parameters: + depth: ((32*(32+p4))/w) + p4: 4 + w: 1 +signals: + in: + - i_clk + - - i_waddr + - (clog2(depth)-1) + - 0 + out: + - - o_waddr + - (clog2(depth)-1) + - 0 diff --git a/tests/data/data_build/clog2/clog2_tester2.v b/tests/data/data_build/clog2/clog2_tester2.v new file mode 100644 index 00000000..b6b90b9c --- /dev/null +++ b/tests/data/data_build/clog2/clog2_tester2.v @@ -0,0 +1,14 @@ +module clog2_tester + #(parameter w=1, + parameter p4=4, + parameter depth=32*(32+p4)/w) + (input wire i_clk, + input wire [$clog2(depth*2)-2:0] i_waddr, + output wire [$clog2(depth*2)-2:0] o_waddr + ); + + always @(posedge i_clk) begin + o_waddr <= i_waddr; + end + +endmodule diff --git a/tests/data/data_build/clog2/clog2_tester2.yaml b/tests/data/data_build/clog2/clog2_tester2.yaml new file mode 100644 index 00000000..68b8d144 --- /dev/null +++ b/tests/data/data_build/clog2/clog2_tester2.yaml @@ -0,0 +1,15 @@ +name: clog2_tester +parameters: + depth: ((32*(32+p4))/w) + p4: 4 + w: 1 +signals: + in: + - i_clk + - - i_waddr + - (clog2(depth*2)-2) + - 0 + out: + - - o_waddr + - (clog2(depth*2)-2) + - 0 diff --git a/tests/tests_build/test_design.py b/tests/tests_build/test_design.py index aef4aa38..80b964e9 100644 --- a/tests/tests_build/test_design.py +++ b/tests/tests_build/test_design.py @@ -9,3 +9,26 @@ def test_design(self): from topwrap.design import build_design_from_yaml build_design_from_yaml(Path("tests/data/data_build/design.yaml"), Path("build")) + + def test_clog2_build(self): + from topwrap.design import build_design_from_yaml + + build_design_from_yaml( + Path("tests/data/data_build/clog2/clog2_design.yaml"), + Path("build"), + [Path("tests/data/data_build/clog2/")], + ) + + with open("build/top.v") as e: + result1 = e.read() + + build_design_from_yaml( + Path("tests/data/data_build/clog2/clog2_design2.yaml"), + Path("build"), + [Path("tests/data/data_build/clog2/")], + ) + + with open("build/top.v") as e: + result2 = e.read() + + assert result1 == result2 diff --git a/tests/tests_parse/conftest.py b/tests/tests_parse/conftest.py index 330d6f8c..3aa5181f 100644 --- a/tests/tests_parse/conftest.py +++ b/tests/tests_parse/conftest.py @@ -20,6 +20,17 @@ def axi_verilog_module() -> VerilogModule: return verilog_modules[0] +@pytest.fixture +def clog2_test_module() -> VerilogModule: + from topwrap.verilog_parser import VerilogModuleGenerator + + verilog_modules = VerilogModuleGenerator().get_modules( + "tests/data/data_build/clog2/clog2_tester.v" + ) + assert len(verilog_modules) == 1 + return verilog_modules[0] + + @pytest.fixture def dependant_verilog_module() -> VerilogModule: from tempfile import NamedTemporaryFile diff --git a/tests/tests_parse/test_ip_desc.py b/tests/tests_parse/test_ip_desc.py index 8dad6ad5..bda13d45 100644 --- a/tests/tests_parse/test_ip_desc.py +++ b/tests/tests_parse/test_ip_desc.py @@ -88,6 +88,14 @@ def ip_core_description(self, axi_verilog_module: VerilogModule) -> IPCoreDescri def expected_output(self) -> IPCoreDescription: return IPCoreDescription.load("tests/data/data_parse/axi_axil_adapter.yaml", False) + @pytest.fixture + def clog2_ip_core_description(self, clog2_test_module: VerilogModule) -> IPCoreDescription: + return clog2_test_module.to_ip_core_description(standard_iface_grouper()) + + @pytest.fixture + def clog2_expected_output(self) -> IPCoreDescription: + return IPCoreDescription.load("tests/data/data_parse/clog2_core.yaml", False) + @pytest.fixture def force_compliance(self): from topwrap.config import config diff --git a/tests/tests_parse/test_parse.py b/tests/tests_parse/test_parse.py index bb5ba3a1..9812022f 100644 --- a/tests/tests_parse/test_parse.py +++ b/tests/tests_parse/test_parse.py @@ -18,6 +18,21 @@ class TestVerilogParse: "CONVERT_NARROW_BURST": 0, } + def test_clog2_parameters(self, clog2_test_module): + assert clog2_test_module.parameters == {"w": 1, "p4": 4, "depth": "((32*(32+p4))/w)"} + + def test_clog2_ports(self, clog2_test_module): + assert clog2_test_module.ports == set( + [ + PortDefinition("i_clk", "0", "0", PortDirection.IN), + PortDefinition("i_waddr", "(clog2(depth)-1)", "0", PortDirection.IN), + PortDefinition("o_waddr", "(clog2(depth)-1)", "0", PortDirection.OUT), + ] + ) + + def test_clog2_module_name(self, clog2_test_module): + assert clog2_test_module.module_name == "clog2_tester" + def test_axi_module_parameters(self, axi_verilog_module): assert axi_verilog_module.parameters == self.AXI_AXIL_ADAPTER_PARAMS diff --git a/topwrap/hdl_module.py b/topwrap/hdl_module.py index 99a1b06c..4dd9a6ce 100644 --- a/topwrap/hdl_module.py +++ b/topwrap/hdl_module.py @@ -6,7 +6,13 @@ from .hdl_parsers_utils import PortDefinition from .interface_grouper import InterfaceGrouper -from .ip_desc import IPCoreDescription, IPCoreInterface, IPCoreIntfPorts, IPCorePorts +from .ip_desc import ( + IPCoreComplexParameter, + IPCoreDescription, + IPCoreInterface, + IPCoreIntfPorts, + IPCorePorts, +) HDLParameter = Union[int, str, Dict[str, int]] @@ -46,9 +52,16 @@ def to_ip_core_description(self, iface_grouper: InterfaceGrouper) -> IPCoreDescr ), ) + p = {} + for pname, par in self.parameters.items(): + if isinstance(par, dict): + p[pname] = IPCoreComplexParameter(width=par["width"], value=par["value"]) + else: + p[pname] = par + return IPCoreDescription( name=self.module_name, signals=IPCorePorts.from_port_def_list(ports - iface_ports), - parameters=self.parameters, + parameters=p, interfaces=ifaces_by_name, ) diff --git a/topwrap/hdl_parsers_utils.py b/topwrap/hdl_parsers_utils.py index 224b697e..95a18bb4 100644 --- a/topwrap/hdl_parsers_utils.py +++ b/topwrap/hdl_parsers_utils.py @@ -3,8 +3,9 @@ from dataclasses import dataclass from enum import Enum from logging import warning +from typing import Any, Dict, Union -from simpleeval import SimpleEval +from simpleeval import DEFAULT_FUNCTIONS, SimpleEval, simple_eval from topwrap.amaranth_helpers import DIR_IN, DIR_INOUT, DIR_OUT @@ -26,14 +27,23 @@ class PortDefinition: direction: PortDirection -def _eval_param(val, params: dict, simpleeval_instance: SimpleEval): +def _eval_param( + val: Union[int, dict, str], params: Dict[str, Any], simpleeval_instance: SimpleEval +): """Function used to calculate parameter value. It is used for evaluating CONCAT and REPL_CONCAT in resolve_ops()""" + if isinstance(val, int): return val if isinstance(val, dict) and val.keys() == {"value", "width"}: return val if isinstance(val, str): + if val not in params.keys(): + keying = DEFAULT_FUNCTIONS.copy() + for k in params.keys(): + keying[k] = params[k] + return str(simple_eval(val, functions=keying)) + return _eval_param(params[val], params, simpleeval_instance) elif val["__class__"] == "HdlValueInt": @@ -71,7 +81,9 @@ def _eval_param(val, params: dict, simpleeval_instance: SimpleEval): ) -def resolve_ops(val, params: dict, simpleeval_instance: SimpleEval): +def resolve_ops( + val: Union[str, int, Dict[str, Any]], params: Dict[str, Any], simpleeval_instance: SimpleEval +): """Get 'val' representation, that will be used in ip core yaml :param val: expression gathered from HdlConvertor data. @@ -88,6 +100,9 @@ def resolve_ops(val, params: dict, simpleeval_instance: SimpleEval): its default value and return as a { 'value': ..., 'width': ... } dict """ + if isinstance(val, str): + val = val.replace("$", "") + if isinstance(val, int) or isinstance(val, str): return val diff --git a/topwrap/ipwrapper.py b/topwrap/ipwrapper.py index 1b3a15ab..20078a8f 100644 --- a/topwrap/ipwrapper.py +++ b/topwrap/ipwrapper.py @@ -1,5 +1,6 @@ # Copyright (c) 2021-2024 Antmicro # SPDX-License-Identifier: Apache-2.0 +import math from itertools import groupby from logging import error from pathlib import Path @@ -8,13 +9,16 @@ from amaranth import Instance, Module, Signal from amaranth.build import Platform from amaranth.hdl.ast import Cat, Const -from simpleeval import simple_eval +from simpleeval import DEFAULT_FUNCTIONS, simple_eval from topwrap.ip_desc import IPCoreComplexParameter, IPCoreDescription from .amaranth_helpers import WrapperPort, port_direction_to_prefix from .wrapper import Wrapper +my_functions = DEFAULT_FUNCTIONS.copy() +my_functions.update(clog2=(lambda n: math.ceil(math.log2(n)))) + def _group_by_internal_name(ports: List[WrapperPort]): """Group ports by their 'internal_name' attribute @@ -58,7 +62,7 @@ def _eval_bounds(bounds, params: dict): for i, item in enumerate(bounds): if isinstance(item, str): try: - result[i] = int(simple_eval(item, names=params)) + result[i] = int(simple_eval(item, names=params, functions=my_functions)) except TypeError: error( "Could not evaluate expression with parameter: " diff --git a/topwrap/verilog_parser.py b/topwrap/verilog_parser.py index 58a41261..3b7ab454 100644 --- a/topwrap/verilog_parser.py +++ b/topwrap/verilog_parser.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from functools import cached_property from logging import error -from typing import Dict, List, Set +from typing import Any, Dict, List, Set from hdlConvertor import HdlConvertor from hdlConvertorAst.language import Language @@ -20,7 +20,7 @@ class VerilogModule(HDLModule): using HdlConvertor. """ - def __init__(self, verilog_file: str, verilog_module: dict): + def __init__(self, verilog_file: str, verilog_module: Dict[str, Any]): super().__init__(verilog_file) self._data = verilog_module @@ -99,8 +99,8 @@ def get_modules(self, file: str) -> List[VerilogModule]: hdl_context = self.conv.parse( [file], Language.VERILOG, [], hierarchyOnly=False, debug=True ) + modules = ToJson().visit_HdlContext(hdl_context) + return [VerilogModule(file, module) for module in modules] except IndexError: error(f"No module found in {file}!") - - modules = ToJson().visit_HdlContext(hdl_context) - return [VerilogModule(file, module) for module in modules] + return []