From 324a404ab29968801c7e898329abdb52f54143a5 Mon Sep 17 00:00:00 2001 From: Jiahan Xie Date: Sun, 5 Jan 2025 01:23:57 -0500 Subject: [PATCH] use morty crate to pickle multiple files --- Cargo.lock | 469 ++++++++++++++++++++++++++++++++++- calyx-backend/Cargo.toml | 2 + calyx-backend/src/verilog.rs | 250 +++++++++++++++++-- 3 files changed, 696 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3037b460b4..c714f5f76c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.80" @@ -232,6 +281,18 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "bitvec" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55f93d0ef3363c364d5976646a38f04cf67cfe1d4c8d160cdea02cab2c116b33" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -276,6 +337,18 @@ version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +[[package]] +name = "bytecount" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8" + +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + [[package]] name = "bytemuck" version = "1.14.3" @@ -324,6 +397,7 @@ dependencies = [ "itertools 0.11.0", "linked-hash-map", "log", + "morty", "petgraph", "quick-xml", "serde", @@ -332,6 +406,7 @@ dependencies = [ "serde_with 3.6.1", "smallvec", "string-interner", + "tempfile", "vast", ] @@ -602,6 +677,46 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "clap" +version = "4.5.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + [[package]] name = "clipboard-win" version = "4.5.0" @@ -613,6 +728,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.52.0", +] + [[package]] name = "component_cells" version = "0.7.1" @@ -697,7 +828,7 @@ checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" dependencies = [ "atty", "cast", - "clap", + "clap 2.34.0", "criterion-plot", "csv", "itertools 0.10.5", @@ -1188,6 +1319,12 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "futures" version = "0.3.30" @@ -1284,6 +1421,15 @@ dependencies = [ "version_check 0.9.4", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -1347,6 +1493,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1487,6 +1639,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1526,6 +1684,19 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "cfg-if", + "ryu", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.153" @@ -1653,6 +1824,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "morty" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6f9bcf39547c7974641e3967c95abc0ae4399b165eb83625244af48d7b22" +dependencies = [ + "anyhow", + "chrono", + "clap 4.5.23", + "colored", + "log", + "petgraph", + "pulldown-cmark", + "rayon", + "serde", + "serde_json", + "simple_logger", + "sv-parser", + "term", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -1683,6 +1875,30 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "nom" +version = "5.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08959a387a676302eebf4ddbcbc611da04285579f76f88ee0506c63b1a61dd4b" +dependencies = [ + "lexical-core", + "memchr", + "version_check 0.9.4", +] + +[[package]] +name = "nom" +version = "6.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2" +dependencies = [ + "bitvec", + "funty", + "lexical-core", + "memchr", + "version_check 0.9.4", +] + [[package]] name = "nom" version = "7.1.3" @@ -1693,6 +1909,116 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom-greedyerror" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133e5024c0b65c4235e3200a3b6e30f3875475f1e452525e1a421b7f2a997c52" +dependencies = [ + "nom 5.1.3", + "nom 6.1.2", + "nom_locate 1.0.0", + "nom_locate 2.1.0", + "nom_locate 3.0.2", +] + +[[package]] +name = "nom-packrat" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5c5a5a7eae83c3c9d53bdfd94e8bb1d700c6bb78f00d25af71263fc07cf477b" +dependencies = [ + "nom-packrat-macros", + "nom_locate 1.0.0", + "nom_locate 3.0.2", +] + +[[package]] +name = "nom-packrat-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fccdfb4771d14a08918cd7b7352de2797ade66a2df9920cee13793e943c3d09" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "nom-recursive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0de2967d4f9065b08596dcfa9be631abc4997951b9e0a93e2279b052370bacc" +dependencies = [ + "nom-recursive-macros", + "nom_locate 1.0.0", + "nom_locate 3.0.2", +] + +[[package]] +name = "nom-recursive-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07744fc6b7423baf7198f9e1200305f27eafe7395289fa7462b63dacd4eac78d" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "nom-tracable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128b58b88f084359e18858edde832830041e0a561d23bb214e656e00972de316" +dependencies = [ + "nom 6.1.2", + "nom-tracable-macros", + "nom_locate 1.0.0", + "nom_locate 3.0.2", +] + +[[package]] +name = "nom-tracable-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8416fc5553b00d217b0381929fbce7368935d609afdee46c844e09f962b379e6" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "nom_locate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f932834fd8e391fc7710e2ba17e8f9f8645d846b55aa63207e17e110a1e1ce35" +dependencies = [ + "bytecount 0.3.2", + "memchr", + "nom 5.1.3", +] + +[[package]] +name = "nom_locate" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a67484adf5711f94f2f28b653bf231bff8e438be33bf5b0f35935a0db4f618a2" +dependencies = [ + "bytecount 0.6.8", + "memchr", + "nom 5.1.3", +] + +[[package]] +name = "nom_locate" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4689294073dda8a54e484212171efdcb6b12b1908fd70c3dc3eec15b8833b06d" +dependencies = [ + "bytecount 0.6.8", + "memchr", + "nom 6.1.2", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1796,6 +2122,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.32.2" @@ -2057,6 +2392,18 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" +dependencies = [ + "bitflags 2.4.2", + "getopts", + "memchr", + "unicase", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2082,6 +2429,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + [[package]] name = "radix_trie" version = "0.2.1" @@ -2543,6 +2896,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" +[[package]] +name = "simple_logger" +version = "4.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e7e46c8c90251d47d08b28b8a419ffb4aede0f87c2eea95e17d1d5bacbf3ef1" +dependencies = [ + "colored", + "log", + "time", + "windows-sys 0.48.0", +] + [[package]] name = "slab" version = "0.4.9" @@ -2625,6 +2990,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +[[package]] +name = "str-concat" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3468939e48401c4fe3cdf5e5cef50951c2808ed549d1467fde249f1fcb602634" + [[package]] name = "string-interner" version = "0.14.0" @@ -2660,13 +3031,87 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", "syn 2.0.90", ] +[[package]] +name = "sv-parser" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6567e4ff8c8f26fbd561e8797f2bf658462b68346d481d9a53f9390873688d" +dependencies = [ + "nom 6.1.2", + "nom-greedyerror", + "sv-parser-error", + "sv-parser-parser", + "sv-parser-pp", + "sv-parser-syntaxtree", +] + +[[package]] +name = "sv-parser-error" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8ab9840f6af470b46a27ec709c5bdba054d0fe57d408ba550d99b6de027b143" +dependencies = [ + "thiserror 1.0.64", +] + +[[package]] +name = "sv-parser-macros" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60599265d86a4647a32e97f5c9ae758c1ac82402a6e0d123347384997148ba6a" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sv-parser-parser" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4628479b05874500c1227228f481e04e9aaf858bc43eecd87fbbf4b6999b7269" +dependencies = [ + "nom 6.1.2", + "nom-greedyerror", + "nom-packrat", + "nom-recursive", + "nom-tracable", + "nom_locate 3.0.2", + "str-concat", + "sv-parser-macros", + "sv-parser-syntaxtree", +] + +[[package]] +name = "sv-parser-pp" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db489604d13d9f630173a477357a0010d310597df4728a35bc54409af72ebb3c" +dependencies = [ + "nom 6.1.2", + "nom-greedyerror", + "sv-parser-error", + "sv-parser-parser", + "sv-parser-syntaxtree", +] + +[[package]] +name = "sv-parser-syntaxtree" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a49debd66a99e1783badfa9780bb96b3a96b92d94ebd58ca4b82f3a32ca498a3" +dependencies = [ + "regex", + "sv-parser-macros", + "walkdir", +] + [[package]] name = "symbol_table" version = "0.2.0" @@ -2722,6 +3167,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempdir" version = "0.3.7" @@ -2837,7 +3288,9 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -3140,6 +3593,12 @@ dependencies = [ "version_check 0.9.4", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -3491,6 +3950,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/calyx-backend/Cargo.toml b/calyx-backend/Cargo.toml index 9ccd175e33..5254a62fe7 100644 --- a/calyx-backend/Cargo.toml +++ b/calyx-backend/Cargo.toml @@ -29,6 +29,8 @@ calyx-ir.workspace = true csv = { version = "1.1", optional = true } vast = "0.3.1" +morty = "0.9.0" +tempfile = "3.3" [dependencies.quick-xml] version = "0.30" diff --git a/calyx-backend/src/verilog.rs b/calyx-backend/src/verilog.rs index 0f0371af53..a46d38f62f 100644 --- a/calyx-backend/src/verilog.rs +++ b/calyx-backend/src/verilog.rs @@ -8,9 +8,12 @@ use calyx_ir::{self as ir, Control, FlatGuard, Group, Guard, GuardRef, RRC}; use calyx_utils::{CalyxResult, Error, OutputFile}; use ir::Nothing; use itertools::Itertools; +use morty::{FileBundle, LibraryBundle}; use std::io; -use std::{collections::HashMap, rc::Rc}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::{collections::HashMap, collections::HashSet, path::PathBuf, rc::Rc}; use std::{fs::File, time::Instant}; +use tempfile::NamedTempFile; use vast::v17::ast as v; /// Implements a simple Verilog backend. The backend only accepts Calyx programs with no control @@ -90,6 +93,145 @@ fn validate_control(ctrl: &ir::Control) -> CalyxResult<()> { } } +/// Each external library has its own specific handler to build all information that Morty needs to do pickling. We can implement each command line argument (see https://github.com/pulp-platform/morty/blob/master/src/main.rs for available arguments) as a trait's method. +trait LibraryHandlerTrait { + /// Add search path(s) for SystemVerilog includes to build a LibraryBundle + fn add_incs(&self) -> CalyxResult>; + /// Directory(s) to search for SystemVerilog modules + fn add_library_dirs(&self) -> CalyxResult>; + /// Define preprocesor macro(s) + fn add_defs(&self) -> CalyxResult>>; + /// Add search path(s) for SystemVerilog includes to build a FileBundle + fn add_stdin_incdirs(&self) -> CalyxResult>; + /// Add export include directories + fn add_export_incdirs(&self) -> CalyxResult>>; + /// Create a map from module name to file path + fn map_module_names_to_file_paths( + &self, + ) -> CalyxResult> { + // a hashmap from 'module name' to 'path' for all libraries. + let mut library_files = HashMap::new(); + // a list of paths for all library files + let mut library_paths: Vec = Vec::new(); + + // we first accumulate all library files from the 'library_dir' and 'library_file' options into + // a vector of paths, and then construct the library hashmap. + let library_dirs: Vec = self.add_library_dirs()?; + for dir in library_dirs { + let entries = std::fs::read_dir(&dir).map_err(|e| { + Error::invalid_file(format!( + "Error accessing library directory `{:?}`: {}", + dir, e + )) + })?; + + for entry in entries { + let entry = entry.map_err(|e| { + Error::invalid_file(format!( + "Error reading entry in directory `{:?}`: {}", + dir, e + )) + })?; + library_paths.push(entry.path()); + } + } + + for p in &library_paths { + // Must have the library extension (.v or .sv). + if morty::has_libext(p) { + if let Some(m) = morty::lib_module(p) { + library_files.insert(m, p.to_owned()); + } + } + } + + Ok(library_files) + } +} + +/// Check if any special library is needed +fn check_library_needed(ctx: &ir::Context) -> bool { + ctx.lib + .extern_paths() + .iter() + .any(|path| path.to_string_lossy().contains("float")) +} + +/// Collect all included files specified by the Calyx source file +fn collect_included_files(ctx: &ir::Context) -> Vec { + ctx.lib + .extern_paths() + .into_iter() + .map(|pb| pb.to_string_lossy().into_owned()) + .collect() +} + +/// Build the library bundle for all libraries needed for pickling +fn build_library_bundle( + ctx: &ir::Context, + calyx_emitted_file: &NamedTempFile, + handlers: &Vec>, +) -> CalyxResult { + let mut included_files = collect_included_files(ctx); + included_files + .push(calyx_emitted_file.path().to_string_lossy().to_string()); // `calyx_emitted_file` is used as part of the source files + let mut include_dirs = Vec::new(); + let mut defines = HashMap::new(); + let mut files = HashMap::new(); + for handler in handlers { + include_dirs.extend(handler.add_incs()?); + defines.extend(handler.add_defs()?); + files.extend(handler.map_module_names_to_file_paths()?); + } + let main_module = String::from(ctx.entrypoint.id.as_str()); + files.insert(main_module, calyx_emitted_file.path().to_path_buf()); + + let include_dirs: Vec = include_dirs + .into_iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + Ok(LibraryBundle { + include_dirs, + defines, + files, + }) +} + +/// Build a list of `FileBundle`s needed for building the syntax tree +fn derive_file_list( + ctx: &ir::Context, + file: &NamedTempFile, + handlers: &Vec>, +) -> CalyxResult> { + let mut file_bundles = Vec::new(); + for handler in handlers { + let stdin_incdirs = handler.add_stdin_incdirs()?; + let include_dirs: Vec = stdin_incdirs + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + let export_incdirs = handler.add_export_incdirs()?; + let defines = handler.add_defs()?; + let mut files = handler.map_module_names_to_file_paths()?; + + let main_module = String::from(ctx.entrypoint.id.as_str()); + files.insert(main_module, file.path().to_path_buf()); + + let mut included_files = collect_included_files(ctx); + for lib_file in files.values() { + included_files.push(String::from(lib_file.to_str().unwrap())); + } + + file_bundles.push(FileBundle { + include_dirs, + export_incdirs, + defines, + files: included_files, + }); + } + Ok(file_bundles) +} + impl Backend for VerilogBackend { fn name(&self) -> &'static str { "verilog" @@ -103,35 +245,42 @@ impl Backend for VerilogBackend { Ok(()) } - /// Generate a "fat" library by copy-pasting all of the extern files. - /// A possible alternative in the future is to use SystemVerilog `include` - /// statement. + /// If no special libraries are needed, generate a "fat" library by copy-pasting all of the extern files. fn link_externs( ctx: &ir::Context, file: &mut OutputFile, ) -> CalyxResult<()> { - let fw = &mut file.get_write(); - for extern_path in &ctx.lib.extern_paths() { - // The extern file is guaranteed to exist by the frontend. - let mut ext = File::open(extern_path).unwrap(); - io::copy(&mut ext, fw).map_err(|err| { - let std::io::Error { .. } = err; - Error::write_error(format!( - "File not found: {}", - file.as_path_string() - )) - })?; - // Add a newline after appending a library file - writeln!(fw)?; - } - for (prim, _) in ctx.lib.prim_inlines() { - emit_prim_inline(prim, fw)?; + // If we need special libraries (like HardFloat), run Morty to pickle the files in the `emit` stage. We postpone linking extern special libraries because Morty needs all emitted information to do pickle. We could soley use Morty, but currently it eliminates the body inside `ifndef-endif`. Also discussed in here: https://github.com/pulp-platform/morty/issues/49 + if !check_library_needed(ctx) { + let fw = &mut file.get_write(); + for extern_path in &ctx.lib.extern_paths() { + let mut ext = File::open(extern_path).unwrap(); + io::copy(&mut ext, fw).map_err(|err| { + let std::io::Error { .. } = err; + Error::write_error(format!( + "File not found: {}", + file.as_path_string() + )) + })?; + writeln!(fw)?; + } } + Ok(()) } fn emit(ctx: &ir::Context, file: &mut OutputFile) -> CalyxResult<()> { - let out = &mut file.get_write(); + // Create a temporary file as an intermediate storage to emit inline primtives and components to. This temporary file will be used as one of the source SystemVerilog file for Morty to do pickle. + let temp_file = tempfile::NamedTempFile::new().map_err(|_| { + Error::write_error("Failed to create a temporary file".to_string()) + })?; + let mut temp_writer = temp_file.as_file(); + + // Write inline primitives + for (prim, _) in ctx.lib.prim_inlines() { + emit_prim_inline(prim, &mut temp_writer)?; + } + let comps = ctx.components.iter().try_for_each(|comp| { // Time the generation of the component. let time = Instant::now(); @@ -140,7 +289,7 @@ impl Backend for VerilogBackend { ctx.bc.synthesis_mode, ctx.bc.enable_verification, ctx.bc.flat_assign, - out, + &mut temp_writer, ); log::info!("Generated `{}` in {:?}", comp.name, time.elapsed()); out @@ -151,7 +300,62 @@ impl Backend for VerilogBackend { "File not found: {}", file.as_path_string() )) - }) + })?; + + if check_library_needed(ctx) { + let handlers: Vec> = Vec::new(); + // Special libraries (like HardFloat) are needed, run Morty to pickle the files + let library_bundle = + build_library_bundle(ctx, &temp_file, &handlers)?; + + let file_list = derive_file_list(ctx, &temp_file, &handlers)?; + let syntax_trees = morty::build_syntax_tree( + &file_list, false, // By default, don't strip comments + false, // By default, don't ignore unparseable + true, // By default, propagate defines from first files to the following files + false, // By default, don't force sequential + ) + .unwrap(); + let top_module = ctx.entrypoint.to_string(); + let _pickled = morty::do_pickle( + None::<&String>, // By default, don't need to prepend a name to all global names + None::<&String>, // By default, don't need to append a name to all global names + HashSet::new(), // By default, don't specify module, interface, package that should not be renamed; instead, let morty to dictate the behavior + HashSet::new(), // By default, don't specify module, interface, package that shouldn't be included in the pickled file list; instead, we let morty to dictate the behavior + library_bundle, + syntax_trees, + Box::new(temp_file.reopen()?) as Box, + Some(&top_module), + true, // By default, keep defines to prevent removal of `define` statements + true, // By default, propagate defines from first files to the following files + false, // By default, keep the time units + ) + .map_err(|err| { + Error::write_error(format!("{}", err.to_string())) + })?; + } + // Rewind to the start of the temporary file so that we can read the content + temp_writer.seek(SeekFrom::Start(0)).map_err(|_| { + Error::write_error( + "Failed to rewind the temporary file".to_string(), + ) + })?; + // Read from the temporary file and write to the user-specified output `file` + let mut temp_content = String::new(); + temp_writer.read_to_string(&mut temp_content).map_err(|_| { + Error::write_error("Failed to read from temporary file".to_string()) + })?; + + let mut final_writer = file.get_write(); + final_writer + .write_all(temp_content.as_bytes()) + .map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!("Write failed: {}", err), + ) + })?; + Ok(()) } }