diff --git a/bin/verilator_splitter b/bin/verilator_splitter new file mode 100755 index 00000000000..a6559eb2a59 --- /dev/null +++ b/bin/verilator_splitter @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import sys +from dataclasses import dataclass, field +from pathlib import Path + +def get_list(fn: Path) -> tuple[int, list[tuple[int, Path]]]: + total_size = 0 + result: list[tuple[int, Path]] = [] + files: list[str] = [] + with fn.open("r") as f: + files = f.readlines() + + for f in files: + f = Path(f.strip()) + size = f.stat().st_size + total_size += size + result.append((size, f)) + files = [] + + return (total_size, result) + + +def main(): + input_list_file = Path(sys.argv[1]) + buckets_count = int(sys.argv[2]) + if buckets_count <= 0: + raise ValueError(f"Arg 2: Expected bucket count, got: {sys.argv[2]}") + output_name_template = sys.argv[3] + if "%" not in output_name_template: + raise ValueError(f"Arg 3: template must contain '%': {sys.argv[3]}") + + total_size, input_list = get_list(input_list_file) + + ideal_bucket_size = total_size // buckets_count + + huge_files_num = 0 + huge_files_size = 0 + for size, _ in input_list: + if size > ideal_bucket_size: + huge_files_num += 1 + huge_files_size += size + + ideal_bucket_size = max(1, total_size - huge_files_size) // max(1, buckets_count - huge_files_num) + + @dataclass(slots = True) + class BucketData: + bucket_id: int + size: int = 0 + filenames: list[str] = field(default_factory=list) + + def __iter__(self): + return iter((self.bucket_id, self.size, self.filenames)) + + buckets: list[BucketData] = [BucketData(i + 1) for i in range(buckets_count)] + for bucket in buckets: + while input_list: + next_size, next_fn = input_list[0] + diff_now = abs(ideal_bucket_size - bucket.size) + diff_next = abs(ideal_bucket_size - bucket.size - next_size) + if bucket.size == 0 or diff_now > diff_next: + bucket.size += next_size + bucket.filenames.append(str(next_fn)) + input_list.pop(0) + else: + break + + while input_list: + next_size, next_fn = input_list[0] + buckets[-1].size += next_size + buckets[-1].filenames.append(str(next_fn)) + input_list.pop(0) + + # for size, fn in input_list: + # buckets[0].size += size + # buckets[0].filenames.append(str(fn)) + # buckets.sort(key = lambda bd: bd.size) + + for bucket_id, size, filenames in sorted(buckets, key = lambda b: b.size, reverse = True): + print(f"Bucket {bucket_id:>2} size: {size:>8}, distance from ideal: {ideal_bucket_size - size:>8}", file=sys.stderr) + output_list_file = Path(output_name_template.replace("%", str(bucket_id))) + print(output_list_file) + with output_list_file.open("w") as f: + f.write("\n".join(filenames) + "\n") + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/include/verilated.mk.in b/include/verilated.mk.in index 59a94486f2f..c2a5bcf9e63 100644 --- a/include/verilated.mk.in +++ b/include/verilated.mk.in @@ -51,6 +51,7 @@ CFG_LDLIBS_THREADS = @CFG_LDLIBS_THREADS@ VERILATOR_COVERAGE = $(PERL) $(VERILATOR_ROOT)/bin/verilator_coverage VERILATOR_INCLUDER = $(PYTHON3) $(VERILATOR_ROOT)/bin/verilator_includer +VERILATOR_SPLITTER = $(PYTHON3) $(VERILATOR_ROOT)/bin/verilator_splitter VERILATOR_CCACHE_REPORT = $(PYTHON3) $(VERILATOR_ROOT)/bin/verilator_ccache_report ###################################################################### @@ -187,8 +188,54 @@ VK_PCH_I_SLOW = $(CFG_CXXFLAGS_PCH_I) $(VM_PREFIX)__pch.h.slow$(CFG_GCH_IF_CLANG ####################################################################### ### Overall Objects Linking -VK_OBJS_FAST = $(addsuffix .o, $(VM_FAST)) -VK_OBJS_SLOW = $(addsuffix .o, $(VM_SLOW)) +#─────────────────────────────────────────────────────────────────────────────── + +empty := +sp := ${empty} ${empty} +define nl := + + +endef + +JOBS_NUM_fast ?= 20 +JOBS_NUM_slow ?= 20 +JOB_IDS_fast := $(shell seq 1 ${JOBS_NUM_fast}) +JOB_IDS_slow := $(shell seq 1 ${JOBS_NUM_slow}) + +#─────────────────────────────────────────────────────────────────────────────── +VK_LISTS_FAST := $(foreach job_id,${JOB_IDS_fast},fast_${job_id}_${JOBS_NUM_fast}.list) +VK_LISTS_SLOW := $(foreach job_id,${JOB_IDS_slow},slow_${job_id}_${JOBS_NUM_slow}.list) + +fast.list: $(addsuffix .cpp,${VM_FAST}) ${vk_${prefix}_${job_id}} $(VM_PREFIX).mk + @echo Generate $@ + $(file >fast.list,$(subst ${sp},${nl},$(addsuffix .cpp,${VM_FAST}))) + +slow.list: $(addsuffix .cpp,${VM_SLOW}) $(VM_PREFIX).mk + @echo Generate $@ + $(file >slow.list,$(subst ${sp},${nl},$(addsuffix .cpp,${VM_SLOW}))) + +$(sort $(subst fast,%,$(VK_LISTS_FAST)) $(subst slow,%,$(VK_LISTS_SLOW))): %.list + @echo Generate $@ + @$(VERILATOR_SPLITTER) $< ${JOBS_NUM_slow} $(patsubst %.list,%,$<)_%_${JOBS_NUM_slow}.list > /dev/null + +define gen_per_process_jobs +vk_${prefix}_${job_id} := $(strip $(file <${prefix}_${job_id}_${JOBS_NUM_${prefix}}.list)) + +${prefix}_${job_id}_${JOBS_NUM_${prefix}}.cpp : private prefix := ${prefix} +${prefix}_${job_id}_${JOBS_NUM_${prefix}}.cpp : private job_id := ${job_id} + +${prefix}_${job_id}_${JOBS_NUM_${prefix}}.cpp : ${prefix}_${job_id}_${JOBS_NUM_${prefix}}.list + @echo Generate $@ + @printf '// Generated from %s\n' $< > $@ + @while read file; do if [[ -n "$${file}" ]]; then printf '#include "%s"\n' $$file; fi; done < ${prefix}_${job_id}_${JOBS_NUM_${prefix}}.list >> $@ + +endef +$(foreach prefix,fast slow,$(foreach job_id,${JOB_IDS_${prefix}},$(eval $(value gen_per_process_jobs)))) + +#─────────────────────────────────────────────────────────────────────────────── + +VK_OBJS_FAST = $(foreach job_id,${JOB_IDS_fast},fast_${job_id}_${JOBS_NUM_fast}.o) +VK_OBJS_SLOW = $(foreach job_id,${JOB_IDS_slow},slow_${job_id}_${JOBS_NUM_slow}.o) VK_USER_OBJS = $(addsuffix .o, $(VM_USER_CLASSES))